diff options
Diffstat (limited to 'drivers/crypto')
21 files changed, 556 insertions, 304 deletions
diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c index a5fd8975f3d3..a6abb701bfc6 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c @@ -8,7 +8,7 @@ * This file add support for AES cipher with 128,192,256 bits keysize in * CBC and ECB mode. * - * You could find a link for the datasheet in Documentation/arm/sunxi/README + * You could find a link for the datasheet in Documentation/arm/sunxi.rst */ #include <linux/crypto.h> diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c index 3e4e4bbda34c..b957061424a1 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c @@ -7,7 +7,7 @@ * * Core file which registers crypto algorithms supported by the CryptoEngine. * - * You could find a link for the datasheet in Documentation/arm/sunxi/README + * You could find a link for the datasheet in Documentation/arm/sunxi.rst */ #include <linux/clk.h> #include <linux/crypto.h> diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c index 84d52fc3a2da..c89cb2ee2496 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c @@ -8,7 +8,7 @@ * This file add support for AES cipher with 128,192,256 bits keysize in * CBC and ECB mode. * - * You could find a link for the datasheet in Documentation/arm/sunxi/README + * You could find a link for the datasheet in Documentation/arm/sunxi.rst */ #include <linux/crypto.h> diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c index a1fb2fbdbe7b..5d9d0fedcb06 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c @@ -7,7 +7,7 @@ * * Core file which registers crypto algorithms supported by the SecuritySystem * - * You could find a link for the datasheet in Documentation/arm/sunxi/README + * You could find a link for the datasheet in Documentation/arm/sunxi.rst */ #include <linux/clk.h> #include <linux/crypto.h> diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 439cd737076e..a2426334be61 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -394,8 +394,7 @@ static int sev_ioctl_do_pek_csr(struct sev_issue_cmd *argp, bool writable) goto cmd; /* allocate a physically contiguous buffer to store the CSR blob */ - if (!access_ok(input.address, input.length) || - input.length > SEV_FW_BLOB_MAX_SIZE) { + if (input.length > SEV_FW_BLOB_MAX_SIZE) { ret = -EFAULT; goto e_free; } @@ -632,12 +631,6 @@ static int sev_ioctl_do_get_id2(struct sev_issue_cmd *argp) if (copy_from_user(&input, (void __user *)argp->data, sizeof(input))) return -EFAULT; - /* Check if we have write access to the userspace buffer */ - if (input.address && - input.length && - !access_ok(input.address, input.length)) - return -EFAULT; - data = kzalloc(sizeof(*data), GFP_KERNEL); if (!data) return -ENOMEM; @@ -753,15 +746,13 @@ static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp, bool writable) goto cmd; /* Allocate a physically contiguous buffer to store the PDH blob. */ - if ((input.pdh_cert_len > SEV_FW_BLOB_MAX_SIZE) || - !access_ok(input.pdh_cert_address, input.pdh_cert_len)) { + if (input.pdh_cert_len > SEV_FW_BLOB_MAX_SIZE) { ret = -EFAULT; goto e_free; } /* Allocate a physically contiguous buffer to store the cert chain blob. */ - if ((input.cert_chain_len > SEV_FW_BLOB_MAX_SIZE) || - !access_ok(input.cert_chain_address, input.cert_chain_len)) { + if (input.cert_chain_len > SEV_FW_BLOB_MAX_SIZE) { ret = -EFAULT; goto e_free; } diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index caf1136e7ef9..f26a7a15551a 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -255,7 +255,7 @@ static void get_aes_decrypt_key(unsigned char *dec_key, return; } for (i = 0; i < nk; i++) - w_ring[i] = be32_to_cpu(*(u32 *)&key[4 * i]); + w_ring[i] = get_unaligned_be32(&key[i * 4]); i = 0; temp = w_ring[nk - 1]; @@ -274,7 +274,7 @@ static void get_aes_decrypt_key(unsigned char *dec_key, } i--; for (k = 0, j = i % nk; k < nk; k++) { - *((u32 *)dec_key + k) = htonl(w_ring[j]); + put_unaligned_be32(w_ring[j], &dec_key[k * 4]); j--; if (j < 0) j += nk; @@ -1053,8 +1053,8 @@ static unsigned int adjust_ctr_overflow(u8 *iv, u32 bytes) u32 temp = be32_to_cpu(*--b); temp = ~temp; - c = (u64)temp + 1; // No of block can processed withou overflow - if ((bytes / AES_BLOCK_SIZE) > c) + c = (u64)temp + 1; // No of block can processed without overflow + if ((bytes / AES_BLOCK_SIZE) >= c) bytes = c * AES_BLOCK_SIZE; return bytes; } @@ -1076,7 +1076,14 @@ static int chcr_update_tweak(struct skcipher_request *req, u8 *iv, keylen = ablkctx->enckey_len / 2; key = ablkctx->key + keylen; - ret = aes_expandkey(&aes, key, keylen); + /* For a 192 bit key remove the padded zeroes which was + * added in chcr_xts_setkey + */ + if (KEY_CONTEXT_CK_SIZE_G(ntohl(ablkctx->key_ctx_hdr)) + == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) + ret = aes_expandkey(&aes, key, keylen - 8); + else + ret = aes_expandkey(&aes, key, keylen); if (ret) return ret; aes_encrypt(&aes, iv, iv); @@ -1157,15 +1164,16 @@ static int chcr_final_cipher_iv(struct skcipher_request *req, static int chcr_handle_cipher_resp(struct skcipher_request *req, unsigned char *input, int err) { + struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chcr_context *ctx = c_ctx(tfm); - struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm)); - struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); - struct sk_buff *skb; struct cpl_fw6_pld *fw6_pld = (struct cpl_fw6_pld *)input; - struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); - struct cipher_wr_param wrparam; + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); + struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm)); struct chcr_dev *dev = c_ctx(tfm)->dev; + struct chcr_context *ctx = c_ctx(tfm); + struct adapter *adap = padap(ctx->dev); + struct cipher_wr_param wrparam; + struct sk_buff *skb; int bytes; if (err) @@ -1196,6 +1204,8 @@ static int chcr_handle_cipher_resp(struct skcipher_request *req, if (unlikely(bytes == 0)) { chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req); + memcpy(req->iv, reqctx->init_iv, IV); + atomic_inc(&adap->chcr_stats.fallback); err = chcr_cipher_fallback(ablkctx->sw_cipher, req->base.flags, req->src, @@ -1247,20 +1257,28 @@ static int process_cipher(struct skcipher_request *req, struct sk_buff **skb, unsigned short op_type) { + struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); unsigned int ivsize = crypto_skcipher_ivsize(tfm); - struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); + struct adapter *adap = padap(c_ctx(tfm)->dev); struct cipher_wr_param wrparam; int bytes, err = -EINVAL; + int subtype; reqctx->processed = 0; reqctx->partial_req = 0; if (!req->iv) goto error; + subtype = get_cryptoalg_subtype(tfm); if ((ablkctx->enckey_len == 0) || (ivsize > AES_BLOCK_SIZE) || (req->cryptlen == 0) || (req->cryptlen % crypto_skcipher_blocksize(tfm))) { + if (req->cryptlen == 0 && subtype != CRYPTO_ALG_SUB_TYPE_XTS) + goto fallback; + else if (req->cryptlen % crypto_skcipher_blocksize(tfm) && + subtype == CRYPTO_ALG_SUB_TYPE_XTS) + goto fallback; pr_err("AES: Invalid value of Key Len %d nbytes %d IV Len %d\n", ablkctx->enckey_len, req->cryptlen, ivsize); goto error; @@ -1301,12 +1319,10 @@ static int process_cipher(struct skcipher_request *req, } else { bytes = req->cryptlen; } - if (get_cryptoalg_subtype(tfm) == - CRYPTO_ALG_SUB_TYPE_CTR) { + if (subtype == CRYPTO_ALG_SUB_TYPE_CTR) { bytes = adjust_ctr_overflow(req->iv, bytes); } - if (get_cryptoalg_subtype(tfm) == - CRYPTO_ALG_SUB_TYPE_CTR_RFC3686) { + if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_RFC3686) { memcpy(reqctx->iv, ablkctx->nonce, CTR_RFC3686_NONCE_SIZE); memcpy(reqctx->iv + CTR_RFC3686_NONCE_SIZE, req->iv, CTR_RFC3686_IV_SIZE); @@ -1314,20 +1330,25 @@ static int process_cipher(struct skcipher_request *req, /* initialize counter portion of counter block */ *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE + CTR_RFC3686_IV_SIZE) = cpu_to_be32(1); + memcpy(reqctx->init_iv, reqctx->iv, IV); } else { memcpy(reqctx->iv, req->iv, IV); + memcpy(reqctx->init_iv, req->iv, IV); } if (unlikely(bytes == 0)) { chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req); +fallback: atomic_inc(&adap->chcr_stats.fallback); err = chcr_cipher_fallback(ablkctx->sw_cipher, req->base.flags, req->src, req->dst, req->cryptlen, - reqctx->iv, + subtype == + CRYPTO_ALG_SUB_TYPE_CTR_RFC3686 ? + reqctx->iv : req->iv, op_type); goto error; } @@ -1442,6 +1463,7 @@ static int chcr_device_init(struct chcr_context *ctx) if (!ctx->dev) { u_ctx = assign_chcr_device(); if (!u_ctx) { + err = -ENXIO; pr_err("chcr device assignment fails\n"); goto out; } @@ -1983,7 +2005,7 @@ static int chcr_ahash_digest(struct ahash_request *req) req_ctx->data_len += params.bfr_len + params.sg_len; if (req->nbytes == 0) { - create_last_hash_block(req_ctx->reqbfr, bs, 0); + create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len); params.more = 1; params.bfr_len = bs; } @@ -2249,12 +2271,28 @@ static int chcr_aes_xts_setkey(struct crypto_skcipher *cipher, const u8 *key, ablkctx->enckey_len = key_len; get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, key_len << 2); context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len) >> 4; - ablkctx->key_ctx_hdr = + /* Both keys for xts must be aligned to 16 byte boundary + * by padding with zeros. So for 24 byte keys padding 8 zeroes. + */ + if (key_len == 48) { + context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len + + 16) >> 4; + memmove(ablkctx->key + 32, ablkctx->key + 24, 24); + memset(ablkctx->key + 24, 0, 8); + memset(ablkctx->key + 56, 0, 8); + ablkctx->enckey_len = 64; + ablkctx->key_ctx_hdr = + FILL_KEY_CTX_HDR(CHCR_KEYCTX_CIPHER_KEY_SIZE_192, + CHCR_KEYCTX_NO_KEY, 1, + 0, context_size); + } else { + ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR((key_len == AES_KEYSIZE_256) ? CHCR_KEYCTX_CIPHER_KEY_SIZE_128 : CHCR_KEYCTX_CIPHER_KEY_SIZE_256, CHCR_KEYCTX_NO_KEY, 1, 0, context_size); + } ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS; return 0; badkey_err: @@ -2555,7 +2593,7 @@ int chcr_aead_dma_map(struct device *dev, int dst_size; dst_size = req->assoclen + req->cryptlen + (op_type ? - -authsize : authsize); + 0 : authsize); if (!req->cryptlen || !dst_size) return 0; reqctx->iv_dma = dma_map_single(dev, reqctx->iv, (IV + reqctx->b0_len), @@ -2602,15 +2640,16 @@ void chcr_aead_dma_unmap(struct device *dev, int dst_size; dst_size = req->assoclen + req->cryptlen + (op_type ? - -authsize : authsize); + 0 : authsize); if (!req->cryptlen || !dst_size) return; dma_unmap_single(dev, reqctx->iv_dma, (IV + reqctx->b0_len), DMA_BIDIRECTIONAL); if (req->src == req->dst) { - dma_unmap_sg(dev, req->src, sg_nents(req->src), - DMA_BIDIRECTIONAL); + dma_unmap_sg(dev, req->src, + sg_nents_for_len(req->src, dst_size), + DMA_BIDIRECTIONAL); } else { dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); @@ -2887,8 +2926,7 @@ static int ccm_format_packet(struct aead_request *req, memcpy(ivptr, req->iv, 16); } if (assoclen) - *((unsigned short *)(reqctx->scratch_pad + 16)) = - htons(assoclen); + put_unaligned_be16(assoclen, &reqctx->scratch_pad[16]); rc = generate_b0(req, ivptr, op_type); /* zero the ctr value */ @@ -2909,7 +2947,7 @@ static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl, unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC; unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan; unsigned int ccm_xtra; - unsigned char tag_offset = 0, auth_offset = 0; + unsigned int tag_offset = 0, auth_offset = 0; unsigned int assoclen; if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) @@ -3162,8 +3200,7 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, } else { memcpy(ivptr, req->iv, GCM_AES_IV_SIZE); } - *((unsigned int *)(ivptr + 12)) = htonl(0x01); - + put_unaligned_be32(0x01, &ivptr[12]); ulptx = (struct ulptx_sgl *)(ivptr + 16); chcr_add_aead_dst_ent(req, phys_cpl, qid); @@ -3701,6 +3738,13 @@ static int chcr_aead_op(struct aead_request *req, return -ENOSPC; } + if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 && + crypto_ipsec_check_assoclen(req->assoclen) != 0) { + pr_err("RFC4106: Invalid value of assoclen %d\n", + req->assoclen); + return -EINVAL; + } + /* Form a WR from req */ skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[reqctx->rxqidx], size); diff --git a/drivers/crypto/chelsio/chcr_algo.h b/drivers/crypto/chelsio/chcr_algo.h index f58c2b5c7fc5..d4f6e010dc79 100644 --- a/drivers/crypto/chelsio/chcr_algo.h +++ b/drivers/crypto/chelsio/chcr_algo.h @@ -389,10 +389,6 @@ static inline void copy_hash_init_values(char *key, int digestsize) } } -static const u8 sgl_lengths[20] = { - 0, 1, 2, 3, 4, 4, 5, 6, 7, 7, 8, 9, 10, 10, 11, 12, 13, 13, 14, 15 -}; - /* Number of len fields(8) * size of one addr field */ #define PHYSDSGL_MAX_LEN_SIZE 16 diff --git a/drivers/crypto/chelsio/chcr_core.c b/drivers/crypto/chelsio/chcr_core.c index ffd4ec0c7374..bd8dac806e7a 100644 --- a/drivers/crypto/chelsio/chcr_core.c +++ b/drivers/crypto/chelsio/chcr_core.c @@ -33,6 +33,13 @@ static int cpl_fw6_pld_handler(struct adapter *adap, unsigned char *input); static void *chcr_uld_add(const struct cxgb4_lld_info *lld); static int chcr_uld_state_change(void *handle, enum cxgb4_state state); +#if defined(CONFIG_CHELSIO_TLS_DEVICE) +static const struct tlsdev_ops chcr_ktls_ops = { + .tls_dev_add = chcr_ktls_dev_add, + .tls_dev_del = chcr_ktls_dev_del, +}; +#endif + #ifdef CONFIG_CHELSIO_IPSEC_INLINE static void update_netdev_features(void); #endif /* CONFIG_CHELSIO_IPSEC_INLINE */ @@ -56,6 +63,9 @@ static struct cxgb4_uld_info chcr_uld_info = { #if defined(CONFIG_CHELSIO_IPSEC_INLINE) || defined(CONFIG_CHELSIO_TLS_DEVICE) .tx_handler = chcr_uld_tx_handler, #endif /* CONFIG_CHELSIO_IPSEC_INLINE || CONFIG_CHELSIO_TLS_DEVICE */ +#if defined(CONFIG_CHELSIO_TLS_DEVICE) + .tlsdev_ops = &chcr_ktls_ops, +#endif }; static void detach_work_fn(struct work_struct *work) @@ -207,11 +217,6 @@ static void *chcr_uld_add(const struct cxgb4_lld_info *lld) } u_ctx->lldi = *lld; chcr_dev_init(u_ctx); - -#ifdef CONFIG_CHELSIO_TLS_DEVICE - if (lld->ulp_crypto & ULP_CRYPTO_KTLS_INLINE) - chcr_enable_ktls(padap(&u_ctx->dev)); -#endif out: return u_ctx; } @@ -348,20 +353,12 @@ static void __exit chcr_crypto_exit(void) list_for_each_entry_safe(u_ctx, tmp, &drv_data.act_dev, entry) { adap = padap(&u_ctx->dev); memset(&adap->chcr_stats, 0, sizeof(adap->chcr_stats)); -#ifdef CONFIG_CHELSIO_TLS_DEVICE - if (u_ctx->lldi.ulp_crypto & ULP_CRYPTO_KTLS_INLINE) - chcr_disable_ktls(adap); -#endif list_del(&u_ctx->entry); kfree(u_ctx); } list_for_each_entry_safe(u_ctx, tmp, &drv_data.inact_dev, entry) { adap = padap(&u_ctx->dev); memset(&adap->chcr_stats, 0, sizeof(adap->chcr_stats)); -#ifdef CONFIG_CHELSIO_TLS_DEVICE - if (u_ctx->lldi.ulp_crypto & ULP_CRYPTO_KTLS_INLINE) - chcr_disable_ktls(adap); -#endif list_del(&u_ctx->entry); kfree(u_ctx); } diff --git a/drivers/crypto/chelsio/chcr_core.h b/drivers/crypto/chelsio/chcr_core.h index 2c09672e00a4..67d77abd6775 100644 --- a/drivers/crypto/chelsio/chcr_core.h +++ b/drivers/crypto/chelsio/chcr_core.h @@ -37,6 +37,7 @@ #define __CHCR_CORE_H__ #include <crypto/algapi.h> +#include <net/tls.h> #include "t4_hw.h" #include "cxgb4.h" #include "t4_msg.h" @@ -223,10 +224,15 @@ int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, int chcr_ipsec_xmit(struct sk_buff *skb, struct net_device *dev); void chcr_add_xfrmops(const struct cxgb4_lld_info *lld); #ifdef CONFIG_CHELSIO_TLS_DEVICE -void chcr_enable_ktls(struct adapter *adap); -void chcr_disable_ktls(struct adapter *adap); int chcr_ktls_cpl_act_open_rpl(struct adapter *adap, unsigned char *input); int chcr_ktls_cpl_set_tcb_rpl(struct adapter *adap, unsigned char *input); int chcr_ktls_xmit(struct sk_buff *skb, struct net_device *dev); +extern int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk, + enum tls_offload_ctx_dir direction, + struct tls_crypto_info *crypto_info, + u32 start_offload_tcp_sn); +extern void chcr_ktls_dev_del(struct net_device *netdev, + struct tls_context *tls_ctx, + enum tls_offload_ctx_dir direction); #endif #endif /* __CHCR_CORE_H__ */ diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h index 542bebae001f..b3fdbdc25acb 100644 --- a/drivers/crypto/chelsio/chcr_crypto.h +++ b/drivers/crypto/chelsio/chcr_crypto.h @@ -302,6 +302,7 @@ struct chcr_skcipher_req_ctx { unsigned int op; u16 imm; u8 iv[CHCR_MAX_CRYPTO_IV_LEN]; + u8 init_iv[CHCR_MAX_CRYPTO_IV_LEN]; u16 txqidx; u16 rxqidx; }; diff --git a/drivers/crypto/chelsio/chcr_ipsec.c b/drivers/crypto/chelsio/chcr_ipsec.c index 25bf6d963066..967babd67a51 100644 --- a/drivers/crypto/chelsio/chcr_ipsec.c +++ b/drivers/crypto/chelsio/chcr_ipsec.c @@ -293,9 +293,6 @@ static bool chcr_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x) if (ipv6_ext_hdr(ipv6_hdr(skb)->nexthdr)) return false; } - /* Inline single pdu */ - if (skb_shinfo(skb)->gso_size) - return false; return true; } @@ -405,7 +402,7 @@ inline void *copy_esn_pktxt(struct sk_buff *skb, xo = xfrm_offload(skb); aadiv->spi = (esphdr->spi); - seqlo = htonl(esphdr->seq_no); + seqlo = ntohl(esphdr->seq_no); seqno = cpu_to_be64(seqlo + ((u64)xo->seq.hi << 32)); memcpy(aadiv->seq_no, &seqno, 8); iv = skb_transport_header(skb) + sizeof(struct ip_esp_hdr); diff --git a/drivers/crypto/chelsio/chcr_ktls.c b/drivers/crypto/chelsio/chcr_ktls.c index cd1769ecdc1c..91dee616d15e 100644 --- a/drivers/crypto/chelsio/chcr_ktls.c +++ b/drivers/crypto/chelsio/chcr_ktls.c @@ -120,12 +120,10 @@ out: static int chcr_ktls_update_connection_state(struct chcr_ktls_info *tx_info, int new_state) { - unsigned long flags; - /* This function can be called from both rx (interrupt context) and tx * queue contexts. */ - spin_lock_irqsave(&tx_info->lock, flags); + spin_lock_bh(&tx_info->lock); switch (tx_info->connection_state) { case KTLS_CONN_CLOSED: tx_info->connection_state = new_state; @@ -169,7 +167,7 @@ static int chcr_ktls_update_connection_state(struct chcr_ktls_info *tx_info, pr_err("unknown KTLS connection state\n"); break; } - spin_unlock_irqrestore(&tx_info->lock, flags); + spin_unlock_bh(&tx_info->lock); return tx_info->connection_state; } @@ -223,6 +221,7 @@ static int chcr_ktls_act_open_req(struct sock *sk, return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te); } +#if IS_ENABLED(CONFIG_IPV6) /* * chcr_ktls_act_open_req6: creates TCB entry for ipv6 connection. * @sk - tcp socket. @@ -272,6 +271,7 @@ static int chcr_ktls_act_open_req6(struct sock *sk, return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te); } +#endif /* #if IS_ENABLED(CONFIG_IPV6) */ /* * chcr_setup_connection: create a TCB entry so that TP will form tcp packets. @@ -292,20 +292,26 @@ static int chcr_setup_connection(struct sock *sk, tx_info->atid = atid; tx_info->ip_family = sk->sk_family; - if (sk->sk_family == AF_INET || - (sk->sk_family == AF_INET6 && !sk->sk_ipv6only && - ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED)) { + if (sk->sk_family == AF_INET) { tx_info->ip_family = AF_INET; ret = chcr_ktls_act_open_req(sk, tx_info, atid); +#if IS_ENABLED(CONFIG_IPV6) } else { - tx_info->ip_family = AF_INET6; - ret = - cxgb4_clip_get(tx_info->netdev, - (const u32 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8, - 1); - if (ret) - goto out; - ret = chcr_ktls_act_open_req6(sk, tx_info, atid); + if (!sk->sk_ipv6only && + ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) { + tx_info->ip_family = AF_INET; + ret = chcr_ktls_act_open_req(sk, tx_info, atid); + } else { + tx_info->ip_family = AF_INET6; + ret = cxgb4_clip_get(tx_info->netdev, + (const u32 *) + &sk->sk_v6_rcv_saddr.s6_addr, + 1); + if (ret) + goto out; + ret = chcr_ktls_act_open_req6(sk, tx_info, atid); + } +#endif } /* if return type is NET_XMIT_CN, msg will be sent but delayed, mark ret @@ -375,9 +381,9 @@ static int chcr_ktls_mark_tcb_close(struct chcr_ktls_info *tx_info) * @tls_cts - tls context. * @direction - TX/RX crypto direction */ -static void chcr_ktls_dev_del(struct net_device *netdev, - struct tls_context *tls_ctx, - enum tls_offload_ctx_dir direction) +void chcr_ktls_dev_del(struct net_device *netdev, + struct tls_context *tls_ctx, + enum tls_offload_ctx_dir direction) { struct chcr_ktls_ofld_ctx_tx *tx_ctx = chcr_get_ktls_tx_context(tls_ctx); @@ -396,11 +402,13 @@ static void chcr_ktls_dev_del(struct net_device *netdev, if (tx_info->l2te) cxgb4_l2t_release(tx_info->l2te); +#if IS_ENABLED(CONFIG_IPV6) /* clear clip entry */ if (tx_info->ip_family == AF_INET6) cxgb4_clip_release(netdev, (const u32 *)&sk->sk_v6_daddr.in6_u.u6_addr8, 1); +#endif /* clear tid */ if (tx_info->tid != -1) { @@ -413,6 +421,8 @@ static void chcr_ktls_dev_del(struct net_device *netdev, atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_connection_close); kvfree(tx_info); tx_ctx->chcr_info = NULL; + /* release module refcount */ + module_put(THIS_MODULE); } /* @@ -424,10 +434,10 @@ static void chcr_ktls_dev_del(struct net_device *netdev, * @direction - TX/RX crypto direction * return: SUCCESS/FAILURE. */ -static int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk, - enum tls_offload_ctx_dir direction, - struct tls_crypto_info *crypto_info, - u32 start_offload_tcp_sn) +int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk, + enum tls_offload_ctx_dir direction, + struct tls_crypto_info *crypto_info, + u32 start_offload_tcp_sn) { struct tls_context *tls_ctx = tls_get_ctx(sk); struct chcr_ktls_ofld_ctx_tx *tx_ctx; @@ -491,12 +501,16 @@ static int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk, goto out2; /* get peer ip */ - if (sk->sk_family == AF_INET || - (sk->sk_family == AF_INET6 && !sk->sk_ipv6only && - ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED)) { + if (sk->sk_family == AF_INET) { memcpy(daaddr, &sk->sk_daddr, 4); +#if IS_ENABLED(CONFIG_IPV6) } else { - memcpy(daaddr, sk->sk_v6_daddr.in6_u.u6_addr8, 16); + if (!sk->sk_ipv6only && + ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) + memcpy(daaddr, &sk->sk_daddr, 4); + else + memcpy(daaddr, sk->sk_v6_daddr.in6_u.u6_addr8, 16); +#endif } /* get the l2t index */ @@ -530,6 +544,12 @@ static int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk, if (ret) goto out2; + /* Driver shouldn't be removed until any single connection exists */ + if (!try_module_get(THIS_MODULE)) { + ret = -EINVAL; + goto out2; + } + atomic64_inc(&adap->chcr_stats.ktls_tx_connection_open); return 0; out2: @@ -539,43 +559,6 @@ out: return ret; } -static const struct tlsdev_ops chcr_ktls_ops = { - .tls_dev_add = chcr_ktls_dev_add, - .tls_dev_del = chcr_ktls_dev_del, -}; - -/* - * chcr_enable_ktls: add NETIF_F_HW_TLS_TX flag in all the ports. - */ -void chcr_enable_ktls(struct adapter *adap) -{ - struct net_device *netdev; - int i; - - for_each_port(adap, i) { - netdev = adap->port[i]; - netdev->features |= NETIF_F_HW_TLS_TX; - netdev->hw_features |= NETIF_F_HW_TLS_TX; - netdev->tlsdev_ops = &chcr_ktls_ops; - } -} - -/* - * chcr_disable_ktls: remove NETIF_F_HW_TLS_TX flag from all the ports. - */ -void chcr_disable_ktls(struct adapter *adap) -{ - struct net_device *netdev; - int i; - - for_each_port(adap, i) { - netdev = adap->port[i]; - netdev->features &= ~NETIF_F_HW_TLS_TX; - netdev->hw_features &= ~NETIF_F_HW_TLS_TX; - netdev->tlsdev_ops = NULL; - } -} - /* * chcr_init_tcb_fields: Initialize tcb fields to handle TCP seq number * handling. @@ -675,41 +658,14 @@ int chcr_ktls_cpl_set_tcb_rpl(struct adapter *adap, unsigned char *input) return 0; } -/* - * chcr_write_cpl_set_tcb_ulp: update tcb values. - * TCB is responsible to create tcp headers, so all the related values - * should be correctly updated. - * @tx_info - driver specific tls info. - * @q - tx queue on which packet is going out. - * @tid - TCB identifier. - * @pos - current index where should we start writing. - * @word - TCB word. - * @mask - TCB word related mask. - * @val - TCB word related value. - * @reply - set 1 if looking for TP response. - * return - next position to write. - */ -static void *chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info, - struct sge_eth_txq *q, u32 tid, - void *pos, u16 word, u64 mask, +static void *__chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info, + u32 tid, void *pos, u16 word, u64 mask, u64 val, u32 reply) { struct cpl_set_tcb_field_core *cpl; struct ulptx_idata *idata; struct ulp_txpkt *txpkt; - void *save_pos = NULL; - u8 buf[48] = {0}; - int left; - left = (void *)q->q.stat - pos; - if (unlikely(left < CHCR_SET_TCB_FIELD_LEN)) { - if (!left) { - pos = q->q.desc; - } else { - save_pos = pos; - pos = buf; - } - } /* ULP_TXPKT */ txpkt = pos; txpkt->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) | ULP_TXPKT_DEST_V(0)); @@ -734,18 +690,54 @@ static void *chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info, idata = (struct ulptx_idata *)(cpl + 1); idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP)); idata->len = htonl(0); + pos = idata + 1; - if (save_pos) { - pos = chcr_copy_to_txd(buf, &q->q, save_pos, - CHCR_SET_TCB_FIELD_LEN); - } else { - /* check again if we are at the end of the queue */ - if (left == CHCR_SET_TCB_FIELD_LEN) + return pos; +} + + +/* + * chcr_write_cpl_set_tcb_ulp: update tcb values. + * TCB is responsible to create tcp headers, so all the related values + * should be correctly updated. + * @tx_info - driver specific tls info. + * @q - tx queue on which packet is going out. + * @tid - TCB identifier. + * @pos - current index where should we start writing. + * @word - TCB word. + * @mask - TCB word related mask. + * @val - TCB word related value. + * @reply - set 1 if looking for TP response. + * return - next position to write. + */ +static void *chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info, + struct sge_eth_txq *q, u32 tid, + void *pos, u16 word, u64 mask, + u64 val, u32 reply) +{ + int left = (void *)q->q.stat - pos; + + if (unlikely(left < CHCR_SET_TCB_FIELD_LEN)) { + if (!left) { pos = q->q.desc; - else - pos = idata + 1; + } else { + u8 buf[48] = {0}; + + __chcr_write_cpl_set_tcb_ulp(tx_info, tid, buf, word, + mask, val, reply); + + return chcr_copy_to_txd(buf, &q->q, pos, + CHCR_SET_TCB_FIELD_LEN); + } } + pos = __chcr_write_cpl_set_tcb_ulp(tx_info, tid, pos, word, + mask, val, reply); + + /* check again if we are at the end of the queue */ + if (left == CHCR_SET_TCB_FIELD_LEN) + pos = q->q.desc; + return pos; } @@ -925,7 +917,9 @@ chcr_ktls_write_tcp_options(struct chcr_ktls_info *tx_info, struct sk_buff *skb, struct fw_eth_tx_pkt_wr *wr; struct cpl_tx_pkt_core *cpl; u32 ctrl, iplen, maclen; +#if IS_ENABLED(CONFIG_IPV6) struct ipv6hdr *ip6; +#endif unsigned int ndesc; struct tcphdr *tcp; int len16, pktlen; |