diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2014-12-13 13:33:26 -0800 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2014-12-13 13:33:26 -0800 |
commit | e3aa91a7cb21a595169b20c64f63ca39a91a0c43 (patch) | |
tree | 6a92a2e595629949a45336c770c2408abba8444d /drivers | |
parent | 78a45c6f067824cf5d0a9fedea7339ac2e28603c (diff) | |
parent | 8606813a6c8997fd3bb805186056d78670eb86ca (diff) |
Merge git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6
Pull crypto update from Herbert Xu:
- The crypto API is now documented :)
- Disallow arbitrary module loading through crypto API.
- Allow get request with empty driver name through crypto_user.
- Allow speed testing of arbitrary hash functions.
- Add caam support for ctr(aes), gcm(aes) and their derivatives.
- nx now supports concurrent hashing properly.
- Add sahara support for SHA1/256.
- Add ARM64 version of CRC32.
- Misc fixes.
* git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6: (77 commits)
crypto: tcrypt - Allow speed testing of arbitrary hash functions
crypto: af_alg - add user space interface for AEAD
crypto: qat - fix problem with coalescing enable logic
crypto: sahara - add support for SHA1/256
crypto: sahara - replace tasklets with kthread
crypto: sahara - add support for i.MX53
crypto: sahara - fix spinlock initialization
crypto: arm - replace memset by memzero_explicit
crypto: powerpc - replace memset by memzero_explicit
crypto: sha - replace memset by memzero_explicit
crypto: sparc - replace memset by memzero_explicit
crypto: algif_skcipher - initialize upon init request
crypto: algif_skcipher - removed unneeded code
crypto: algif_skcipher - Fixed blocking recvmsg
crypto: drbg - use memzero_explicit() for clearing sensitive data
crypto: drbg - use MODULE_ALIAS_CRYPTO
crypto: include crypto- module prefix in template
crypto: user - add MODULE_ALIAS
crypto: sha-mb - remove a bogus NULL check
crytpo: qat - Fix 64 bytes requests
...
Diffstat (limited to 'drivers')
35 files changed, 3092 insertions, 598 deletions
diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig index 91a04ae8003c..de57b38809c7 100644 --- a/drivers/char/hw_random/Kconfig +++ b/drivers/char/hw_random/Kconfig @@ -64,7 +64,7 @@ config HW_RANDOM_AMD config HW_RANDOM_ATMEL tristate "Atmel Random Number Generator support" - depends on ARCH_AT91 && HAVE_CLK + depends on ARCH_AT91 && HAVE_CLK && OF default HW_RANDOM ---help--- This driver provides kernel-side support for the Random Number diff --git a/drivers/char/hw_random/atmel-rng.c b/drivers/char/hw_random/atmel-rng.c index 851bc7e20ad2..0bb0b2120a63 100644 --- a/drivers/char/hw_random/atmel-rng.c +++ b/drivers/char/hw_random/atmel-rng.c @@ -67,7 +67,7 @@ static int atmel_trng_probe(struct platform_device *pdev) if (IS_ERR(trng->clk)) return PTR_ERR(trng->clk); - ret = clk_enable(trng->clk); + ret = clk_prepare_enable(trng->clk); if (ret) return ret; @@ -95,7 +95,7 @@ static int atmel_trng_remove(struct platform_device *pdev) hwrng_unregister(&trng->rng); writel(TRNG_KEY, trng->base + TRNG_CR); - clk_disable(trng->clk); + clk_disable_unprepare(trng->clk); return 0; } @@ -105,7 +105,7 @@ static int atmel_trng_suspend(struct device *dev) { struct atmel_trng *trng = dev_get_drvdata(dev); - clk_disable(trng->clk); + clk_disable_unprepare(trng->clk); return 0; } @@ -114,7 +114,7 @@ static int atmel_trng_resume(struct device *dev) { struct atmel_trng *trng = dev_get_drvdata(dev); - return clk_enable(trng->clk); + return clk_prepare_enable(trng->clk); } static const struct dev_pm_ops atmel_trng_pm_ops = { @@ -123,6 +123,12 @@ static const struct dev_pm_ops atmel_trng_pm_ops = { }; #endif /* CONFIG_PM */ +static const struct of_device_id atmel_trng_dt_ids[] = { + { .compatible = "atmel,at91sam9g45-trng" }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, atmel_trng_dt_ids); + static struct platform_driver atmel_trng_driver = { .probe = atmel_trng_probe, .remove = atmel_trng_remove, @@ -132,6 +138,7 @@ static struct platform_driver atmel_trng_driver = { #ifdef CONFIG_PM .pm = &atmel_trng_pm_ops, #endif /* CONFIG_PM */ + .of_match_table = atmel_trng_dt_ids, }, }; diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c index aa30a25c8d49..1500cfd799a7 100644 --- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -281,7 +281,6 @@ static ssize_t hwrng_attr_available_show(struct device *dev, char *buf) { int err; - ssize_t ret = 0; struct hwrng *rng; err = mutex_lock_interruptible(&rng_mutex); @@ -289,16 +288,13 @@ static ssize_t hwrng_attr_available_show(struct device *dev, return -ERESTARTSYS; buf[0] = '\0'; list_for_each_entry(rng, &rng_list, list) { - strncat(buf, rng->name, PAGE_SIZE - ret - 1); - ret += strlen(rng->name); - strncat(buf, " ", PAGE_SIZE - ret - 1); - ret++; + strlcat(buf, rng->name, PAGE_SIZE); + strlcat(buf, " ", PAGE_SIZE); } - strncat(buf, "\n", PAGE_SIZE - ret - 1); - ret++; + strlcat(buf, "\n", PAGE_SIZE); mutex_unlock(&rng_mutex); - return ret; + return strlen(buf); } static DEVICE_ATTR(rng_current, S_IRUGO | S_IWUSR, diff --git a/drivers/crypto/bfin_crc.c b/drivers/crypto/bfin_crc.c index b099e33cb073..e96eddc0e0b3 100644 --- a/drivers/crypto/bfin_crc.c +++ b/drivers/crypto/bfin_crc.c @@ -21,13 +21,13 @@ #include <linux/scatterlist.h> #include <linux/dma-mapping.h> #include <linux/delay.h> -#include <linux/unaligned/access_ok.h> #include <linux/crypto.h> #include <linux/cryptohash.h> #include <crypto/scatterwalk.h> #include <crypto/algapi.h> #include <crypto/hash.h> #include <crypto/internal/hash.h> +#include <asm/unaligned.h> #include <asm/dma.h> #include <asm/portmux.h> diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index a80ea853701d..3187400daf31 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -60,6 +60,7 @@ #define CAAM_CRA_PRIORITY 3000 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ + CTR_RFC3686_NONCE_SIZE + \ SHA512_DIGEST_SIZE * 2) /* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ #define CAAM_MAX_IV_LENGTH 16 @@ -70,17 +71,34 @@ #define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 18 * CAAM_CMD_SZ) #define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 7 * CAAM_CMD_SZ) +/* Note: Nonce is counted in enckeylen */ +#define DESC_AEAD_CTR_RFC3686_LEN (6 * CAAM_CMD_SZ) + #define DESC_AEAD_NULL_BASE (3 * CAAM_CMD_SZ) #define DESC_AEAD_NULL_ENC_LEN (DESC_AEAD_NULL_BASE + 14 * CAAM_CMD_SZ) #define DESC_AEAD_NULL_DEC_LEN (DESC_AEAD_NULL_BASE + 17 * CAAM_CMD_SZ) +#define DESC_GCM_BASE (3 * CAAM_CMD_SZ) +#define DESC_GCM_ENC_LEN (DESC_GCM_BASE + 23 * CAAM_CMD_SZ) +#define DESC_GCM_DEC_LEN (DESC_GCM_BASE + 19 * CAAM_CMD_SZ) + +#define DESC_RFC4106_BASE (3 * CAAM_CMD_SZ) +#define DESC_RFC4106_ENC_LEN (DESC_RFC4106_BASE + 15 * CAAM_CMD_SZ) +#define DESC_RFC4106_DEC_LEN (DESC_RFC4106_BASE + 14 * CAAM_CMD_SZ) +#define DESC_RFC4106_GIVENC_LEN (DESC_RFC4106_BASE + 21 * CAAM_CMD_SZ) + +#define DESC_RFC4543_BASE (3 * CAAM_CMD_SZ) +#define DESC_RFC4543_ENC_LEN (DESC_RFC4543_BASE + 25 * CAAM_CMD_SZ) +#define DESC_RFC4543_DEC_LEN (DESC_RFC4543_BASE + 27 * CAAM_CMD_SZ) +#define DESC_RFC4543_GIVENC_LEN (DESC_RFC4543_BASE + 30 * CAAM_CMD_SZ) + #define DESC_ABLKCIPHER_BASE (3 * CAAM_CMD_SZ) #define DESC_ABLKCIPHER_ENC_LEN (DESC_ABLKCIPHER_BASE + \ 20 * CAAM_CMD_SZ) #define DESC_ABLKCIPHER_DEC_LEN (DESC_ABLKCIPHER_BASE + \ 15 * CAAM_CMD_SZ) -#define DESC_MAX_USED_BYTES (DESC_AEAD_GIVENC_LEN + \ +#define DESC_MAX_USED_BYTES (DESC_RFC4543_GIVENC_LEN + \ CAAM_MAX_KEY_SIZE) #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) @@ -128,11 +146,13 @@ static inline void aead_append_src_dst(u32 *desc, u32 msg_type) /* * For aead encrypt and decrypt, read iv for both classes */ -static inline void aead_append_ld_iv(u32 *desc, int ivsize) +static inline void aead_append_ld_iv(u32 *desc, int ivsize, int ivoffset) { - append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | - LDST_CLASS_1_CCB | ivsize); - append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | ivsize); + append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + (ivoffset << LDST_OFFSET_SHIFT)); + append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | + (ivoffset << MOVE_OFFSET_SHIFT) | ivsize); } /* @@ -178,35 +198,60 @@ struct caam_ctx { }; static void append_key_aead(u32 *desc, struct caam_ctx *ctx, - int keys_fit_inline) + int keys_fit_inline, bool is_rfc3686) { + u32 *nonce; + unsigned int enckeylen = ctx->enckeylen; + + /* + * RFC3686 specific: + * | ctx->key = {AUTH_KEY, ENC_KEY, NONCE} + * | enckeylen = encryption key size + nonce size + */ + if (is_rfc3686) + enckeylen -= CTR_RFC3686_NONCE_SIZE; + if (keys_fit_inline) { append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, ctx->split_key_len, CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC); append_key_as_imm(desc, (void *)ctx->key + - ctx->split_key_pad_len, ctx->enckeylen, - ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); + ctx->split_key_pad_len, enckeylen, + enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); } else { append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC); append_key(desc, ctx->key_dma + ctx->split_key_pad_len, - ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); + enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); + } + + /* Load Counter into CONTEXT1 reg */ + if (is_rfc3686) { + nonce = (u32 *)((void *)ctx->key + ctx->split_key_pad_len + + enckeylen); + append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB | + LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); + append_move(desc, + MOVE_SRC_OUTFIFO | + MOVE_DEST_CLASS1CTX | + (16 << MOVE_OFFSET_SHIFT) | + (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); } } static void init_sh_desc_key_aead(u32 *desc, struct caam_ctx *ctx, - int keys_fit_inline) + int keys_fit_inline, bool is_rfc3686) { u32 *key_jump_cmd; - init_sh_desc(desc, HDR_SHARE_SERIAL); + /* Note: Context registers are saved. */ + init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); /* Skip if already shared */ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_SHRD); - append_key_aead(desc, ctx, keys_fit_inline); + append_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); set_jump_tgt_here(desc, key_jump_cmd); } @@ -406,10 +451,17 @@ static int aead_set_sh_desc(struct crypto_aead *aead) { struct aead_tfm *tfm = &aead->base.crt_aead; struct caam_ctx *ctx = crypto_aead_ctx(aead); + struct crypto_tfm *ctfm = crypto_aead_tfm(aead); + const char *alg_name = crypto_tfm_alg_name(ctfm); struct device *jrdev = ctx->jrdev; - bool keys_fit_inline = false; + bool keys_fit_inline; u32 geniv, moveiv; + u32 ctx1_iv_off = 0; u32 *desc; + const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == + OP_ALG_AAI_CTR_MOD128); + const bool is_rfc3686 = (ctr_mode && + (strstr(alg_name, "rfc3686") != NULL)); if (!ctx->authsize) return 0; @@ -419,18 +471,36 @@ static int aead_set_sh_desc(struct crypto_aead *aead) return aead_null_set_sh_desc(aead); /* + * AES-CTR needs to load IV in CONTEXT1 reg + * at an offset of 128bits (16bytes) + * CONTEXT1[255:128] = IV + */ + if (ctr_mode) + ctx1_iv_off = 16; + + /* + * RFC3686 specific: + * CONTEXT1[255:128] = {NONCE, IV, COUNTER} + */ + if (is_rfc3686) + ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; + + /* * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ + keys_fit_inline = false; if (DESC_AEAD_ENC_LEN + DESC_JOB_IO_LEN + - ctx->split_key_pad_len + ctx->enckeylen <= + ctx->split_key_pad_len + ctx->enckeylen + + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= CAAM_DESC_BYTES_MAX) keys_fit_inline = true; /* aead_encrypt shared descriptor */ desc = ctx->sh_desc_enc; - init_sh_desc_key_aead(desc, ctx, keys_fit_inline); + /* Note: Context registers are saved. */ + init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); /* Class 2 operation */ append_operation(desc, ctx->class2_alg_type | @@ -448,7 +518,15 @@ static int aead_set_sh_desc(struct crypto_aead *aead) /* read assoc before reading payload */ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | KEY_VLF); - aead_append_ld_iv(desc, tfm->ivsize); + aead_append_ld_iv(desc, tfm->ivsize, ctx1_iv_off); + + /* Load Counter into CONTEXT1 reg */ + if (is_rfc3686) + append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | + LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << + LDST_OFFSET_SHIFT)); /* Class 1 operation */ append_operation(desc, ctx->class1_alg_type | @@ -482,14 +560,16 @@ static int aead_set_sh_desc(struct crypto_aead *aead) */ keys_fit_inline = false; if (DESC_AEAD_DEC_LEN + DESC_JOB_IO_LEN + - ctx->split_key_pad_len + ctx->enckeylen <= + ctx->split_key_pad_len + ctx->enckeylen + + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= CAAM_DESC_BYTES_MAX) keys_fit_inline = true; /* aead_decrypt shared descriptor */ desc = ctx->sh_desc_dec; - init_sh_desc_key_aead(desc, ctx, keys_fit_inline); + /* Note: Context registers are saved. */ + init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); /* Class 2 operation */ append_operation(desc, ctx->class2_alg_type | @@ -506,9 +586,22 @@ static int aead_set_sh_desc(struct crypto_aead *aead) append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | KEY_VLF); - aead_append_ld_iv(desc, tfm->ivsize); + aead_append_ld_iv(desc, tfm->ivsize, ctx1_iv_off); - append_dec_op1(desc, ctx->class1_alg_type); + /* Load Counter into CONTEXT1 reg */ + if (is_rfc3686) + append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | + LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << + LDST_OFFSET_SHIFT)); + + /* Choose operation */ + if (ctr_mode) + append_operation(desc, ctx->class1_alg_type | + OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT); + else + append_dec_op1(desc, ctx->class1_alg_type); /* Read and write cryptlen bytes */ append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); @@ -538,14 +631,16 @@ static int aead_set_sh_desc(struct crypto_aead *aead) */ keys_fit_inline = false; if (DESC_AEAD_GIVENC_LEN + DESC_JOB_IO_LEN + - ctx->split_key_pad_len + ctx->enckeylen <= + ctx->split_key_pad_len + ctx->enckeylen + + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= CAAM_DESC_BYTES_MAX) keys_fit_inline = true; /* aead_givencrypt shared descriptor */ desc = ctx->sh_desc_givenc; - init_sh_desc_key_aead(desc, ctx, keys_fit_inline); + /* Note: Context registers are saved. */ + init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); /* Generate IV */ geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | @@ -554,13 +649,16 @@ static int aead_set_sh_desc(struct crypto_aead *aead) append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); - append_move(desc, MOVE_SRC_INFIFO | - MOVE_DEST_CLASS1CTX | (tfm->ivsize << MOVE_LEN_SHIFT)); + append_move(desc, MOVE_WAITCOMP | + MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX | + (ctx1_iv_off << MOVE_OFFSET_SHIFT) | + (tfm->ivsize << MOVE_LEN_SHIFT)); append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); /* Copy IV to class 1 context */ - append_move(desc, MOVE_SRC_CLASS1CTX | - MOVE_DEST_OUTFIFO | (tfm->ivsize << MOVE_LEN_SHIFT)); + append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO | + (ctx1_iv_off << MOVE_OFFSET_SHIFT) | + (tfm->ivsize << MOVE_LEN_SHIFT)); /* Return to encryption */ append_operation(desc, ctx->class2_alg_type | @@ -576,7 +674,7 @@ static int aead_set_sh_desc(struct crypto_aead *aead) append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | KEY_VLF); - /* Copy iv from class 1 ctx to class 2 fifo*/ + /* Copy iv from outfifo to class 2 fifo */ moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 | NFIFOENTRY_DTYPE_MSG | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT); append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB | @@ -584,6 +682,14 @@ static int aead_set_sh_desc(struct crypto_aead *aead) append_load_imm_u32(desc, tfm->ivsize, LDST_CLASS_2_CCB | LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM); + /* Load Counter into CONTEXT1 reg */ + if (is_rfc3686) + append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | + LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << + LDST_OFFSET_SHIFT)); + /* Class 1 operation */ append_operation(desc, ctx->class1_alg_type | OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); @@ -630,6 +736,912 @@ static int aead_setauthsize(struct crypto_aead *authenc, return 0; } +static int gcm_set_sh_desc(struct crypto_aead *aead) +{ + struct aead_tfm *tfm = &aead->base.crt_aead; + struct caam_ctx *ctx = crypto_aead_ctx(aead); + struct device *jrdev = ctx->jrdev; + bool keys_fit_inline = false; + u32 *key_jump_cmd, *zero_payload_jump_cmd, + *zero_assoc_jump_cmd1, *zero_assoc_jump_cmd2; + u32 *desc; + + if (!ctx->enckeylen || !ctx->authsize) + return 0; + + /* + * AES GCM encrypt shared descriptor + * Job Descriptor and Shared Descriptor + * must fit into the 64-word Descriptor h/w Buffer + */ + if (DESC_GCM_ENC_LEN + DESC_JOB_IO_LEN + + ctx->enckeylen <= CAAM_DESC_BYTES_MAX) + keys_fit_inline = true; + + desc = ctx->sh_desc_enc; + + init_sh_desc(desc, HDR_SHARE_SERIAL); + + /* skip key loading if they are loaded due to sharing */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD | JUMP_COND_SELF); + if (keys_fit_inline) + append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, + ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); + else + append_key(desc, ctx->key_dma, ctx->enckeylen, + CLASS_1 | KEY_DEST_CLASS_REG); + set_jump_tgt_here(desc, key_jump_cmd); + + /* class 1 operation */ + append_operation(desc, ctx->class1_alg_type | + OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); + + /* cryptlen = seqoutlen - authsize */ + append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); + + /* assoclen + cryptlen = seqinlen - ivsize */ + append_math_sub_imm_u32(desc, REG2, SEQINLEN, IMM, tfm->ivsize); + + /* assoclen = (assoclen + cryptlen) - cryptlen */ + append_math_sub(desc, REG1, REG2, REG3, CAAM_CMD_SZ); + + /* if cryptlen is ZERO jump to zero-payload commands */ + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | + JUMP_COND_MATH_Z); + /* read IV */ + append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); + + /* if assoclen is ZERO, skip reading the assoc data */ + append_math_add(desc, VARSEQINLEN, ZERO, REG1, CAAM_CMD_SZ); + zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | + JUMP_COND_MATH_Z); + + /* read assoc data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); + set_jump_tgt_here(desc, zero_assoc_jump_cmd1); + + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + + /* write encrypted data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); + + /* read payload data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); + + /* jump the zero-payload commands */ + append_jump(desc, JUMP_TEST_ALL | 7); + + /* zero-payload commands */ + set_jump_tgt_here(desc, zero_payload_jump_cmd); + + /* if assoclen is ZERO, jump to IV reading - is the only input data */ + append_math_add(desc, VARSEQINLEN, ZERO, REG1, CAAM_CMD_SZ); + zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL | + JUMP_COND_MATH_Z); + /* read IV */ + append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); + + /* read assoc data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1); + + /* jump to ICV writing */ + append_jump(desc, JUMP_TEST_ALL | 2); + + /* read IV - is the only input data */ + set_jump_tgt_here(desc, zero_assoc_jump_cmd2); + append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | + FIFOLD_TYPE_LAST1); + + /* write ICV */ + append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT); + + ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, + desc_bytes(desc), + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { + dev_err(jrdev, "unable to map shared descriptor\n"); + return -ENOMEM; + } +#ifdef DEBUG + print_hex_dump(KERN_ERR, "gcm enc shdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); +#endif + + /* + * Job Descriptor and Shared Descriptors + * must all fit into the 64-word Descriptor h/w Buffer + */ + keys_fit_inline = false; + if (DESC_GCM_DEC_LEN + DESC_JOB_IO_LEN + + ctx->enckeylen <= CAAM_DESC_BYTES_MAX) + keys_fit_inline = true; + + desc = ctx->sh_desc_dec; + + init_sh_desc(desc, HDR_SHARE_SERIAL); + + /* skip key loading if they are loaded due to sharing */ + key_jump_cmd = append_jump(desc, JUMP_JSL | + JUMP_TEST_ALL | JUMP_COND_SHRD | + JUMP_COND_SELF); + if (keys_fit_inline) + append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, + ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); + else + append_key(desc, ctx->key_dma, ctx->enckeylen, + CLASS_1 | KEY_DEST_CLASS_REG); + set_jump_tgt_here(desc, key_jump_cmd); + + /* class 1 operation */ + append_operation(desc, ctx->class1_alg_type | + OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); + + /* assoclen + cryptlen = seqinlen - ivsize - icvsize */ + append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM, + ctx->authsize + tfm->ivsize); + + /* assoclen = (assoclen + cryptlen) - cryptlen */ + append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); + append_math_sub(desc, REG1, REG3, REG2, CAAM_CMD_SZ); + + /* read IV */ + append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); + + /* jump to zero-payload command if cryptlen is zero */ + append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); + zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | + JUMP_COND_MATH_Z); + + append_math_add(desc, VARSEQINLEN, ZERO, REG1, CAAM_CMD_SZ); + /* if asoclen is ZERO, skip reading assoc data */ + zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | + JUMP_COND_MATH_Z); + /* read assoc data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); + set_jump_tgt_here(desc, zero_assoc_jump_cmd1); + + append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); + + /* store encrypted data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); + + /* read payload data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); + + /* jump the zero-payload commands */ + append_jump(desc, JUMP_TEST_ALL | 4); + + /* zero-payload command */ + set_jump_tgt_here(desc, zero_payload_jump_cmd); + + /* if assoclen is ZERO, jump to ICV reading */ + append_math_add(desc, VARSEQINLEN, ZERO, REG1, CAAM_CMD_SZ); + zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL | + JUMP_COND_MATH_Z); + /* read assoc data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); + set_jump_tgt_here(desc, zero_assoc_jump_cmd2); + + /* read ICV */ + append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); + + ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, + desc_bytes(desc), + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { + dev_err(jrdev, "unable to map shared descriptor\n"); + return -ENOMEM; + } +#ifdef DEBUG + print_hex_dump(KERN_ERR, "gcm dec shdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); +#endif + + return 0; +} + +static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) +{ + struct caam_ctx *ctx = crypto_aead_ctx(authenc); + + ctx->authsize = authsize; + gcm_set_sh_desc(authenc); + + return 0; +} + +static int rfc4106_set_sh_desc(struct crypto_aead *aead) +{ + struct aead_tfm *tfm = &aead->base.crt_aead; + struct caam_ctx *ctx = crypto_aead_ctx(aead); + struct device *jrdev = ctx->jrdev; + bool keys_fit_inline = false; + u32 *key_jump_cmd, *move_cmd, *write_iv_cmd; + u32 *desc; + u32 geniv; + + if (!ctx->enckeylen || !ctx->authsize) + return 0; + + /* + * RFC4106 encrypt shared descriptor + * Job Descriptor and Shared Descriptor + * must fit into the 64-word Descriptor h/w Buffer + */ + if (DESC_RFC4106_ENC_LEN + DESC_JOB_IO_LEN + + ctx->enckeylen <= CAAM_DESC_BYTES_MAX) + keys_fit_inline = true; + + desc = ctx->sh_desc_enc; + + init_sh_desc(desc, HDR_SHARE_SERIAL); + + /* Skip key loading if it is loaded due to sharing */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + if (keys_fit_inline) + append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, + ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); + else + append_key(desc, ctx->key_dma, ctx->enckeylen, + CLASS_1 | KEY_DEST_CLASS_REG); + set_jump_tgt_here(desc, key_jump_cmd); + + /* Class 1 operation */ + append_operation(desc, ctx->class1_alg_type | + OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); + + /* cryptlen = seqoutlen - authsize */ + append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + + /* assoclen + cryptlen = seqinlen - ivsize */ + append_math_sub_imm_u32(desc, REG2, SEQINLEN, IMM, tfm->ivsize); + + /* assoclen = (assoclen + cryptlen) - cryptlen */ + append_math_sub(desc, VARSEQINLEN, REG2, REG3, CAAM_CMD_SZ); + + /* Read Salt */ + append_fifo_load_as_imm(desc, (void *)(ctx->key + ctx->enckeylen), + 4, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_IV); + /* Read AES-GCM-ESP IV */ + append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); + + /* Read assoc data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); + + /* Will read cryptlen bytes */ + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + + /* Write encrypted data */ + append_seq_f |