From b29cb8d645da87d7d1ff28a7317c541ffccfa66c Mon Sep 17 00:00:00 2001 From: Tero Kristo Date: Wed, 27 May 2020 15:24:23 +0300 Subject: crypto: omap-aes - avoid spamming console with self tests Running the self test suite for omap-aes with extra tests enabled causes huge spam with the tag message wrong indicators. With self tests, this is fine as there are some tests that purposedly pass bad data to the driver. Also, returning -EBADMSG from the driver is enough, so remove the dev_err message completely. Signed-off-by: Tero Kristo Signed-off-by: Herbert Xu --- drivers/crypto/omap-aes-gcm.c | 1 - 1 file changed, 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/omap-aes-gcm.c b/drivers/crypto/omap-aes-gcm.c index 32dc00dc570b..9f937bdc53a7 100644 --- a/drivers/crypto/omap-aes-gcm.c +++ b/drivers/crypto/omap-aes-gcm.c @@ -77,7 +77,6 @@ static void omap_aes_gcm_done_task(struct omap_aes_dev *dd) tag = (u8 *)rctx->auth_tag; for (i = 0; i < dd->authsize; i++) { if (tag[i]) { - dev_err(dd->dev, "GCM decryption: Tag Message is wrong\n"); ret = -EBADMSG; } } -- cgit v1.2.3 From 8dc43636e39a89ec51188b012a789625fa83bbe0 Mon Sep 17 00:00:00 2001 From: Tero Kristo Date: Wed, 27 May 2020 15:24:24 +0300 Subject: crypto: omap-sham - force kernel driver usage for sha algos As the hardware acceleration for the omap-sham algos is not available from userspace, force kernel driver usage. Without this flag in place, openssl 1.1 implementation thinks it can accelerate sha algorithms on omap devices directly from userspace. Signed-off-by: Tero Kristo Signed-off-by: Herbert Xu --- drivers/crypto/omap-sham.c | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index 063ad5d03f33..34ecace14360 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -1571,7 +1571,8 @@ static struct ahash_alg algs_sha224_sha256[] = { .cra_name = "sha224", .cra_driver_name = "omap-sha224", .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC | + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_ctxsize = sizeof(struct omap_sham_ctx), @@ -1592,7 +1593,8 @@ static struct ahash_alg algs_sha224_sha256[] = { .cra_name = "sha256", .cra_driver_name = "omap-sha256", .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC | + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_ctxsize = sizeof(struct omap_sham_ctx), @@ -1614,7 +1616,8 @@ static struct ahash_alg algs_sha224_sha256[] = { .cra_name = "hmac(sha224)", .cra_driver_name = "omap-hmac-sha224", .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC | + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_ctxsize = sizeof(struct omap_sham_ctx) + @@ -1637,7 +1640,8 @@ static struct ahash_alg algs_sha224_sha256[] = { .cra_name = "hmac(sha256)", .cra_driver_name = "omap-hmac-sha256", .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC | + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_ctxsize = sizeof(struct omap_sham_ctx) + @@ -1662,7 +1666,8 @@ static struct ahash_alg algs_sha384_sha512[] = { .cra_name = "sha384", .cra_driver_name = "omap-sha384", .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC | + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = SHA384_BLOCK_SIZE, .cra_ctxsize = sizeof(struct omap_sham_ctx), @@ -1683,7 +1688,8 @@ static struct ahash_alg algs_sha384_sha512[] = { .cra_name = "sha512", .cra_driver_name = "omap-sha512", .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC | + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_ctxsize = sizeof(struct omap_sham_ctx), @@ -1705,7 +1711,8 @@ static struct ahash_alg algs_sha384_sha512[] = { .cra_name = "hmac(sha384)", .cra_driver_name = "omap-hmac-sha384", .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC | + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = SHA384_BLOCK_SIZE, .cra_ctxsize = sizeof(struct omap_sham_ctx) + @@ -1728,7 +1735,8 @@ static struct ahash_alg algs_sha384_sha512[] = { .cra_name = "hmac(sha512)", .cra_driver_name = "omap-hmac-sha512", .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC | + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_ctxsize = sizeof(struct omap_sham_ctx) + -- cgit v1.2.3 From 7e34e0bbc6449458a8de675814ffa22dec003698 Mon Sep 17 00:00:00 2001 From: Tero Kristo Date: Wed, 27 May 2020 15:24:25 +0300 Subject: crypto: omap-crypto - fix userspace copied buffer access In case buffers are copied from userspace, directly accessing the page will most likely fail because it hasn't been mapped into the kernel memory space. Fix the issue by forcing a kmap / kunmap within the cleanup functionality. Signed-off-by: Tero Kristo Signed-off-by: Herbert Xu --- drivers/crypto/omap-crypto.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/omap-crypto.c b/drivers/crypto/omap-crypto.c index cc88b7362bc2..94b2dba90f0d 100644 --- a/drivers/crypto/omap-crypto.c +++ b/drivers/crypto/omap-crypto.c @@ -178,11 +178,17 @@ static void omap_crypto_copy_data(struct scatterlist *src, amt = min(src->length - srco, dst->length - dsto); amt = min(len, amt); - srcb = sg_virt(src) + srco; - dstb = sg_virt(dst) + dsto; + srcb = kmap_atomic(sg_page(src)) + srco + src->offset; + dstb = kmap_atomic(sg_page(dst)) + dsto + dst->offset; memcpy(dstb, srcb, amt); + if (!PageSlab(sg_page(dst))) + flush_kernel_dcache_page(sg_page(dst)); + + kunmap_atomic(srcb); + kunmap_atomic(dstb); + srco += amt; dsto += amt; len -= amt; -- cgit v1.2.3 From 6395166d7a19019d5e9574eb9ecdaf0028abb887 Mon Sep 17 00:00:00 2001 From: Tero Kristo Date: Wed, 27 May 2020 15:24:26 +0300 Subject: crypto: omap-sham - huge buffer access fixes The ctx internal buffer can only hold buflen amount of data, don't try to copy over more than that. Also, initialize the context sg pointer if we only have data in the context internal buffer, this can happen when closing a hash with certain data amounts. Signed-off-by: Tero Kristo Signed-off-by: Herbert Xu --- drivers/crypto/omap-sham.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index 34ecace14360..5efc66ccef46 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -750,8 +750,15 @@ static int omap_sham_align_sgs(struct scatterlist *sg, int offset = rctx->offset; int bufcnt = rctx->bufcnt; - if (!sg || !sg->length || !nbytes) + if (!sg || !sg->length || !nbytes) { + if (bufcnt) { + sg_init_table(rctx->sgl, 1); + sg_set_buf(rctx->sgl, rctx->dd->xmit_buf, bufcnt); + rctx->sg = rctx->sgl; + } + return 0; + } new_len = nbytes; @@ -895,7 +902,7 @@ static int omap_sham_prepare_request(struct ahash_request *req, bool update) if (hash_later < 0) hash_later = 0; - if (hash_later) { + if (hash_later && hash_later <= rctx->buflen) { scatterwalk_map_and_copy(rctx->buffer, req->src, req->nbytes - hash_later, -- cgit v1.2.3 From 63832a0c6fe180aa34d2a77052aa501be6281c99 Mon Sep 17 00:00:00 2001 From: Tero Kristo Date: Wed, 27 May 2020 15:24:27 +0300 Subject: crypto: omap-sham - fix very small data size handling With very small data sizes, the whole data can end up in the xmit buffer. This code path does not set the sg_len properly which causes the core dma framework to crash. Fix by adding the proper size in place. Also, the data length must be a multiple of block-size, so extend the DMA data size while here. Signed-off-by: Tero Kristo Signed-off-by: Herbert Xu --- drivers/crypto/omap-sham.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'drivers/crypto') diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index 5efc66ccef46..45bc551ebc20 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -752,9 +752,11 @@ static int omap_sham_align_sgs(struct scatterlist *sg, if (!sg || !sg->length || !nbytes) { if (bufcnt) { + bufcnt = DIV_ROUND_UP(bufcnt, bs) * bs; sg_init_table(rctx->sgl, 1); sg_set_buf(rctx->sgl, rctx->dd->xmit_buf, bufcnt); rctx->sg = rctx->sgl; + rctx->sg_len = 1; } return 0; -- cgit v1.2.3 From 9ef4e6e5e31403860081b995f26dc0d4b26356c2 Mon Sep 17 00:00:00 2001 From: Tero Kristo Date: Wed, 27 May 2020 15:24:28 +0300 Subject: crypto: omap-aes - prevent unregistering algorithms twice Most of the OMAP family SoCs contain two instances for AES core, which causes the remove callbacks to be also done twice when driver is removed. Fix the algorithm unregister callbacks to take into account the number of algorithms still registered to avoid removing these twice. Signed-off-by: Tero Kristo Signed-off-by: Herbert Xu --- drivers/crypto/omap-aes.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index 824ddf2a66ff..b5aff20c5900 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -1269,13 +1269,17 @@ static int omap_aes_remove(struct platform_device *pdev) spin_unlock(&list_lock); for (i = dd->pdata->algs_info_size - 1; i >= 0; i--) - for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) + for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) { crypto_unregister_skcipher( &dd->pdata->algs_info[i].algs_list[j]); + dd->pdata->algs_info[i].registered--; + } - for (i = dd->pdata->aead_algs_info->size - 1; i >= 0; i--) { + for (i = dd->pdata->aead_algs_info->registered - 1; i >= 0; i--) { aalg = &dd->pdata->aead_algs_info->algs_list[i]; crypto_unregister_aead(aalg); + dd->pdata->aead_algs_info->registered--; + } crypto_engine_exit(dd->engine); -- cgit v1.2.3 From 281c377872ff5d15d80df25fc4df02d2676c7cde Mon Sep 17 00:00:00 2001 From: Tero Kristo Date: Wed, 27 May 2020 15:24:29 +0300 Subject: crypto: omap-sham - add proper load balancing support for multicore The current implementation of the multiple accelerator core support for OMAP SHA does not work properly. It always picks up the first probed accelerator core if this is available, and rest of the book keeping also gets confused if there are two cores available. Add proper load balancing support for SHA, and also fix any bugs related to the multicore support while doing it. Signed-off-by: Tero Kristo Signed-off-by: Herbert Xu --- drivers/crypto/omap-sham.c | 64 ++++++++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 33 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index 45bc551ebc20..82691a057d2a 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -168,8 +168,6 @@ struct omap_sham_hmac_ctx { }; struct omap_sham_ctx { - struct omap_sham_dev *dd; - unsigned long flags; /* fallback stuff */ @@ -934,27 +932,35 @@ static int omap_sham_update_dma_stop(struct omap_sham_dev *dd) return 0; } +struct omap_sham_dev *omap_sham_find_dev(struct omap_sham_reqctx *ctx) +{ + struct omap_sham_dev *dd; + + if (ctx->dd) + return ctx->dd; + + spin_lock_bh(&sham.lock); + dd = list_first_entry(&sham.dev_list, struct omap_sham_dev, list); + list_move_tail(&dd->list, &sham.dev_list); + ctx->dd = dd; + spin_unlock_bh(&sham.lock); + + return dd; +} + static int omap_sham_init(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm); struct omap_sham_reqctx *ctx = ahash_request_ctx(req); - struct omap_sham_dev *dd = NULL, *tmp; + struct omap_sham_dev *dd; int bs = 0; - spin_lock_bh(&sham.lock); - if (!tctx->dd) { - list_for_each_entry(tmp, &sham.dev_list, list) { - dd = tmp; - break; - } - tctx->dd = dd; - } else { - dd = tctx->dd; - } - spin_unlock_bh(&sham.lock); + ctx->dd = NULL; - ctx->dd = dd; + dd = omap_sham_find_dev(ctx); + if (!dd) + return -ENODEV; ctx->flags = 0; @@ -1224,8 +1230,7 @@ err1: static int omap_sham_enqueue(struct ahash_request *req, unsigned int op) { struct omap_sham_reqctx *ctx = ahash_request_ctx(req); - struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm); - struct omap_sham_dev *dd = tctx->dd; + struct omap_sham_dev *dd = ctx->dd; ctx->op = op; @@ -1235,7 +1240,7 @@ static int omap_sham_enqueue(struct ahash_request *req, unsigned int op) static int omap_sham_update(struct ahash_request *req) { struct omap_sham_reqctx *ctx = ahash_request_ctx(req); - struct omap_sham_dev *dd = ctx->dd; + struct omap_sham_dev *dd = omap_sham_find_dev(ctx); if (!req->nbytes) return 0; @@ -1328,21 +1333,8 @@ static int omap_sham_setkey(struct crypto_ahash *tfm, const u8 *key, struct omap_sham_hmac_ctx *bctx = tctx->base; int bs = crypto_shash_blocksize(bctx->shash); int ds = crypto_shash_digestsize(bctx->shash); - struct omap_sham_dev *dd = NULL, *tmp; int err, i; - spin_lock_bh(&sham.lock); - if (!tctx->dd) { - list_for_each_entry(tmp, &sham.dev_list, list) { - dd = tmp; - break; - } - tctx->dd = dd; - } else { - dd = tctx->dd; - } - spin_unlock_bh(&sham.lock); - err = crypto_shash_setkey(tctx->fallback, key, keylen); if (err) return err; @@ -1359,7 +1351,7 @@ static int omap_sham_setkey(struct crypto_ahash *tfm, const u8 *key, memset(bctx->ipad + keylen, 0, bs - keylen); - if (!test_bit(FLAGS_AUTO_XOR, &dd->flags)) { + if (!test_bit(FLAGS_AUTO_XOR, &sham.flags)) { memcpy(bctx->opad, bctx->ipad, bs); for (i = 0; i < bs; i++) { @@ -2171,6 +2163,7 @@ static int omap_sham_probe(struct platform_device *pdev) } dd->flags |= dd->pdata->flags; + sham.flags |= dd->pdata->flags; pm_runtime_use_autosuspend(dev); pm_runtime_set_autosuspend_delay(dev, DEFAULT_AUTOSUSPEND_DELAY); @@ -2198,6 +2191,9 @@ static int omap_sham_probe(struct platform_device *pdev) spin_unlock(&sham.lock); for (i = 0; i < dd->pdata->algs_info_size; i++) { + if (dd->pdata->algs_info[i].registered) + break; + for (j = 0; j < dd->pdata->algs_info[i].size; j++) { struct ahash_alg *alg; @@ -2249,9 +2245,11 @@ static int omap_sham_remove(struct platform_device *pdev) list_del(&dd->list); spin_unlock(&sham.lock); for (i = dd->pdata->algs_info_size - 1; i >= 0; i--) - for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) + for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) { crypto_unregister_ahash( &dd->pdata->algs_info[i].algs_list[j]); + dd->pdata->algs_info[i].registered--; + } tasklet_kill(&dd->done_task); pm_runtime_disable(&pdev->dev); -- cgit v1.2.3 From 320bdbd816156f9ca07e5fed7bfb449f2908dda7 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sat, 30 May 2020 15:35:37 +0200 Subject: crypto: cavium/nitrox - Fix 'nitrox_get_first_device()' when ndevlist is fully iterated When a list is completely iterated with 'list_for_each_entry(x, ...)', x is not NULL at the end. While at it, remove a useless initialization of the ndev variable. It is overridden by 'list_for_each_entry'. Fixes: f2663872f073 ("crypto: cavium - Register the CNN55XX supported crypto algorithms.") Cc: Signed-off-by: Christophe JAILLET Signed-off-by: Herbert Xu --- drivers/crypto/cavium/nitrox/nitrox_main.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c index 788c6607078b..cee2a2713038 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_main.c +++ b/drivers/crypto/cavium/nitrox/nitrox_main.c @@ -278,7 +278,7 @@ static void nitrox_remove_from_devlist(struct nitrox_device *ndev) struct nitrox_device *nitrox_get_first_device(void) { - struct nitrox_device *ndev = NULL; + struct nitrox_device *ndev; mutex_lock(&devlist_lock); list_for_each_entry(ndev, &ndevlist, list) { @@ -286,7 +286,7 @@ struct nitrox_device *nitrox_get_first_device(void) break; } mutex_unlock(&devlist_lock); - if (!ndev) + if (&ndev->list == &ndevlist) return NULL; refcount_inc(&ndev->refcnt); -- cgit v1.2.3