APPS & TEST: Eliminate as much use of EVP_PKEY_size() as possible

Some uses were going against documented recommendations.

Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/10798)

2 files changed, 10 insertions, 11 deletions

diff --git a/apps/dgst.c b/apps/dgst.c
index 21f0f01787..7a81cb28dc 100644
--- a/apps/dgst.c
+++ b/apps/dgst.c
@@ -541,11 +541,16 @@ int do_fp(BIO *out, unsigned char *buf, BIO *bp, int sep, int binout,
     }
     if (key != NULL) {
         EVP_MD_CTX *ctx;
-        int pkey_len;
+        size_t tmplen;
+
         BIO_get_md_ctx(bp, &ctx);
-        pkey_len = EVP_PKEY_size(key);
-        if (pkey_len > BUFSIZE) {
-            len = pkey_len;
+        if (!EVP_DigestSignFinal(ctx, NULL, &tmplen)) {
+            BIO_printf(bio_err, "Error Signing Data\n");
+            ERR_print_errors(bio_err);
+            goto end;
+        }
+        if (tmplen > BUFSIZE) {
+            len = tmplen;
             sigbuf = app_malloc(len, "Signature buffer");
             buf = sigbuf;
         }
diff --git a/test/evp_extra_test.c b/test/evp_extra_test.c
index e7e73cd150..5f2bcc1a51 100644
--- a/test/evp_extra_test.c
+++ b/test/evp_extra_test.c
@@ -582,10 +582,7 @@ static int test_EVP_DigestSignInit(int tst)
 
     /* Determine the size of the signature. */
     if (!TEST_true(EVP_DigestSignFinal(md_ctx, NULL, &sig_len))
-            || !TEST_size_t_le(sig_len, (size_t)EVP_PKEY_size(pkey)))
-        goto out;
-
-    if (!TEST_ptr(sig = OPENSSL_malloc(sig_len))
+            || !TEST_ptr(sig = OPENSSL_malloc(sig_len))
             || !TEST_true(EVP_DigestSignFinal(md_ctx, sig, &sig_len)))
         goto out;
 
@@ -919,9 +916,6 @@ static int test_EVP_SM2(void)
     if (!TEST_true(EVP_DigestSignFinal(md_ctx, NULL, &sig_len)))
         goto done;
 
-    if (!TEST_size_t_eq(sig_len, (size_t)EVP_PKEY_size(pkey)))
-        goto done;
-
     if (!TEST_ptr(sig = OPENSSL_malloc(sig_len)))
         goto done;