APPS & TEST: Eliminate as much use of EVP_PKEY_size() as possible

Some uses were going against documented recommendations.

Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/10798)

1 files changed, 9 insertions, 4 deletions

diff --git a/apps/dgst.c b/apps/dgst.c
index 21f0f01787..7a81cb28dc 100644
--- a/apps/dgst.c
+++ b/apps/dgst.c
@@ -541,11 +541,16 @@ int do_fp(BIO *out, unsigned char *buf, BIO *bp, int sep, int binout,
     }
     if (key != NULL) {
         EVP_MD_CTX *ctx;
-        int pkey_len;
+        size_t tmplen;
+
         BIO_get_md_ctx(bp, &ctx);
-        pkey_len = EVP_PKEY_size(key);
-        if (pkey_len > BUFSIZE) {
-            len = pkey_len;
+        if (!EVP_DigestSignFinal(ctx, NULL, &tmplen)) {
+            BIO_printf(bio_err, "Error Signing Data\n");
+            ERR_print_errors(bio_err);
+            goto end;
+        }
+        if (tmplen > BUFSIZE) {
+            len = tmplen;
             sigbuf = app_malloc(len, "Signature buffer");
             buf = sigbuf;
         }