Fix some memory leaks in the openssl app

In some error cases the normal cleanup did not
happen, but instead an exit(1) which caused some
memory leaks, as reported in #22049.

Reviewed-by: Tom Cosgrove <tom.cosgrove@arm.com>
Reviewed-by: Nicola Tuveri <nic.tuv@gmail.com>
(Merged from https://github.com/openssl/openssl/pull/22055)

8 files changed, 18 insertions, 4 deletions

diff --git a/apps/dgst.c b/apps/dgst.c
index fe05b312d7..28123f813f 100644
--- a/apps/dgst.c
+++ b/apps/dgst.c
@@ -321,6 +321,8 @@ int dgst_main(int argc, char **argv)
         sigkey = app_keygen(mac_ctx, mac_name, 0, 0 /* not verbose */);
         /* Verbose output would make external-tests gost-engine fail */
         EVP_PKEY_CTX_free(mac_ctx);
+        if (sigkey == NULL)
+            goto end;
     }
 
     if (hmac_key != NULL) {
diff --git a/apps/dhparam.c b/apps/dhparam.c
index a41e70fe38..d14c569503 100644
--- a/apps/dhparam.c
+++ b/apps/dhparam.c
@@ -233,6 +233,8 @@ int dhparam_main(int argc, char **argv)
         }
 
         tmppkey = app_paramgen(ctx, alg);
+        if (tmppkey == NULL)
+            goto end;
         EVP_PKEY_CTX_free(ctx);
         ctx = NULL;
         if (dsaparam) {
diff --git a/apps/dsaparam.c b/apps/dsaparam.c
index 4eb157042e..8bd2e1361b 100644
--- a/apps/dsaparam.c
+++ b/apps/dsaparam.c
@@ -232,6 +232,8 @@ int dsaparam_main(int argc, char **argv)
             goto end;
         }
         pkey = app_keygen(ctx, "DSA", numbits, verbose);
+        if (pkey == NULL)
+            goto end;
         assert(private);
         if (outformat == FORMAT_ASN1)
             i = i2d_PrivateKey_bio(out, pkey);
diff --git a/apps/gendsa.c b/apps/gendsa.c
index bd8aecedbd..b6d1d0f5b3 100644
--- a/apps/gendsa.c
+++ b/apps/gendsa.c
@@ -148,6 +148,8 @@ int gendsa_main(int argc, char **argv)
         goto end;
     }
     pkey = app_keygen(ctx, "DSA", nbits, verbose);
+    if (pkey == NULL)
+        goto end;
 
     assert(private);
     if (!PEM_write_bio_PrivateKey(out, pkey, enc, NULL, 0, NULL, passout)) {
diff --git a/apps/genpkey.c b/apps/genpkey.c
index 5a59dae681..080f1f6075 100644
--- a/apps/genpkey.c
+++ b/apps/genpkey.c
@@ -234,6 +234,8 @@ int genpkey_main(int argc, char **argv)
 
     pkey = do_param ? app_paramgen(ctx, algname)
                     : app_keygen(ctx, algname, 0, 0 /* not verbose */);
+    if (pkey == NULL)
+        goto end;
 
     if (do_param) {
         rv = PEM_write_bio_Parameters(out, pkey);
diff --git a/apps/genrsa.c b/apps/genrsa.c
index 3151de646b..f71bc6eeb1 100644
--- a/apps/genrsa.c
+++ b/apps/genrsa.c
@@ -204,6 +204,8 @@ opthelp:
         goto end;
     }
     pkey = app_keygen(ctx, "RSA", num, verbose);
+    if (pkey == NULL)
+        goto end;
 
     if (verbose) {
         BIGNUM *e = NULL;
diff --git a/apps/lib/apps.c b/apps/lib/apps.c
index 8aad9a1ef7..6b2a4b86ce 100644
--- a/apps/lib/apps.c
+++ b/apps/lib/apps.c
@@ -3417,8 +3417,8 @@ EVP_PKEY *app_keygen(EVP_PKEY_CTX *ctx, const char *alg, int bits, int verbose)
         BIO_printf(bio_err, "Warning: generating random key material may take a long time\n"
                    "if the system has a poor entropy source\n");
     if (EVP_PKEY_keygen(ctx, &res) <= 0)
-        app_bail_out("%s: Error generating %s key\n", opt_getprog(),
-                     alg != NULL ? alg : "asymmetric");
+        BIO_printf(bio_err, "%s: Error generating %s key\n", opt_getprog(),
+                   alg != NULL ? alg : "asymmetric");
     return res;
 }
 
@@ -3430,8 +3430,8 @@ EVP_PKEY *app_paramgen(EVP_PKEY_CTX *ctx, const char *alg)
         BIO_printf(bio_err, "Warning: generating random key parameters may take a long time\n"
                    "if the system has a poor entropy source\n");
     if (EVP_PKEY_paramgen(ctx, &res) <= 0)
-        app_bail_out("%s: Generating %s key parameters failed\n",
-                     opt_getprog(), alg != NULL ? alg : "asymmetric");
+        BIO_printf(bio_err, "%s: Generating %s key parameters failed\n",
+                   opt_getprog(), alg != NULL ? alg : "asymmetric");
     return res;
 }
 
diff --git a/apps/req.c b/apps/req.c
index c4c9ba292c..3ce2b38496 100644
--- a/apps/req.c
+++ b/apps/req.c
@@ -663,6 +663,8 @@ int req_main(int argc, char **argv)
             EVP_PKEY_CTX_set_cb(genctx, progress_cb);
 
         pkey = app_keygen(genctx, keyalgstr, newkey_len, verbose);
+        if (pkey == NULL)
+            goto end;
 
         EVP_PKEY_CTX_free(genctx);
         genctx = NULL;