Fix pkeyutl/rsautl empty encrypt-input/decrypt-output handling

Also fix option processing in pkeyutl to allow use of (formerly)
"out-of-order" switches that were needless implementation limitations.

RT2018

Reviewed-by: Rich Salz <rsalz@openssl.org>

4 files changed, 63 insertions, 43 deletions

diff --git a/apps/apps.c b/apps/apps.c
index 2e778054ca..b1dd97038f 100644
--- a/apps/apps.c
+++ b/apps/apps.c
@@ -2442,7 +2442,11 @@ int bio_to_mem(unsigned char **out, int maxlen, BIO *in)
         else
             len = 1024;
         len = BIO_read(in, tbuf, len);
-        if (len <= 0)
+        if (len < 0) {
+            BIO_free(mem);
+            return -1;
+        }
+        if (len == 0)
             break;
         if (BIO_write(mem, tbuf, len) != len) {
             BIO_free(mem);
@@ -2459,7 +2463,7 @@ int bio_to_mem(unsigned char **out, int maxlen, BIO *in)
     return ret;
 }
 
-int pkey_ctrl_string(EVP_PKEY_CTX *ctx, char *value)
+int pkey_ctrl_string(EVP_PKEY_CTX *ctx, const char *value)
 {
     int rv;
     char *stmp, *vtmp = NULL;
diff --git a/apps/apps.h b/apps/apps.h
index 8276e70869..19bf5cc333 100644
--- a/apps/apps.h
+++ b/apps/apps.h
@@ -321,7 +321,7 @@ int args_verify(char ***pargs, int *pargc,
                 int *badarg, BIO *err, X509_VERIFY_PARAM **pm);
 void policies_print(BIO *out, X509_STORE_CTX *ctx);
 int bio_to_mem(unsigned char **out, int maxlen, BIO *in);
-int pkey_ctrl_string(EVP_PKEY_CTX *ctx, char *value);
+int pkey_ctrl_string(EVP_PKEY_CTX *ctx, const char *value);
 int init_gen_str(BIO *err, EVP_PKEY_CTX **pctx,
                  const char *algname, ENGINE *e, int do_param);
 int do_X509_sign(BIO *err, X509 *x, EVP_PKEY *pkey, const EVP_MD *md,
diff --git a/apps/pkeyutl.c b/apps/pkeyutl.c
index c8d513b44a..e47206c40a 100644
--- a/apps/pkeyutl.c
+++ b/apps/pkeyutl.c
@@ -73,7 +73,7 @@ static void usage(void);
 #define PROG pkeyutl_main
 
 static EVP_PKEY_CTX *init_ctx(int *pkeysize,
-                              char *keyfile, int keyform, int key_type,
+                              const char *keyfile, int keyform, int key_type,
                               char *passargin, int pkey_op, ENGINE *e,
                               int   impl);
 
@@ -99,10 +99,12 @@ int MAIN(int argc, char **argv)
     char *passargin = NULL;
     int keysize = -1;
     int engine_impl = 0;
-
     unsigned char *buf_in = NULL, *buf_out = NULL, *sig = NULL;
-    size_t buf_outlen;
+    size_t buf_outlen = 0;
     int buf_inlen = 0, siglen = -1;
+    const char *inkey = NULL;
+    const char *peerkey = NULL;
+    STACK_OF(OPENSSL_STRING) *pkeyopts = NULL;
 
     int ret = 1, rv = -1;
 
@@ -136,21 +138,13 @@ int MAIN(int argc, char **argv)
         } else if (!strcmp(*argv, "-inkey")) {
             if (--argc < 1)
                 badarg = 1;
-            else {
-                ctx = init_ctx(&keysize,
-                               *(++argv), keyform, key_type,
-                               passargin, pkey_op, e, engine_impl);
-                if (!ctx) {
-                    BIO_puts(bio_err, "Error initializing context\n");
-                    ERR_print_errors(bio_err);
-                    badarg = 1;
-                }
-            }
+            else
+                inkey = *++argv;
         } else if (!strcmp(*argv, "-peerkey")) {
             if (--argc < 1)
                 badarg = 1;
-            else if (!setup_peer(bio_err, ctx, peerform, *(++argv), e))
-                badarg = 1;
+            else
+                peerkey = *++argv;
         } else if (!strcmp(*argv, "-passin")) {
             if (--argc < 1)
                 badarg = 1;
@@ -191,23 +185,21 @@ int MAIN(int argc, char **argv)
             pkey_op = EVP_PKEY_OP_VERIFY;
         else if (!strcmp(*argv, "-verifyrecover"))
             pkey_op = EVP_PKEY_OP_VERIFYRECOVER;
-        else if (!strcmp(*argv, "-rev"))
-            rev = 1;
         else if (!strcmp(*argv, "-encrypt"))
             pkey_op = EVP_PKEY_OP_ENCRYPT;
         else if (!strcmp(*argv, "-decrypt"))
             pkey_op = EVP_PKEY_OP_DECRYPT;
         else if (!strcmp(*argv, "-derive"))
             pkey_op = EVP_PKEY_OP_DERIVE;
+        else if (!strcmp(*argv, "-rev"))
+            rev = 1;
         else if (strcmp(*argv, "-pkeyopt") == 0) {
             if (--argc < 1)
                 badarg = 1;
-            else if (!ctx) {
-                BIO_puts(bio_err, "-pkeyopt command before -inkey\n");
-                badarg = 1;
-            } else if (pkey_ctrl_string(ctx, *(++argv)) <= 0) {
-                BIO_puts(bio_err, "parameter setting error\n");
-                ERR_print_errors(bio_err);
+            else if ((pkeyopts == NULL &&
+                     (pkeyopts = sk_OPENSSL_STRING_new_null()) == NULL) ||
+                    sk_OPENSSL_STRING_push(pkeyopts, *++argv) == 0) {
+                BIO_puts(bio_err, "out of memory\n");
                 goto end;
             }
         } else
@@ -220,10 +212,37 @@ int MAIN(int argc, char **argv)
         argv++;
     }
 
-    if (!ctx) {
+    if (inkey == NULL ||
+        (peerkey != NULL && pkey_op != EVP_PKEY_OP_DERIVE)) {
         usage();
         goto end;
     }
+    ctx = init_ctx(&keysize, inkey, keyform, key_type,
+                   passargin, pkey_op, e, engine_impl);
+    if (!ctx) {
+        BIO_puts(bio_err, "Error initializing context\n");
+        ERR_print_errors(bio_err);
+        goto end;
+    }
+    if (peerkey != NULL && !setup_peer(bio_err, ctx, peerform, peerkey, e)) {
+        BIO_puts(bio_err, "Error setting up peer key\n");
+        ERR_print_errors(bio_err);
+        goto end;
+    }
+    if (pkeyopts != NULL) {
+        int num = sk_OPENSSL_STRING_num(pkeyopts);
+        int i;
+
+        for (i = 0; i < num; ++i) {
+            const char *opt = sk_OPENSSL_STRING_value(pkeyopts, i);
+
+            if (pkey_ctrl_string(ctx, opt) <= 0) {
+                BIO_puts(bio_err, "parameter setting error\n");
+                ERR_print_errors(bio_err);
+                goto end;
+            }
+        }
+    }
 
     if (sigfile && (pkey_op != EVP_PKEY_OP_VERIFY)) {
         BIO_puts(bio_err, "Signature file specified for non verify\n");
@@ -273,7 +292,7 @@ int MAIN(int argc, char **argv)
         }
         siglen = bio_to_mem(&sig, keysize * 10, sigbio);
         BIO_free(sigbio);
-        if (siglen <= 0) {
+        if (siglen < 0) {
             BIO_printf(bio_err, "Error reading signature data\n");
             goto end;
         }
@@ -282,7 +301,7 @@ int MAIN(int argc, char **argv)
     if (in) {
         /* Read the input data */
         buf_inlen = bio_to_mem(&buf_in, keysize * 10, in);
-        if (buf_inlen <= 0) {
+        if (buf_inlen < 0) {
             BIO_printf(bio_err, "Error reading input Data\n");
             exit(1);
         }
@@ -310,7 +329,7 @@ int MAIN(int argc, char **argv)
     } else {
         rv = do_keyop(ctx, pkey_op, NULL, (size_t *)&buf_outlen,
                       buf_in, (size_t)buf_inlen);
-        if (rv > 0) {
+        if (rv > 0 && buf_outlen != 0) {
             buf_out = OPENSSL_malloc(buf_outlen);
             if (!buf_out)
                 rv = -1;
@@ -340,12 +359,14 @@ int MAIN(int argc, char **argv)
         EVP_PKEY_CTX_free(ctx);
     BIO_free(in);
     BIO_free_all(out);
-    if (buf_in)
+    if (buf_in != NULL)
         OPENSSL_free(buf_in);
-    if (buf_out)
+    if (buf_out != NULL)
         OPENSSL_free(buf_out);
-    if (sig)
+    if (sig != NULL)
         OPENSSL_free(sig);
+    if (pkeyopts != NULL)
+        sk_OPENSSL_STRING_free(pkeyopts);
     return ret;
 }
 
@@ -380,7 +401,7 @@ static void usage()
 }
 
 static EVP_PKEY_CTX *init_ctx(int *pkeysize,
-                              char *keyfile, int keyform, int key_type,
+                              const char *keyfile, int keyform, int key_type,
                               char *passargin, int pkey_op, ENGINE *e,
                               int   engine_impl)
 {
@@ -484,14 +505,9 @@ static int setup_peer(BIO *err, EVP_PKEY_CTX *ctx, int peerform,
     EVP_PKEY *peer = NULL;
     ENGINE* engine = NULL;
     int ret;
-    if (!ctx) {
-        BIO_puts(err, "-peerkey command before -inkey\n");
-        return 0;
-    }
 
     if (peerform == FORMAT_ENGINE)
-      engine = e;
-
+        engine = e;
     peer = load_pubkey(bio_err, file, peerform, 0, NULL, engine, "Peer Key");
 
     if (!peer) {
diff --git a/apps/rsautl.c b/apps/rsautl.c
index d642f9ad97..5b6f849ea7 100644
--- a/apps/rsautl.c
+++ b/apps/rsautl.c
@@ -250,7 +250,7 @@ int MAIN(int argc, char **argv)
 
     if (outfile) {
         if (!(out = BIO_new_file(outfile, "wb"))) {
-            BIO_printf(bio_err, "Error Reading Output File\n");
+            BIO_printf(bio_err, "Error Writing Output File\n");
             ERR_print_errors(bio_err);
             goto end;
         }
@@ -276,7 +276,7 @@ int MAIN(int argc, char **argv)
 
     /* Read the input data */
     rsa_inlen = BIO_read(in, rsa_in, keysize * 2);
-    if (rsa_inlen <= 0) {
+    if (rsa_inlen < 0) {
         BIO_printf(bio_err, "Error reading input Data\n");
         exit(1);
     }
@@ -311,7 +311,7 @@ int MAIN(int argc, char **argv)
 
     }
 
-    if (rsa_outlen <= 0) {
+    if (rsa_outlen < 0) {
         BIO_printf(bio_err, "RSA operation error\n");
         ERR_print_errors(bio_err);
         goto end;