Big apps cleanup (option-parsing, etc)

This is merges the old "rsalz-monolith" branch over to master.  The biggest
change is that option parsing switch from cascasding 'else if strcmp("-foo")'
to a utility routine and somethin akin to getopt.  Also, an error in the
command line no longer prints the full summary; use -help (or --help :)
for that.  There have been many other changes and code-cleanup, see
bullet list below.

Special thanks to Matt for the long and detailed code review.

TEMPORARY:
        For now, comment out CRYPTO_mem_leaks() at end of main

Tickets closed:
        RT3515: Use 3DES in pkcs12 if built with no-rc2
        RT1766: s_client -reconnect and -starttls broke
        RT2932: Catch write errors
        RT2604: port should be 'unsigned short'
        RT2983: total_bytes undeclared #ifdef RENEG
        RT1523: Add -nocert to fix output in x509 app
        RT3508: Remove unused variable introduced by b09eb24
        RT3511: doc fix; req default serial is random
        RT1325,2973: Add more extensions to c_rehash
        RT2119,3407: Updated to dgst.pod
        RT2379: Additional typo fix
        RT2693: Extra include of string.h
        RT2880: HFS is case-insensitive filenames
        RT3246: req command prints version number wrong

Other changes; incompatibilities marked with *:
        Add SCSV support
        Add -misalign to speed command
        Make dhparam, dsaparam, ecparam, x509 output C in proper style
        Make some internal ocsp.c functions void
        Only display cert usages with -help in verify
        Use global bio_err, remove "BIO*err" parameter from functions
        For filenames, - always means stdin (or stdout as appropriate)
        Add aliases for -des/aes "wrap" ciphers.
        *Remove support for IISSGC (server gated crypto)
        *The undocumented OCSP -header flag is now "-header name=value"
        *Documented the OCSP -header flag

Reviewed-by: Matt Caswell <matt@openssl.org>

1 files changed, 165 insertions, 205 deletions

diff --git a/apps/pkeyutl.c b/apps/pkeyutl.c
index 1028686738..942ba05ded 100644
--- a/apps/pkeyutl.c
+++ b/apps/pkeyutl.c
@@ -66,200 +66,194 @@
 #define KEY_PUBKEY      2
 #define KEY_CERT        3
 
-static void usage(void);
-
-#undef PROG
-
-#define PROG pkeyutl_main
-
 static EVP_PKEY_CTX *init_ctx(int *pkeysize,
                               char *keyfile, int keyform, int key_type,
-                              char *passargin, int pkey_op, ENGINE *e);
+                              char *passinarg, int pkey_op, ENGINE *e);
 
-static int setup_peer(BIO *err, EVP_PKEY_CTX *ctx, int peerform,
-                      const char *file);
+static int setup_peer(EVP_PKEY_CTX *ctx, int peerform, const char *file);
 
 static int do_keyop(EVP_PKEY_CTX *ctx, int pkey_op,
                     unsigned char *out, size_t *poutlen,
                     unsigned char *in, size_t inlen);
 
-int MAIN(int argc, char **);
+typedef enum OPTION_choice {
+    OPT_ERR = -1, OPT_EOF = 0, OPT_HELP,
+    OPT_ENGINE, OPT_IN, OPT_OUT,
+    OPT_PUBIN, OPT_CERTIN, OPT_ASN1PARSE, OPT_HEXDUMP, OPT_SIGN,
+    OPT_VERIFY, OPT_VERIFYRECOVER, OPT_REV, OPT_ENCRYPT, OPT_DECRYPT,
+    OPT_DERIVE, OPT_SIGFILE, OPT_INKEY, OPT_PEERKEY, OPT_PASSIN,
+    OPT_PEERFORM, OPT_KEYFORM, OPT_PKEYOPT
+} OPTION_CHOICE;
+
+OPTIONS pkeyutl_options[] = {
+    {"help", OPT_HELP, '-', "Display this summary"},
+    {"in", OPT_IN, '<', "Input file"},
+    {"out", OPT_OUT, '>', "Output file"},
+    {"pubin", OPT_PUBIN, '-', "Input is a public key"},
+    {"certin", OPT_CERTIN, '-', "Input is a cert with a public key"},
+    {"asn1parse", OPT_ASN1PARSE, '-'},
+    {"hexdump", OPT_HEXDUMP, '-', "Hex dump output"},
+    {"sign", OPT_SIGN, '-', "Sign with private key"},
+    {"verify", OPT_VERIFY, '-', "Verify with public key"},
+    {"verifyrecover", OPT_VERIFYRECOVER, '-',
+     "Verify with public key, recover original data"},
+    {"rev", OPT_REV, '-'},
+    {"encrypt", OPT_ENCRYPT, '-', "Encrypt with public key"},
+    {"decrypt", OPT_DECRYPT, '-', "Decrypt with private key"},
+    {"derive", OPT_DERIVE, '-', "Derive shared secret"},
+    {"sigfile", OPT_SIGFILE, '<', "Signature file (verify operation only)"},
+    {"inkey", OPT_INKEY, 's', "Input key"},
+    {"peerkey", OPT_PEERKEY, 's'},
+    {"passin", OPT_PASSIN, 's', "Pass phrase source"},
+    {"peerform", OPT_PEERFORM, 'F'},
+    {"keyform", OPT_KEYFORM, 'F', "Private key format - default PEM"},
+    {"pkeyopt", OPT_PKEYOPT, 's', "Public key options as opt:value"},
+#ifndef OPENSSL_NO_ENGINE
+    {"engine", OPT_ENGINE, 's', "Use engine, possibly a hardware device"},
+#endif
+    {NULL}
+};
 
-int MAIN(int argc, char **argv)
+int pkeyutl_main(int argc, char **argv)
 {
     BIO *in = NULL, *out = NULL;
-    char *infile = NULL, *outfile = NULL, *sigfile = NULL;
     ENGINE *e = NULL;
-    int pkey_op = EVP_PKEY_OP_SIGN, key_type = KEY_PRIVKEY;
-    int keyform = FORMAT_PEM, peerform = FORMAT_PEM;
-    char badarg = 0, rev = 0;
-    char hexdump = 0, asn1parse = 0;
     EVP_PKEY_CTX *ctx = NULL;
-    char *passargin = NULL;
-    int keysize = -1;
-
+    char *infile = NULL, *outfile = NULL, *sigfile = NULL, *passinarg = NULL;
+    char hexdump = 0, asn1parse = 0, rev = 0, *prog;
     unsigned char *buf_in = NULL, *buf_out = NULL, *sig = NULL;
-    size_t buf_outlen;
-    int buf_inlen = 0, siglen = -1;
-
+    OPTION_CHOICE o;
+    int buf_inlen = 0, siglen = -1, keyform = FORMAT_PEM, peerform =
+        FORMAT_PEM;
+    int keysize = -1, pkey_op = EVP_PKEY_OP_SIGN, key_type = KEY_PRIVKEY;
     int ret = 1, rv = -1;
+    size_t buf_outlen;
 
-    argc--;
-    argv++;
-
-    if (!bio_err)
-        bio_err = BIO_new_fp(stderr, BIO_NOCLOSE);
-
-    if (!load_config(bio_err, NULL))
-        goto end;
-    ERR_load_crypto_strings();
-    OpenSSL_add_all_algorithms();
-
-    while (argc >= 1) {
-        if (!strcmp(*argv, "-in")) {
-            if (--argc < 1)
-                badarg = 1;
-            else
-                infile = *(++argv);
-        } else if (!strcmp(*argv, "-out")) {
-            if (--argc < 1)
-                badarg = 1;
-            else
-                outfile = *(++argv);
-        } else if (!strcmp(*argv, "-sigfile")) {
-            if (--argc < 1)
-                badarg = 1;
-            else
-                sigfile = *(++argv);
-        } else if (!strcmp(*argv, "-inkey")) {
-            if (--argc < 1)
-                badarg = 1;
-            else {
-                ctx = init_ctx(&keysize,
-                               *(++argv), keyform, key_type,
-                               passargin, pkey_op, e);
-                if (!ctx) {
-                    BIO_puts(bio_err, "Error initializing context\n");
-                    ERR_print_errors(bio_err);
-                    badarg = 1;
-                }
+    prog = opt_init(argc, argv, pkeyutl_options);
+    while ((o = opt_next()) != OPT_EOF) {
+        switch (o) {
+        case OPT_EOF:
+        case OPT_ERR:
+ opthelp:
+            BIO_printf(bio_err, "%s: Use -help for summary.\n", prog);
+            goto end;
+        case OPT_HELP:
+            opt_help(pkeyutl_options);
+            ret = 0;
+            goto end;
+        case OPT_IN:
+            infile = opt_arg();
+            break;
+        case OPT_OUT:
+            outfile = opt_arg();
+            break;
+        case OPT_SIGFILE:
+            sigfile = opt_arg();
+            break;
+        case OPT_INKEY:
+            ctx = init_ctx(&keysize, opt_arg(), keyform, key_type,
+                           passinarg, pkey_op, e);
+            if (ctx == NULL) {
+                BIO_puts(bio_err, "%s: Error initializing context\n");
+                ERR_print_errors(bio_err);
+                goto opthelp;
             }
-        } else if (!strcmp(*argv, "-peerkey")) {
-            if (--argc < 1)
-                badarg = 1;
-            else if (!setup_peer(bio_err, ctx, peerform, *(++argv)))
-                badarg = 1;
-        } else if (!strcmp(*argv, "-passin")) {
-            if (--argc < 1)
-                badarg = 1;
-            else
-                passargin = *(++argv);
-        } else if (strcmp(*argv, "-peerform") == 0) {
-            if (--argc < 1)
-                badarg = 1;
-            else
-                peerform = str2fmt(*(++argv));
-        } else if (strcmp(*argv, "-keyform") == 0) {
-            if (--argc < 1)
-                badarg = 1;
-            else
-                keyform = str2fmt(*(++argv));
-        }
+            break;
+        case OPT_PEERKEY:
+            if (!setup_peer(ctx, peerform, opt_arg()))
+                goto opthelp;
+            break;
+        case OPT_PASSIN:
+            passinarg = opt_arg();
+            break;
+        case OPT_PEERFORM:
+            if (!opt_format(opt_arg(), OPT_FMT_PEMDER, &peerform))
+                goto opthelp;
+            break;
+        case OPT_KEYFORM:
+            if (!opt_format(opt_arg(), OPT_FMT_PEMDER, &keyform))
+                goto opthelp;
+            break;
 #ifndef OPENSSL_NO_ENGINE
-        else if (!strcmp(*argv, "-engine")) {
-            if (--argc < 1)
-                badarg = 1;
-            else
-                e = setup_engine(bio_err, *(++argv), 0);
-        }
+        case OPT_ENGINE:
+            e = setup_engine(opt_arg(), 0);
+            break;
 #endif
-        else if (!strcmp(*argv, "-pubin"))
+        case OPT_PUBIN:
             key_type = KEY_PUBKEY;
-        else if (!strcmp(*argv, "-certin"))
+            break;
+        case OPT_CERTIN:
             key_type = KEY_CERT;
-        else if (!strcmp(*argv, "-asn1parse"))
+            break;
+        case OPT_ASN1PARSE:
             asn1parse = 1;
-        else if (!strcmp(*argv, "-hexdump"))
+            break;
+        case OPT_HEXDUMP:
             hexdump = 1;
-        else if (!strcmp(*argv, "-sign"))
+            break;
+        case OPT_SIGN:
             pkey_op = EVP_PKEY_OP_SIGN;
-        else if (!strcmp(*argv, "-verify"))
+            break;
+        case OPT_VERIFY:
             pkey_op = EVP_PKEY_OP_VERIFY;
-        else if (!strcmp(*argv, "-verifyrecover"))
+            break;
+        case OPT_VERIFYRECOVER:
             pkey_op = EVP_PKEY_OP_VERIFYRECOVER;
-        else if (!strcmp(*argv, "-rev"))
+            break;
+        case OPT_REV:
             rev = 1;
-        else if (!strcmp(*argv, "-encrypt"))
+        case OPT_ENCRYPT:
             pkey_op = EVP_PKEY_OP_ENCRYPT;
-        else if (!strcmp(*argv, "-decrypt"))
+            break;
+        case OPT_DECRYPT:
             pkey_op = EVP_PKEY_OP_DECRYPT;
-        else if (!strcmp(*argv, "-derive"))
+            break;
+        case OPT_DERIVE:
             pkey_op = EVP_PKEY_OP_DERIVE;
-        else if (strcmp(*argv, "-pkeyopt") == 0) {
-            if (--argc < 1)
-                badarg = 1;
-            else if (!ctx) {
-                BIO_puts(bio_err, "-pkeyopt command before -inkey\n");
-                badarg = 1;
-            } else if (pkey_ctrl_string(ctx, *(++argv)) <= 0) {
-                BIO_puts(bio_err, "parameter setting error\n");
+            break;
+        case OPT_PKEYOPT:
+            if (ctx == NULL) {
+                BIO_printf(bio_err,
+                           "%s: Must have -inkey before -pkeyopt\n", prog);
+                goto opthelp;
+            }
+            if (pkey_ctrl_string(ctx, opt_arg()) <= 0) {
+                BIO_printf(bio_err, "%s: Can't set parameter:\n", prog);
                 ERR_print_errors(bio_err);
                 goto end;
             }
-        } else
-            badarg = 1;
-        if (badarg) {
-            usage();
-            goto end;
+            break;
         }
-        argc--;
-        argv++;
     }
+    argc = opt_num_rest();
+    argv = opt_rest();
 
-    if (!ctx) {
-        usage();
-        goto end;
-    }
+    if (ctx == NULL)
+        goto opthelp;
 
     if (sigfile && (pkey_op != EVP_PKEY_OP_VERIFY)) {
-        BIO_puts(bio_err, "Signature file specified for non verify\n");
+        BIO_printf(bio_err,
+                   "%s: Signature file specified for non verify\n", prog);
         goto end;
     }
 
     if (!sigfile && (pkey_op == EVP_PKEY_OP_VERIFY)) {
-        BIO_puts(bio_err, "No signature file specified for verify\n");
+        BIO_printf(bio_err,
+                   "%s: No signature file specified for verify\n", prog);
         goto end;
     }
 
 /* FIXME: seed PRNG only if needed */
-    app_RAND_load_file(NULL, bio_err, 0);
+    app_RAND_load_file(NULL, 0);
 
     if (pkey_op != EVP_PKEY_OP_DERIVE) {
-        if (infile) {
-            if (!(in = BIO_new_file(infile, "rb"))) {
-                BIO_puts(bio_err, "Error Opening Input File\n");
-                ERR_print_errors(bio_err);
-                goto end;
-            }
-        } else
-            in = BIO_new_fp(stdin, BIO_NOCLOSE);
-    }
-
-    if (outfile) {
-        if (!(out = BIO_new_file(outfile, "wb"))) {
-            BIO_printf(bio_err, "Error Creating Output File\n");
-            ERR_print_errors(bio_err);
+        in = bio_open_default(infile, "rb");
+        if (in == NULL)
             goto end;
-        }
-    } else {
-        out = BIO_new_fp(stdout, BIO_NOCLOSE);
-#ifdef OPENSSL_SYS_VMS
-        {
-            BIO *tmpbio = BIO_new(BIO_f_linebuffer());
-            out = BIO_push(tmpbio, out);
-        }
-#endif
     }
+    out = bio_open_default(outfile, "wb");
+    if (out == NULL)
+        goto end;
 
     if (sigfile) {
         BIO *sigbio = BIO_new_file(sigfile, "rb");
@@ -297,32 +291,30 @@ int MAIN(int argc, char **argv)
     if (pkey_op == EVP_PKEY_OP_VERIFY) {
         rv = EVP_PKEY_verify(ctx, sig, (size_t)siglen,
                              buf_in, (size_t)buf_inlen);
-        if (rv == 0)
-            BIO_puts(out, "Signature Verification Failure\n");
-        else if (rv == 1)
+        if (rv == 1) {
             BIO_puts(out, "Signature Verified Successfully\n");
-        if (rv >= 0)
-            goto end;
-    } else {
-        rv = do_keyop(ctx, pkey_op, NULL, (size_t *)&buf_outlen,
-                      buf_in, (size_t)buf_inlen);
-        if (rv > 0) {
-            buf_out = OPENSSL_malloc(buf_outlen);
-            if (!buf_out)
-                rv = -1;
-            else
-                rv = do_keyop(ctx, pkey_op,
-                              buf_out, (size_t *)&buf_outlen,
-                              buf_in, (size_t)buf_inlen);
-        }
+            ret = 0;
+        } else
+            BIO_puts(out, "Signature Verification Failure\n");
+        goto end;
+    }
+    rv = do_keyop(ctx, pkey_op, NULL, (size_t *)&buf_outlen,
+                  buf_in, (size_t)buf_inlen);
+    if (rv > 0) {
+        buf_out = OPENSSL_malloc(buf_outlen);
+        if (!buf_out)
+            rv = -1;
+        else
+            rv = do_keyop(ctx, pkey_op,
+                          buf_out, (size_t *)&buf_outlen,
+                          buf_in, (size_t)buf_inlen);
     }
-
     if (rv <= 0) {
-        BIO_printf(bio_err, "Public Key operation error\n");
         ERR_print_errors(bio_err);
         goto end;
     }
     ret = 0;
+
     if (asn1parse) {
         if (!ASN1_parse_dump(out, buf_out, buf_outlen, 1, -1))
             ERR_print_errors(bio_err);
@@ -344,38 +336,9 @@ int MAIN(int argc, char **argv)
     return ret;
 }
 
-static void usage()
-{
-    BIO_printf(bio_err, "Usage: pkeyutl [options]\n");
-    BIO_printf(bio_err, "-in file        input file\n");
-    BIO_printf(bio_err, "-out file       output file\n");
-    BIO_printf(bio_err,
-               "-sigfile file signature file (verify operation only)\n");
-    BIO_printf(bio_err, "-inkey file     input key\n");
-    BIO_printf(bio_err, "-keyform arg    private key format - default PEM\n");
-    BIO_printf(bio_err, "-pubin          input is a public key\n");
-    BIO_printf(bio_err,
-               "-certin         input is a certificate carrying a public key\n");
-    BIO_printf(bio_err, "-pkeyopt X:Y    public key options\n");
-    BIO_printf(bio_err, "-sign           sign with private key\n");
-    BIO_printf(bio_err, "-verify         verify with public key\n");
-    BIO_printf(bio_err,
-               "-verifyrecover  verify with public key, recover original data\n");
-    BIO_printf(bio_err, "-encrypt        encrypt with public key\n");
-    BIO_printf(bio_err, "-decrypt        decrypt with private key\n");
-    BIO_printf(bio_err, "-derive         derive shared secret\n");
-    BIO_printf(bio_err, "-hexdump        hex dump output\n");
-#ifndef OPENSSL_NO_ENGINE
-    BIO_printf(bio_err,
-               "-engine e       use engine e, possibly a hardware device.\n");
-#endif
-    BIO_printf(bio_err, "-passin arg     pass phrase source\n");
-
-}
-
 static EVP_PKEY_CTX *init_ctx(int *pkeysize,
                               char *keyfile, int keyform, int key_type,
-                              char *passargin, int pkey_op, ENGINE *e)
+                              char *passinarg, int pkey_op, ENGINE *e)
 {
     EVP_PKEY *pkey = NULL;
     EVP_PKEY_CTX *ctx = NULL;
@@ -388,23 +351,21 @@ static EVP_PKEY_CTX *init_ctx(int *pkeysize,
         BIO_printf(bio_err, "A private key is needed for this operation\n");
         goto end;
     }
-    if (!app_passwd(bio_err, passargin, NULL, &passin, NULL)) {
+    if (!app_passwd(passinarg, NULL, &passin, NULL)) {
         BIO_printf(bio_err, "Error getting password\n");
         goto end;
     }
     switch (key_type) {
     case KEY_PRIVKEY:
-        pkey = load_key(bio_err, keyfile, keyform, 0,
-                        passin, e, "Private Key");
+        pkey = load_key(keyfile, keyform, 0, passin, e, "Private Key");
         break;
 
     case KEY_PUBKEY:
-        pkey = load_pubkey(bio_err, keyfile, keyform, 0,
-                           NULL, e, "Public Key");
+        pkey = load_pubkey(keyfile, keyform, 0, NULL, e, "Public Key");
         break;
 
     case KEY_CERT:
-        x = load_cert(bio_err, keyfile, keyform, NULL, e, "Certificate");
+        x = load_cert(keyfile, keyform, NULL, e, "Certificate");
         if (x) {
             pkey = X509_get_pubkey(x);
             X509_free(x);
@@ -465,21 +426,20 @@ static EVP_PKEY_CTX *init_ctx(int *pkeysize,
 
 }
 
-static int setup_peer(BIO *err, EVP_PKEY_CTX *ctx, int peerform,
-                      const char *file)
+static int setup_peer(EVP_PKEY_CTX *ctx, int peerform, const char *file)
 {
     EVP_PKEY *peer = NULL;
     int ret;
     if (!ctx) {
-        BIO_puts(err, "-peerkey command before -inkey\n");
+        BIO_puts(bio_err, "-peerkey command before -inkey\n");
         return 0;
     }
 
-    peer = load_pubkey(bio_err, file, peerform, 0, NULL, NULL, "Peer Key");
+    peer = load_pubkey(file, peerform, 0, NULL, NULL, "Peer Key");
 
     if (!peer) {
         BIO_printf(bio_err, "Error reading peer key %s\n", file);
-        ERR_print_errors(err);
+        ERR_print_errors(bio_err);
         return 0;
     }
 
@@ -487,7 +447,7 @@ static int setup_peer(BIO *err, EVP_PKEY_CTX *ctx, int peerform,
 
     EVP_PKEY_free(peer);
     if (ret <= 0)
-        ERR_print_errors(err);
+        ERR_print_errors(bio_err);
     return ret;
 }