Big apps cleanup (option-parsing, etc)

This is merges the old "rsalz-monolith" branch over to master.  The biggest
change is that option parsing switch from cascasding 'else if strcmp("-foo")'
to a utility routine and somethin akin to getopt.  Also, an error in the
command line no longer prints the full summary; use -help (or --help :)
for that.  There have been many other changes and code-cleanup, see
bullet list below.

Special thanks to Matt for the long and detailed code review.

TEMPORARY:
        For now, comment out CRYPTO_mem_leaks() at end of main

Tickets closed:
        RT3515: Use 3DES in pkcs12 if built with no-rc2
        RT1766: s_client -reconnect and -starttls broke
        RT2932: Catch write errors
        RT2604: port should be 'unsigned short'
        RT2983: total_bytes undeclared #ifdef RENEG
        RT1523: Add -nocert to fix output in x509 app
        RT3508: Remove unused variable introduced by b09eb24
        RT3511: doc fix; req default serial is random
        RT1325,2973: Add more extensions to c_rehash
        RT2119,3407: Updated to dgst.pod
        RT2379: Additional typo fix
        RT2693: Extra include of string.h
        RT2880: HFS is case-insensitive filenames
        RT3246: req command prints version number wrong

Other changes; incompatibilities marked with *:
        Add SCSV support
        Add -misalign to speed command
        Make dhparam, dsaparam, ecparam, x509 output C in proper style
        Make some internal ocsp.c functions void
        Only display cert usages with -help in verify
        Use global bio_err, remove "BIO*err" parameter from functions
        For filenames, - always means stdin (or stdout as appropriate)
        Add aliases for -des/aes "wrap" ciphers.
        *Remove support for IISSGC (server gated crypto)
        *The undocumented OCSP -header flag is now "-header name=value"
        *Documented the OCSP -header flag

Reviewed-by: Matt Caswell <matt@openssl.org>

1 files changed, 121 insertions, 157 deletions

diff --git a/apps/rsautl.c b/apps/rsautl.c
index d642f9ad97..04667469f6 100644
--- a/apps/rsautl.c
+++ b/apps/rsautl.c
@@ -1,4 +1,3 @@
-/* rsautl.c */
 /*
  * Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL project
  * 2000.
@@ -75,150 +74,162 @@
 # define KEY_PUBKEY      2
 # define KEY_CERT        3
 
-static void usage(void);
-
-# undef PROG
-
-# define PROG rsautl_main
-
-int MAIN(int argc, char **);
+typedef enum OPTION_choice {
+    OPT_ERR = -1, OPT_EOF = 0, OPT_HELP,
+    OPT_ENGINE, OPT_IN, OPT_OUT, OPT_ASN1PARSE, OPT_HEXDUMP,
+    OPT_RAW, OPT_OAEP, OPT_SSL, OPT_PKCS, OPT_X931,
+    OPT_SIGN, OPT_VERIFY, OPT_REV, OPT_ENCRYPT, OPT_DECRYPT,
+    OPT_PUBIN, OPT_CERTIN, OPT_INKEY, OPT_PASSIN, OPT_KEYFORM
+} OPTION_CHOICE;
+
+OPTIONS rsautl_options[] = {
+    {"help", OPT_HELP, '-', "Display this summary"},
+    {"in", OPT_IN, '<', "Input file"},
+    {"out", OPT_OUT, '>', "Output file"},
+    {"inkey", OPT_INKEY, '<', "Input key"},
+    {"keyform", OPT_KEYFORM, 'F', "Private key format - default PEM"},
+    {"pubin", OPT_PUBIN, '-', "Input is an RSA public"},
+    {"certin", OPT_CERTIN, '-', "Input is a cert carrying an RSA public key"},
+    {"ssl", OPT_SSL, '-', "Use SSL v2 padding"},
+    {"raw", OPT_RAW, '-', "Use no padding"},
+    {"pkcs", OPT_PKCS, '-', "Use PKCS#1 v1.5 padding (default)"},
+    {"oaep", OPT_OAEP, '-', "Use PKCS#1 OAEP"},
+    {"sign", OPT_SIGN, '-', "Sign with private key"},
+    {"verify", OPT_VERIFY, '-', "Verify with public key"},
+    {"asn1parse", OPT_ASN1PARSE, '-'},
+    {"hexdump", OPT_HEXDUMP, '-', "Hex dump output"},
+    {"x931", OPT_X931, '-', "Use ANSI X9.31 padding"},
+    {"rev", OPT_REV, '-'},
+    {"encrypt", OPT_ENCRYPT, '-', "Encrypt with public key"},
+    {"decrypt", OPT_DECRYPT, '-', "Decrypt with private key"},
+    {"passin", OPT_PASSIN, 's', "Pass phrase source"},
+# ifndef OPENSSL_NO_ENGINE
+    {"engine", OPT_ENGINE, 's', "Use engine, possibly a hardware device"},
+# endif
+    {NULL}
+};
 
-int MAIN(int argc, char **argv)
+int rsautl_main(int argc, char **argv)
 {
-    ENGINE *e = NULL;
     BIO *in = NULL, *out = NULL;
-    char *infile = NULL, *outfile = NULL;
-# ifndef OPENSSL_NO_ENGINE
-    char *engine = NULL;
-# endif
-    char *keyfile = NULL;
-    char rsa_mode = RSA_VERIFY, key_type = KEY_PRIVKEY;
-    int keyform = FORMAT_PEM;
-    char need_priv = 0, badarg = 0, rev = 0;
-    char hexdump = 0, asn1parse = 0;
-    X509 *x;
+    ENGINE *e = NULL;
     EVP_PKEY *pkey = NULL;
     RSA *rsa = NULL;
-    unsigned char *rsa_in = NULL, *rsa_out = NULL, pad;
-    char *passargin = NULL, *passin = NULL;
-    int rsa_inlen, rsa_outlen = 0;
-    int keysize;
-
-    int ret = 1;
-
-    argc--;
-    argv++;
-
-    if (!bio_err)
-        bio_err = BIO_new_fp(stderr, BIO_NOCLOSE);
-
-    if (!load_config(bio_err, NULL))
-        goto end;
-    ERR_load_crypto_strings();
-    OpenSSL_add_all_algorithms();
-    pad = RSA_PKCS1_PADDING;
-
-    while (argc >= 1) {
-        if (!strcmp(*argv, "-in")) {
-            if (--argc < 1)
-                badarg = 1;
-            else
-                infile = *(++argv);
-        } else if (!strcmp(*argv, "-out")) {
-            if (--argc < 1)
-                badarg = 1;
-            else
-                outfile = *(++argv);
-        } else if (!strcmp(*argv, "-inkey")) {
-            if (--argc < 1)
-                badarg = 1;
-            else
-                keyfile = *(++argv);
-        } else if (!strcmp(*argv, "-passin")) {
-            if (--argc < 1)
-                badarg = 1;
-            else
-                passargin = *(++argv);
-        } else if (strcmp(*argv, "-keyform") == 0) {
-            if (--argc < 1)
-                badarg = 1;
-            else
-                keyform = str2fmt(*(++argv));
-# ifndef OPENSSL_NO_ENGINE
-        } else if (!strcmp(*argv, "-engine")) {
-            if (--argc < 1)
-                badarg = 1;
-            else
-                engine = *(++argv);
-# endif
-        } else if (!strcmp(*argv, "-pubin")) {
-            key_type = KEY_PUBKEY;
-        } else if (!strcmp(*argv, "-certin")) {
-            key_type = KEY_CERT;
-        } else if (!strcmp(*argv, "-asn1parse"))
+    X509 *x;
+    char *engine = NULL, *infile = NULL, *outfile = NULL, *keyfile = NULL;
+    char *passinarg = NULL, *passin = NULL, *prog;
+    char rsa_mode = RSA_VERIFY, key_type = KEY_PRIVKEY;
+    unsigned char *rsa_in = NULL, *rsa_out = NULL, pad = RSA_PKCS1_PADDING;
+    int rsa_inlen, keyformat = FORMAT_PEM, keysize, ret = 1;
+    int rsa_outlen = 0, hexdump = 0, asn1parse = 0, need_priv = 0, rev = 0;
+    OPTION_CHOICE o;
+
+    prog = opt_init(argc, argv, rsautl_options);
+    while ((o = opt_next()) != OPT_EOF) {
+        switch (o) {
+        case OPT_EOF:
+        case OPT_ERR:
+ opthelp:
+            BIO_printf(bio_err, "%s: Use -help for summary.\n", prog);
+            goto end;
+        case OPT_HELP:
+            opt_help(rsautl_options);
+            ret = 0;
+            goto end;
+        case OPT_KEYFORM:
+            if (!opt_format(opt_arg(), OPT_FMT_PEMDER, &keyformat))
+                goto opthelp;
+            break;
+        case OPT_IN:
+            infile = opt_arg();
+            break;
+        case OPT_OUT:
+            outfile = opt_arg();
+            break;
+        case OPT_ENGINE:
+            engine = opt_arg();
+            break;
+        case OPT_ASN1PARSE:
             asn1parse = 1;
-        else if (!strcmp(*argv, "-hexdump"))
+            break;
+        case OPT_HEXDUMP:
             hexdump = 1;
-        else if (!strcmp(*argv, "-raw"))
+            break;
+        case OPT_RAW:
             pad = RSA_NO_PADDING;
-        else if (!strcmp(*argv, "-oaep"))
+            break;
+        case OPT_OAEP:
             pad = RSA_PKCS1_OAEP_PADDING;
-        else if (!strcmp(*argv, "-ssl"))
+            break;
+        case OPT_SSL:
             pad = RSA_SSLV23_PADDING;
-        else if (!strcmp(*argv, "-pkcs"))
+            break;
+        case OPT_PKCS:
             pad = RSA_PKCS1_PADDING;
-        else if (!strcmp(*argv, "-x931"))
+            break;
+        case OPT_X931:
             pad = RSA_X931_PADDING;
-        else if (!strcmp(*argv, "-sign")) {
+            break;
+        case OPT_SIGN:
             rsa_mode = RSA_SIGN;
             need_priv = 1;
-        } else if (!strcmp(*argv, "-verify"))
+            break;
+        case OPT_VERIFY:
             rsa_mode = RSA_VERIFY;
-        else if (!strcmp(*argv, "-rev"))
+            break;
+        case OPT_REV:
             rev = 1;
-        else if (!strcmp(*argv, "-encrypt"))
+            break;
+        case OPT_ENCRYPT:
             rsa_mode = RSA_ENCRYPT;
-        else if (!strcmp(*argv, "-decrypt")) {
+            break;
+        case OPT_DECRYPT:
             rsa_mode = RSA_DECRYPT;
             need_priv = 1;
-        } else
-            badarg = 1;
-        if (badarg) {
-            usage();
-            goto end;
+            break;
+        case OPT_PUBIN:
+            key_type = KEY_PUBKEY;
+            break;
+        case OPT_CERTIN:
+            key_type = KEY_CERT;
+            break;
+        case OPT_INKEY:
+            keyfile = opt_arg();
+            break;
+        case OPT_PASSIN:
+            passinarg = opt_arg();
+            break;
         }
-        argc--;
-        argv++;
     }
+    argc = opt_num_rest();
+    argv = opt_rest();
 
     if (need_priv && (key_type != KEY_PRIVKEY)) {
         BIO_printf(bio_err, "A private key is needed for this operation\n");
         goto end;
     }
 # ifndef OPENSSL_NO_ENGINE
-    e = setup_engine(bio_err, engine, 0);
+    e = setup_engine(engine, 0);
 # endif
-    if (!app_passwd(bio_err, passargin, NULL, &passin, NULL)) {
+    if (!app_passwd(passinarg, NULL, &passin, NULL)) {
         BIO_printf(bio_err, "Error getting password\n");
         goto end;
     }
 
 /* FIXME: seed PRNG only if needed */
-    app_RAND_load_file(NULL, bio_err, 0);
+    app_RAND_load_file(NULL, 0);
 
     switch (key_type) {
     case KEY_PRIVKEY:
-        pkey = load_key(bio_err, keyfile, keyform, 0,
-                        passin, e, "Private Key");
+        pkey = load_key(keyfile, keyformat, 0, passin, e, "Private Key");
         break;
 
     case KEY_PUBKEY:
-        pkey = load_pubkey(bio_err, keyfile, keyform, 0,
-                           NULL, e, "Public Key");
+        pkey = load_pubkey(keyfile, keyformat, 0, NULL, e, "Public Key");
         break;
 
     case KEY_CERT:
-        x = load_cert(bio_err, keyfile, keyform, NULL, e, "Certificate");
+        x = load_cert(keyfile, keyformat, NULL, e, "Certificate");
         if (x) {
             pkey = X509_get_pubkey(x);
             X509_free(x);
@@ -239,30 +250,12 @@ int MAIN(int argc, char **argv)
         goto end;
     }
 
-    if (infile) {
-        if (!(in = BIO_new_file(infile, "rb"))) {
-            BIO_printf(bio_err, "Error Reading Input File\n");
-            ERR_print_errors(bio_err);
-            goto end;
-        }
-    } else
-        in = BIO_new_fp(stdin, BIO_NOCLOSE);
-
-    if (outfile) {
-        if (!(out = BIO_new_file(outfile, "wb"))) {
-            BIO_printf(bio_err, "Error Reading Output File\n");
-            ERR_print_errors(bio_err);
-            goto end;
-        }
-    } else {
-        out = BIO_new_fp(stdout, BIO_NOCLOSE);
-# ifdef OPENSSL_SYS_VMS
-        {
-            BIO *tmpbio = BIO_new(BIO_f_linebuffer());
-            out = BIO_push(tmpbio, out);
-        }
-# endif
-    }
+    in = bio_open_default(infile, "rb");
+    if (in == NULL)
+        goto end;
+    out = bio_open_default(outfile, "wb");
+    if (out == NULL)
+        goto end;
 
     keysize = RSA_size(rsa);
 
@@ -270,7 +263,6 @@ int MAIN(int argc, char **argv)
     rsa_out = OPENSSL_malloc(keysize);
     if (!rsa_in || !rsa_out) {
         BIO_printf(bio_err, "Out of memory\n");
-        ERR_print_errors(bio_err);
         goto end;
     }
 
@@ -278,7 +270,7 @@ int MAIN(int argc, char **argv)
     rsa_inlen = BIO_read(in, rsa_in, keysize * 2);
     if (rsa_inlen <= 0) {
         BIO_printf(bio_err, "Error reading input Data\n");
-        exit(1);
+        goto end;
     }
     if (rev) {
         int i;
@@ -338,34 +330,6 @@ int MAIN(int argc, char **argv)
     return ret;
 }
 
-static void usage()
-{
-    BIO_printf(bio_err, "Usage: rsautl [options]\n");
-    BIO_printf(bio_err, "-in file        input file\n");
-    BIO_printf(bio_err, "-out file       output file\n");
-    BIO_printf(bio_err, "-inkey file     input key\n");
-    BIO_printf(bio_err, "-keyform arg    private key format - default PEM\n");
-    BIO_printf(bio_err, "-pubin          input is an RSA public\n");
-    BIO_printf(bio_err,
-               "-certin         input is a certificate carrying an RSA public key\n");
-    BIO_printf(bio_err, "-ssl            use SSL v2 padding\n");
-    BIO_printf(bio_err, "-raw            use no padding\n");
-    BIO_printf(bio_err,
-               "-pkcs           use PKCS#1 v1.5 padding (default)\n");
-    BIO_printf(bio_err, "-oaep           use PKCS#1 OAEP\n");
-    BIO_printf(bio_err, "-sign           sign with private key\n");
-    BIO_printf(bio_err, "-verify         verify with public key\n");
-    BIO_printf(bio_err, "-encrypt        encrypt with public key\n");
-    BIO_printf(bio_err, "-decrypt        decrypt with private key\n");
-    BIO_printf(bio_err, "-hexdump        hex dump output\n");
-# ifndef OPENSSL_NO_ENGINE
-    BIO_printf(bio_err,
-               "-engine e       use engine e, possibly a hardware device.\n");
-    BIO_printf(bio_err, "-passin arg    pass phrase source\n");
-# endif
-
-}
-
 #else                           /* !OPENSSL_NO_RSA */
 
 # if PEDANTIC