Make the handling of output and input formats consistent

Most of all, we needed to sort out which ones are binary and which
ones are text, and make sure they are treated accordingly and
consistently so

Reviewed-by: Tim Hudson <tjh@openssl.org>

18 files changed, 37 insertions, 41 deletions

diff --git a/apps/apps.h b/apps/apps.h
index a134b8da98..cd70948313 100644
--- a/apps/apps.h
+++ b/apps/apps.h
@@ -163,8 +163,9 @@ int app_load_modules(const CONF *config);
 void unbuffer(FILE *fp);
 
 /* Often used in calls to bio_open_default. */
-# define RB(xformat)  ((xformat) == FORMAT_ASN1 ? "rb" : "r")
-# define WB(xformat)  ((xformat) == FORMAT_ASN1 ? "wb" : "w")
+# define RB(xformat)  (((xformat) & B_FORMAT_TEXT) ? "rb" : "r")
+# define WB(xformat)  (((xformat) & B_FORMAT_TEXT) ? "wb" : "w")
+# define AB(xformat)  (((xformat) & B_FORMAT_TEXT) ? "ab" : "a")
 
 /*
  * Common verification options.
@@ -535,19 +536,20 @@ void print_cert_checks(BIO *bio, X509 *x,
 void store_setup_crl_download(X509_STORE *st);
 
 /* See OPT_FMT_xxx, above. */
+# define B_FORMAT_TEXT   0x8000
 # define FORMAT_UNDEF    0
 # define FORMAT_ASN1     1
-# define FORMAT_TEXT     2
-# define FORMAT_PEM      3
+# define FORMAT_TEXT    (2 | B_FORMAT_TEXT)
+# define FORMAT_PEM     (3 | B_FORMAT_TEXT)
 # define FORMAT_PKCS12   5
-# define FORMAT_SMIME    6
+# define FORMAT_SMIME   (6 | B_FORMAT_TEXT)
 # define FORMAT_ENGINE   7
-# define FORMAT_PEMRSA   9      /* PEM RSAPubicKey format */
-# define FORMAT_ASN1RSA  10     /* DER RSAPubicKey format */
-# define FORMAT_MSBLOB   11     /* MS Key blob format */
-# define FORMAT_PVK      12     /* MS PVK file format */
-# define FORMAT_HTTP     13     /* Download using HTTP */
-# define FORMAT_NSS      14     /* NSS keylog format */
+# define FORMAT_PEMRSA  (9 | B_FORMAT_TEXT)     /* PEM RSAPubicKey format */
+# define FORMAT_ASN1RSA  10                     /* DER RSAPubicKey format */
+# define FORMAT_MSBLOB   11                     /* MS Key blob format */
+# define FORMAT_PVK      12                     /* MS PVK file format */
+# define FORMAT_HTTP     13                     /* Download using HTTP */
+# define FORMAT_NSS      14                     /* NSS keylog format */
 
 # define EXT_COPY_NONE   0
 # define EXT_COPY_ADD    1
diff --git a/apps/asn1pars.c b/apps/asn1pars.c
index 24b55681f0..8881ad4e33 100644
--- a/apps/asn1pars.c
+++ b/apps/asn1pars.c
@@ -197,7 +197,7 @@ int asn1parse_main(int argc, char **argv)
         BIO_free(in);
     }
 
-    if ((in = bio_open_default(infile, "r")) == NULL)
+    if ((in = bio_open_default(infile, RB(informat))) == NULL)
         goto end;
 
     if (derfile && (derout = bio_open_default(derfile, "wb")) == NULL)
diff --git a/apps/ca.c b/apps/ca.c
index 5cd8002067..ce09155f28 100644
--- a/apps/ca.c
+++ b/apps/ca.c
@@ -793,7 +793,7 @@ end_of_options:
             extensions = "default";
     }
 
-        /*****************************************************************/
+    /*****************************************************************/
     if (req || gencrl) {
         Sout = bio_open_default(outfile, "w");
         if (Sout == NULL)
diff --git a/apps/cms.c b/apps/cms.c
index e40686b5d4..599f21774e 100644
--- a/apps/cms.c
+++ b/apps/cms.c
@@ -690,16 +690,14 @@ int cms_main(int argc, char **argv)
         flags &= ~CMS_DETACHED;
 
     if (operation & SMIME_OP) {
-        if (outformat == FORMAT_ASN1)
-            outmode = "wb";
+        outmode = WB(outformat);
     } else {
         if (flags & CMS_BINARY)
             outmode = "wb";
     }
 
     if (operation & SMIME_IP) {
-        if (informat == FORMAT_ASN1)
-            inmode = "rb";
+        inmode = RB(informat);
     } else {
         if (flags & CMS_BINARY)
             inmode = "rb";
diff --git a/apps/crl.c b/apps/crl.c
index c0bf8749d2..1ea0c319c8 100644
--- a/apps/crl.c
+++ b/apps/crl.c
@@ -346,7 +346,7 @@ int crl_main(int argc, char **argv)
             }
         }
     }
-    out = bio_open_default(outfile, "w");
+    out = bio_open_default(outfile, WB(outformat));
     if (out == NULL)
         goto end;
 
diff --git a/apps/dhparam.c b/apps/dhparam.c
index 2e5ce2c6d3..0640cf808b 100644
--- a/apps/dhparam.c
+++ b/apps/dhparam.c
@@ -352,7 +352,7 @@ int dhparam_main(int argc, char **argv)
         /* dh != NULL */
     }
 
-    out = bio_open_default(outfile, "w");
+    out = bio_open_default(outfile, WB(outformat));
     if (out == NULL)
         goto end;
 
diff --git a/apps/dsa.c b/apps/dsa.c
index 9998bfe30a..4fca852638 100644
--- a/apps/dsa.c
+++ b/apps/dsa.c
@@ -225,7 +225,7 @@ int dsa_main(int argc, char **argv)
         goto end;
     }
 
-    out = bio_open_owner(outfile, "w", private);
+    out = bio_open_owner(outfile, WB(outformat), private);
     if (out == NULL)
         goto end;
 
diff --git a/apps/dsaparam.c b/apps/dsaparam.c
index 8d48313d2b..d61bb70a70 100644
--- a/apps/dsaparam.c
+++ b/apps/dsaparam.c
@@ -195,10 +195,10 @@ int dsaparam_main(int argc, char **argv)
     }
     private = genkey ? 1 : 0;
 
-    in = bio_open_default(infile, "r");
+    in = bio_open_default(infile, RB(informat));
     if (in == NULL)
         goto end;
-    out = bio_open_owner(outfile, "w", private);
+    out = bio_open_owner(outfile, WB(outformat), private);
     if (out == NULL)
         goto end;
 
diff --git a/apps/enc.c b/apps/enc.c
index 18fcb9505d..3b8d7eb265 100644
--- a/apps/enc.c
+++ b/apps/enc.c
@@ -330,7 +330,7 @@ int enc_main(int argc, char **argv)
         unbuffer(stdin);
         in = dup_bio_in();
     } else
-        in = bio_open_default(infile, "r");
+        in = bio_open_default(infile, base64 ? "r" : "rb");
     if (in == NULL)
         goto end;
 
@@ -366,7 +366,7 @@ int enc_main(int argc, char **argv)
         }
     }
 
-    out = bio_open_default(outfile, "w");
+    out = bio_open_default(outfile, base64 ? "w" : "wb");
     if (out == NULL)
         goto end;
 
diff --git a/apps/genpkey.c b/apps/genpkey.c
index dbbedaa35e..c29b1947a6 100644
--- a/apps/genpkey.c
+++ b/apps/genpkey.c
@@ -184,7 +184,7 @@ int genpkey_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    out = bio_open_owner(outfile, "wb", private);
+    out = bio_open_owner(outfile, WB(outformat), private);
     if (out == NULL)
         goto end;
 
diff --git a/apps/pkcs8.c b/apps/pkcs8.c
index b120b93aa9..e3cb7750e1 100644
--- a/apps/pkcs8.c
+++ b/apps/pkcs8.c
@@ -239,10 +239,10 @@ int pkcs8_main(int argc, char **argv)
     if ((pbe_nid == -1) && !cipher)
         pbe_nid = NID_pbeWithMD5AndDES_CBC;
 
-    in = bio_open_default(infile, "rb");
+    in = bio_open_default(infile, RB(informat));
     if (in == NULL)
         goto end;
-    out = bio_open_owner(outfile, "wb", private);
+    out = bio_open_owner(outfile, WB(outformat), private);
     if (out == NULL)
         goto end;
 
diff --git a/apps/pkey.c b/apps/pkey.c
index 80c2e154dd..9ef228f3a0 100644
--- a/apps/pkey.c
+++ b/apps/pkey.c
@@ -172,7 +172,7 @@ int pkey_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    out = bio_open_owner(outfile, "wb", private);
+    out = bio_open_owner(outfile, WB(outformat), private);
     if (out == NULL)
         goto end;
 
diff --git a/apps/rand.c b/apps/rand.c
index 432e784496..a5aa5d9410 100644
--- a/apps/rand.c
+++ b/apps/rand.c
@@ -134,7 +134,7 @@ int rand_main(int argc, char **argv)
         BIO_printf(bio_err, "%ld semi-random bytes loaded\n",
                    app_RAND_load_files(inrand));
 
-    out = bio_open_default(outfile, "w");
+    out = bio_open_default(outfile, base64 ? "w" : "wb");
     if (out == NULL)
         goto end;
 
diff --git a/apps/req.c b/apps/req.c
index 59cc6b4664..bae3eeca0c 100644
--- a/apps/req.c
+++ b/apps/req.c
@@ -764,7 +764,8 @@ int req_main(int argc, char **argv)
 
     out = bio_open_default(outfile,
                            keyout != NULL && outfile != NULL &&
-                           strcmp(keyout, outfile) == 0 ? "a" : "w");
+                           strcmp(keyout, outfile) == 0
+                           ? AB(outformat) : WB(outformat));
     if (out == NULL)
         goto end;
 
diff --git a/apps/rsa.c b/apps/rsa.c
index c7ad44b75d..01cc6ea18d 100644
--- a/apps/rsa.c
+++ b/apps/rsa.c
@@ -292,7 +292,7 @@ int rsa_main(int argc, char **argv)
         goto end;
     }
 
-    out = bio_open_owner(outfile, "w", private);
+    out = bio_open_owner(outfile, WB(outformat), private);
     if (out == NULL)
         goto end;
 
diff --git a/apps/sess_id.c b/apps/sess_id.c
index 681654335a..e743791d23 100644
--- a/apps/sess_id.c
+++ b/apps/sess_id.c
@@ -160,10 +160,7 @@ int sess_id_main(int argc, char **argv)
     }
 
     if (!noout || text) {
-        const char* modeflag = "w";
-        if (outformat == FORMAT_ASN1 || outformat == FORMAT_NSS)
-            modeflag = "wb";
-        out = bio_open_default(outfile, modeflag);
+        out = bio_open_default(outfile, WB(outformat));
         if (out == NULL)
             goto end;
     }
diff --git a/apps/smime.c b/apps/smime.c
index 45898de4c1..d597ebf534 100644
--- a/apps/smime.c
+++ b/apps/smime.c
@@ -427,16 +427,14 @@ int smime_main(int argc, char **argv)
         flags &= ~PKCS7_DETACHED;
 
     if (operation & SMIME_OP) {
-        if (outformat == FORMAT_ASN1)
-            outmode = "wb";
+        outmode = WB(outformat);
     } else {
         if (flags & PKCS7_BINARY)
             outmode = "wb";
     }
 
     if (operation & SMIME_IP) {
-        if (informat == FORMAT_ASN1)
-            inmode = "rb";
+        inmode = RB(informat);
     } else {
         if (flags & PKCS7_BINARY)
             inmode = "rb";
diff --git a/apps/x509.c b/apps/x509.c
index acce9e9ddd..8020e8a086 100644
--- a/apps/x509.c
+++ b/apps/x509.c
@@ -496,7 +496,7 @@ int x509_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    out = bio_open_default(outfile, "w");
+    out = bio_open_default(outfile, WB(outformat));
     if (out == NULL)
         goto end;
 
@@ -556,7 +556,7 @@ int x509_main(int argc, char **argv)
             BIO_printf(bio_err, "We need a private key to sign with\n");
             goto end;
         }
-        in = bio_open_default(infile, "r");
+        in = bio_open_default(infile, RB(informat));
         if (in == NULL)
             goto end;
         req = PEM_read_bio_X509_REQ(in, NULL, NULL, NULL);