Change the way apps open their input and output files

The different apps had the liberty to decide whether they would open their
input and output files in binary mode or not, which could be confusing if
two different apps were handling the same type of file in different ways.

The solution is to centralise the decision of low level file organisation,
and that the apps would use a selection of formats to state the intent of
the file.

Reviewed-by: Tim Hudson <tjh@openssl.org>

1 files changed, 16 insertions, 14 deletions

diff --git a/apps/apps.h b/apps/apps.h
index cd70948313..c34d22ed1d 100644
--- a/apps/apps.h
+++ b/apps/apps.h
@@ -154,19 +154,14 @@ extern BIO *bio_out;
 extern BIO *bio_err;
 BIO *dup_bio_in(void);
 BIO *dup_bio_out(void);
-BIO *bio_open_owner(const char *filename, const char *mode, int private);
-BIO *bio_open_default(const char *filename, const char *mode);
-BIO *bio_open_default_quiet(const char *filename, const char *mode);
+BIO *bio_open_owner(const char *filename, int format, int private);
+BIO *bio_open_default(const char *filename, char mode, int format);
+BIO *bio_open_default_quiet(const char *filename, char mode, int format);
 CONF *app_load_config(const char *filename);
 CONF *app_load_config_quiet(const char *filename);
 int app_load_modules(const CONF *config);
 void unbuffer(FILE *fp);
 
-/* Often used in calls to bio_open_default. */
-# define RB(xformat)  (((xformat) & B_FORMAT_TEXT) ? "rb" : "r")
-# define WB(xformat)  (((xformat) & B_FORMAT_TEXT) ? "wb" : "w")
-# define AB(xformat)  (((xformat) & B_FORMAT_TEXT) ? "ab" : "a")
-
 /*
  * Common verification options.
  */
@@ -536,14 +531,21 @@ void print_cert_checks(BIO *bio, X509 *x,
 void store_setup_crl_download(X509_STORE *st);
 
 /* See OPT_FMT_xxx, above. */
+/* On some platforms, it's important to distinguish between text and binary
+ * files.  On some, there might even be specific file formats for different
+ * contents.  The FORMAT_xxx macros are meant to express an intent with the
+ * file being read or created.
+ */
 # define B_FORMAT_TEXT   0x8000
 # define FORMAT_UNDEF    0
-# define FORMAT_ASN1     1
-# define FORMAT_TEXT    (2 | B_FORMAT_TEXT)
-# define FORMAT_PEM     (3 | B_FORMAT_TEXT)
-# define FORMAT_PKCS12   5
-# define FORMAT_SMIME   (6 | B_FORMAT_TEXT)
-# define FORMAT_ENGINE   7
+# define FORMAT_TEXT    (1 | B_FORMAT_TEXT)     /* Generic text */
+# define FORMAT_BINARY   2                      /* Generic binary */
+# define FORMAT_BASE64  (3 | B_FORMAT_TEXT)     /* Base64 */
+# define FORMAT_ASN1     4                      /* ASN.1/DER */
+# define FORMAT_PEM     (5 | B_FORMAT_TEXT)
+# define FORMAT_PKCS12   6
+# define FORMAT_SMIME   (7 | B_FORMAT_TEXT)
+# define FORMAT_ENGINE   8                      /* Not really a file format */
 # define FORMAT_PEMRSA  (9 | B_FORMAT_TEXT)     /* PEM RSAPubicKey format */
 # define FORMAT_ASN1RSA  10                     /* DER RSAPubicKey format */
 # define FORMAT_MSBLOB   11                     /* MS Key blob format */