apps/cms.c: Correct -sign output and -verify input with -binary

Also add related warnings on irrelevant use of -nodetach and -content options.

Fixes #15347

Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/15358)

1 files changed, 20 insertions, 5 deletions

diff --git a/apps/cms.c b/apps/cms.c
index e9fe29ab8e..da00ece93b 100644
--- a/apps/cms.c
+++ b/apps/cms.c
@@ -287,10 +287,11 @@ static void warn_binary(const char *file)
                 BIO_printf(bio_err, "Warning: input file '%s' contains %s"
                            " character; better use -binary option\n",
                            file, *cur == '\0' ? "NUL" : "8-bit");
-                break;
+                goto end;
             }
         }
     }
+ end:
     BIO_free(bio);
 }
 
@@ -320,7 +321,8 @@ int cms_main(int argc, char **argv)
     char *originatorfile = NULL, *recipfile = NULL, *ciphername = NULL;
     char *to = NULL, *from = NULL, *subject = NULL, *prog;
     cms_key_param *key_first = NULL, *key_param = NULL;
-    int flags = CMS_DETACHED, noout = 0, print = 0, keyidx = -1, vpmtouched = 0;
+    int flags = CMS_DETACHED, binary_files = 0;
+    int noout = 0, print = 0, keyidx = -1, vpmtouched = 0;
     int informat = FORMAT_SMIME, outformat = FORMAT_SMIME;
     int operation = 0, ret = 1, rr_print = 0, rr_allorfirst = -1;
     int verify_retcode = 0, rctformat = FORMAT_SMIME, keyform = FORMAT_UNDEF;
@@ -811,14 +813,26 @@ int cms_main(int argc, char **argv)
 
     ret = 2;
 
-    if (!(operation & SMIME_SIGNERS))
+    if ((operation & SMIME_SIGNERS) == 0) {
+        if ((flags & CMS_DETACHED) == 0)
+            BIO_printf(bio_err,
+                       "Warning: -nodetach option is ignored for non-signing operation\n");
+
         flags &= ~CMS_DETACHED;
+    }
+    if ((operation & SMIME_IP) == 0 && contfile != NULL)
+        BIO_printf(bio_err,
+                   "Warning: -contfile option is ignored for the given operation\n");
 
     if ((flags & CMS_BINARY) != 0) {
         if (!(operation & SMIME_OP))
             outformat = FORMAT_BINARY;
         if (!(operation & SMIME_IP))
             informat = FORMAT_BINARY;
+        if ((operation & SMIME_SIGNERS) != 0 && (flags & CMS_DETACHED) != 0)
+            binary_files = 1;
+        if ((operation & SMIME_IP) != 0 && contfile == NULL)
+            binary_files = 1;
     }
 
     if (operation == SMIME_ENCRYPT) {
@@ -902,7 +916,7 @@ int cms_main(int argc, char **argv)
     if ((flags & CMS_BINARY) == 0)
         warn_binary(infile);
     in = bio_open_default(infile, 'r',
-                          (flags & CMS_BINARY) != 0 ? FORMAT_BINARY : informat);
+                          binary_files ? FORMAT_BINARY : informat);
     if (in == NULL)
         goto end;
 
@@ -945,7 +959,8 @@ int cms_main(int argc, char **argv)
             goto end;
     }
 
-    out = bio_open_default(outfile, 'w', outformat);
+    out = bio_open_default(outfile, 'w',
+                           binary_files ? FORMAT_BINARY : outformat);
     if (out == NULL)
         goto end;