apps/cms.c: Correct -sign output and -verify input with -binary

Also add related warnings on irrelevant use of -nodetach and -content options.

Fixes #15347

Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/15358)

5 files changed, 33 insertions, 17 deletions

diff --git a/apps/cms.c b/apps/cms.c
index e9fe29ab8e..da00ece93b 100644
--- a/apps/cms.c
+++ b/apps/cms.c
@@ -287,10 +287,11 @@ static void warn_binary(const char *file)
                 BIO_printf(bio_err, "Warning: input file '%s' contains %s"
                            " character; better use -binary option\n",
                            file, *cur == '\0' ? "NUL" : "8-bit");
-                break;
+                goto end;
             }
         }
     }
+ end:
     BIO_free(bio);
 }
 
@@ -320,7 +321,8 @@ int cms_main(int argc, char **argv)
     char *originatorfile = NULL, *recipfile = NULL, *ciphername = NULL;
     char *to = NULL, *from = NULL, *subject = NULL, *prog;
     cms_key_param *key_first = NULL, *key_param = NULL;
-    int flags = CMS_DETACHED, noout = 0, print = 0, keyidx = -1, vpmtouched = 0;
+    int flags = CMS_DETACHED, binary_files = 0;
+    int noout = 0, print = 0, keyidx = -1, vpmtouched = 0;
     int informat = FORMAT_SMIME, outformat = FORMAT_SMIME;
     int operation = 0, ret = 1, rr_print = 0, rr_allorfirst = -1;
     int verify_retcode = 0, rctformat = FORMAT_SMIME, keyform = FORMAT_UNDEF;
@@ -811,14 +813,26 @@ int cms_main(int argc, char **argv)
 
     ret = 2;
 
-    if (!(operation & SMIME_SIGNERS))
+    if ((operation & SMIME_SIGNERS) == 0) {
+        if ((flags & CMS_DETACHED) == 0)
+            BIO_printf(bio_err,
+                       "Warning: -nodetach option is ignored for non-signing operation\n");
+
         flags &= ~CMS_DETACHED;
+    }
+    if ((operation & SMIME_IP) == 0 && contfile != NULL)
+        BIO_printf(bio_err,
+                   "Warning: -contfile option is ignored for the given operation\n");
 
     if ((flags & CMS_BINARY) != 0) {
         if (!(operation & SMIME_OP))
             outformat = FORMAT_BINARY;
         if (!(operation & SMIME_IP))
             informat = FORMAT_BINARY;
+        if ((operation & SMIME_SIGNERS) != 0 && (flags & CMS_DETACHED) != 0)
+            binary_files = 1;
+        if ((operation & SMIME_IP) != 0 && contfile == NULL)
+            binary_files = 1;
     }
 
     if (operation == SMIME_ENCRYPT) {
@@ -902,7 +916,7 @@ int cms_main(int argc, char **argv)
     if ((flags & CMS_BINARY) == 0)
         warn_binary(infile);
     in = bio_open_default(infile, 'r',
-                          (flags & CMS_BINARY) != 0 ? FORMAT_BINARY : informat);
+                          binary_files ? FORMAT_BINARY : informat);
     if (in == NULL)
         goto end;
 
@@ -945,7 +959,8 @@ int cms_main(int argc, char **argv)
             goto end;
     }
 
-    out = bio_open_default(outfile, 'w', outformat);
+    out = bio_open_default(outfile, 'w',
+                           binary_files ? FORMAT_BINARY : outformat);
     if (out == NULL)
         goto end;
 
diff --git a/doc/man1/openssl-cms.pod.in b/doc/man1/openssl-cms.pod.in
index 6e0f86804a..c63a7f330b 100644
--- a/doc/man1/openssl-cms.pod.in
+++ b/doc/man1/openssl-cms.pod.in
@@ -507,8 +507,8 @@ will be written to this file if the verification was successful.
 
 =item B<-content> I<filename>
 
-This specifies a file containing the detached content, this is only
-useful with the B<-verify> command. This is only usable if the CMS
+This specifies a file containing the detached content for operations taking
+S/MIME input, such as the B<-verify> command. This is only usable if the CMS
 structure is using the detached signature form where the content is
 not included. This option will override any content if the input format
 is S/MIME and it uses the multipart/signed MIME content type.
diff --git a/smcont.signed_ b/smcont.signed_
new file mode 100644
index 0000000000..59701f31d4
--- /dev/null
+++ b/smcont.signed_
diff --git a/test/recipes/80-test_cms.t b/test/recipes/80-test_cms.t
index 7896bc9b12..193c738a5d 100644
--- a/test/recipes/80-test_cms.t
+++ b/test/recipes/80-test_cms.t
@@ -818,7 +818,6 @@ subtest "CMS binary input tests\n" => sub {
     my $cert = srctop_file("test", "certs", "ee-self-signed.pem");
     my $key = srctop_file("test", "certs", "ee-key.pem");
 
-    plan skip_all => "Binary input tests currently disabled on Windows" if $^O =~ /^MSWin32$/;
     plan tests => 11;
 
     ok(run(app(["openssl", "cms", "-sign", "-md", "sha256",
@@ -829,29 +828,31 @@ subtest "CMS binary input tests\n" => sub {
                 "-binary", "-in", $signed, "-out", $verified])),
        "verify binary input with -binary");
     is(compare($input, $verified), 0, "binary input retained with -binary");
+
     ok(run(app(["openssl", "cms", "-sign", "-md", "sha256",
                 "-signer", $cert, "-inkey", $key,
-                "-in", $input, "-out", $signed])),
+                "-in", $input, "-out", $signed.".nobin"])),
        "sign binary input without -binary");
     ok(run(app(["openssl", "cms", "-verify", "-CAfile", $cert,
-                "-in", $signed, "-out", $verified])),
+                "-in", $signed.".nobin", "-out", $verified.".nobin"])),
        "verify binary input without -binary");
-    is(compare($input, $verified), 1, "binary input not retained without -binary");
+    is(compare($input, $verified.".nobin"), 1, "binary input not retained without -binary");
     ok(!run(app(["openssl", "cms", "-verify", "-CAfile", $cert, "-crlfeol",
-                "-binary", "-in", $signed, "-out", $verified])),
+                "-binary", "-in", $signed, "-out", $verified.".crlfeol"])),
        "verify binary input wrong crlfeol");
 
     ok(run(app(["openssl", "cms", "-sign", "-md", "sha256", "-crlfeol",
                 "-signer", $cert, "-inkey", $key,
                 "-binary", "-in", $input, "-out", $signed.".crlf"])),
-       "sign binary input crlfeol");
+       "sign binary input with -binary -crlfeol");
     ok(run(app(["openssl", "cms", "-verify", "-CAfile", $cert, "-crlfeol",
                 "-binary", "-in", $signed.".crlf", "-out", $verified.".crlf"])),
-       "verify binary input crlfeol");
-    is(compare($input, $verified.".crlf"), 0);
+       "verify binary input with -binary -crlfeol");
+    is(compare($input, $verified.".crlf"), 0,
+       "binary input retained with -binary -crlfeol");
     ok(!run(app(["openssl", "cms", "-verify", "-CAfile", $cert,
-                "-binary", "-in", $signed.".crlf", "-out", $verified.".crlf"])),
-       "verify binary input missing crlfeol");
+                "-binary", "-in", $signed.".crlf", "-out", $verified.".crlf2"])),
+       "verify binary input with -binary missing -crlfeol");
 };
 
 sub check_availability {
diff --git a/test/smcont.bin b/test/smcont.bin
index 2a5ce10224..96e5c57485 100644
--- a/test/smcont.bin
+++ b/test/smcont.bin