apps/cms.c: Make -sign and -verify handle binary input

Fixes #8940

Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/12959)

2 files changed, 44 insertions, 3 deletions

diff --git a/test/recipes/80-test_cms.t b/test/recipes/80-test_cms.t
index a371f21ad8..0e20b807c8 100644
--- a/test/recipes/80-test_cms.t
+++ b/test/recipes/80-test_cms.t
@@ -12,7 +12,7 @@ use warnings;
 
 use POSIX;
 use File::Spec::Functions qw/catfile/;
-use File::Compare qw/compare_text/;
+use File::Compare qw/compare_text compare/;
 use OpenSSL::Test qw/:DEFAULT srctop_dir srctop_file bldtop_dir bldtop_file/;
 
 use OpenSSL::Test::Utils;
@@ -50,8 +50,7 @@ my ($no_des, $no_dh, $no_dsa, $no_ec, $no_ec2m, $no_rc2, $no_zlib)
 
 $no_rc2 = 1 if disabled("legacy");
 
-plan tests =>
-    + 10;
+plan tests => 11;
 
 unless ($no_fips) {
     @config = ( "-config", srctop_file("test", "fips-and-base.cnf") );
@@ -812,6 +811,48 @@ subtest "CAdES ko tests\n" => sub {
     }
 };
 
+subtest "CMS binary input tests\n" => sub {
+    my $input = srctop_file("test", "smcont.bin");
+    my $signed = "smcont.signed";
+    my $verified = "smcont.verified";
+    my $cert = srctop_file("test", "certs", "ee-self-signed.pem");
+    my $key = srctop_file("test", "certs", "ee-key.pem");
+
+    plan tests => 11;
+
+    ok(run(app(["openssl", "cms", "-sign", "-md", "sha256",
+                "-signer", $cert, "-inkey", $key,
+                "-binary", "-in", $input, "-out", $signed])),
+       "sign binary input with -binary");
+    ok(run(app(["openssl", "cms", "-verify", "-CAfile", $cert,
+                "-binary", "-in", $signed, "-out", $verified])),
+       "verify binary input with -binary");
+    is(compare($input, $verified), 0, "binary input retained with -binary");
+    ok(run(app(["openssl", "cms", "-sign", "-md", "sha256",
+                "-signer", $cert, "-inkey", $key,
+                "-in", $input, "-out", $signed])),
+       "sign binary input without -binary");
+    ok(run(app(["openssl", "cms", "-verify", "-CAfile", $cert,
+                "-in", $signed, "-out", $verified])),
+       "verify binary input without -binary");
+    is(compare($input, $verified), 1, "binary input not retained without -binary");
+    ok(!run(app(["openssl", "cms", "-verify", "-CAfile", $cert, "-crlfeol",
+                "-binary", "-in", $signed, "-out", $verified])),
+       "verify binary input wrong crlfeol");
+
+    ok(run(app(["openssl", "cms", "-sign", "-md", "sha256", "-crlfeol",
+                "-signer", $cert, "-inkey", $key,
+                "-binary", "-in", $input, "-out", $signed.".crlf"])),
+       "sign binary input crlfeol");
+    ok(run(app(["openssl", "cms", "-verify", "-CAfile", $cert, "-crlfeol",
+                "-binary", "-in", $signed.".crlf", "-out", $verified.".crlf"])),
+       "verify binary input crlfeol");
+    is(compare($input, $verified.".crlf"), 0);
+    ok(!run(app(["openssl", "cms", "-verify", "-CAfile", $cert,
+                "-binary", "-in", $signed.".crlf", "-out", $verified.".crlf"])),
+       "verify binary input missing crlfeol");
+};
+
 sub check_availability {
     my $tnam = shift;
 
diff --git a/test/smcont.bin b/test/smcont.bin
new file mode 100644
index 0000000000..2a5ce10224
--- /dev/null
+++ b/test/smcont.bin