cms: avoid intermittent test failure

If you decrypt a random input using RSAES-PKCS-v1_5, then there is a
non-negligible chance that the result will look like a valid plaintext
(that is why RSAES-PKCS-v1_5 shouldn't be used anymore).  This was the
cause of an intermittent failure in a test that did a cms-encrypt
operation targetting multiple recipients.

The failure happened during key-only decrypt.  The recipient decrypts
every RSA ciphertext -- only one is supposed to decrypt successfully,
which would reveal the right content-key.  Occassionally, more than
one decrypted successfully.

Update the test by specifying the recipient cert in the decrypt op
(this avoids looping over all RSA ciphertexts).

Add a new test to get coverage for key-only decrypt, but use RSA-OAEP
during the encrypt op.

Fixes https://github.com/openssl/project/issues/380

Testing:

  $ make TESTS='test_cms' test

Reviewed-by: Neil Horman <nhorman@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/23055)

(cherry picked from commit ffed597882baf2f07274e7eaa8f3c4fa9fa74ac1)

3 files changed, 103 insertions, 4 deletions

diff --git a/test/recipes/80-test_cms.t b/test/recipes/80-test_cms.t
index 75d6f52389..0062fb9dcb 100644
--- a/test/recipes/80-test_cms.t
+++ b/test/recipes/80-test_cms.t
@@ -50,7 +50,7 @@ my ($no_des, $no_dh, $no_dsa, $no_ec, $no_ec2m, $no_rc2, $no_zlib)
 
 $no_rc2 = 1 if disabled("legacy");
 
-plan tests => 18;
+plan tests => 19;
 
 ok(run(test(["pkcs7_test"])), "test pkcs7");
 
@@ -222,13 +222,15 @@ my @smime_pkcs7_tests = (
       \&final_compare
     ],
 
-    [ "enveloped content test streaming S/MIME format, DES, 3 recipients, key only used",
+    [ "enveloped content test streaming S/MIME format, DES, 3 recipients, cert and key files used",
       [ "{cmd1}", @defaultprov, "-encrypt", "-in", $smcont,
         "-stream", "-out", "{output}.cms",
         $smrsa1,
         catfile($smdir, "smrsa2.pem"),
-        catfile($smdir, "smrsa3.pem") ],
-      [ "{cmd2}", @defaultprov, "-decrypt", "-inkey", catfile($smdir, "smrsa3.pem"),
+        catfile($smdir, "smrsa3-cert.pem") ],
+      [ "{cmd2}", @defaultprov, "-decrypt",
+	"-recip", catfile($smdir, "smrsa3-cert.pem"),
+	"-inkey", catfile($smdir, "smrsa3-key.pem"),
         "-in", "{output}.cms", "-out", "{output}.txt" ],
       \&final_compare
     ],
@@ -1071,3 +1073,51 @@ with({ exit_checker => sub { return shift == 3; } },
 		   ])),
 	   "Check for failure when cipher does not have an assigned OID (issue#22225)");
      });
+
+# Test encrypt to three recipients, and decrypt using key-only;
+# i.e. do not follow the recommended practice of providing the
+# recipient cert in the decrypt op.
+#
+# Use RSAES-OAEP for key-transport, not RSAES-PKCS-v1_5.
+#
+# Because the cert is not provided during decrypt, all RSA ciphertexts
+# are decrypted in turn, and when/if there is a valid decryption, it
+# is assumed the correct content-key has been recovered.
+#
+# That process may fail with RSAES-PKCS-v1_5 b/c there is a
+# non-negligible chance that decrypting a random input using
+# RSAES-PKCS-v1_5 can result in a valid plaintext (so two content-keys
+# could be recovered and the wrong one might be used).
+#
+# See https://github.com/openssl/project/issues/380
+subtest "encrypt to three recipients with RSA-OAEP, key only decrypt" => sub {
+    plan tests => 3;
+
+    my $pt = srctop_file("test", "smcont.txt");
+    my $ct = "smtst.cms";
+    my $ptpt = "smtst.txt";
+
+    ok(run(app(['openssl', 'cms',
+		@defaultprov,
+		'-encrypt',
+		'-in', $pt,
+		'-out', $ct,
+		'-stream',
+		'-recip', catfile($smdir, "smrsa1.pem"),
+		'-keyopt', 'rsa_padding_mode:oaep',
+		'-recip', catfile($smdir, "smrsa2.pem"),
+		'-keyopt', 'rsa_padding_mode:oaep',
+		'-recip', catfile($smdir, "smrsa3-cert.pem"),
+		'-keyopt', 'rsa_padding_mode:oaep',
+	       ])),
+       "encrypt to three recipients with RSA-OAEP (avoid openssl/project issue#380)");
+    ok(run(app(['openssl', 'cms',
+		@defaultprov,
+		'-decrypt',
+		'-in', $ct,
+		'-out', $ptpt,
+		'-inkey', catfile($smdir, "smrsa3-key.pem"),
+	       ])),
+       "decrypt with key only");
+    is(compare($pt, $ptpt), 0, "compare original message with decrypted ciphertext");
+};
diff --git a/test/smime-certs/smrsa3-cert.pem b/test/smime-certs/smrsa3-cert.pem
new file mode 100644
index 0000000000..70004acb86
--- /dev/null
+++ b/test/smime-certs/smrsa3-cert.pem
@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDeTCCAmGgAwIBAgIUIDyc//j/LoNDesZTGbPBoVarv4EwDQYJKoZIhvcNAQEL
+BQAwRDELMAkGA1UEBhMCVUsxFjAUBgNVBAoMDU9wZW5TU0wgR3JvdXAxHTAbBgNV
+BAMMFFRlc3QgUy9NSU1FIFJTQSBSb290MCAXDTIyMDYwMjE1MzMxM1oYDzIxMjIw
+NTA5MTUzMzEzWjBFMQswCQYDVQQGEwJVSzEWMBQGA1UECgwNT3BlblNTTCBHcm91
+cDEeMBwGA1UEAwwVVGVzdCBTL01JTUUgRUUgUlNBICMzMIIBIjANBgkqhkiG9w0B
+AQEFAAOCAQ8AMIIBCgKCAQEA+QP7d56K4/9eu7aChtWILYNxvqWeDcJeWvX5Z5vC
+XUjFuUxBD9U0rw1SBLgFYu8aqAJ+oXsqaGjJARifgKEqPUe7pnYYatr55lhTbHR+
+qA88p1V4sclEaPNWKzd7J/V3eeYr04kqWV5XYhAq9k9AWLzsNIePe2z7OoGPS6oK
+wRzWFRd5RYXTpmFr/tqknbYvtYFd7duKb9QqytgHV+RKXXeY0fnjZ7frLmaqDwtI
+U3DY7MyS3Hw2BVx72vQXBNA364HGEpqEgVOdzI7et0wpSumaFXDye714xUR53L7N
+f3fp3PQXS/RbBiNXs7KUsHCR6nsdsIKO+sg66gxOLNt6zwIDAQABo2AwXjAMBgNV
+HRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIF4DAdBgNVHQ4EFgQUN9pGq/UFS3o50rTi
+V+AYgAk+3R4wHwYDVR0jBBgwFoAUFcETIWviVV+nah1XINbP86lzZFkwDQYJKoZI
+hvcNAQELBQADggEBAGcOh380/6aJqMpYBssuf2CB3DX/hGKdvEF7fF8iNSfl5HHq
+112kHl3MhbL9Th/safJq9sLDJqjXRNdVCUJJbU4YI2P2gsi04paC0qxWxMLtzQLd
+CE7ki2xH94Fuu/dThbpzZBABROO1RrdI24GDGt9t4Gf0WVkobmT/zNlwGppKTIB2
+iV/Ug30iKr/C49UzwUIa+XXXujkjPTmGSnrKwVQNxQh81rb+iTL7GEnNuqDsatHW
+ZyLS2SaVdG5tMqDkITPMDGjehUzJcAbVc8Bv4m8Ukuov3uDj2Doc6MxlvrVkV0AE
+BcSCb/bWQJJ/X4LQZlx9cMk4NINxV9UeFPZOefg=
+-----END CERTIFICATE-----
diff --git a/test/smime-certs/smrsa3-key.pem b/test/smime-certs/smrsa3-key.pem
new file mode 100644
index 0000000000..216d70b61b
--- /dev/null
+++ b/test/smime-certs/smrsa3-key.pem
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQD5A/t3norj/167
+toKG1Ygtg3G+pZ4Nwl5a9flnm8JdSMW5TEEP1TSvDVIEuAVi7xqoAn6heypoaMkB
+GJ+AoSo9R7umdhhq2vnmWFNsdH6oDzynVXixyURo81YrN3sn9Xd55ivTiSpZXldi
+ECr2T0BYvOw0h497bPs6gY9LqgrBHNYVF3lFhdOmYWv+2qSdti+1gV3t24pv1CrK
+2AdX5Epdd5jR+eNnt+suZqoPC0hTcNjszJLcfDYFXHva9BcE0DfrgcYSmoSBU53M
+jt63TClK6ZoVcPJ7vXjFRHncvs1/d+nc9BdL9FsGI1ezspSwcJHqex2wgo76yDrq
+DE4s23rPAgMBAAECggEAEDi+VWD5VUpjD5zWOoPQiRDGBJBhtMAKkl6okxEmXvWb
+Xz3STFnjHgA1JFHW3bRU9BHI9k8vSHmnlnkfKb3V/ZX5IHNcKCHb/x9NBak+QLVQ
+0zLtfE9vxiTC0B/oac+MPaiD4hYFQ81pFwK6VS0Poi8ZCBJtOkRqfUvsyV8zZrgh
+/6cs4mwOVyZPFRgF9eWXYv7PJz8pNRizhII0iv9H/r2I3DzsZLPCg7c29mP+I/SG
+A7Pl82UXjtOc0KurGY2M5VheZjxJT/k/FLMkWY2GS5n6dfcyzsVSKb25HoeuvQsI
+vs1mKs+Onbobdc17hCcKVJzbi3DwXs5XDhrEzfHccQKBgQD88uBxVCRV31PsCN6I
+pKxQDGgz+1BqPqe7KMRiZI7HgDUK0eCM3/oG089/jsBtJcSxnScLSVNBjQ+xGiFi
+YCD4icQoJSzpqJyR6gDq5lTHASAe+9LWRW771MrtyACQWNXowYEyu8AjekrZkCUS
+wIKVpw57oWykzIoS7ixZsJ8gxwKBgQD8BPWqJEsLiQvOlS5E/g88eV1KTpxm9Xs+
+BbwsDXZ7m4Iw5lYaUu5CwBB/2jkGGRl8Q/EfAdUT7gXv3t6x5b1qMXaIczmRGYto
+NuI3AH2MPxAa7lg5TgBgie1r7PKwyPMfG3CtDx6n8W5sexgJpbIy5u7E+U6d8s1o
+c7EcsefduQKBgCkHJAx9v18GWFBip+W2ABUDzisQSlzRSNd8p03mTZpiWzgkDq4K
+7j0JQhDIkMGjbKH6gYi9Hfn17WOmf1+7g92MSvrP/NbxeGPadsejEIEu14zu/6Wt
+oXDLdRbYZ+8B2cBlEpWuCl42yck8Lic6fnPTou++oSah3otvglYR5d2lAoGACd8L
+3FE1m0sP6lSPjmZBJIZAcDOqDqJY5HIHD9arKGZL8CxlfPx4lqa9PrTGfQWoqORk
+YmmI9hHhq6aYJHGyPKGZWfjhbVyJyFg1/h+Hy2GA+P0S+ZOjkiR050BNtTz5wOMr
+Q6wO8FcVkywzIdWaqEHBYne9a5RiFVBKxKv3QAkCgYBxmCBKajFkMVb4Uc55WqJs
+Add0mctGgmZ1l5vq81eWe3wjM8wgfJgaD3Q3gwx2ABUX/R+OsVWSh4o5ZR86sYoz
+TviknBHF8GeDLjpT49+04fEaz336J2JOptF9zIpz7ZK1nrOEjzaZGtumReVjUP7X
+fNcb5iDYqZRzD8ixBbLxUw==
+-----END PRIVATE KEY-----