Make relevant tests more sensitive to 'no-fips'

This applies to test/recipes/30-test_evp.t and
test/recipes/30-test_evp_fetch_prov.t.

Additionally, we make test/recipes/30-test_evp_fetch_prov.t data
driven, to make test number planning more automated, and to separate
what is unique from what is common to all the test cases.

[extended tests]

Reviewed-by: Matthias St. Pierre <Matthias.St.Pierre@ncp-e.com>
(Merged from https://github.com/openssl/openssl/pull/10047)

2 files changed, 118 insertions, 63 deletions

diff --git a/test/recipes/30-test_evp.t b/test/recipes/30-test_evp.t
index 52f9c2882a..e60f82415b 100644
--- a/test/recipes/30-test_evp.t
+++ b/test/recipes/30-test_evp.t
@@ -21,12 +21,15 @@ use lib srctop_dir('Configurations');
 use lib bldtop_dir('.');
 use platform;
 
+my $no_fips = disabled('fips') || ($ENV{NO_FIPS} // 0);
+my $no_legacy = disabled('legacy') || ($ENV{NO_LEGACY} // 0);
+
 # Default config depends on if the legacy module is built or not
-my $defaultcnf = disabled('legacy') ? 'default.cnf' : 'default-and-legacy.cnf';
+my $defaultcnf = $no_legacy ? 'default.cnf' : 'default-and-legacy.cnf';
 
 my @configs = ( $defaultcnf );
 # Only add the FIPS config if the FIPS module has been built
-push @configs, 'fips.cnf' unless disabled('fips');
+push @configs, 'fips.cnf' unless $no_fips;
 
 my @files = qw( evpciph.txt evpdigest.txt );
 my @defltfiles = qw( evpencod.txt evpkdf.txt evppkey_kdf.txt evpmac.txt
@@ -50,17 +53,24 @@ push @defltfiles, @desfiles unless disabled("des");
 my @rc4files = qw( evpciph_rc4.txt );
 push @defltfiles, @rc4files unless disabled("rc4");
 
-plan tests => (scalar(@configs) * scalar(@files)) + scalar(@defltfiles) + 1;
-
-my $infile = bldtop_file('providers', platform->dso('fips'));
-$ENV{OPENSSL_MODULES} = bldtop_dir("providers");
-$ENV{OPENSSL_CONF_INCLUDE} = bldtop_dir("providers");
-
-ok(run(app(['openssl', 'fipsinstall', '-out', bldtop_file('providers', 'fipsinstall.conf'),
-            '-module', $infile,
-            '-provider_name', 'fips', '-mac_name', 'HMAC',
-            '-macopt', 'digest:SHA256', '-macopt', 'hexkey:00',
-            '-section_name', 'fips_sect'])), "fipinstall");
+plan tests =>
+    ($no_fips ? 0 : 1)          # FIPS install test
+    + (scalar(@configs) * scalar(@files))
+    + scalar(@defltfiles);
+
+unless ($no_fips) {
+    my $infile = bldtop_file('providers', platform->dso('fips'));
+    $ENV{OPENSSL_MODULES} = bldtop_dir("providers");
+    $ENV{OPENSSL_CONF_INCLUDE} = bldtop_dir("providers");
+
+    ok(run(app(['openssl', 'fipsinstall',
+                '-out', bldtop_file('providers', 'fipsinstall.conf'),
+                '-module', $infile,
+                '-provider_name', 'fips', '-mac_name', 'HMAC',
+                '-macopt', 'digest:SHA256', '-macopt', 'hexkey:00',
+                '-section_name', 'fips_sect'])),
+       "fipinstall");
+}
 
 foreach (@configs) {
     $ENV{OPENSSL_CONF} = srctop_file("test", $_);
diff --git a/test/recipes/30-test_evp_fetch_prov.t b/test/recipes/30-test_evp_fetch_prov.t
index 4aa1a1019d..d7a44ec306 100644
--- a/test/recipes/30-test_evp_fetch_prov.t
+++ b/test/recipes/30-test_evp_fetch_prov.t
@@ -20,60 +20,105 @@ use lib srctop_dir('Configurations');
 use lib bldtop_dir('.');
 use platform;
 
-my @types = ( "digest", "cipher" );
+my $no_fips = disabled('fips') || ($ENV{NO_FIPS} // 0);
 
-plan tests => 2 + 16 * scalar(@types);
+my @types = ( "digest", "cipher" );
 
 $ENV{OPENSSL_MODULES} = bldtop_dir("providers");
 $ENV{OPENSSL_CONF_INCLUDE} = bldtop_dir("providers");
 
-my $infile = bldtop_file('providers', platform->dso('fips'));
-ok(run(app(['openssl', 'fipsinstall', '-out', bldtop_file('providers', 'fipsinstall.conf'),
-            '-module', $infile,
-            '-provider_name', 'fips', '-mac_name', 'HMAC',
-            '-macopt', 'digest:SHA256', '-macopt', 'hexkey:00',
-            '-section_name', 'fips_sect'])), "fipinstall");
+my @setups = ();
+my @testdata = (
+    { config    => srctop_file("test", "default.cnf"),
+      providers => [ 'default' ],
+      tests  => [ { providers => [] },
+                  { },
+                  { args      => [ '-property', 'default=yes' ],
+                    message   => 'using property "default=yes"' },
+                  { args      => [ '-property', 'fips=no' ],
+                    message   => 'using property "fips=no"' },
+                  { args      => [ '-property', 'default=no', '-fetchfail' ],
+                    message   =>
+                        'using property "default=no" is expected to fail' },
+                  { args      => [ '-property', 'fips=yes', '-fetchfail' ],
+                    message   =>
+                        'using property "fips=yes" is expected to fail' } ] }
+);
+
+unless ($no_fips) {
+    push @setups, {
+        cmd     => app(['openssl', 'fipsinstall',
+                        '-out', bldtop_file('providers', 'fipsinstall.conf'),
+                        '-module', bldtop_file('providers', platform->dso('fips')),
+                        '-provider_name', 'fips', '-mac_name', 'HMAC',
+                        '-macopt', 'digest:SHA256', '-macopt', 'hexkey:00',
+                        '-section_name', 'fips_sect']),
+        message => "fipinstall"
+    };
+    push @testdata, (
+        { config    => srctop_file("test", "fips.cnf"),
+          providers => [ 'fips' ],
+          tests     => [
+              { args    => [ '-property', '' ] },
+              { args    => [ '-property', 'fips=yes' ],
+                message => 'using property "fips=yes"' },
+              { args    => [ '-property', 'default=no' ],
+                message => 'using property "default = no"' },
+              { args      => [ '-property', 'default=yes', '-fetchfail' ],
+                message   =>
+                    'using property "default=yes" is expected to fail' },
+              { args      => [ '-property', 'fips=no', '-fetchfail' ],
+                message   =>
+                    'using property "fips=no" is expected to fail' } ] },
+        { config    => srctop_file("test", "default-and-fips.cnf"),
+          providers => [ 'default', 'fips' ],
+          tests     => [
+              { args    => [ '-property', '' ] },
+              { args      => [ '-property', 'default=no' ],
+                message   => 'using property "default=no"' },
+              { args      => [ '-property', 'default=yes' ],
+                message   => 'using property "default=yes"' },
+              { args      => [ '-property', 'fips=no' ],
+                message   => 'using property "fips=no"' },
+              { args      => [ '-property', 'fips=yes' ],
+                message   => 'using property "fips=yes"' } ] }
+    );
+}
+
+my $testcount = 0;
+foreach (@testdata) {
+    $testcount += scalar @{$_->{tests}};
+}
+
+plan tests => 1 + scalar @setups + $testcount * scalar(@types);
 
-# Do implicit fetch using the default context
 ok(run(test(["evp_fetch_prov_test", "-defaultctx"])),
-    "running evp_fetch_prov_test using implicit fetch using the default libctx");
-
-foreach my $alg(@types) {
-   $ENV{OPENSSL_CONF} = srctop_file("test", "default.cnf");
-   ok(run(test(["evp_fetch_prov_test", "-type", "$alg"])),
-       "running evp_fetch_prov_test using implicit fetch using a created libctx");
-   ok(run(test(["evp_fetch_prov_test", "-type", "$alg", "default"])),
-       "running evp_fetch_prov_test with implicit fetch using default provider loaded");
-   ok(run(test(["evp_fetch_prov_test", "-type", "$alg", "-property", "default=yes", "default"])),
-       "running evp_fetch_prov_test with $alg fetch 'default=yes' using default provider loaded");
-   ok(run(test(["evp_fetch_prov_test", "-type", "$alg", "-property", "fips=no", "default"])),
-       "running evp_fetch_prov_test with $alg fetch 'fips=no' using default provider loaded");
-   ok(run(test(["evp_fetch_prov_test", "-type", "$alg", "-property", "default=no", "-fetchfail", "default"])),
-       "running evp_fetch_prov_test with $alg fetch 'default=no' using default provider loaded should fail");
-   ok(run(test(["evp_fetch_prov_test", "-type", "$alg", "-property", "fips=yes", "-fetchfail", "default"])),
-       "running evp_fetch_prov_test with $alg fetch 'fips=yes' using default provider loaded should fail");
-
-   $ENV{OPENSSL_CONF} = srctop_file("test", "fips.cnf");
-   ok(run(test(["evp_fetch_prov_test", "-type", "$alg", "-property", "", "fips"])),
-       "running evp_fetch_prov_test with $alg fetch '' using loaded fips provider");
-   ok(run(test(["evp_fetch_prov_test", "-type", "$alg", "-property", "fips=yes", "fips"])),
-       "running evp_fetch_prov_test with $alg fetch 'fips=yes' using loaded fips provider");
-   ok(run(test(["evp_fetch_prov_test", "-type", "$alg", "-property", "default=no", "fips"])),
-       "running evp_fetch_prov_test with $alg fetch 'default=no' using loaded fips provider");
-   ok(run(test(["evp_fetch_prov_test", "-type", "$alg", "-property", "default=yes", "-fetchfail", "fips"])),
-       "running evp_fetch_prov_test with $alg fetch 'default=yes' using loaded fips provider should fail");
-   ok(run(test(["evp_fetch_prov_test", "-type", "$alg", "-property", "fips=no", "-fetchfail", "fips"])),
-       "running evp_fetch_prov_test with $alg fetch 'fips=no' using loaded fips provider should fail");
-
-   $ENV{OPENSSL_CONF} = srctop_file("test", "default-and-fips.cnf");
-   ok(run(test(["evp_fetch_prov_test", "-type", "$alg", "-property", "", "default", "fips"])),
-       "running evp_fetch_prov_test with $alg fetch '' using loaded default & fips provider");
-   ok(run(test(["evp_fetch_prov_test", "-type", "$alg", "-property", "default=no", "default", "fips"])),
-       "running evp_fetch_prov_test with $alg fetch 'default=no' using loaded default & fips provider");
-   ok(run(test(["evp_fetch_prov_test", "-type", "$alg", "-property", "default=yes", "default", "fips"])),
-       "running evp_fetch_prov_test with $alg fetch 'default=yes' using loaded default & fips provider");
-   ok(run(test(["evp_fetch_prov_test", "-type", "$alg", "-property", "fips=no", "default", "fips"])),
-       "running evp_fetch_prov_test with $alg fetch 'fips=no' using loaded default & fips provider");
-   ok(run(test(["evp_fetch_prov_test", "-type", "$alg", "-property", "fips=yes", "default", "fips"])),
-       "running evp_fetch_prov_test with $alg fetch 'fips=yes' using loaded default & fips provider");
-}
\ No newline at end of file
+   "running evp_fetch_prov_test using the default libctx");
+
+foreach my $setup (@setups) {
+    ok(run($setup->{cmd}), $setup->{message});
+}
+
+foreach my $alg (@types) {
+    foreach my $testcase (@testdata) {
+        $ENV{OPENSSL_CONF} = $testcase->{config};
+        foreach my $test (@{$testcase->{tests}}) {
+            my @testproviders =
+                @{ $test->{providers} // $testcase->{providers} };
+            my $testprovstr = @testproviders
+                ? ' and loaded providers ' . join(' & ',
+                                                  map { "'$_'" } @testproviders)
+                : '';
+            my @testargs = @{ $test->{args} // [] };
+            my $testmsg =
+                defined $test->{message} ? ' '.$test->{message} : '';
+
+            my $message =
+                "running evp_fetch_prov_test with $alg$testprovstr$testmsg";
+
+            ok(run(test(["evp_fetch_prov_test", "-type", "$alg",
+                         @testargs, @testproviders])),
+               $message);
+        }
+    }
+}