diff options
Diffstat (limited to 'test')
-rw-r--r-- | test/README-dev.md (renamed from test/README) | 115 | ||||
-rw-r--r-- | test/README-external.md (renamed from test/README.external) | 0 | ||||
-rw-r--r-- | test/README.md | 4 | ||||
-rw-r--r-- | test/data.txt | 4 | ||||
-rw-r--r-- | test/data2.txt | 188 | ||||
-rw-r--r-- | test/recipes/04-test_pem_data/NOTES.txt (renamed from test/recipes/04-test_pem_data/NOTES) | 0 | ||||
-rw-r--r-- | test/recipes/20-test_dgst.t | 4 | ||||
-rw-r--r-- | test/recipes/20-test_pkeyutl.t | 4 | ||||
-rw-r--r-- | test/recipes/95-test_external_krb5.t | 2 |
9 files changed, 256 insertions, 65 deletions
diff --git a/test/README b/test/README-dev.md index 9094d9a38d..56114fdc34 100644 --- a/test/README +++ b/test/README-dev.md @@ -1,44 +1,46 @@ +Guidelines for test developers +============================== + How to add recipes -================== +------------------ For any test that you want to perform, you write a script located in -test/recipes/, named {nn}-test_{name}.t, where {nn} is a two digit number and -{name} is a unique name of your choice. +`test/recipes/`, named `{nn}-test_{name}.t`, +where `{nn}` is a two digit number and +`{name}` is a unique name of your choice. Please note that if a test involves a new testing executable, you will need to -do some additions in test/build.info. Please refer to the section "Changes to -test/build.info" below. - +do some additions in test/build.info. Please refer to the section +["Changes to test/build.info"](README.md#changes-to-testbuildinfo) below. Naming conventions -================= - -A test executable is named test/{name}test.c - -A test recipe is named test/recipes/{nn}-test_{name}.t, where {nn} is a two -digit number and {name} is a unique name of your choice. - -The number {nn} is (somewhat loosely) grouped as follows: - -00-04 sanity, internal and essential API tests -05-09 individual symmetric cipher algorithms -10-14 math (bignum) -15-19 individual asymmetric cipher algorithms -20-24 openssl commands (some otherwise not tested) -25-29 certificate forms, generation and verification -30-35 engine and evp -60-79 APIs: - 60 X509 subsystem - 61 BIO subsystem - 65 CMP subsystem - 70 PACKET layer -80-89 "larger" protocols (CA, CMS, OCSP, SSL, TSA) -90-98 misc -99 most time consuming tests [such as test_fuzz] - +------------------ + +A test executable is named `test/{name}test.c` + +A test recipe is named `test/recipes/{nn}-test_{name}.t`, where `{nn}` is a two +digit number and `{name}` is a unique name of your choice. + +The number `{nn}` is (somewhat loosely) grouped as follows: + + 00-04 sanity, internal and essential API tests + 05-09 individual symmetric cipher algorithms + 10-14 math (bignum) + 15-19 individual asymmetric cipher algorithms + 20-24 openssl commands (some otherwise not tested) + 25-29 certificate forms, generation and verification + 30-35 engine and evp + 60-79 APIs: + 60 X509 subsystem + 61 BIO subsystem + 65 CMP subsystem + 70 PACKET layer + 80-89 "larger" protocols (CA, CMS, OCSP, SSL, TSA) + 90-98 misc + 99 most time consuming tests [such as test_fuzz] A recipe that just runs a test executable -========================================= +----------------------------------------- A script that just runs a program looks like this: @@ -48,21 +50,20 @@ A script that just runs a program looks like this: simple_test("test_{name}", "{name}test", "{name}"); -{name} is the unique name you have chosen for your test. - -The second argument to `simple_test' is the test executable, and `simple_test' -expects it to be located in test/ +`{name}` is the unique name you have chosen for your test. -For documentation on OpenSSL::Test::Simple, do -`perldoc util/perl/OpenSSL/Test/Simple.pm'. +The second argument to `simple_test` is the test executable, and `simple_test` +expects it to be located in `test/` +For documentation on `OpenSSL::Test::Simple`, +do `perldoc util/perl/OpenSSL/Test/Simple.pm`. A recipe that runs a more complex test -====================================== +-------------------------------------- For more complex tests, you will need to read up on Test::More and -OpenSSL::Test. Test::More is normally preinstalled, do `man Test::More' for -documentation. For OpenSSL::Test, do `perldoc util/perl/OpenSSL/Test.pm'. +OpenSSL::Test. Test::More is normally preinstalled, do `man Test::More` for +documentation. For OpenSSL::Test, do `perldoc util/perl/OpenSSL/Test.pm`. A script to start from could be this: @@ -89,26 +90,25 @@ A script to start from could be this: # test feature 2 } - Changes to test/build.info -========================== +-------------------------- Whenever a new test involves a new test executable you need to do the following (at all times, replace {NAME} and {name} with the name of your test): -* add {name} to the list of programs under PROGRAMS_NO_INST + * add `{name}` to the list of programs under `PROGRAMS_NO_INST` -* create a three line description of how to build the test, you will have -to modify the include paths and source files if you don't want to use the -basic test framework: + * create a three line description of how to build the test, you will have + to modify the include paths and source files if you don't want to use the + basic test framework: - SOURCE[{name}]={name}.c - INCLUDE[{name}]=.. ../include ../apps/include - DEPEND[{name}]=../libcrypto libtestutil.a + SOURCE[{name}]={name}.c + INCLUDE[{name}]=.. ../include ../apps/include + DEPEND[{name}]=../libcrypto libtestutil.a Generic form of C test executables -================================== +---------------------------------- #include "testutil.h" @@ -133,12 +133,12 @@ Generic form of C test executables return 1; /* Indicate success */ } -You should use the TEST_xxx macros provided by testutil.h to test all failure +You should use the `TEST_xxx` macros provided by `testutil.h` to test all failure conditions. These macros produce an error message in a standard format if the condition is not met (and nothing if the condition is met). Additional -information can be presented with the TEST_info macro that takes a printf -format string and arguments. TEST_error is useful for complicated conditions, -it also takes a printf format string and argument. In all cases the TEST_xxx +information can be presented with the `TEST_info` macro that takes a `printf` +format string and arguments. `TEST_error` is useful for complicated conditions, +it also takes a `printf` format string and argument. In all cases the `TEST_xxx` macros are guaranteed to evaluate their arguments exactly once. This means that expressions with side effects are allowed as parameters. Thus, @@ -152,6 +152,5 @@ works fine and can be used in place of: The former produces a more meaningful message on failure than the latter. Note that the test infrastructure automatically sets up all required environment -variables (such as OPENSSL_MODULES, OPENSSL_CONF etc) for the tests. Individual -tests may choose to override the default settings as required. - +variables (such as `OPENSSL_MODULES`, `OPENSSL_CONF`, etc.) for the tests. +Individual tests may choose to override the default settings as required. diff --git a/test/README.external b/test/README-external.md index 8069ce8d00..8069ce8d00 100644 --- a/test/README.external +++ b/test/README-external.md diff --git a/test/README.md b/test/README.md index a9e0c827d1..f9058a0026 100644 --- a/test/README.md +++ b/test/README.md @@ -1,5 +1,5 @@ -Test OpenSSL -============ +Using OpenSSL Tests +=================== After a successful build, and before installing, the libraries should be tested. Run: diff --git a/test/data.txt b/test/data.txt new file mode 100644 index 0000000000..3442255280 --- /dev/null +++ b/test/data.txt @@ -0,0 +1,4 @@ +TEST DATA + +Please note that if a test involves a new testing executable, +you will need to do some additions in test/build.info. diff --git a/test/data2.txt b/test/data2.txt new file mode 100644 index 0000000000..c4f9691a26 --- /dev/null +++ b/test/data2.txt @@ -0,0 +1,188 @@ +TEST DATA2 + +Running external test suites with OpenSSL +========================================= + +It is possible to integrate external test suites into OpenSSL's "make test". +This capability is considered a developer option and does not work on all +platforms. + + + +The BoringSSL test suite +======================== + +In order to run the BoringSSL tests with OpenSSL, first checkout the BoringSSL +source code into an appropriate directory. This can be done in two ways: + +1) Separately from the OpenSSL checkout using: + + $ git clone https://boringssl.googlesource.com/boringssl boringssl + + The BoringSSL tests are only confirmed to work at a specific commit in the + BoringSSL repository. Later commits may or may not pass the test suite: + + $ cd boringssl + $ git checkout 490469f850e + +2) Using the already configured submodule settings in OpenSSL: + + $ git submodule update --init + +Configure the OpenSSL source code to enable the external tests: + +$ cd ../openssl +$ ./config enable-ssl3 enable-ssl3-method enable-weak-ssl-ciphers \ + enable-external-tests + +Note that using other config options than those given above may cause the tests +to fail. + +Run the OpenSSL tests by providing the path to the BoringSSL test runner in the +BORING_RUNNER_DIR environment variable: + +$ BORING_RUNNER_DIR=/path/to/boringssl/ssl/test/runner make test + +Note that the test suite may change directory while running so the path provided +should be absolute and not relative to the current working directory. + +To see more detailed output you can run just the BoringSSL tests with the +verbose option: + +$ VERBOSE=1 BORING_RUNNER_DIR=/path/to/boringssl/ssl/test/runner make \ + TESTS="test_external_boringssl" test + + +Test failures and suppressions +------------------------------ + +A large number of the BoringSSL tests are known to fail. A test could fail +because of many possible reasons. For example: + +- A bug in OpenSSL +- Different interpretations of standards +- Assumptions about the way BoringSSL works that do not apply to OpenSSL +- The test uses APIs added to BoringSSL that are not present in OpenSSL +- etc + +In order to provide a "clean" baseline run with all the tests passing a config +file has been provided that suppresses the running of tests that are known to +fail. These suppressions are held in the file "test/ossl_shim/ossl_config.json" +within the OpenSSL source code. + +The community is encouraged to contribute patches which reduce the number of +suppressions that are currently present. + + +Python PYCA/Cryptography test suite +=================================== + +This python test suite runs cryptographic tests with a local OpenSSL build as +the implementation. + +First checkout the PYCA/Cryptography module into ./pyca-cryptography using: + +$ git submodule update --init + +Then configure/build OpenSSL compatible with the python module: + +$ ./config shared enable-external-tests +$ make + +The tests will run in a python virtual environment which requires virtualenv +to be installed. + +$ make test VERBOSE=1 TESTS=test_external_pyca + +Test failures and suppressions +------------------------------ + +Some tests target older (<=1.0.2) versions so will not run. Other tests target +other crypto implementations so are not relevant. Currently no tests fail. + + +krb5 test suite +=============== + +Much like the PYCA/Cryptography test suite, this builds and runs the krb5 +tests against the local OpenSSL build. + +You will need a git checkout of krb5 at the top level: + +$ git clone https://github.com/krb5/krb5 + +krb5's master has to pass this same CI, but a known-good version is +krb5-1.15.1-final if you want to be sure. + +$ cd krb5 +$ git checkout krb5-1.15.1-final +$ cd .. + +OpenSSL must be built with external tests enabled: + +$ ./config enable-external-tests +$ make + +krb5's tests will then be run as part of the rest of the suite, or can be +explicitly run (with more debugging): + +$ VERBOSE=1 make TESTS=test_external_krb5 test + +Test-failures suppressions +-------------------------- + +krb5 will automatically adapt its test suite to account for the configuration +of your system. Certain tests may require more installed packages to run. No +tests are expected to fail. + + +GOST engine test suite +=============== + +Much like the PYCA/Cryptography test suite, this builds and runs the GOST engine +tests against the local OpenSSL build. + +You will need a git checkout of gost-engine at the top level: + +$ git submodule update --init + +Then configure/build OpenSSL enabling external tests: + +$ ./config shared enable-external-tests +$ make + +GOST engine requires CMake for the build process. + +GOST engine tests will then be run as part of the rest of the suite, or can be +explicitly run (with more debugging): + +$ make test VERBOSE=1 TESTS=test_external_gost_engine + +Updating test suites +==================== + +To update the commit for any of the above test suites: + +- Make sure the submodules are cloned locally: + + $ git submodule update --init --recursive + +- Enter subdirectory and pull from the repository (use a specific branch/tag if required): + + $ cd <submodule-dir> + $ git pull origin master + +- Go to root directory, there should be a new git status: + + $ cd ../ + $ git status + ... + # modified: <submodule-dir> (new commits) + ... + +- Add/commit/push the update + + git add <submodule-dir> + git commit -m "Updated <submodule> to latest commit" + git push + diff --git a/test/recipes/04-test_pem_data/NOTES b/test/recipes/04-test_pem_data/NOTES.txt index baafd1560c..baafd1560c 100644 --- a/test/recipes/04-test_pem_data/NOTES +++ b/test/recipes/04-test_pem_data/NOTES.txt diff --git a/test/recipes/20-test_dgst.t b/test/recipes/20-test_dgst.t index c10da5a9f4..0b7ab2d5d1 100644 --- a/test/recipes/20-test_dgst.t +++ b/test/recipes/20-test_dgst.t @@ -24,8 +24,8 @@ sub tsignverify { my $privkey = shift; my $pubkey = shift; - my $data_to_sign = srctop_file('test', 'README'); - my $other_data = srctop_file('test', 'README.external'); + my $data_to_sign = srctop_file('test', 'data.txt'); + my $other_data = srctop_file('test', 'data2.txt'); my $sigfile = basename($privkey, '.pem') . '.sig'; plan tests => 4; diff --git a/test/recipes/20-test_pkeyutl.t b/test/recipes/20-test_pkeyutl.t index 753995428d..543038cab9 100644 --- a/test/recipes/20-test_pkeyutl.t +++ b/test/recipes/20-test_pkeyutl.t @@ -74,8 +74,8 @@ sub tsignverify { my $pubkey = shift; my @extraopts = @_; - my $data_to_sign = srctop_file('test', 'README'); - my $other_data = srctop_file('test', 'README.external'); + my $data_to_sign = srctop_file('test', 'data.txt'); + my $other_data = srctop_file('test', 'data2.txt'); my $sigfile = basename($privkey, '.pem') . '.sig'; my @args = (); diff --git a/test/recipes/95-test_external_krb5.t b/test/recipes/95-test_external_krb5.t index c431d966b3..ad262da2d9 100644 --- a/test/recipes/95-test_external_krb5.t +++ b/test/recipes/95-test_external_krb5.t @@ -17,7 +17,7 @@ setup("test_external_krb5"); plan skip_all => "No external tests in this configuration" if disabled("external-tests"); plan skip_all => "krb5 not available" - if ! -f srctop_file("krb5", "README"); + if ! -f srctop_file("krb5", "data.txt"); plan tests => 1; |