Test skip option.

Provide C test cases with the option to skip tests and subtests.

Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/8695)

5 files changed, 74 insertions, 33 deletions

diff --git a/test/test_test.c b/test/test_test.c
index 3741e17122..ba66a158ad 100644
--- a/test/test_test.c
+++ b/test/test_test.c
@@ -531,6 +531,25 @@ static int test_bn_output(int n)
     return 1;
 }
 
+static int test_skip_one(void)
+{
+    return TEST_skip("skip test");
+}
+
+static int test_skip_many(int n)
+{
+    return TEST_skip("skip tests: %d", n);
+}
+
+static int test_skip_null(void)
+{
+    /*
+     * This is not a recommended way of skipping a test, a reason or
+     * description should be included.
+     */
+    return TEST_skip(NULL);
+}
+
 int setup_tests(void)
 {
     ADD_TEST(test_int);
@@ -553,5 +572,8 @@ int setup_tests(void)
     ADD_TEST(test_single_eval);
     ADD_TEST(test_output);
     ADD_ALL_TESTS(test_bn_output, OSSL_NELEM(bn_output_tests));
+    ADD_TEST(test_skip_one);
+    ADD_TEST(test_skip_null);
+    ADD_ALL_TESTS(test_skip_many, 3);
     return 1;
 }
diff --git a/test/testutil.h b/test/testutil.h
index 7f5db601cd..3a5c4866da 100644
--- a/test/testutil.h
+++ b/test/testutil.h
@@ -346,6 +346,9 @@ void test_info(const char *file, int line, const char *desc, ...)
     PRINTF_FORMAT(3, 4);
 void test_info_c90(const char *desc, ...) PRINTF_FORMAT(1, 2);
 void test_note(const char *desc, ...) PRINTF_FORMAT(1, 2);
+int test_skip(const char *file, int line, const char *desc, ...)
+    PRINTF_FORMAT(3, 4);
+int test_skip_c90(const char *desc, ...) PRINTF_FORMAT(1, 2);
 void test_openssl_errors(void);
 void test_perror(const char *s);
 
@@ -463,9 +466,11 @@ void test_perror(const char *s);
 # if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L
 #  define TEST_error         test_error_c90
 #  define TEST_info          test_info_c90
+#  define TEST_skip          test_skip_c90
 # else
 #  define TEST_error(...)    test_error(__FILE__, __LINE__, __VA_ARGS__)
 #  define TEST_info(...)     test_info(__FILE__, __LINE__, __VA_ARGS__)
+#  define TEST_skip(...)     test_skip(__FILE__, __LINE__, __VA_ARGS__)
 # endif
 # define TEST_note           test_note
 # define TEST_openssl_errors test_openssl_errors
diff --git a/test/testutil/driver.c b/test/testutil/driver.c
index 10d74e2ea4..2da83ed727 100644
--- a/test/testutil/driver.c
+++ b/test/testutil/driver.c
@@ -280,20 +280,20 @@ void set_test_title(const char *title)
     test_title = title == NULL ? NULL : strdup(title);
 }
 
-PRINTF_FORMAT(2, 3) static void test_verdict(int pass, const char *extra, ...)
+PRINTF_FORMAT(2, 3) static void test_verdict(int verdict,
+                                             const char *description, ...)
 {
     va_list ap;
 
     test_flush_stdout();
     test_flush_stderr();
 
-    test_printf_stdout("%*s%s", level, "", pass ? "ok" : "not ok");
-    if (extra != NULL) {
-        test_printf_stdout(" ");
-        va_start(ap, extra);
-        test_vprintf_stdout(extra, ap);
-        va_end(ap);
-    }
+    test_printf_stdout("%*s%s ", level, "", verdict != 0 ? "ok" : "not ok");
+    va_start(ap, description);
+    test_vprintf_stdout(description, ap);
+    va_end(ap);
+    if (verdict == TEST_SKIP_CODE)
+        test_printf_stdout(" # skipped");
     test_printf_stdout("\n");
     test_flush_stdout();
 }
@@ -349,20 +349,14 @@ int run_tests(const char *test_prog_name)
             }
             test_flush_stdout();
         } else if (all_tests[i].num == -1) {
-            int ret = 0;
-
             set_test_title(all_tests[i].test_case_name);
-            ret = all_tests[i].test_fn();
-            verdict = 1;
-            if (!ret) {
-                verdict = 0;
-                ++num_failed;
-            }
+            verdict = all_tests[i].test_fn();
             test_verdict(verdict, "%d - %s", ii + 1, test_title);
-            finalize(ret);
+            finalize(verdict != 0);
         } else {
             int num_failed_inner = 0;
 
+            verdict = TEST_SKIP_CODE;
             level += 4;
             if (all_tests[i].subtest && single_iter == -1) {
                 test_printf_stdout("%*s# Subtest: %s\n", level, "",
@@ -381,39 +375,34 @@ int run_tests(const char *test_prog_name)
                 while (jstep == 0 || gcd(all_tests[i].num, jstep) != 1);
 
             for (jj = 0; jj < all_tests[i].num; jj++) {
-                int ret;
+                int v;
 
                 j = (j + jstep) % all_tests[i].num;
                 if (single_iter != -1 && ((jj + 1) != single_iter))
                     continue;
                 set_test_title(NULL);
-                ret = all_tests[i].param_test_fn(j);
+                v = all_tests[i].param_test_fn(j);
 
-                if (!ret)
+                if (v == 0) {
                     ++num_failed_inner;
+                    verdict = 0;
+                } else if (v != TEST_SKIP_CODE && verdict != 0) {
+                    verdict = 1;
+                }
 
-                finalize(ret);
+                finalize(v != 0);
 
                 if (all_tests[i].subtest) {
-                    verdict = 1;
-                    if (!ret) {
-                        verdict = 0;
-                        ++num_failed_inner;
-                    }
                     if (test_title != NULL)
-                        test_verdict(verdict, "%d - %s", jj + 1, test_title);
+                        test_verdict(v, "%d - %s", jj + 1, test_title);
                     else
-                        test_verdict(verdict, "%d - iteration %d",
-                                     jj + 1, j + 1);
+                        test_verdict(v, "%d - iteration %d", jj + 1, j + 1);
                 }
             }
 
             level -= 4;
-            verdict = 1;
-            if (num_failed_inner) {
-                verdict = 0;
+            if (verdict == 0)
                 ++num_failed;
-            }
             test_verdict(verdict, "%d - %s", ii + 1,
                          all_tests[i].test_case_name);
         }
diff --git a/test/testutil/tests.c b/test/testutil/tests.c
index 05b0688880..56177cd999 100644
--- a/test/testutil/tests.c
+++ b/test/testutil/tests.c
@@ -157,6 +157,29 @@ void test_note(const char *fmt, ...)
     test_flush_stderr();
 }
 
+
+int test_skip(const char *file, int line, const char *desc, ...)
+{
+    va_list ap;
+
+    va_start(ap, desc);
+    test_fail_message_va("SKIP", file, line, NULL, NULL, NULL, NULL, desc, ap);
+    va_end(ap);
+    return TEST_SKIP_CODE;
+}
+
+int test_skip_c90(const char *desc, ...)
+{
+    va_list ap;
+
+    va_start(ap, desc);
+    test_fail_message_va("SKIP", NULL, -1, NULL, NULL, NULL, NULL, desc, ap);
+    va_end(ap);
+    test_printf_stderr("\n");
+    return TEST_SKIP_CODE;
+}
+
+
 void test_openssl_errors(void)
 {
     ERR_print_errors_cb(openssl_error_cb, NULL);
diff --git a/test/testutil/tu_local.h b/test/testutil/tu_local.h
index 049d7b1bc2..ede4546948 100644
--- a/test/testutil/tu_local.h
+++ b/test/testutil/tu_local.h
@@ -12,6 +12,8 @@
 #include <openssl/bio.h>
 #include "../testutil.h"
 
+#define TEST_SKIP_CODE  123
+
 int subtest_level(void);
 int openssl_error_cb(const char *str, size_t len, void *u);
 const BIO_METHOD *BIO_f_tap(void);