TEST: Create test specific output directories

We had all tests run with test/test-runs/ as working directory, and
tests cleaned up after themselves...  which is well and good, until
you want to have a look at what went wrong when a complex test fails,
and you have to recreate everything it does manually.

To remedy this, we have OpenSSL::Test create the result directory
dynamically (and cleaning it up first if it's already there) and let
the test recipe have that as working directory.

Test recipes are now encouraged to name their diverse output files
uniquely, and not to clean them up, to allow a developer to have a
look at the files that were produced.

With continuous integration that allows this, the result directories
could also be archived and be left as a build artifact.

Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/11080)

3 files changed, 5 insertions, 22 deletions

diff --git a/Configurations/descrip.mms.tmpl b/Configurations/descrip.mms.tmpl
index d379a8230b..15fefd2502 100644
--- a/Configurations/descrip.mms.tmpl
+++ b/Configurations/descrip.mms.tmpl
@@ -28,14 +28,6 @@
       (my $x = shift) =~ s|\]$|...]|;
       $x
   }
-  sub move {
-      my $f = catdir(@_);
-      my $b = abs2rel(rel2abs("."),rel2abs($f));
-      $sourcedir = catdir($b,$sourcedir)
-          if !file_name_is_absolute($sourcedir);
-      $builddir = catdir($b,$builddir)
-          if !file_name_is_absolute($builddir);
-      "";
   }
 
   # Because we need to make two computations of these data,
@@ -439,11 +431,8 @@ all : build_sw build_docs
 test : tests
 {- dependmagic('tests'); -} : build_programs_nodep, build_modules_nodep
         @ ! {- output_off() if $disabled{tests}; "" -}
-        SET DEFAULT [.test]{- move("test") -}
-        CREATE/DIR [.test-runs]
         DEFINE SRCTOP {- sourcedir() -}
         DEFINE BLDTOP {- builddir() -}
-        DEFINE RESULT_D {- builddir(qw(test test-runs)) -}
         DEFINE OPENSSL_ENGINES {- builddir("engines") -}
         DEFINE OPENSSL_MODULES {- builddir("providers") -}
         IF "$(VERBOSE)" .NES. "" THEN DEFINE VERBOSE "$(VERBOSE)"
@@ -452,7 +441,6 @@ test : tests
         DEASSIGN OPENSSL_ENGINES
         DEASSIGN BLDTOP
         DEASSIGN SRCTOP
-        SET DEFAULT [-]{- move("..") -}
         @ ! {- if ($disabled{tests}) { output_on(); } else { output_off(); } "" -}
         @ WRITE SYS$OUTPUT "Tests are not supported with your chosen Configure options"
         @ ! {- output_on() if !$disabled{tests}; "" -}
diff --git a/Configurations/unix-Makefile.tmpl b/Configurations/unix-Makefile.tmpl
index 6777bb8de9..55c1b12573 100644
--- a/Configurations/unix-Makefile.tmpl
+++ b/Configurations/unix-Makefile.tmpl
@@ -464,16 +464,13 @@ all: build_sw build_docs
 test: tests
 {- dependmagic('tests'); -}: build_programs_nodep build_modules_nodep link-utils
 	@ : {- output_off() if $disabled{tests}; "" -}
-	( cd test; \
-	  mkdir -p test-runs; \
-	  SRCTOP=../$(SRCDIR) \
-	  BLDTOP=../$(BLDDIR) \
-	  RESULT_D=test-runs \
+	( SRCTOP=$(SRCDIR) \
+	  BLDTOP=$(BLDDIR) \
 	  PERL="$(PERL)" \
 	  EXE_EXT={- platform->binext() -} \
-	  OPENSSL_ENGINES=`cd ../$(BLDDIR)/engines 2>/dev/null && pwd` \
-	  OPENSSL_MODULES=`cd ../$(BLDDIR)/providers 2>/dev/null && pwd` \
-	  $(PERL) ../$(SRCDIR)/test/run_tests.pl $(TESTS) )
+	  OPENSSL_ENGINES=`cd $(BLDDIR)/engines 2>/dev/null && pwd` \
+	  OPENSSL_MODULES=`cd $(BLDDIR)/providers 2>/dev/null && pwd` \
+	  $(PERL) $(SRCDIR)/test/run_tests.pl $(TESTS) )
 	@ : {- if ($disabled{tests}) { output_on(); } else { output_off(); } "" -}
 	@echo "Tests are not supported with your chosen Configure options"
 	@ : {- output_on() if !$disabled{tests}; "" -}
diff --git a/Configurations/windows-makefile.tmpl b/Configurations/windows-makefile.tmpl
index 275c93ebc1..074e8f74c5 100644
--- a/Configurations/windows-makefile.tmpl
+++ b/Configurations/windows-makefile.tmpl
@@ -387,10 +387,8 @@ all: build_sw build_docs
 test: tests
 {- dependmagic('tests'); -}: build_programs_nodep build_modules_nodep
 	@{- output_off() if $disabled{tests}; "" -}
-	-mkdir $(BLDDIR)\test\test-runs
 	set SRCTOP=$(SRCDIR)
 	set BLDTOP=$(BLDDIR)
-	set RESULT_D=$(BLDDIR)\test\test-runs
 	set PERL=$(PERL)
 	set OPENSSL_ENGINES=$(MAKEDIR)\engines
 	set OPENSSL_MODULES=$(MAKEDIR)\providers