patch 8.1.1476: no statistics displayed after running testsv8.1.1476

Problem:    No statistics displayed after running tests.
Solution:   Summarize the test results. (Christian Brabandt, closes #4391)
            Also make it possible to report a skipped file.

8 files changed, 84 insertions, 9 deletions

diff --git a/src/Makefile b/src/Makefile
index 1347499eb1..65398d0184 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -2117,7 +2117,8 @@ types.vim: $(TAGS_FILES)
 # TESTING
 #
 # Execute the test scripts and the unittests.
-test check: scripttests unittests test_libvterm
+# Do the scripttests first, so that the summary shows last.
+test check: unittests test_libvterm scripttests
 
 # Execute the test scripts.  Run these after compiling Vim, before installing.
 # This doesn't depend on $(VIMTARGET), because that won't work when configure
diff --git a/src/testdir/Makefile b/src/testdir/Makefile
index 45c520d2df..0abf66137d 100644
--- a/src/testdir/Makefile
+++ b/src/testdir/Makefile
@@ -12,7 +12,7 @@ SCRIPTSOURCE = ../../runtime
 # Comment out this line to see the verbose output of tests.
 #
 # Catches SwapExists to avoid hanging at the ATTENTION prompt.
-REDIR_TEST_TO_NULL = --cmd 'au SwapExists * let v:swapchoice = "e"' > /dev/null
+# REDIR_TEST_TO_NULL = --cmd 'au SwapExists * let v:swapchoice = "e"' > /dev/null
 
 # Uncomment this line to use valgrind for memory leaks and extra warnings.
 #   The output goes into a file "valgrind.testN"
@@ -49,10 +49,12 @@ gui:	nolog $(SCRIPTS_FIRST) $(SCRIPTS) $(SCRIPTS_GUI) newtests report
 benchmark: $(SCRIPTS_BENCH)
 
 report:
+	$(RUN_VIMTEST) $(NO_INITS) -S summarize.vim messages $(REDIR_TEST_TO_NULL)
 	@echo
 	@echo 'Test results:'
+	@cat test_result.log
 	@/bin/sh -c "if test -f test.log; \
-		then cat test.log; echo TEST FAILURE; exit 1; \
+		then echo TEST FAILURE; exit 1; \
 		else echo ALL DONE; \
 		fi"
 
@@ -77,7 +79,10 @@ RM_ON_START = tiny.vim small.vim mbyte.vim mzscheme.vim test.ok benchmark.out
 RUN_VIM = VIMRUNTIME=$(SCRIPTSOURCE); export VIMRUNTIME; $(VALGRIND) $(VIMPROG) -f $(GUI_FLAG) -u unix.vim $(NO_INITS) -s dotest.in
 
 clean:
-	-rm -rf *.out *.failed *.res *.rej *.orig opt_test.vim test.log messages $(RM_ON_RUN) $(RM_ON_START) valgrind.*
+	-rm -rf *.out *.failed *.res *.rej *.orig
+	-rm opt_test.vim test.log test_result.log messages
+	-rm $(RM_ON_RUN) $(RM_ON_START)
+	-rm valgrind.*
 
 test1.out: test1.in
 	-rm -rf $*.failed $(RM_ON_RUN) $(RM_ON_START) wrongtermsize
diff --git a/src/testdir/runtest.vim b/src/testdir/runtest.vim
index 2a1b5ea5cc..02d4d20e34 100644
--- a/src/testdir/runtest.vim
+++ b/src/testdir/runtest.vim
@@ -268,6 +268,9 @@ if expand('%') =~ 'test_vimscript.vim'
 else
   try
     source %
+  catch /^\cskipped/
+    call add(s:messages, '    Skipped')
+    call add(s:skipped, 'SKIPPED ' . expand('%') . ': ' . substitute(v:exception, '^\S*\s\+', '',  ''))
   catch
     let s:fail += 1
     call add(s:errors, 'Caught exception: ' . v:exception . ' @ ' . v:throwpoint)
diff --git a/src/testdir/summarize.vim b/src/testdir/summarize.vim
new file mode 100644
index 0000000000..43dbce0f44
--- /dev/null
+++ b/src/testdir/summarize.vim
@@ -0,0 +1,60 @@
+if 1
+  " This is executed with the eval feature
+  set nocp
+  func Count(match, type)
+    if a:type ==# 'executed'
+      let g:executed += (a:match+0)
+    elseif a:type ==# 'failed'
+      let g:failed = a:match+0
+    elseif a:type ==# 'skipped'
+      let g:skipped += 1
+      call extend(g:skipped_output, ["\t".a:match])
+    endif
+  endfunc
+
+  let g:executed = 0
+  let g:skipped = 0
+  let g:failed = 0
+  let g:skipped_output = []
+  let g:failed_output = []
+  let output = [""]
+
+  try
+    " This uses the :s command to just fetch and process the output of the
+    " tests, it doesn't acutally replay anything
+    %s/^Executed\s\+\zs\d\+\ze\s\+tests/\=Count(submatch(0),'executed')/egn
+    %s/^SKIPPED \zs.*/\=Count(submatch(0), 'skipped')/egn
+    %s/^\(\d\+\)\s\+FAILED:/\=Count(submatch(1), 'failed')/egn
+
+    call extend(output, ["Skipped:"]) 
+    call extend(output, skipped_output)
+
+    call extend(output, [
+          \ "",
+          \ "-------------------------------",
+          \ printf("Executed: %5d Tests", g:executed),
+          \ printf(" Skipped: %5d Tests", g:skipped),
+          \ printf("  %s: %5d Tests", g:failed == 0 ? 'Failed' : 'FAILED', g:failed),
+          \ "",
+          \ ]) 
+    if filereadable('test.log')
+      " outputs and indents the failed test result
+      call extend(output, ["", "Failures: "])
+      let failed_output = filter(readfile('test.log'), { v,k -> !empty(k)})
+      call extend(output, map(failed_output, { v,k -> "\t".k}))
+      " Add a final newline
+      call extend(output, [""])
+    endif
+
+  catch  " Catch-all
+  finally
+    call writefile(output, 'test_result.log')  " overwrites an existing file
+    q!
+  endtry
+endif
+
+" This is executed without the eval feature
+%d
+r test.log
+w test_result.log
+q!
diff --git a/src/testdir/test_arabic.vim b/src/testdir/test_arabic.vim
index d0211fee22..450c6f98f5 100644
--- a/src/testdir/test_arabic.vim
+++ b/src/testdir/test_arabic.vim
@@ -3,7 +3,7 @@
 " functional tests that check the shaping works with real text.
 
 if !has('arabic')
-  finish
+  throw 'Skipped: arabic feature missing'
 endif
 
 source view_util.vim
diff --git a/src/testdir/test_autochdir.vim b/src/testdir/test_autochdir.vim
index 90538fbc09..1f99858efb 100644
--- a/src/testdir/test_autochdir.vim
+++ b/src/testdir/test_autochdir.vim
@@ -1,7 +1,7 @@
 " Test 'autochdir' behavior
 
 if !exists("+autochdir")
-  finish
+  throw 'Skipped: autochdir feature missing'
 endif
 
 func Test_set_filename()
diff --git a/src/testdir/test_balloon.vim b/src/testdir/test_balloon.vim
index 39c2602be6..37fc57eb5c 100644
--- a/src/testdir/test_balloon.vim
+++ b/src/testdir/test_balloon.vim
@@ -1,11 +1,15 @@
 " Tests for 'balloonevalterm'.
 
-" Tests that only work in the terminal.
-if has('balloon_eval_term') && !has('gui_running')
+if !has('balloon_eval_term')
+  throw 'Skipped: balloon_eval_term feature missing'
+endif
+
+" A few tests only work in the terminal.
+if !has('gui_running')
 
 source screendump.vim
 if !CanRunVimInTerminal()
-  finish
+  throw 'Skipped: cannot run Vim in a terminal window'
 endif
 
 let s:common_script =<< [CODE]
diff --git a/src/version.c b/src/version.c
index 8f707b0263..d0d2c28fc8 100644
--- a/src/version.c
+++ b/src/version.c
@@ -768,6 +768,8 @@ static char *(features[]) =
 static int included_patches[] =
 {   /* Add new patch number below this line */
 /**/
+    1476,
+/**/
     1475,
 /**/
     1474,