From 9c0cec65f891492314caadeef87a50251a21e630 Mon Sep 17 00:00:00 2001 From: Bram Moolenaar Date: Thu, 6 Jun 2019 13:38:15 +0200 Subject: patch 8.1.1476: no statistics displayed after running tests Problem: No statistics displayed after running tests. Solution: Summarize the test results. (Christian Brabandt, closes #4391) Also make it possible to report a skipped file. --- src/testdir/Makefile | 11 +++++--- src/testdir/runtest.vim | 3 +++ src/testdir/summarize.vim | 60 ++++++++++++++++++++++++++++++++++++++++++ src/testdir/test_arabic.vim | 2 +- src/testdir/test_autochdir.vim | 2 +- src/testdir/test_balloon.vim | 10 ++++--- 6 files changed, 80 insertions(+), 8 deletions(-) create mode 100644 src/testdir/summarize.vim (limited to 'src/testdir') diff --git a/src/testdir/Makefile b/src/testdir/Makefile index 45c520d2df..0abf66137d 100644 --- a/src/testdir/Makefile +++ b/src/testdir/Makefile @@ -12,7 +12,7 @@ SCRIPTSOURCE = ../../runtime # Comment out this line to see the verbose output of tests. # # Catches SwapExists to avoid hanging at the ATTENTION prompt. -REDIR_TEST_TO_NULL = --cmd 'au SwapExists * let v:swapchoice = "e"' > /dev/null +# REDIR_TEST_TO_NULL = --cmd 'au SwapExists * let v:swapchoice = "e"' > /dev/null # Uncomment this line to use valgrind for memory leaks and extra warnings. # The output goes into a file "valgrind.testN" @@ -49,10 +49,12 @@ gui: nolog $(SCRIPTS_FIRST) $(SCRIPTS) $(SCRIPTS_GUI) newtests report benchmark: $(SCRIPTS_BENCH) report: + $(RUN_VIMTEST) $(NO_INITS) -S summarize.vim messages $(REDIR_TEST_TO_NULL) @echo @echo 'Test results:' + @cat test_result.log @/bin/sh -c "if test -f test.log; \ - then cat test.log; echo TEST FAILURE; exit 1; \ + then echo TEST FAILURE; exit 1; \ else echo ALL DONE; \ fi" @@ -77,7 +79,10 @@ RM_ON_START = tiny.vim small.vim mbyte.vim mzscheme.vim test.ok benchmark.out RUN_VIM = VIMRUNTIME=$(SCRIPTSOURCE); export VIMRUNTIME; $(VALGRIND) $(VIMPROG) -f $(GUI_FLAG) -u unix.vim $(NO_INITS) -s dotest.in clean: - -rm -rf *.out *.failed *.res *.rej *.orig opt_test.vim test.log messages $(RM_ON_RUN) $(RM_ON_START) valgrind.* + -rm -rf *.out *.failed *.res *.rej *.orig + -rm opt_test.vim test.log test_result.log messages + -rm $(RM_ON_RUN) $(RM_ON_START) + -rm valgrind.* test1.out: test1.in -rm -rf $*.failed $(RM_ON_RUN) $(RM_ON_START) wrongtermsize diff --git a/src/testdir/runtest.vim b/src/testdir/runtest.vim index 2a1b5ea5cc..02d4d20e34 100644 --- a/src/testdir/runtest.vim +++ b/src/testdir/runtest.vim @@ -268,6 +268,9 @@ if expand('%') =~ 'test_vimscript.vim' else try source % + catch /^\cskipped/ + call add(s:messages, ' Skipped') + call add(s:skipped, 'SKIPPED ' . expand('%') . ': ' . substitute(v:exception, '^\S*\s\+', '', '')) catch let s:fail += 1 call add(s:errors, 'Caught exception: ' . v:exception . ' @ ' . v:throwpoint) diff --git a/src/testdir/summarize.vim b/src/testdir/summarize.vim new file mode 100644 index 0000000000..43dbce0f44 --- /dev/null +++ b/src/testdir/summarize.vim @@ -0,0 +1,60 @@ +if 1 + " This is executed with the eval feature + set nocp + func Count(match, type) + if a:type ==# 'executed' + let g:executed += (a:match+0) + elseif a:type ==# 'failed' + let g:failed = a:match+0 + elseif a:type ==# 'skipped' + let g:skipped += 1 + call extend(g:skipped_output, ["\t".a:match]) + endif + endfunc + + let g:executed = 0 + let g:skipped = 0 + let g:failed = 0 + let g:skipped_output = [] + let g:failed_output = [] + let output = [""] + + try + " This uses the :s command to just fetch and process the output of the + " tests, it doesn't acutally replay anything + %s/^Executed\s\+\zs\d\+\ze\s\+tests/\=Count(submatch(0),'executed')/egn + %s/^SKIPPED \zs.*/\=Count(submatch(0), 'skipped')/egn + %s/^\(\d\+\)\s\+FAILED:/\=Count(submatch(1), 'failed')/egn + + call extend(output, ["Skipped:"]) + call extend(output, skipped_output) + + call extend(output, [ + \ "", + \ "-------------------------------", + \ printf("Executed: %5d Tests", g:executed), + \ printf(" Skipped: %5d Tests", g:skipped), + \ printf(" %s: %5d Tests", g:failed == 0 ? 'Failed' : 'FAILED', g:failed), + \ "", + \ ]) + if filereadable('test.log') + " outputs and indents the failed test result + call extend(output, ["", "Failures: "]) + let failed_output = filter(readfile('test.log'), { v,k -> !empty(k)}) + call extend(output, map(failed_output, { v,k -> "\t".k})) + " Add a final newline + call extend(output, [""]) + endif + + catch " Catch-all + finally + call writefile(output, 'test_result.log') " overwrites an existing file + q! + endtry +endif + +" This is executed without the eval feature +%d +r test.log +w test_result.log +q! diff --git a/src/testdir/test_arabic.vim b/src/testdir/test_arabic.vim index d0211fee22..450c6f98f5 100644 --- a/src/testdir/test_arabic.vim +++ b/src/testdir/test_arabic.vim @@ -3,7 +3,7 @@ " functional tests that check the shaping works with real text. if !has('arabic') - finish + throw 'Skipped: arabic feature missing' endif source view_util.vim diff --git a/src/testdir/test_autochdir.vim b/src/testdir/test_autochdir.vim index 90538fbc09..1f99858efb 100644 --- a/src/testdir/test_autochdir.vim +++ b/src/testdir/test_autochdir.vim @@ -1,7 +1,7 @@ " Test 'autochdir' behavior if !exists("+autochdir") - finish + throw 'Skipped: autochdir feature missing' endif func Test_set_filename() diff --git a/src/testdir/test_balloon.vim b/src/testdir/test_balloon.vim index 39c2602be6..37fc57eb5c 100644 --- a/src/testdir/test_balloon.vim +++ b/src/testdir/test_balloon.vim @@ -1,11 +1,15 @@ " Tests for 'balloonevalterm'. -" Tests that only work in the terminal. -if has('balloon_eval_term') && !has('gui_running') +if !has('balloon_eval_term') + throw 'Skipped: balloon_eval_term feature missing' +endif + +" A few tests only work in the terminal. +if !has('gui_running') source screendump.vim if !CanRunVimInTerminal() - finish + throw 'Skipped: cannot run Vim in a terminal window' endif let s:common_script =<< [CODE] -- cgit v1.2.3