neovim/src/nvim/testdir/runtest.vim

460 lines
13 KiB
VimL
Raw Normal View History

" This script is sourced while editing the .vim file with the tests.
" When the script is successful the .res file will be created.
" Errors are appended to the test.log file.
"
" To execute only specific test functions, add a second argument. It will be
" matched against the names of the Test_ function. E.g.:
" ../vim -u NONE -S runtest.vim test_channel.vim open_delay
" The output can be found in the "messages" file.
"
" If the environment variable $TEST_FILTER is set then only test functions
" matching this pattern are executed. E.g. for sh/bash:
" export TEST_FILTER=Test_channel
" For csh:
" setenv TEST_FILTER Test_channel
"
" To ignore failure for tests that are known to fail in a certain environment,
" set $TEST_MAY_FAIL to a comma separated list of function names. E.g. for
" sh/bash:
" export TEST_MAY_FAIL=Test_channel_one,Test_channel_other
" The failure report will then not be included in the test.log file and
" "make test" will not fail.
"
" The test script may contain anything, only functions that start with
" "Test_" are special. These will be invoked and should contain assert
" functions. See test_assert.vim for an example.
"
" It is possible to source other files that contain "Test_" functions. This
" can speed up testing, since Vim does not need to restart. But be careful
" that the tests do not interfere with each other.
"
" If an error cannot be detected properly with an assert function add the
" error to the v:errors list:
" call add(v:errors, 'test foo failed: Cannot find xyz')
"
" If preparation for each Test_ function is needed, define a SetUp function.
" It will be called before each Test_ function.
"
" If cleanup after each Test_ function is needed, define a TearDown function.
" It will be called after each Test_ function.
"
" When debugging a test it can be useful to add messages to v:errors:
" call add(v:errors, "this happened")
" Check that the screen size is at least 24 x 80 characters.
if &lines < 24 || &columns < 80
let error = 'Screen size too small! Tests require at least 24 lines with 80 characters, got ' .. &lines .. ' lines with ' .. &columns .. ' characters'
echoerr error
split test.log
$put =error
write
split messages
call append(line('$'), '')
call append(line('$'), 'From ' . expand('%') . ':')
call append(line('$'), error)
write
qa!
endif
if has('reltime')
let s:start_time = reltime()
endif
" Common with all tests on all systems.
source setup.vim
" For consistency run all tests with 'nocompatible' set.
" This also enables use of line continuation.
set nocp viminfo+=nviminfo
" Use utf-8 by default, instead of whatever the system default happens to be.
" Individual tests can overrule this at the top of the file.
set encoding=utf-8
" REDIR_TEST_TO_NULL has a very permissive SwapExists autocommand which is for
" the test_name.vim file itself. Replace it here with a more restrictive one,
" so we still catch mistakes.
let s:test_script_fname = expand('%')
au! SwapExists * call HandleSwapExists()
func HandleSwapExists()
if exists('g:ignoreSwapExists')
return
endif
" Ignore finding a swap file for the test script (the user might be
" editing it and do ":make test_name") and the output file.
" Report finding another swap file and chose 'q' to avoid getting stuck.
if expand('<afile>') == 'messages' || expand('<afile>') =~ s:test_script_fname
let v:swapchoice = 'e'
else
call assert_report('Unexpected swap file: ' .. v:swapname)
let v:swapchoice = 'q'
endif
endfunc
" Avoid stopping at the "hit enter" prompt
set nomore
" Output all messages in English.
lang mess C
" Nvim: append runtime from build dir, which contains the generated doc/tags.
let &runtimepath .= ','.expand($BUILD_DIR).'/runtime/'
" Always use forward slashes.
set shellslash
let s:t_bold = &t_md
let s:t_normal = &t_me
if has('win32')
" avoid prompt that is long or contains a line break
let $PROMPT = '$P$G'
endif
" Prepare for calling test_garbagecollect_now().
let v:testing = 1
" Support function: get the alloc ID by name.
2018-10-15 15:26:01 -07:00
func GetAllocId(name)
exe 'split ' . s:srcdir . '/alloc.h'
let top = search('typedef enum')
if top == 0
call add(v:errors, 'typedef not found in alloc.h')
endif
let lnum = search('aid_' . a:name . ',')
if lnum == 0
call add(v:errors, 'Alloc ID ' . a:name . ' not defined')
endif
close
return lnum - top - 1
endfunc
func RunTheTest(test)
echo 'Executing ' . a:test
if has('reltime')
let func_start = reltime()
endif
" Avoid stopping at the "hit enter" prompt
set nomore
" Avoid a three second wait when a message is about to be overwritten by the
" mode message.
set noshowmode
" Some tests wipe out buffers. To be consistent, always wipe out all
" buffers.
%bwipe!
" The test may change the current directory. Save and restore the
" directory after executing the test.
let save_cwd = getcwd()
if exists("*SetUp")
try
call SetUp()
catch
call add(v:errors, 'Caught exception in SetUp() before ' . a:test . ': ' . v:exception . ' @ ' . v:throwpoint)
endtry
endif
if a:test =~ 'Test_nocatch_'
" Function handles errors itself. This avoids skipping commands after the
" error.
exe 'call ' . a:test
else
try
let s:test = a:test
au VimLeavePre * call EarlyExit(s:test)
exe 'call ' . a:test
au! VimLeavePre
catch /^\cskipped/
call add(s:messages, ' Skipped')
call add(s:skipped, 'SKIPPED ' . a:test . ': ' . substitute(v:exception, '^\S*\s\+', '', ''))
catch
call add(v:errors, 'Caught exception in ' . a:test . ': ' . v:exception . ' @ ' . v:throwpoint)
endtry
endif
" In case 'insertmode' was set and something went wrong, make sure it is
" reset to avoid trouble with anything else.
set noinsertmode
if exists("*TearDown")
try
call TearDown()
catch
call add(v:errors, 'Caught exception in TearDown() after ' . a:test . ': ' . v:exception . ' @ ' . v:throwpoint)
endtry
endif
" Clear any autocommands and put back the catch-all for SwapExists.
au!
au SwapExists * call HandleSwapExists()
" Close any extra tab pages and windows and make the current one not modified.
while tabpagenr('$') > 1
quit!
endwhile
while 1
let wincount = winnr('$')
if wincount == 1
break
endif
bwipe!
if wincount == winnr('$')
" Did not manage to close a window.
only!
break
endif
endwhile
exe 'cd ' . save_cwd
let message = 'Executed ' . a:test
if has('reltime')
let message ..= repeat(' ', 50 - len(message))
let time = reltime(func_start)
if has('float') && reltimefloat(time) > 0.1
let message = s:t_bold .. message
endif
let message ..= ' in ' .. reltimestr(time) .. ' seconds'
if has('float') && reltimefloat(time) > 0.1
let message ..= s:t_normal
endif
endif
call add(s:messages, message)
let s:done += 1
endfunc
func AfterTheTest(func_name)
if len(v:errors) > 0
if match(s:may_fail_list, '^' .. a:func_name) >= 0
let s:fail_expected += 1
call add(s:errors_expected, 'Found errors in ' . s:test . ':')
call extend(s:errors_expected, v:errors)
else
let s:fail += 1
call add(s:errors, 'Found errors in ' . s:test . ':')
call extend(s:errors, v:errors)
endif
let v:errors = []
endif
endfunc
func EarlyExit(test)
" It's OK for the test we use to test the quit detection.
if a:test != 'Test_zz_quit_detected()'
call add(v:errors, 'Test caused Vim to exit: ' . a:test)
endif
call FinishTesting()
endfunc
" This function can be called by a test if it wants to abort testing.
func FinishTesting()
call AfterTheTest('')
" Don't write viminfo on exit.
set viminfo=
" Clean up files created by setup.vim
call delete('XfakeHOME', 'rf')
if s:fail == 0 && s:fail_expected == 0
" Success, create the .res file so that make knows it's done.
exe 'split ' . fnamemodify(g:testname, ':r') . '.res'
write
endif
if len(s:errors) > 0
" Append errors to test.log
split test.log
call append(line('$'), '')
call append(line('$'), 'From ' . g:testname . ':')
call append(line('$'), s:errors)
write
endif
if s:done == 0
if s:filtered > 0
let message = "NO tests match $TEST_FILTER: '" .. $TEST_FILTER .. "'"
else
let message = 'NO tests executed'
endif
else
if s:filtered > 0
call add(s:messages, "Filtered " .. s:filtered .. " tests with $TEST_FILTER")
endif
let message = 'Executed ' . s:done . (s:done > 1 ? ' tests' : ' test')
endif
if s:done > 0 && has('reltime')
let message = s:t_bold .. message .. repeat(' ', 40 - len(message))
let message ..= ' in ' .. reltimestr(reltime(s:start_time)) .. ' seconds'
let message ..= s:t_normal
endif
echo message
call add(s:messages, message)
if s:fail > 0
let message = s:fail . ' FAILED:'
echo message
call add(s:messages, message)
call extend(s:messages, s:errors)
endif
if s:fail_expected > 0
let message = s:fail_expected . ' FAILED (matching $TEST_MAY_FAIL):'
echo message
call add(s:messages, message)
call extend(s:messages, s:errors_expected)
endif
" Add SKIPPED messages
call extend(s:messages, s:skipped)
" Append messages to the file "messages"
split messages
call append(line('$'), '')
call append(line('$'), 'From ' . g:testname . ':')
call append(line('$'), s:messages)
write
qall!
endfunc
" Source the test script. First grab the file name, in case the script
" navigates away. g:testname can be used by the tests.
let g:testname = expand('%')
let s:done = 0
let s:fail = 0
let s:fail_expected = 0
let s:errors = []
let s:errors_expected = []
let s:messages = []
vim-patch:0 completion-related patches #5745 #5829 vim-patch:8.0.0058 Problem: Positioning of the popup menu is not good. Solution: Position it better. (Hirohito Higashi) https://github.com/vim/vim/commit/91e44a3305ef6bf2d43496c351dcff0a45c6bfb8 vim-patch:8.0.0099 Problem: Popup menu always appears above the cursor when it is in the lower half of the screen. (Matt Gardner) Solution: Compute the available space better. (Hirohito Higashi, closes vim/vim#1241) https://github.com/vim/vim/commit/73095288da839f7c738a49baa109773e76106806 vim-patch:8.0.0127 Problem: Cancelling completion still inserts text when formatting is done for 'textwidth'. (lacygoill) Solution: Don't format when CTRL-E was typed. (Hirohito Higashi, closes vim/vim#1312) https://github.com/vim/vim/commit/73fd4988866c3adc15b5d093efdf5e8cf70d093d vim-patch:7.4.2188 Problem: Completion does not work properly with some plugins. Solution: Revert the part related to typing CTRL-E. (closes vim/vim#972) https://github.com/vim/vim/commit/c9fb77c69244870a97384152f20845665c19fe39 vim-patch:7.4.2146 Problem: Not enough testing for popup menu. CTRL-E does not always work properly. Solution: Add more tests. When using CTRL-E check if the popup menu is visible. (Christian Brabandt) https://github.com/vim/vim/commit/472472898ab71ac80a86fedc37f8eb91461788dd vim-patch:7.4.2147 Problem: test_alot fails. Solution: Close window. https://github.com/vim/vim/commit/abb71fbd399772d467aaa7b34b958b0f975c7e65 vim-patch:7.4.2149 Problem: If a test leaves a window open a following test may fail. Solution: Always close extra windows after running a test. https://github.com/vim/vim/commit/7cba71d7e3576639679b6a3aedeeb1ac07f7f2f5 vim-patch:7.4.2321 Problem: When a test is commented out we forget about it. Solution: Let a test throw an exception with "Skipped" and list skipped test functions. (Christian Brabandt) https://github.com/vim/vim/commit/dac1947bb366ef43cd6da95acc730554e76d8b84 vim-patch:7.4.2331 Problem: Using CTRL-X CTRL-V to complete a command line from Insert mode does not work after entering an expression on the command line. Solution: Don't use "ccline" when not actually using a command line. (test by Hirohito Higashi) https://github.com/vim/vim/commit/33a80eeb859a78ba93432da6fa585786cfd77249 vim-patch:8.0.0008 Problem: Popup complete test is disabled. Solution: Enable the test and change the assert. (Hirohito Higashi) https://github.com/vim/vim/commit/9e02cfa226b2577ec867b544a1a450a428a19880 vim-patch:8.0.0047 Problem: Crash when using the preview window from an unnamed buffer. (lifepillar) Solution: Do not clear the wrong buffer. (closes vim/vim#1200) https://github.com/vim/vim/commit/50e5376926dc2ec4a26a7a16f8f0f3213c4afdf0 vim-patch:8.0.0053 Problem: No test for what 8.0.0047 fixes. Solution: Add a test. (Hirohito Higashi) https://github.com/vim/vim/commit/60ef3e81f4a54d9f7ee617d57021f0811ec8ada5
2016-12-10 01:02:02 -07:00
let s:skipped = []
if expand('%') =~ 'test_vimscript.vim'
" this test has intentional errors, don't use try/catch.
source %
else
try
source %
catch /^\cskipped/
call add(s:messages, ' Skipped')
call add(s:skipped, 'SKIPPED ' . expand('%') . ': ' . substitute(v:exception, '^\S*\s\+', '', ''))
catch
let s:fail += 1
call add(s:errors, 'Caught exception: ' . v:exception . ' @ ' . v:throwpoint)
endtry
endif
" Names of flaky tests.
let s:flaky_tests = [
\ 'Test_autocmd_SafeState()',
\ 'Test_cursorhold_insert()',
\ 'Test_exit_callback_interval()',
\ 'Test_map_timeout_with_timer_interrupt()',
2017-06-04 19:09:19 -07:00
\ 'Test_oneshot()',
\ 'Test_out_cb()',
\ 'Test_paused()',
\ 'Test_popup_and_window_resize()',
\ 'Test_quoteplus()',
\ 'Test_quotestar()',
\ 'Test_reltime()',
\ 'Test_repeat_many()',
\ 'Test_repeat_three()',
\ 'Test_state()',
\ 'Test_stop_all_in_callback()',
\ 'Test_term_mouse_double_click_to_create_tab()',
\ 'Test_term_mouse_multiple_clicks_to_visually_select()',
\ 'Test_terminal_composing_unicode()',
\ 'Test_terminal_redir_file()',
\ 'Test_terminal_tmap()',
\ 'Test_termwinscroll()',
\ 'Test_with_partial_callback()',
\ ]
" Pattern indicating a common flaky test failure.
let s:flaky_errors_re = 'StopVimInTerminal\|VerifyScreenDump'
" Locate Test_ functions and execute them.
redir @q
silent function /^Test_
redir END
let s:tests = split(substitute(@q, 'function \(\k*()\)', '\1', 'g'))
" If there is an extra argument filter the function names against it.
if argc() > 1
let s:tests = filter(s:tests, 'v:val =~ argv(1)')
endif
" If the environment variable $TEST_FILTER is set then filter the function
" names against it.
let s:filtered = 0
if $TEST_FILTER != ''
let s:filtered = len(s:tests)
let s:tests = filter(s:tests, 'v:val =~ $TEST_FILTER')
let s:filtered -= len(s:tests)
endif
let s:may_fail_list = []
if $TEST_MAY_FAIL != ''
" Split the list at commas and add () to make it match s:test.
let s:may_fail_list = split($TEST_MAY_FAIL, ',')->map({i, v -> v .. '()'})
endif
" Execute the tests in alphabetical order.
for s:test in sort(s:tests)
" Silence, please!
set belloff=all
let prev_error = ''
let total_errors = []
let run_nr = 1
call RunTheTest(s:test)
" Repeat a flaky test. Give up when:
" - it fails again with the same message
" - it fails five times (with a different mesage)
if len(v:errors) > 0
\ && (index(s:flaky_tests, s:test) >= 0
\ || v:errors[0] =~ s:flaky_errors_re)
while 1
call add(s:messages, 'Found errors in ' . s:test . ':')
call extend(s:messages, v:errors)
call add(total_errors, 'Run ' . run_nr . ':')
call extend(total_errors, v:errors)
if run_nr == 5 || prev_error == v:errors[0]
call add(total_errors, 'Flaky test failed too often, giving up')
let v:errors = total_errors
break
endif
call add(s:messages, 'Flaky test failed, running it again')
" Flakiness is often caused by the system being very busy. Sleep a
" couple of seconds to have a higher chance of succeeding the second
" time.
sleep 2
let prev_error = v:errors[0]
let v:errors = []
let run_nr += 1
call RunTheTest(s:test)
if len(v:errors) == 0
" Test passed on rerun.
break
endif
endwhile
endif
call AfterTheTest(s:test)
endfor
call FinishTesting()
" vim: shiftwidth=2 sts=2 expandtab