2015-12-12 20:19:54 -07:00
|
|
|
" This script is sourced while editing the .vim file with the tests.
|
|
|
|
" When the script is successful the .res file will be created.
|
|
|
|
" Errors are appended to the test.log file.
|
|
|
|
"
|
2016-07-19 22:40:04 -07:00
|
|
|
" To execute only specific test functions, add a second argument. It will be
|
|
|
|
" matched against the names of the Test_ function. E.g.:
|
2018-02-03 04:11:07 -07:00
|
|
|
" ../vim -u NONE -S runtest.vim test_channel.vim open_delay
|
2016-07-19 22:40:04 -07:00
|
|
|
" The output can be found in the "messages" file.
|
|
|
|
"
|
2015-12-12 20:19:54 -07:00
|
|
|
" The test script may contain anything, only functions that start with
|
|
|
|
" "Test_" are special. These will be invoked and should contain assert
|
|
|
|
" functions. See test_assert.vim for an example.
|
|
|
|
"
|
|
|
|
" It is possible to source other files that contain "Test_" functions. This
|
|
|
|
" can speed up testing, since Vim does not need to restart. But be careful
|
|
|
|
" that the tests do not interfere with each other.
|
|
|
|
"
|
|
|
|
" If an error cannot be detected properly with an assert function add the
|
|
|
|
" error to the v:errors list:
|
|
|
|
" call add(v:errors, 'test foo failed: Cannot find xyz')
|
|
|
|
"
|
|
|
|
" If preparation for each Test_ function is needed, define a SetUp function.
|
|
|
|
" It will be called before each Test_ function.
|
|
|
|
"
|
|
|
|
" If cleanup after each Test_ function is needed, define a TearDown function.
|
|
|
|
" It will be called after each Test_ function.
|
2016-09-21 09:00:50 -07:00
|
|
|
"
|
|
|
|
" When debugging a test it can be useful to add messages to v:errors:
|
2019-01-06 17:52:34 -07:00
|
|
|
" call add(v:errors, "this happened")
|
2016-09-21 09:00:50 -07:00
|
|
|
|
2015-12-12 20:19:54 -07:00
|
|
|
|
|
|
|
" Check that the screen size is at least 24 x 80 characters.
|
|
|
|
if &lines < 24 || &columns < 80
|
2019-08-17 15:29:13 -07:00
|
|
|
let error = 'Screen size too small! Tests require at least 24 lines with 80 characters, got ' .. &lines .. ' lines with ' .. &columns .. ' characters'
|
2015-12-12 20:19:54 -07:00
|
|
|
echoerr error
|
|
|
|
split test.log
|
|
|
|
$put =error
|
2019-06-15 17:24:38 -07:00
|
|
|
write
|
|
|
|
split messages
|
2019-08-17 15:29:13 -07:00
|
|
|
call append(line('$'), '')
|
|
|
|
call append(line('$'), 'From ' . expand('%') . ':')
|
2019-06-15 17:24:38 -07:00
|
|
|
call append(line('$'), error)
|
|
|
|
write
|
|
|
|
qa!
|
2015-12-12 20:19:54 -07:00
|
|
|
endif
|
|
|
|
|
2017-01-16 04:50:43 -07:00
|
|
|
" Common with all tests on all systems.
|
|
|
|
source setup.vim
|
|
|
|
|
2018-02-02 13:44:07 -07:00
|
|
|
" For consistency run all tests with 'nocompatible' set.
|
2016-04-14 12:14:08 -07:00
|
|
|
" This also enables use of line continuation.
|
2018-02-02 13:44:07 -07:00
|
|
|
set nocp viminfo+=nviminfo
|
2016-04-14 12:14:08 -07:00
|
|
|
|
2018-02-01 15:32:43 -07:00
|
|
|
" Use utf-8 or latin1 by default, instead of whatever the system default
|
2016-11-22 13:33:13 -07:00
|
|
|
" happens to be. Individual tests can overrule this at the top of the file.
|
|
|
|
if has('multi_byte')
|
|
|
|
set encoding=utf-8
|
|
|
|
else
|
|
|
|
set encoding=latin1
|
|
|
|
endif
|
|
|
|
|
2019-08-13 16:56:59 -07:00
|
|
|
" REDIR_TEST_TO_NULL has a very permissive SwapExists autocommand which is for
|
|
|
|
" the test_name.vim file itself. Replace it here with a more restrictive one,
|
|
|
|
" so we still catch mistakes.
|
|
|
|
let s:test_script_fname = expand('%')
|
|
|
|
au! SwapExists * call HandleSwapExists()
|
|
|
|
func HandleSwapExists()
|
|
|
|
" Only ignore finding a swap file for the test script (the user might be
|
|
|
|
" editing it and do ":make test_name") and the output file.
|
|
|
|
if expand('<afile>') == 'messages' || expand('<afile>') =~ s:test_script_fname
|
|
|
|
let v:swapchoice = 'e'
|
|
|
|
endif
|
|
|
|
endfunc
|
|
|
|
|
2018-02-11 09:42:18 -07:00
|
|
|
" Avoid stopping at the "hit enter" prompt
|
|
|
|
set nomore
|
|
|
|
|
2016-04-14 12:14:08 -07:00
|
|
|
" Output all messages in English.
|
|
|
|
lang mess C
|
|
|
|
|
2016-07-06 06:13:48 -07:00
|
|
|
" Always use forward slashes.
|
2019-08-02 18:44:01 -07:00
|
|
|
set shellslash
|
2016-07-06 06:13:48 -07:00
|
|
|
|
2018-02-02 13:44:07 -07:00
|
|
|
" Prepare for calling test_garbagecollect_now().
|
2017-01-05 17:13:23 -07:00
|
|
|
let v:testing = 1
|
|
|
|
|
2018-02-02 13:44:07 -07:00
|
|
|
" Support function: get the alloc ID by name.
|
2018-10-15 15:26:01 -07:00
|
|
|
func GetAllocId(name)
|
2018-02-02 13:44:07 -07:00
|
|
|
exe 'split ' . s:srcdir . '/alloc.h'
|
|
|
|
let top = search('typedef enum')
|
|
|
|
if top == 0
|
|
|
|
call add(v:errors, 'typedef not found in alloc.h')
|
|
|
|
endif
|
|
|
|
let lnum = search('aid_' . a:name . ',')
|
|
|
|
if lnum == 0
|
|
|
|
call add(v:errors, 'Alloc ID ' . a:name . ' not defined')
|
|
|
|
endif
|
|
|
|
close
|
|
|
|
return lnum - top - 1
|
|
|
|
endfunc
|
|
|
|
|
2018-10-15 15:26:01 -07:00
|
|
|
func CanRunVimInTerminal()
|
|
|
|
" Nvim: always false, we use Lua screen-tests instead.
|
|
|
|
return 0
|
|
|
|
endfunc
|
|
|
|
|
2017-06-05 20:17:54 -07:00
|
|
|
func RunTheTest(test)
|
2016-09-21 09:00:50 -07:00
|
|
|
echo 'Executing ' . a:test
|
2018-02-03 04:11:07 -07:00
|
|
|
|
2018-02-11 09:39:15 -07:00
|
|
|
" Avoid stopping at the "hit enter" prompt
|
|
|
|
set nomore
|
|
|
|
|
|
|
|
" Avoid a three second wait when a message is about to be overwritten by the
|
|
|
|
" mode message.
|
|
|
|
set noshowmode
|
|
|
|
|
2018-02-03 04:11:07 -07:00
|
|
|
" Some tests wipe out buffers. To be consistent, always wipe out all
|
|
|
|
" buffers.
|
|
|
|
%bwipe!
|
|
|
|
|
2018-02-11 15:04:32 -07:00
|
|
|
" The test may change the current directory. Save and restore the
|
|
|
|
" directory after executing the test.
|
|
|
|
let save_cwd = getcwd()
|
|
|
|
|
2016-07-11 02:01:22 -07:00
|
|
|
if exists("*SetUp")
|
2017-07-26 18:56:25 -07:00
|
|
|
try
|
|
|
|
call SetUp()
|
|
|
|
catch
|
|
|
|
call add(v:errors, 'Caught exception in SetUp() before ' . a:test . ': ' . v:exception . ' @ ' . v:throwpoint)
|
|
|
|
endtry
|
2016-07-11 02:01:22 -07:00
|
|
|
endif
|
|
|
|
|
2016-09-21 09:00:50 -07:00
|
|
|
call add(s:messages, 'Executing ' . a:test)
|
2016-07-11 02:01:22 -07:00
|
|
|
let s:done += 1
|
2018-02-11 08:51:52 -07:00
|
|
|
|
|
|
|
if a:test =~ 'Test_nocatch_'
|
|
|
|
" Function handles errors itself. This avoids skipping commands after the
|
|
|
|
" error.
|
2016-09-21 09:00:50 -07:00
|
|
|
exe 'call ' . a:test
|
2018-02-11 08:51:52 -07:00
|
|
|
else
|
|
|
|
try
|
2019-06-15 17:52:27 -07:00
|
|
|
let s:test = a:test
|
|
|
|
au VimLeavePre * call EarlyExit(s:test)
|
2018-02-11 08:51:52 -07:00
|
|
|
exe 'call ' . a:test
|
2019-06-15 17:52:27 -07:00
|
|
|
au! VimLeavePre
|
2018-02-11 08:51:52 -07:00
|
|
|
catch /^\cskipped/
|
|
|
|
call add(s:messages, ' Skipped')
|
|
|
|
call add(s:skipped, 'SKIPPED ' . a:test . ': ' . substitute(v:exception, '^\S*\s\+', '', ''))
|
|
|
|
catch
|
|
|
|
call add(v:errors, 'Caught exception in ' . a:test . ': ' . v:exception . ' @ ' . v:throwpoint)
|
|
|
|
endtry
|
|
|
|
endif
|
2016-07-11 02:01:22 -07:00
|
|
|
|
2019-01-06 17:52:34 -07:00
|
|
|
" In case 'insertmode' was set and something went wrong, make sure it is
|
|
|
|
" reset to avoid trouble with anything else.
|
|
|
|
set noinsertmode
|
|
|
|
|
2016-07-11 02:01:22 -07:00
|
|
|
if exists("*TearDown")
|
2017-07-26 18:56:25 -07:00
|
|
|
try
|
|
|
|
call TearDown()
|
|
|
|
catch
|
|
|
|
call add(v:errors, 'Caught exception in TearDown() after ' . a:test . ': ' . v:exception . ' @ ' . v:throwpoint)
|
|
|
|
endtry
|
2016-07-11 02:01:22 -07:00
|
|
|
endif
|
2016-12-28 12:54:30 -07:00
|
|
|
|
2019-08-17 15:29:13 -07:00
|
|
|
" Clear any autocommands and put back the catch-all for SwapExists.
|
2018-02-03 04:11:07 -07:00
|
|
|
au!
|
2019-08-13 16:56:59 -07:00
|
|
|
au SwapExists * call HandleSwapExists()
|
2018-02-03 04:11:07 -07:00
|
|
|
|
2018-02-03 04:09:37 -07:00
|
|
|
" Close any extra tab pages and windows and make the current one not modified.
|
|
|
|
while tabpagenr('$') > 1
|
2018-02-03 04:11:07 -07:00
|
|
|
quit!
|
2018-02-03 04:09:37 -07:00
|
|
|
endwhile
|
|
|
|
|
2017-03-14 02:10:45 -07:00
|
|
|
while 1
|
|
|
|
let wincount = winnr('$')
|
|
|
|
if wincount == 1
|
|
|
|
break
|
|
|
|
endif
|
2016-12-28 12:54:30 -07:00
|
|
|
bwipe!
|
2017-03-14 02:10:45 -07:00
|
|
|
if wincount == winnr('$')
|
|
|
|
" Did not manage to close a window.
|
|
|
|
only!
|
|
|
|
break
|
|
|
|
endif
|
2016-12-28 12:54:30 -07:00
|
|
|
endwhile
|
2018-02-11 15:04:32 -07:00
|
|
|
|
|
|
|
exe 'cd ' . save_cwd
|
2016-07-11 02:01:22 -07:00
|
|
|
endfunc
|
|
|
|
|
2017-06-05 20:17:54 -07:00
|
|
|
func AfterTheTest()
|
|
|
|
if len(v:errors) > 0
|
|
|
|
let s:fail += 1
|
|
|
|
call add(s:errors, 'Found errors in ' . s:test . ':')
|
|
|
|
call extend(s:errors, v:errors)
|
|
|
|
let v:errors = []
|
|
|
|
endif
|
|
|
|
endfunc
|
|
|
|
|
2019-06-15 17:52:27 -07:00
|
|
|
func EarlyExit(test)
|
|
|
|
" It's OK for the test we use to test the quit detection.
|
|
|
|
if a:test != 'Test_zz_quit_detected()'
|
|
|
|
call add(v:errors, 'Test caused Vim to exit: ' . a:test)
|
|
|
|
endif
|
|
|
|
|
|
|
|
call FinishTesting()
|
|
|
|
endfunc
|
|
|
|
|
2017-06-05 20:17:54 -07:00
|
|
|
" This function can be called by a test if it wants to abort testing.
|
|
|
|
func FinishTesting()
|
|
|
|
call AfterTheTest()
|
|
|
|
|
|
|
|
" Don't write viminfo on exit.
|
|
|
|
set viminfo=
|
|
|
|
|
2018-07-24 17:52:08 -07:00
|
|
|
" Clean up files created by setup.vim
|
|
|
|
call delete('XfakeHOME', 'rf')
|
|
|
|
|
2017-06-05 20:17:54 -07:00
|
|
|
if s:fail == 0
|
|
|
|
" Success, create the .res file so that make knows it's done.
|
|
|
|
exe 'split ' . fnamemodify(g:testname, ':r') . '.res'
|
|
|
|
write
|
|
|
|
endif
|
|
|
|
|
|
|
|
if len(s:errors) > 0
|
|
|
|
" Append errors to test.log
|
|
|
|
split test.log
|
|
|
|
call append(line('$'), '')
|
|
|
|
call append(line('$'), 'From ' . g:testname . ':')
|
|
|
|
call append(line('$'), s:errors)
|
|
|
|
write
|
|
|
|
endif
|
|
|
|
|
2019-06-11 19:31:12 -07:00
|
|
|
if s:done == 0
|
|
|
|
let message = 'NO tests executed'
|
|
|
|
else
|
|
|
|
let message = 'Executed ' . s:done . (s:done > 1 ? ' tests' : ' test')
|
|
|
|
endif
|
2017-06-05 20:17:54 -07:00
|
|
|
echo message
|
|
|
|
call add(s:messages, message)
|
|
|
|
if s:fail > 0
|
|
|
|
let message = s:fail . ' FAILED:'
|
|
|
|
echo message
|
|
|
|
call add(s:messages, message)
|
|
|
|
call extend(s:messages, s:errors)
|
|
|
|
endif
|
|
|
|
|
|
|
|
" Add SKIPPED messages
|
|
|
|
call extend(s:messages, s:skipped)
|
|
|
|
|
|
|
|
" Append messages to the file "messages"
|
|
|
|
split messages
|
|
|
|
call append(line('$'), '')
|
|
|
|
call append(line('$'), 'From ' . g:testname . ':')
|
|
|
|
call append(line('$'), s:messages)
|
|
|
|
write
|
|
|
|
|
|
|
|
qall!
|
|
|
|
endfunc
|
|
|
|
|
2015-12-12 20:19:54 -07:00
|
|
|
" Source the test script. First grab the file name, in case the script
|
2016-09-21 09:00:50 -07:00
|
|
|
" navigates away. g:testname can be used by the tests.
|
|
|
|
let g:testname = expand('%')
|
|
|
|
let s:done = 0
|
|
|
|
let s:fail = 0
|
|
|
|
let s:errors = []
|
|
|
|
let s:messages = []
|
2016-12-10 01:02:02 -07:00
|
|
|
let s:skipped = []
|
2017-06-05 20:05:28 -07:00
|
|
|
if expand('%') =~ 'test_vimscript.vim'
|
2016-09-21 09:00:50 -07:00
|
|
|
" this test has intentional s:errors, don't use try/catch.
|
2016-01-10 13:31:33 -07:00
|
|
|
source %
|
2016-04-14 12:43:53 -07:00
|
|
|
else
|
|
|
|
try
|
|
|
|
source %
|
|
|
|
catch
|
2016-09-21 09:00:50 -07:00
|
|
|
let s:fail += 1
|
|
|
|
call add(s:errors, 'Caught exception: ' . v:exception . ' @ ' . v:throwpoint)
|
2016-04-14 12:43:53 -07:00
|
|
|
endtry
|
|
|
|
endif
|
2015-12-12 20:19:54 -07:00
|
|
|
|
2016-07-11 02:01:22 -07:00
|
|
|
" Names of flaky tests.
|
2019-07-07 18:37:37 -07:00
|
|
|
let s:flaky_tests = [
|
2019-06-18 16:19:20 -07:00
|
|
|
\ 'Test_cursorhold_insert()',
|
2018-02-04 01:08:13 -07:00
|
|
|
\ 'Test_exit_callback_interval()',
|
2019-08-16 18:22:16 -07:00
|
|
|
\ 'Test_map_timeout_with_timer_interrupt()',
|
2017-06-04 19:09:19 -07:00
|
|
|
\ 'Test_oneshot()',
|
2018-02-11 09:11:28 -07:00
|
|
|
\ 'Test_out_cb()',
|
2018-02-11 09:10:34 -07:00
|
|
|
\ 'Test_paused()',
|
2018-03-25 08:20:59 -07:00
|
|
|
\ 'Test_quoteplus()',
|
2018-05-19 19:16:50 -07:00
|
|
|
\ 'Test_quotestar()',
|
2018-02-11 09:03:31 -07:00
|
|
|
\ 'Test_reltime()',
|
2019-08-16 01:11:03 -07:00
|
|
|
\ 'Test_repeat_many()',
|
2018-08-05 14:55:36 -07:00
|
|
|
\ 'Test_repeat_three()',
|
2019-08-16 13:06:58 -07:00
|
|
|
\ 'Test_stop_all_in_callback()',
|
2018-02-11 08:51:52 -07:00
|
|
|
\ 'Test_terminal_composing_unicode()',
|
2018-02-11 08:54:55 -07:00
|
|
|
\ 'Test_terminal_redir_file()',
|
2018-02-11 09:10:34 -07:00
|
|
|
\ 'Test_terminal_tmap()',
|
2018-02-03 04:11:07 -07:00
|
|
|
\ 'Test_with_partial_callback()',
|
2017-05-16 12:26:54 -07:00
|
|
|
\ ]
|
2016-07-11 02:01:22 -07:00
|
|
|
|
2019-07-07 18:37:37 -07:00
|
|
|
" Pattern indicating a common flaky test failure.
|
2019-07-07 18:41:35 -07:00
|
|
|
let s:flaky_errors_re = 'StopVimInTerminal\|VerifyScreenDump'
|
2019-07-07 18:37:37 -07:00
|
|
|
|
2015-12-12 20:19:54 -07:00
|
|
|
" Locate Test_ functions and execute them.
|
|
|
|
redir @q
|
2016-06-12 21:05:42 -07:00
|
|
|
silent function /^Test_
|
2015-12-12 20:19:54 -07:00
|
|
|
redir END
|
2016-09-21 09:00:50 -07:00
|
|
|
let s:tests = split(substitute(@q, 'function \(\k*()\)', '\1', 'g'))
|
2015-12-12 20:19:54 -07:00
|
|
|
|
2016-07-19 22:40:04 -07:00
|
|
|
" If there is an extra argument filter the function names against it.
|
|
|
|
if argc() > 1
|
2016-09-21 09:00:50 -07:00
|
|
|
let s:tests = filter(s:tests, 'v:val =~ argv(1)')
|
2016-07-19 22:40:04 -07:00
|
|
|
endif
|
|
|
|
|
2016-05-24 16:53:50 -07:00
|
|
|
" Execute the tests in alphabetical order.
|
2016-09-21 09:00:50 -07:00
|
|
|
for s:test in sort(s:tests)
|
2018-02-01 17:42:17 -07:00
|
|
|
" Silence, please!
|
|
|
|
set belloff=all
|
2019-06-08 04:49:32 -07:00
|
|
|
let prev_error = ''
|
|
|
|
let total_errors = []
|
|
|
|
let run_nr = 1
|
2018-02-01 17:42:17 -07:00
|
|
|
|
2016-09-21 09:00:50 -07:00
|
|
|
call RunTheTest(s:test)
|
2015-12-12 20:19:54 -07:00
|
|
|
|
2019-06-08 04:49:32 -07:00
|
|
|
" Repeat a flaky test. Give up when:
|
|
|
|
" - it fails again with the same message
|
|
|
|
" - it fails five times (with a different mesage)
|
2019-07-07 18:37:37 -07:00
|
|
|
if len(v:errors) > 0
|
|
|
|
\ && (index(s:flaky_tests, s:test) >= 0
|
|
|
|
\ || v:errors[0] =~ s:flaky_errors_re)
|
2019-06-08 04:49:32 -07:00
|
|
|
while 1
|
|
|
|
call add(s:messages, 'Found errors in ' . s:test . ':')
|
|
|
|
call extend(s:messages, v:errors)
|
2018-02-11 09:08:40 -07:00
|
|
|
|
2019-06-08 04:49:32 -07:00
|
|
|
call add(total_errors, 'Run ' . run_nr . ':')
|
|
|
|
call extend(total_errors, v:errors)
|
2018-02-11 09:09:26 -07:00
|
|
|
|
2019-06-08 04:49:32 -07:00
|
|
|
if run_nr == 5 || prev_error == v:errors[0]
|
|
|
|
call add(total_errors, 'Flaky test failed too often, giving up')
|
|
|
|
let v:errors = total_errors
|
|
|
|
break
|
|
|
|
endif
|
|
|
|
|
|
|
|
call add(s:messages, 'Flaky test failed, running it again')
|
|
|
|
|
|
|
|
" Flakiness is often caused by the system being very busy. Sleep a
|
|
|
|
" couple of seconds to have a higher chance of succeeding the second
|
|
|
|
" time.
|
|
|
|
sleep 2
|
|
|
|
|
|
|
|
let prev_error = v:errors[0]
|
|
|
|
let v:errors = []
|
|
|
|
let run_nr += 1
|
|
|
|
|
|
|
|
call RunTheTest(s:test)
|
|
|
|
|
|
|
|
if len(v:errors) == 0
|
|
|
|
" Test passed on rerun.
|
|
|
|
break
|
|
|
|
endif
|
|
|
|
endwhile
|
2015-12-12 20:19:54 -07:00
|
|
|
endif
|
|
|
|
|
2017-06-05 20:17:54 -07:00
|
|
|
call AfterTheTest()
|
2015-12-12 20:19:54 -07:00
|
|
|
endfor
|
|
|
|
|
2017-06-05 20:17:54 -07:00
|
|
|
call FinishTesting()
|
2016-01-10 13:38:54 -07:00
|
|
|
|
2017-06-05 20:17:54 -07:00
|
|
|
" vim: shiftwidth=2 sts=2 expandtab
|