test: Replace vroom by lua/busted for functional tests

The 'lupa' python package provides a simple way to seamless integrate lua and
python code.

This commit replaces vroom by a python script that exposes the 'neovim' package
to a lua state, and invokes busted to run functional tests. This is a temporary
solution that will enable writing functional tests using lua/bused while a lua
client library is not available.

The reason for dropping vroom is flexibility: Lua/busted has a nice DSL-style
syntax while also providing the customization power of a full programming
language. Another reason is to use a single framework for unit/functional tests.

Two other changes were performed in this commit:

- Instead of "gcc-unittest/gcc-ia32", the travis builds for gcc are now
  identified by "gcc/gcc-32". They will run unit/functional tests for both 64
  and 32 bits.
- Old integration tests(in src/nvim/testdir) are now ran by the 'oldtest' target

.ci/clang-asan.sh

@@ -1,9 +1,9 @@
 . "$CI_SCRIPTS/common.sh"
 
-install_vroom
-
 set_environment /opt/neovim-deps/64
 
+install_functional_test_deps
+
 sudo pip install cpp-coveralls
 
 clang_version=3.4
@@ -26,8 +26,9 @@ export UBSAN_OPTIONS="log_path=$tmpdir/ubsan" # not sure if this works
 
 install_dir="$(pwd)/dist"
 $MAKE_CMD cmake CMAKE_EXTRA_FLAGS="-DTRAVIS_CI_BUILD=ON -DCMAKE_INSTALL_PREFIX=$install_dir -DUSE_GCOV=ON"
-$MAKE_CMD
+$MAKE_CMD test
-if ! $MAKE_CMD test; then
+asan_check "$tmpdir"
+if ! $MAKE_CMD oldtest; then
 	reset
 	asan_check "$tmpdir"
 	exit 1

.ci/common.sh

@@ -1,4 +1,8 @@
 valgrind_check() {
+	# For some strange reason, now we need to give ubuntu some time to flush it's
+	# FS cache in order to see valgrind logs, even though the script executes
+	# synchronously
+	sleep 1
 	(
 	cd $1
 	set -- valgrind-[*] valgrind-*
@@ -31,6 +35,8 @@ valgrind_check() {
 }
 
 asan_check() {
+	# See valgrind_check
+	sleep 1
 	(
 	cd $1
 	set -- [*]san.[*] *san.*
@@ -65,14 +71,10 @@ install_prebuilt_deps() {
 	fi
 }
 
-install_vroom() {
+install_functional_test_deps() {
-	(
 	sudo pip install git+https://github.com/neovim/python-client.git
-	git clone git://github.com/google/vroom
+	# Pass -E to let pip use PKG_CONFIG_PATH for luajit
-	cd vroom
+	sudo -E pip install lupa
-	python setup.py build
-	sudo python setup.py install
-	)
 }
 
 tmpdir="$(pwd)/tmp"

.ci/gcc-32.sh

@@ -1,6 +1,9 @@
 . "$CI_SCRIPTS/common.sh"
 
-install_vroom
+# To install lupa, a temporarary functional test dependency, we require the
+# 64-bit luajit since travis version of python is 64-bit.
+export PKG_CONFIG_PATH="/opt/neovim-deps/64/usr/lib/pkgconfig"
+install_functional_test_deps
 
 set_environment /opt/neovim-deps/32
 
@@ -22,5 +25,6 @@ CMAKE_EXTRA_FLAGS="-DTRAVIS_CI_BUILD=ON \
 	-DFIND_LIBRARY_USE_LIB64_PATHS=OFF \
 	-DCMAKE_IGNORE_PATH=/lib:/usr/lib:/usr/local/lib \
 	-DCMAKE_TOOLCHAIN_FILE=cmake/i386-linux-gnu.toolchain.cmake"
+
 $MAKE_CMD CMAKE_EXTRA_FLAGS="${CMAKE_EXTRA_FLAGS}" unittest
 $MAKE_CMD test

.ci/gcc-unittest.sh

@@ -1,11 +0,0 @@
-. "$CI_SCRIPTS/common.sh"
-
-set_environment /opt/neovim-deps/64
-
-sudo pip install cpp-coveralls
-
-export CC=gcc
-export SKIP_EXEC=1
-$MAKE_CMD CMAKE_EXTRA_FLAGS="-DTRAVIS_CI_BUILD=ON -DUSE_GCOV=ON" unittest
-
-coveralls --encoding iso-8859-1 || echo 'coveralls upload failed.'

.ci/gcc.sh

@@ -0,0 +1,20 @@
+. "$CI_SCRIPTS/common.sh"
+
+set_environment /opt/neovim-deps/64
+
+install_functional_test_deps
+
+sudo pip install cpp-coveralls
+
+sudo apt-get install valgrind
+
+export VALGRIND=1
+export VALGRIND_LOG="$tmpdir/valgrind-%p.log"
+CMAKE_EXTRA_FLAGS="-DTRAVIS_CI_BUILD=ON -DUSE_GCOV=ON"
+
+$MAKE_CMD CMAKE_EXTRA_FLAGS="${CMAKE_EXTRA_FLAGS}" unittest
+$MAKE_CMD test
+valgrind_check "$tmpdir"
+$MAKE_CMD oldtest
+
+coveralls --encoding iso-8859-1 || echo 'coveralls upload failed.'

.travis.yml

@@ -8,8 +8,8 @@ env:
     - secure: "QEz92NyItkzQu52kCFD928jEwUYnA2OIgSyeNrp+Y3gm5rOmSZerY8hGiXyNZxocap9+qIPCapRRYU3ZYKWZPeucWMLN3aIjxAFdhugKbnmNYE1jFugb6b8N3SxiX/3206NHXlYaz0OZhh6OBAFmPUXamJC8OrWVgPNPo7wv4UQ="
   matrix:
     - CI_TARGET=clang-asan
-    - CI_TARGET=gcc-ia32
+    - CI_TARGET=gcc
-    - CI_TARGET=gcc-unittest
+    - CI_TARGET=gcc-32
     - CI_TARGET=clint
     - CI_TARGET=api-python
     - CI_TARGET=coverity

CMakeLists.txt

@@ -211,6 +211,26 @@ if(BUSTED_PRG)
       -DBUSTED_OUTPUT_TYPE=${BUSTED_OUTPUT_TYPE}
       -DTEST_DIR=${CMAKE_CURRENT_SOURCE_DIR}/test
       -DBUILD_DIR=${CMAKE_BINARY_DIR}
-      -P ${CMAKE_MODULE_PATH}/RunUnittests.cmake
+      -DTEST_TYPE=unit
+      -P ${CMAKE_MODULE_PATH}/RunTests.cmake
     DEPENDS nvim-test unittest-headers)
+
+  # For the functional tests we need the full path to the real busted script,
+  # which will be included by run-functional-tests.py.
+  get_filename_component(LUA_PRG_DIR ${LUA_PRG} PATH)
+  get_filename_component(LUA_PREFIX_DIR ${LUA_PRG_DIR} PATH)
+  file(GLOB_RECURSE BUSTED_REAL_PRG
+    ${LUA_PREFIX_DIR}/lib/luarocks/rocks/busted/*busted)
+
+  add_custom_target(test
+    COMMAND ${CMAKE_COMMAND}
+      -DBUSTED_PRG=${PROJECT_SOURCE_DIR}/scripts/run-functional-tests.py
+      -DBUSTED_REAL_PRG=${BUSTED_REAL_PRG}
+      -DWORKING_DIR=${CMAKE_CURRENT_SOURCE_DIR}
+      -DBUSTED_OUTPUT_TYPE=${BUSTED_OUTPUT_TYPE}
+      -DTEST_DIR=${CMAKE_CURRENT_SOURCE_DIR}/test
+      -DBUILD_DIR=${CMAKE_BINARY_DIR}
+      -DTEST_TYPE=functional
+      -P ${CMAKE_MODULE_PATH}/RunTests.cmake
+    DEPENDS nvim) 
 endif()

Makefile

@@ -74,9 +74,11 @@ endif
 	mkdir -p build
 	touch $@
 
-test: | nvim
+oldtest: | nvim
 	+$(SINGLE_MAKE) -C src/nvim/testdir $(MAKEOVERRIDES)
-	PATH="$$(pwd)/build/bin:$$PATH" vroom --neovim --crawl test
+
+test: | nvim
+	+$(BUILD_CMD) -C build test
 
 unittest: | nvim
 	+$(BUILD_CMD) -C build unittest

cmake/RunTests.cmake

@@ -0,0 +1,23 @@
+get_filename_component(BUSTED_DIR ${BUSTED_PRG} PATH)
+set(ENV{PATH} "${BUSTED_DIR}:$ENV{PATH}")
+if(DEFINED ENV{TEST_FILE})
+  set(TEST_DIR $ENV{TEST_FILE})
+endif()
+
+if(TEST_TYPE STREQUAL "functional")
+  execute_process(
+    COMMAND python ${BUSTED_PRG} ${BUSTED_REAL_PRG} -v -o
+      ${BUSTED_OUTPUT_TYPE} --lpath=${BUILD_DIR}/?.lua ${TEST_DIR}/legacy
+    WORKING_DIRECTORY ${WORKING_DIR}
+    RESULT_VARIABLE res)
+else()
+  execute_process(
+    COMMAND ${BUSTED_PRG} -v -o ${BUSTED_OUTPUT_TYPE}
+      --lpath=${BUILD_DIR}/?.lua ${TEST_DIR}/unit
+    WORKING_DIRECTORY ${WORKING_DIR}
+    RESULT_VARIABLE res)
+endif()
+
+if(NOT res EQUAL 0)
+  message(FATAL_ERROR "Unit tests failed.")
+endif()

cmake/RunUnittests.cmake

@@ -1,14 +0,0 @@
-get_filename_component(BUSTED_DIR ${BUSTED_PRG} PATH)
-set(ENV{PATH} "${BUSTED_DIR}:$ENV{PATH}")
-if(DEFINED ENV{TEST_FILE})
-  set(TEST_DIR $ENV{TEST_FILE})
-endif()
-
-execute_process(
-  COMMAND ${BUSTED_PRG} -v -o ${BUSTED_OUTPUT_TYPE} --lpath=${BUILD_DIR}/?.lua ${TEST_DIR}
-  WORKING_DIRECTORY ${WORKING_DIR}
-  RESULT_VARIABLE res)
-
-if(NOT res EQUAL 0)
-  message(FATAL_ERROR "Unit tests failed.")
-endif()

scripts/run-functional-tests.py

@@ -0,0 +1,81 @@
+# Run functional tests using lua, busted and the python client
+
+import os
+import sys
+import textwrap
+
+from lupa import LuaRuntime
+from neovim import Nvim, spawn_session
+
+
+# Extract arguments
+busted_script = sys.argv[1]
+busted_argv = sys.argv[2:]
+
+# Setup a lua state for running busted
+lua = LuaRuntime(unpack_returned_tuples=True)
+lua_globals = lua.globals()
+
+# helper to transform iterables into lua tables
+list_to_table = lua.eval('''
+function(l)
+  local t = {}
+  for i, item in python.enumerate(l) do t[i + 1] = item end
+  return t
+end
+''')
+
+dict_to_table = lua.eval('''
+function(d)
+  local t = {}
+  for k, v in python.iterex(d.items()) do t[k] = v end
+  return t
+end
+''')
+
+nvim_prog = os.environ.get('NVIM_PROG', 'build/bin/nvim')
+nvim_argv = [nvim_prog, '-u', 'NONE', '--embed']
+
+if 'VALGRIND' in os.environ:
+    log_file = os.environ.get('VALGRIND_LOG', 'valgrind-%p.log')
+    valgrind_argv = ['valgrind', '-q', '--tool=memcheck', '--leak-check=yes',
+                     '--track-origins=yes', '--suppressions=.valgrind.supp',
+                     '--log-file={0}'.format(log_file)]
+    if 'VALGRIND_GDB' in os.environ:
+        valgrind_argv += ['--vgdb=yes', '--vgdb-error=0']
+    nvim_argv = valgrind_argv + nvim_argv
+
+session = spawn_session(nvim_argv)
+nvim = Nvim.from_session(session)
+
+def nvim_command(cmd):
+    nvim.command(cmd)
+
+def nvim_feed(input, mode=''):
+    nvim.feedkeys(input)
+
+def buffer_slice(start=None, stop=None, buffer_idx=None):
+    rv = '\n'.join(nvim.buffers[buffer_idx or 0][start:stop])
+    return rv
+
+def nvim_replace_termcodes(input, *opts):
+    return nvim.replace_termcodes(input, *opts)
+
+expose = [
+    nvim_command,
+    nvim_feed,
+    nvim_replace_termcodes,
+    buffer_slice,
+    textwrap.dedent,
+]
+
+for fn in expose:
+    lua_globals[fn.__name__] = fn
+
+# Set 'arg' global to let busted parse arguments
+lua_globals['arg'] = list_to_table(busted_argv)
+
+# Read the busted script and execute in the lua state
+with open(busted_script) as f:
+    busted_setup = f.read()
+lua.execute(busted_setup)

test/functional/helpers.lua

@@ -0,0 +1,84 @@
+local function clear()
+  nvim_command('call BeforeEachTest()')
+end
+
+local function feed(...)
+  for _, v in ipairs({...}) do
+    nvim_feed(nvim_replace_termcodes(dedent(v)))
+  end
+end
+
+local function rawfeed(...)
+  for _, v in ipairs({...}) do
+    nvim_feed(dedent(v), 'nt')
+  end
+end
+
+local function insert(...)
+  nvim_feed('i', 'nt')
+  rawfeed(...)
+  nvim_feed(nvim_replace_termcodes('<ESC>'), 'nt')
+end
+
+local function execute(...)
+  for _, v in ipairs({...}) do
+    if v:sub(1, 1) ~= '/' then
+      -- not a search command, prefix with colon
+      nvim_feed(':', 'nt')
+    end
+    nvim_feed(v, 'nt')
+    nvim_feed(nvim_replace_termcodes('<CR>'), 'nt')
+  end
+end
+
+local function expect(contents, first, last, buffer_index)
+  return assert.are.same(dedent(contents),
+                         buffer_slice(first, last, buffer_idx))
+end
+
+rawfeed([[:function BeforeEachTest()
+  set all&
+  redir => groups
+  silent augroup
+  redir END
+  for group in split(groups)
+    exe 'augroup '.group
+    autocmd!
+    augroup END
+  endfor
+  autocmd!
+  tabnew
+  let curbufnum = eval(bufnr('%'))
+  redir => buflist
+  silent ls!
+  redir END
+  let bufnums = []
+  for buf in split(buflist, '\n')
+    let bufnum = eval(split(buf, '[ u]')[0])
+    if bufnum != curbufnum
+      call add(bufnums, bufnum)
+    endif
+  endfor
+  if len(bufnums) > 0
+    exe 'silent bwipeout! '.join(bufnums, ' ')
+  endif
+  silent tabonly
+  for k in keys(g:)
+    exe 'unlet g:'.k
+  endfor
+  filetype plugin indent off
+  mapclear
+  mapclear!
+  abclear
+  comclear
+endfunction
+]])
+
+return {
+  clear = clear,
+  rawfeed = rawfeed,
+  insert = insert,
+  feed = feed,
+  execute = execute,
+  expect = expect 
+}

test/legacy/002_filename_recognition_spec.lua

@@ -0,0 +1,42 @@
+-- Test if URLs are recognized as filenames by commands such as "gf". Here
+-- we'll use `expand("<cfile>")` since "gf" would need to open the file.
+
+local helpers = require('test.functional.helpers')
+local clear, feed, insert = helpers.clear, helpers.feed, helpers.insert
+local execute, expect = helpers.execute, helpers.expect
+
+describe('filename recognition', function()
+  setup(clear)
+
+  it('is working', function()
+    -- insert some lines containing URLs
+    insert([[
+      first test for URL://machine.name/tmp/vimtest2a and other text
+      second test for URL://machine.name/tmp/vimtest2b. And other text
+      third test for URL:\\machine.name\vimtest2c and other text
+      fourth test for URL:\\machine.name\tmp\vimtest2d, and other text]])
+
+    -- Go to the first URL and append it to the beginning
+    execute('/^first', '/tmp', 'call append(0, expand("<cfile>"))')
+
+    -- Repeat for the second URL
+    -- this time, navigate to the word "URL" instead of "tmp"
+    execute('/^second', '/URL', 'call append(1, expand("<cfile>"))')
+
+    -- Repeat for the remaining URLs. This time, the 'isfname' option must be
+    -- set to allow '\' in filenames
+    execute('set isf=@,48-57,/,.,-,_,+,,,$,:,~,\\')
+    execute('/^third', '/name', 'call append(2, expand("<cfile>"))')
+    execute('/^fourth', '/URL', 'call append(3, expand("<cfile>"))')
+
+    -- Delete the initial text, which now starts at line 5
+    feed('5GdG')
+
+    -- The buffer should now contain:
+    expect([[
+      URL://machine.name/tmp/vimtest2a
+      URL://machine.name/tmp/vimtest2b
+      URL:\\machine.name\vimtest2c
+      URL:\\machine.name\tmp\vimtest2d]])
+  end)
+end)

test/legacy/test2-filename-recognition.vroom

@@ -1,40 +0,0 @@
-Test if URLs are recognized as filenames by commands such as "gf". Here
-we'll use `expand("<cfile>")` since "gf" would need to open the file.
-
-Insert some URLs:
-
-  % first test for URL://machine.name/tmp/vimtest2a and other text<cr>
-  % second test for URL://machine.name/tmp/vimtest2b. And other text<cr>
-  % third test for URL:\\machine.name\vimtest2c and other text<cr>
-  % fourth test for URL:\\machine.name\tmp\vimtest2d, and other text
-
-Go to the first URL:
-
-  :/^first<cr>/tmp
-
-Append the url to the beginning of the buffer:
-
-  :call append(0, expand("<cfile>"))
-
-Repeat for the second URL, but navigate to the 'URL' word:
-
-  :/^second<cr>/URL<cr>:call append(1, expand("<cfile>"))
-
-Repeat for the other two, but first the 'isfname' option must be set to
-allow '\' in filenames
-
-  :set isf=@,48-57,/,.,-,_,+,,,$,:,~,\
-  :/^third<cr>/name<cr>:call append(2, expand("<cfile>"))
-  :/^fourth<cr>/URL<cr>:call append(3, expand("<cfile>"))
-
-Delete the initial text which starts at line 5:
-
-  > 5GdG
-
-Result:
-
-  URL://machine.name/tmp/vimtest2a
-  URL://machine.name/tmp/vimtest2b
-  URL:\\machine.name\vimtest2c
-  URL:\\machine.name\tmp\vimtest2d
-  @end