From 4ea5e63aa8c866b4fcc9d10f1a26078d2517f96a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Linse?= Date: Sun, 9 Jun 2019 13:26:48 +0200 Subject: [PATCH] tree-sitter: add basic testing on ci build tree-sitter c parser on ci for testing purposes --- ci/build.ps1 | 22 ++++++++ ci/install.sh | 20 +++++++ ci/run_tests.sh | 2 + runtime/lua/vim/tree_sitter.lua | 8 +-- runtime/plugin/ts_test.vim | 2 +- src/nvim/lua/executor.c | 2 +- src/nvim/lua/vim.lua | 3 ++ test/functional/lua/tree_sitter_spec.lua | 67 ++++++++++++++++++++++++ 8 files changed, 121 insertions(+), 5 deletions(-) create mode 100644 test/functional/lua/tree_sitter_spec.lua diff --git a/ci/build.ps1 b/ci/build.ps1 index d533d7b4e0..4e1a69376b 100644 --- a/ci/build.ps1 +++ b/ci/build.ps1 @@ -94,6 +94,28 @@ npm.cmd install -g neovim Get-Command -CommandType Application neovim-node-host.cmd npm.cmd link neovim +#npm.cmd install -g tree-sitter-cli +#npm.cmd link tree-sitter-cli + +mkdir c:\treesitter +$env:TREE_SITTER_DIR = "c:\treesitter" +#$env:PATH = "c:\treesitter;$env:PATH" +$client = new-object System.Net.WebClient +cd c:\treesitter + +if ($bits -eq 32) { + $client.DownloadFile("https://github.com/tree-sitter/tree-sitter/releases/download/0.15.5/tree-sitter-windows-x86.gz","c:\treesitter\tree-sitter-cli.gz") +} +elseif ($bits -eq 64) { + $client.DownloadFile("https://github.com/tree-sitter/tree-sitter/releases/download/0.15.5/tree-sitter-windows-x64.gz","c:\treesitter\tree-sitter-cli.gz") +} +python -c "import gzip, shutil; f1,f2 = gzip.open('tree-sitter-cli.gz', 'rb'), open('tree-sitter.exe', 'wb'); shutil.copyfileobj(f1, f2); f2.close()" + +$client.DownloadFile("https://codeload.github.com/tree-sitter/tree-sitter-c/zip/v0.15.2","c:\treesitter\tree_sitter_c.zip") +Expand-Archive c:\treesitter\tree_sitter_c.zip -DestinationPath c:\treesitter\ +cd c:\treesitter\tree-sitter-c-0.15.2 +c:\treesitter\tree-sitter.exe test + function convertToCmakeArgs($vars) { return $vars.GetEnumerator() | foreach { "-D$($_.Key)=$($_.Value)" } } diff --git a/ci/install.sh b/ci/install.sh index cda9a11f08..b96cf3c073 100755 --- a/ci/install.sh +++ b/ci/install.sh @@ -24,3 +24,23 @@ gem install --no-document --version ">= 0.8.0" neovim echo "Install neovim npm package" npm install -g neovim npm link neovim + +echo "Install tree-sitter npm package" +npm install -g tree-sitter-cli +npm link tree-sitter-cli + +echo "Install tree-sitter c parser" +curl "https://codeload.github.com/tree-sitter/tree-sitter-c/tar.gz/v0.15.2" -o tree_sitter_c.tar.gz +tar xf tree_sitter_c.tar.gz +cd tree-sitter-c-0.15.2 +export TREE_SITTER_DIR=$HOME/tree-sitter-build/ +mkdir -p $TREE_SITTER_DIR/bin + +if [[ "$BUILD_32BIT" != "ON" ]]; then + # builds c parser in $HOME/tree-sitter-build/bin/c.(so|dylib) + tree-sitter test +else + # no tree-sitter binary for 32bit linux, so fake it (no tree-sitter unit tests) + cd src/ + gcc -m32 -o $TREE_SITTER_DIR/bin/c.so -shared parser.c -I. +fi diff --git a/ci/run_tests.sh b/ci/run_tests.sh index c175910da5..6b2f69293c 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -19,6 +19,8 @@ exit_suite --continue enter_suite tests +export TREE_SITTER_DIR=$HOME/tree-sitter-build/ + if test "$CLANG_SANITIZER" != "TSAN" ; then # Additional threads are only created when the builtin UI starts, which # doesn't happen in the unit/functional tests diff --git a/runtime/lua/vim/tree_sitter.lua b/runtime/lua/vim/tree_sitter.lua index a7830bc312..1b5f416b67 100644 --- a/runtime/lua/vim/tree_sitter.lua +++ b/runtime/lua/vim/tree_sitter.lua @@ -32,11 +32,13 @@ local function change_cb(self, ev, bufnr, tick, start_row, oldstopline, stop_row self.valid = false end -local function create_parser(bufnr) +local function create_parser(bufnr, ft) if bufnr == 0 then bufnr = a.nvim_get_current_buf() end - local ft = a.nvim_buf_get_option(bufnr, "filetype") + if ft == nil then + ft = a.nvim_buf_get_option(bufnr, "filetype") + end local self = setmetatable({bufnr=bufnr, valid=false}, Parser) self._parser = vim._create_ts_parser(ft) self:parse_tree() @@ -51,5 +53,5 @@ end -- TODO: weak table with reusable parser per buffer. -return {create_parser=create_parser} +return {create_parser=create_parser, add_language=vim._ts_add_language} diff --git a/runtime/plugin/ts_test.vim b/runtime/plugin/ts_test.vim index 76318163f6..9420c2c9d3 100644 --- a/runtime/plugin/ts_test.vim +++ b/runtime/plugin/ts_test.vim @@ -6,7 +6,7 @@ func! TSTest() return end " TODO: module! - lua theparser = require'vim.tree_sitter'.create_parser(0) + lua theparser = vim.tree_sitter.create_parser(0) lua require'tree_sitter_demo' let g:has_ts = v:true endfunc diff --git a/src/nvim/lua/executor.c b/src/nvim/lua/executor.c index ae53bfce6a..23c4fcabbc 100644 --- a/src/nvim/lua/executor.c +++ b/src/nvim/lua/executor.c @@ -841,5 +841,5 @@ static void nlua_add_treesitter(lua_State *const lstate) FUNC_ATTR_NONNULL_ALL lua_setfield(lstate, -2, "_create_ts_parser"); lua_pushcfunction(lstate, ts_lua_register_lang); - lua_setfield(lstate, -2, "ts_add_language"); + lua_setfield(lstate, -2, "_ts_add_language"); } diff --git a/src/nvim/lua/vim.lua b/src/nvim/lua/vim.lua index b1a684b977..c38926fe24 100644 --- a/src/nvim/lua/vim.lua +++ b/src/nvim/lua/vim.lua @@ -232,6 +232,9 @@ local function __index(t, key) if key == 'inspect' then t.inspect = require('vim.inspect') return t.inspect + elseif key == 'tree_sitter' then + t.tree_sitter = require('vim.tree_sitter') + return t.tree_sitter elseif require('vim.shared')[key] ~= nil then -- Expose all `vim.shared` functions on the `vim` module. t[key] = require('vim.shared')[key] diff --git a/test/functional/lua/tree_sitter_spec.lua b/test/functional/lua/tree_sitter_spec.lua new file mode 100644 index 0000000000..e25eb47a60 --- /dev/null +++ b/test/functional/lua/tree_sitter_spec.lua @@ -0,0 +1,67 @@ +-- Test suite for testing interactions with API bindings +local helpers = require('test.functional.helpers')(after_each) + +local meths = helpers.meths +local clear = helpers.clear +local eq = helpers.eq +local insert = helpers.insert +local meth_pcall = helpers.meth_pcall +local exec_lua = helpers.exec_lua +local iswin = helpers.iswin + +before_each(clear) + +describe('tree-sitter API', function() + -- error tests not requiring a parser library + it('handles basic errors', function() + --eq({false, 'Error executing lua: vim.schedule: expected function'}, + -- meth_pcall(meths.execute_lua, "parser = vim.tree_sitter.create_parser(0, 'nosuchlang')", {})) + + + + end) + + local ts_path = os.getenv("TREE_SITTER_DIR") + + describe('with C parser', function() + if ts_path == nil then + it("works", function() pending("TREE_SITTER_PATH not set, skipping tree-sitter parser tests") end) + return + end + + before_each(function() + -- TODO the .so/.dylib/.dll thingie + local path = ts_path .. '/bin/c'..(iswin() and '.dll' or '.so') + exec_lua([[ + local path = ... + vim.tree_sitter.add_language(path,'c') + + ]], path) + end) + + it('parses buffer', function() + insert([[ + int main() { + int x = 3; + }]]) + + exec_lua([[ + parser = vim.tree_sitter.create_parser(0, "c") + tree = parser:parse_tree() + root = tree:root() + ]]) + + --eq("", exec_lua("return tostring(parser)")) + eq("", exec_lua("return tostring(tree)")) + eq("", exec_lua("return tostring(root)")) + eq({0,0,3,0}, exec_lua("return {root:range()}")) + + eq(1, exec_lua("return root:child_count()")) + exec_lua("child = root:child(0)") + eq("", exec_lua("return tostring(child)")) + eq({0,0,2,1}, exec_lua("return {child:range()}")) + end) + + end) +end) +