tree-sitter: add basic testing on ci

build tree-sitter c parser on ci for testing purposes

ci/build.ps1

@@ -94,6 +94,28 @@ npm.cmd install -g neovim
 Get-Command -CommandType Application neovim-node-host.cmd
 npm.cmd link neovim
 
+#npm.cmd install -g tree-sitter-cli
+#npm.cmd link tree-sitter-cli
+
+mkdir c:\treesitter
+$env:TREE_SITTER_DIR = "c:\treesitter"
+#$env:PATH = "c:\treesitter;$env:PATH"
+$client = new-object System.Net.WebClient
+cd c:\treesitter
+
+if ($bits -eq 32) {
+  $client.DownloadFile("https://github.com/tree-sitter/tree-sitter/releases/download/0.15.5/tree-sitter-windows-x86.gz","c:\treesitter\tree-sitter-cli.gz")
+}
+elseif ($bits -eq 64) {
+  $client.DownloadFile("https://github.com/tree-sitter/tree-sitter/releases/download/0.15.5/tree-sitter-windows-x64.gz","c:\treesitter\tree-sitter-cli.gz")
+}
+python -c "import gzip, shutil; f1,f2 = gzip.open('tree-sitter-cli.gz', 'rb'),  open('tree-sitter.exe', 'wb'); shutil.copyfileobj(f1, f2); f2.close()"
+
+$client.DownloadFile("https://codeload.github.com/tree-sitter/tree-sitter-c/zip/v0.15.2","c:\treesitter\tree_sitter_c.zip")
+Expand-Archive c:\treesitter\tree_sitter_c.zip -DestinationPath c:\treesitter\
+cd c:\treesitter\tree-sitter-c-0.15.2
+c:\treesitter\tree-sitter.exe test
+
 function convertToCmakeArgs($vars) {
   return $vars.GetEnumerator() | foreach { "-D$($_.Key)=$($_.Value)" }
 }

ci/install.sh

@@ -24,3 +24,23 @@ gem install --no-document --version ">= 0.8.0" neovim
 echo "Install neovim npm package"
 npm install -g neovim
 npm link neovim
+
+echo "Install tree-sitter npm package"
+npm install -g tree-sitter-cli
+npm link tree-sitter-cli
+
+echo "Install tree-sitter c parser"
+curl "https://codeload.github.com/tree-sitter/tree-sitter-c/tar.gz/v0.15.2" -o tree_sitter_c.tar.gz
+tar xf tree_sitter_c.tar.gz
+cd tree-sitter-c-0.15.2
+export TREE_SITTER_DIR=$HOME/tree-sitter-build/
+mkdir -p $TREE_SITTER_DIR/bin
+
+if [[ "$BUILD_32BIT" != "ON" ]]; then
+  # builds c parser in $HOME/tree-sitter-build/bin/c.(so|dylib)
+  tree-sitter test
+else
+  # no tree-sitter binary for 32bit linux, so fake it (no tree-sitter unit tests)
+  cd src/
+  gcc -m32 -o $TREE_SITTER_DIR/bin/c.so -shared parser.c -I.
+fi

ci/run_tests.sh

@@ -19,6 +19,8 @@ exit_suite --continue
 
 enter_suite tests
 
+export TREE_SITTER_DIR=$HOME/tree-sitter-build/
+
 if test "$CLANG_SANITIZER" != "TSAN" ; then
   # Additional threads are only created when the builtin UI starts, which
   # doesn't happen in the unit/functional tests

runtime/lua/vim/tree_sitter.lua

@@ -32,11 +32,13 @@ local function change_cb(self, ev, bufnr, tick, start_row, oldstopline, stop_row
   self.valid = false
 end
 
-local function create_parser(bufnr)
+local function create_parser(bufnr, ft)
   if bufnr == 0 then
     bufnr = a.nvim_get_current_buf()
   end
-  local ft = a.nvim_buf_get_option(bufnr, "filetype")
+  if ft == nil then
+    ft = a.nvim_buf_get_option(bufnr, "filetype")
+  end
   local self = setmetatable({bufnr=bufnr, valid=false}, Parser)
   self._parser = vim._create_ts_parser(ft)
   self:parse_tree()
@@ -51,5 +53,5 @@ end
 
 -- TODO: weak table with reusable parser per buffer.
 
-return {create_parser=create_parser}
+return {create_parser=create_parser, add_language=vim._ts_add_language}
 

runtime/plugin/ts_test.vim

@@ -6,7 +6,7 @@ func! TSTest()
     return
   end
   " TODO: module!
-  lua theparser = require'vim.tree_sitter'.create_parser(0)
+  lua theparser = vim.tree_sitter.create_parser(0)
   lua require'tree_sitter_demo'
   let g:has_ts = v:true
 endfunc

src/nvim/lua/executor.c

@@ -841,5 +841,5 @@ static void nlua_add_treesitter(lua_State *const lstate) FUNC_ATTR_NONNULL_ALL
   lua_setfield(lstate, -2, "_create_ts_parser");
 
   lua_pushcfunction(lstate, ts_lua_register_lang);
-  lua_setfield(lstate, -2, "ts_add_language");
+  lua_setfield(lstate, -2, "_ts_add_language");
 }

src/nvim/lua/vim.lua

@@ -232,6 +232,9 @@ local function __index(t, key)
   if key == 'inspect' then
     t.inspect = require('vim.inspect')
     return t.inspect
+  elseif key == 'tree_sitter' then
+    t.tree_sitter = require('vim.tree_sitter')
+    return t.tree_sitter
   elseif require('vim.shared')[key] ~= nil then
     -- Expose all `vim.shared` functions on the `vim` module.
     t[key] = require('vim.shared')[key]

test/functional/lua/tree_sitter_spec.lua

@@ -0,0 +1,67 @@
+-- Test suite for testing interactions with API bindings
+local helpers = require('test.functional.helpers')(after_each)
+
+local meths = helpers.meths
+local clear = helpers.clear
+local eq = helpers.eq
+local insert = helpers.insert
+local meth_pcall = helpers.meth_pcall
+local exec_lua = helpers.exec_lua
+local iswin = helpers.iswin
+
+before_each(clear)
+
+describe('tree-sitter API', function()
+  -- error tests not requiring a parser library
+  it('handles basic errors', function()
+    --eq({false, 'Error executing lua: vim.schedule: expected function'},
+    --   meth_pcall(meths.execute_lua, "parser = vim.tree_sitter.create_parser(0, 'nosuchlang')", {}))
+
+
+
+  end)
+
+  local ts_path = os.getenv("TREE_SITTER_DIR")
+
+  describe('with C parser', function()
+    if ts_path == nil then
+      it("works", function() pending("TREE_SITTER_PATH not set, skipping tree-sitter parser tests") end)
+      return
+    end
+
+    before_each(function()
+      -- TODO the .so/.dylib/.dll thingie
+      local path = ts_path .. '/bin/c'..(iswin() and '.dll' or '.so')
+      exec_lua([[
+        local path = ...
+        vim.tree_sitter.add_language(path,'c')
+
+      ]], path)
+    end)
+
+    it('parses buffer', function()
+      insert([[
+        int main() {
+          int x = 3;
+        }]])
+
+      exec_lua([[
+        parser = vim.tree_sitter.create_parser(0, "c")
+        tree = parser:parse_tree()
+        root = tree:root()
+      ]])
+
+      --eq("<parser>", exec_lua("return tostring(parser)"))
+      eq("<tree>", exec_lua("return tostring(tree)"))
+      eq("<node translation_unit>", exec_lua("return tostring(root)"))
+      eq({0,0,3,0}, exec_lua("return {root:range()}"))
+
+      eq(1, exec_lua("return root:child_count()"))
+      exec_lua("child = root:child(0)")
+      eq("<node function_definition>", exec_lua("return tostring(child)"))
+      eq({0,0,2,1}, exec_lua("return {child:range()}"))
+    end)
+
+  end)
+end)
+