From 88c8803aa167806f32272e5f344af00b98a91f01 Mon Sep 17 00:00:00 2001 From: Christian Clason Date: Sat, 1 Jul 2023 11:19:31 +0200 Subject: [PATCH 1/2] feat(treesitter): add python parser and queries --- cmake.deps/cmake/BuildTreesitterParsers.cmake | 2 +- cmake.deps/deps.txt | 2 + runtime/doc/news.txt | 3 +- runtime/queries/python/folds.scm | 28 ++ runtime/queries/python/highlights.scm | 345 ++++++++++++++++++ 5 files changed, 378 insertions(+), 2 deletions(-) create mode 100644 runtime/queries/python/folds.scm create mode 100644 runtime/queries/python/highlights.scm diff --git a/cmake.deps/cmake/BuildTreesitterParsers.cmake b/cmake.deps/cmake/BuildTreesitterParsers.cmake index 89c1ec5cc9..50968d4643 100644 --- a/cmake.deps/cmake/BuildTreesitterParsers.cmake +++ b/cmake.deps/cmake/BuildTreesitterParsers.cmake @@ -34,7 +34,7 @@ function(BuildTSParser) CMAKE_CACHE_ARGS ${DEPS_CMAKE_CACHE_ARGS}) endfunction() -foreach(lang c lua vim vimdoc query) +foreach(lang c lua vim vimdoc query python) BuildTSParser(LANG ${lang}) endforeach() BuildTSParser(LANG markdown CMAKE_FILE MarkdownParserCMakeLists.txt) diff --git a/cmake.deps/deps.txt b/cmake.deps/deps.txt index fd0b4aa400..c1ebb1215e 100644 --- a/cmake.deps/deps.txt +++ b/cmake.deps/deps.txt @@ -54,6 +54,8 @@ TREESITTER_VIMDOC_URL https://github.com/neovim/tree-sitter-vimdoc/archive/v2.0. TREESITTER_VIMDOC_SHA256 61e165df29778dc0c9277c2a7bc67447cc4e1bed36ca916a2f476dd25ce3260e TREESITTER_QUERY_URL https://github.com/nvim-treesitter/tree-sitter-query/archive/v0.1.0.tar.gz TREESITTER_QUERY_SHA256 e2b806f80e8bf1c4f4e5a96248393fe6622fc1fc6189d6896d269658f67f914c +TREESITTER_PYTHON_URL https://github.com/tree-sitter/tree-sitter-python/archive/36f9e33d52b7572536ac1a8af8d7e78363ad52c3.tar.gz +TREESITTER_PYTHON_SHA256 ac92b4759c363fe284a1bfd9df2d204b5efa166042a1484eefdf8e03b3f37d57 TREESITTER_MARKDOWN_URL https://github.com/MDeiml/tree-sitter-markdown/archive/936cc84289f6de83c263ae8e659fb342867ceb16.tar.gz TREESITTER_MARKDOWN_SHA256 4f2315930dc2c1bd42971a0b728cf4dafc57830c61f8abe3e2548cf230968713 TREESITTER_URL https://github.com/tree-sitter/tree-sitter/archive/91e4d940169a0c0b024560632ef53c4f119117ca.tar.gz diff --git a/runtime/doc/news.txt b/runtime/doc/news.txt index 5ee9cb7a14..c76686dda1 100644 --- a/runtime/doc/news.txt +++ b/runtime/doc/news.txt @@ -101,7 +101,8 @@ The following new APIs and features were added. • Implemented LSP inlay hints: |vim.lsp.inlay_hint()| https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_inlayHint -• Bundled Markdown parser for treesitter highlighting and folding. +• Bundled treesitter parser and queries (highlight, folds) for Markdown and +Python. ============================================================================== CHANGED FEATURES *news-changed* diff --git a/runtime/queries/python/folds.scm b/runtime/queries/python/folds.scm new file mode 100644 index 0000000000..78e1e2c00d --- /dev/null +++ b/runtime/queries/python/folds.scm @@ -0,0 +1,28 @@ +[ + (function_definition) + (class_definition) + + (while_statement) + (for_statement) + (if_statement) + (with_statement) + (try_statement) + (match_statement) + + (import_from_statement) + (parameters) + (argument_list) + + (parenthesized_expression) + (generator_expression) + (list_comprehension) + (set_comprehension) + (dictionary_comprehension) + + (tuple) + (list) + (set) + (dictionary) + + (string) +] @fold diff --git a/runtime/queries/python/highlights.scm b/runtime/queries/python/highlights.scm new file mode 100644 index 0000000000..c18b748674 --- /dev/null +++ b/runtime/queries/python/highlights.scm @@ -0,0 +1,345 @@ +;; From tree-sitter-python licensed under MIT License +; Copyright (c) 2016 Max Brunsfeld + +; Variables +(identifier) @variable + +; Reset highlighting in f-string interpolations +(interpolation) @none + +;; Identifier naming conventions +((identifier) @type + (#lua-match? @type "^[A-Z].*[a-z]")) +((identifier) @constant + (#lua-match? @constant "^[A-Z][A-Z_0-9]*$")) + +((identifier) @constant.builtin + (#lua-match? @constant.builtin "^__[a-zA-Z0-9_]*__$")) + +((identifier) @constant.builtin + (#any-of? @constant.builtin + ;; https://docs.python.org/3/library/constants.html + "NotImplemented" + "Ellipsis" + "quit" + "exit" + "copyright" + "credits" + "license")) + +((attribute + attribute: (identifier) @field) + (#lua-match? @field "^[%l_].*$")) + +((assignment + left: (identifier) @type.definition + (type (identifier) @_annotation)) + (#eq? @_annotation "TypeAlias")) + +((assignment + left: (identifier) @type.definition + right: (call + function: (identifier) @_func)) + (#any-of? @_func "TypeVar" "NewType")) + +; Function calls + +(call + function: (identifier) @function.call) + +(call + function: (attribute + attribute: (identifier) @method.call)) + +((call + function: (identifier) @constructor) + (#lua-match? @constructor "^%u")) + +((call + function: (attribute + attribute: (identifier) @constructor)) + (#lua-match? @constructor "^%u")) + +;; Decorators + +((decorator "@" @attribute) + (#set! "priority" 101)) + +(decorator + (identifier) @attribute) +(decorator + (attribute + attribute: (identifier) @attribute)) +(decorator + (call (identifier) @attribute)) +(decorator + (call (attribute + attribute: (identifier) @attribute))) + +((decorator + (identifier) @attribute.builtin) + (#any-of? @attribute.builtin "classmethod" "property")) + +;; Builtin functions + +((call + function: (identifier) @function.builtin) + (#any-of? @function.builtin + "abs" "all" "any" "ascii" "bin" "bool" "breakpoint" "bytearray" "bytes" "callable" "chr" "classmethod" + "compile" "complex" "delattr" "dict" "dir" "divmod" "enumerate" "eval" "exec" "filter" "float" "format" + "frozenset" "getattr" "globals" "hasattr" "hash" "help" "hex" "id" "input" "int" "isinstance" "issubclass" + "iter" "len" "list" "locals" "map" "max" "memoryview" "min" "next" "object" "oct" "open" "ord" "pow" + "print" "property" "range" "repr" "reversed" "round" "set" "setattr" "slice" "sorted" "staticmethod" "str" + "sum" "super" "tuple" "type" "vars" "zip" "__import__")) + +;; Function definitions + +(function_definition + name: (identifier) @function) + +(type (identifier) @type) +(type + (subscript + (identifier) @type)) ; type subscript: Tuple[int] + +((call + function: (identifier) @_isinstance + arguments: (argument_list + (_) + (identifier) @type)) + (#eq? @_isinstance "isinstance")) + +;; Normal parameters +(parameters + (identifier) @parameter) +;; Lambda parameters +(lambda_parameters + (identifier) @parameter) +(lambda_parameters + (tuple_pattern + (identifier) @parameter)) +; Default parameters +(keyword_argument + name: (identifier) @parameter) +; Naming parameters on call-site +(default_parameter + name: (identifier) @parameter) +(typed_parameter + (identifier) @parameter) +(typed_default_parameter + (identifier) @parameter) +; Variadic parameters *args, **kwargs +(parameters + (list_splat_pattern ; *args + (identifier) @parameter)) +(parameters + (dictionary_splat_pattern ; **kwargs + (identifier) @parameter)) + + +;; Literals + +(none) @constant.builtin +[(true) (false)] @boolean +((identifier) @variable.builtin + (#eq? @variable.builtin "self")) +((identifier) @variable.builtin + (#eq? @variable.builtin "cls")) + +(integer) @number +(float) @float + +(comment) @comment @spell + +((module . (comment) @preproc) + (#lua-match? @preproc "^#!/")) + +(string) @string +(escape_sequence) @string.escape + +; doc-strings + +(module . (expression_statement (string) @string.documentation @spell)) + +(class_definition + body: + (block + . (expression_statement (string) @string.documentation @spell))) + +(function_definition + body: + (block + . (expression_statement (string) @string.documentation @spell))) + +; Tokens + +[ + "-" + "-=" + ":=" + "!=" + "*" + "**" + "**=" + "*=" + "/" + "//" + "//=" + "/=" + "&" + "&=" + "%" + "%=" + "^" + "^=" + "+" + "+=" + "<" + "<<" + "<<=" + "<=" + "<>" + "=" + "==" + ">" + ">=" + ">>" + ">>=" + "@" + "@=" + "|" + "|=" + "~" + "->" +] @operator + +; Keywords +[ + "and" + "in" + "is" + "not" + "or" + "is not" + "not in" + + "del" +] @keyword.operator + +[ + "def" + "lambda" +] @keyword.function + +[ + "assert" + "class" + "exec" + "global" + "nonlocal" + "pass" + "print" + "with" + "as" +] @keyword + +[ + "async" + "await" +] @keyword.coroutine + +[ + "return" + "yield" +] @keyword.return +(yield "from" @keyword.return) + +(future_import_statement + "from" @include + "__future__" @constant.builtin) +(import_from_statement "from" @include) +"import" @include + +(aliased_import "as" @include) + +["if" "elif" "else" "match" "case"] @conditional + +["for" "while" "break" "continue"] @repeat + +[ + "try" + "except" + "except*" + "raise" + "finally" +] @exception + +(raise_statement "from" @exception) + +(try_statement + (else_clause + "else" @exception)) + +["(" ")" "[" "]" "{" "}"] @punctuation.bracket + +(interpolation + "{" @punctuation.special + "}" @punctuation.special) + +(type_conversion) @function.macro + +["," "." ":" ";" (ellipsis)] @punctuation.delimiter + +;; Class definitions + +(class_definition name: (identifier) @type) + +(class_definition + body: (block + (function_definition + name: (identifier) @method))) + +(class_definition + superclasses: (argument_list + (identifier) @type)) + +((class_definition + body: (block + (expression_statement + (assignment + left: (identifier) @field)))) + (#lua-match? @field "^%l.*$")) +((class_definition + body: (block + (expression_statement + (assignment + left: (_ + (identifier) @field))))) + (#lua-match? @field "^%l.*$")) + +((class_definition + (block + (function_definition + name: (identifier) @constructor))) + (#any-of? @constructor "__new__" "__init__")) + +((identifier) @type.builtin + (#any-of? @type.builtin + ;; https://docs.python.org/3/library/exceptions.html + "BaseException" "Exception" "ArithmeticError" "BufferError" "LookupError" "AssertionError" "AttributeError" + "EOFError" "FloatingPointError" "GeneratorExit" "ImportError" "ModuleNotFoundError" "IndexError" "KeyError" + "KeyboardInterrupt" "MemoryError" "NameError" "NotImplementedError" "OSError" "OverflowError" "RecursionError" + "ReferenceError" "RuntimeError" "StopIteration" "StopAsyncIteration" "SyntaxError" "IndentationError" "TabError" + "SystemError" "SystemExit" "TypeError" "UnboundLocalError" "UnicodeError" "UnicodeEncodeError" "UnicodeDecodeError" + "UnicodeTranslateError" "ValueError" "ZeroDivisionError" "EnvironmentError" "IOError" "WindowsError" + "BlockingIOError" "ChildProcessError" "ConnectionError" "BrokenPipeError" "ConnectionAbortedError" + "ConnectionRefusedError" "ConnectionResetError" "FileExistsError" "FileNotFoundError" "InterruptedError" + "IsADirectoryError" "NotADirectoryError" "PermissionError" "ProcessLookupError" "TimeoutError" "Warning" + "UserWarning" "DeprecationWarning" "PendingDeprecationWarning" "SyntaxWarning" "RuntimeWarning" + "FutureWarning" "ImportWarning" "UnicodeWarning" "BytesWarning" "ResourceWarning" + ;; https://docs.python.org/3/library/stdtypes.html + "bool" "int" "float" "complex" "list" "tuple" "range" "str" + "bytes" "bytearray" "memoryview" "set" "frozenset" "dict" "type" "object")) + +;; Error +(ERROR) @error From cb0a1a10b22463664b4aecd3aa25e262692c0cab Mon Sep 17 00:00:00 2001 From: Christian Clason Date: Sat, 1 Jul 2023 11:22:19 +0200 Subject: [PATCH 2/2] feat(treesitter): add bash parser and queries --- cmake.deps/cmake/BuildTreesitterParsers.cmake | 2 +- cmake.deps/deps.txt | 2 + runtime/doc/news.txt | 4 +- runtime/queries/bash/folds.scm | 8 + runtime/queries/bash/highlights.scm | 145 ++++++++++++++++++ 5 files changed, 158 insertions(+), 3 deletions(-) create mode 100644 runtime/queries/bash/folds.scm create mode 100644 runtime/queries/bash/highlights.scm diff --git a/cmake.deps/cmake/BuildTreesitterParsers.cmake b/cmake.deps/cmake/BuildTreesitterParsers.cmake index 50968d4643..e54544740e 100644 --- a/cmake.deps/cmake/BuildTreesitterParsers.cmake +++ b/cmake.deps/cmake/BuildTreesitterParsers.cmake @@ -34,7 +34,7 @@ function(BuildTSParser) CMAKE_CACHE_ARGS ${DEPS_CMAKE_CACHE_ARGS}) endfunction() -foreach(lang c lua vim vimdoc query python) +foreach(lang c lua vim vimdoc query python bash) BuildTSParser(LANG ${lang}) endforeach() BuildTSParser(LANG markdown CMAKE_FILE MarkdownParserCMakeLists.txt) diff --git a/cmake.deps/deps.txt b/cmake.deps/deps.txt index c1ebb1215e..a62bad20c6 100644 --- a/cmake.deps/deps.txt +++ b/cmake.deps/deps.txt @@ -56,6 +56,8 @@ TREESITTER_QUERY_URL https://github.com/nvim-treesitter/tree-sitter-query/archiv TREESITTER_QUERY_SHA256 e2b806f80e8bf1c4f4e5a96248393fe6622fc1fc6189d6896d269658f67f914c TREESITTER_PYTHON_URL https://github.com/tree-sitter/tree-sitter-python/archive/36f9e33d52b7572536ac1a8af8d7e78363ad52c3.tar.gz TREESITTER_PYTHON_SHA256 ac92b4759c363fe284a1bfd9df2d204b5efa166042a1484eefdf8e03b3f37d57 +TREESITTER_BASH_URL https://github.com/tree-sitter/tree-sitter-bash/archive/493646764e7ad61ce63ce3b8c59ebeb37f71b841.tar.gz +TREESITTER_BASH_SHA256 99ebe9f2886efecc1a5e9e1360d804a1b49ad89976a66bb5c3871539cca5fb7e TREESITTER_MARKDOWN_URL https://github.com/MDeiml/tree-sitter-markdown/archive/936cc84289f6de83c263ae8e659fb342867ceb16.tar.gz TREESITTER_MARKDOWN_SHA256 4f2315930dc2c1bd42971a0b728cf4dafc57830c61f8abe3e2548cf230968713 TREESITTER_URL https://github.com/tree-sitter/tree-sitter/archive/91e4d940169a0c0b024560632ef53c4f119117ca.tar.gz diff --git a/runtime/doc/news.txt b/runtime/doc/news.txt index c76686dda1..58c338b225 100644 --- a/runtime/doc/news.txt +++ b/runtime/doc/news.txt @@ -101,8 +101,8 @@ The following new APIs and features were added. • Implemented LSP inlay hints: |vim.lsp.inlay_hint()| https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_inlayHint -• Bundled treesitter parser and queries (highlight, folds) for Markdown and -Python. +• Bundled treesitter parser and queries (highlight, folds) for Markdown, +Python, and Bash. ============================================================================== CHANGED FEATURES *news-changed* diff --git a/runtime/queries/bash/folds.scm b/runtime/queries/bash/folds.scm new file mode 100644 index 0000000000..851c67eed4 --- /dev/null +++ b/runtime/queries/bash/folds.scm @@ -0,0 +1,8 @@ +[ + (function_definition) + (if_statement) + (case_statement) + (for_statement) + (while_statement) + (c_style_for_statement) +] @fold diff --git a/runtime/queries/bash/highlights.scm b/runtime/queries/bash/highlights.scm new file mode 100644 index 0000000000..23bf03e697 --- /dev/null +++ b/runtime/queries/bash/highlights.scm @@ -0,0 +1,145 @@ +(simple_expansion) @none +(expansion + "${" @punctuation.special + "}" @punctuation.special) @none +[ + "(" + ")" + "((" + "))" + "{" + "}" + "[" + "]" + "[[" + "]]" + ] @punctuation.bracket + +[ + ";" + ";;" + (heredoc_start) + ] @punctuation.delimiter + +[ + "$" +] @punctuation.special + +[ + ">" + ">>" + "<" + "<<" + "&" + "&&" + "|" + "||" + "=" + "=~" + "==" + "!=" + ] @operator + +[ + (string) + (raw_string) + (ansi_c_string) + (heredoc_body) +] @string @spell + +(variable_assignment (word) @string) + +[ + "if" + "then" + "else" + "elif" + "fi" + "case" + "in" + "esac" + ] @conditional + +[ + "for" + "do" + "done" + "select" + "until" + "while" + ] @repeat + +[ + "declare" + "export" + "local" + "readonly" + "unset" + ] @keyword + +"function" @keyword.function + +(special_variable_name) @constant + +; trap -l +((word) @constant.builtin + (#match? @constant.builtin "^SIG(HUP|INT|QUIT|ILL|TRAP|ABRT|BUS|FPE|KILL|USR[12]|SEGV|PIPE|ALRM|TERM|STKFLT|CHLD|CONT|STOP|TSTP|TT(IN|OU)|URG|XCPU|XFSZ|VTALRM|PROF|WINCH|IO|PWR|SYS|RTMIN([+]([1-9]|1[0-5]))?|RTMAX(-([1-9]|1[0-4]))?)$")) + +((word) @boolean + (#any-of? @boolean "true" "false")) + +(comment) @comment @spell +(test_operator) @string + +(command_substitution + [ "$(" ")" ] @punctuation.bracket) + +(process_substitution + [ "<(" ")" ] @punctuation.bracket) + + +(function_definition + name: (word) @function) + +(command_name (word) @function.call) + +((command_name (word) @function.builtin) + (#any-of? @function.builtin + "alias" "bg" "bind" "break" "builtin" "caller" "cd" + "command" "compgen" "complete" "compopt" "continue" + "coproc" "dirs" "disown" "echo" "enable" "eval" + "exec" "exit" "fc" "fg" "getopts" "hash" "help" + "history" "jobs" "kill" "let" "logout" "mapfile" + "popd" "printf" "pushd" "pwd" "read" "readarray" + "return" "set" "shift" "shopt" "source" "suspend" + "test" "time" "times" "trap" "type" "typeset" + "ulimit" "umask" "unalias" "wait")) + +(command + argument: [ + (word) @parameter + (concatenation (word) @parameter) + ]) + +((word) @number + (#lua-match? @number "^[0-9]+$")) + +(file_redirect + descriptor: (file_descriptor) @operator + destination: (word) @parameter) + +(expansion + [ "${" "}" ] @punctuation.bracket) + +(variable_name) @variable + +((variable_name) @constant + (#lua-match? @constant "^[A-Z][A-Z_0-9]*$")) + +(case_item + value: (word) @parameter) + +(regex) @string.regex + +((program . (comment) @preproc) + (#lua-match? @preproc "^#!/"))