feat(lsp)!: replace snippet parser by lpeg grammar

This commit is contained in:
Maria José Solano 2023-10-01 09:54:04 -07:00 committed by Christian Clason
parent f736b075d3
commit eb1f0e8fcc
5 changed files with 122 additions and 672 deletions

View File

@ -68,6 +68,10 @@ The following changes may require adaptations in user config or plugins.
• Float window support hide and show by setting `hide` on `nvim_open_win` and
`nvim_win_set_config`.
• |vim.lsp.util.parse_snippet()| will now strictly follow the snippet grammar
defined by LSP, and hence previously parsed snippets might now be considered
invalid input.
==============================================================================
NEW FEATURES *news-features*

View File

@ -1,500 +0,0 @@
local P = {}
---Take characters until the target characters (The escape sequence is '\' + char)
---@param targets string[] The character list for stop consuming text.
---@param specials string[] If the character isn't contained in targets/specials, '\' will be left.
P.take_until = function(targets, specials)
targets = targets or {}
specials = specials or {}
return function(input, pos)
local new_pos = pos
local raw = {}
local esc = {}
while new_pos <= #input do
local c = string.sub(input, new_pos, new_pos)
if c == '\\' then
table.insert(raw, '\\')
new_pos = new_pos + 1
c = string.sub(input, new_pos, new_pos)
if not vim.list_contains(targets, c) and not vim.list_contains(specials, c) then
table.insert(esc, '\\')
end
table.insert(raw, c)
table.insert(esc, c)
new_pos = new_pos + 1
else
if vim.list_contains(targets, c) then
break
end
table.insert(raw, c)
table.insert(esc, c)
new_pos = new_pos + 1
end
end
if new_pos == pos then
return P.unmatch(pos)
end
return {
parsed = true,
value = {
raw = table.concat(raw, ''),
esc = table.concat(esc, ''),
},
pos = new_pos,
}
end
end
P.unmatch = function(pos)
return {
parsed = false,
value = nil,
pos = pos,
}
end
P.map = function(parser, map)
return function(input, pos)
local result = parser(input, pos)
if result.parsed then
return {
parsed = true,
value = map(result.value),
pos = result.pos,
}
end
return P.unmatch(pos)
end
end
P.lazy = function(factory)
return function(input, pos)
return factory()(input, pos)
end
end
P.token = function(token)
return function(input, pos)
local maybe_token = string.sub(input, pos, pos + #token - 1)
if token == maybe_token then
return {
parsed = true,
value = maybe_token,
pos = pos + #token,
}
end
return P.unmatch(pos)
end
end
P.pattern = function(p)
return function(input, pos)
local maybe_match = string.match(string.sub(input, pos), '^' .. p)
if maybe_match then
return {
parsed = true,
value = maybe_match,
pos = pos + #maybe_match,
}
end
return P.unmatch(pos)
end
end
P.many = function(parser)
return function(input, pos)
local values = {}
local new_pos = pos
while new_pos <= #input do
local result = parser(input, new_pos)
if not result.parsed then
break
end
table.insert(values, result.value)
new_pos = result.pos
end
if #values > 0 then
return {
parsed = true,
value = values,
pos = new_pos,
}
end
return P.unmatch(pos)
end
end
P.any = function(...)
local parsers = { ... }
return function(input, pos)
for _, parser in ipairs(parsers) do
local result = parser(input, pos)
if result.parsed then
return result
end
end
return P.unmatch(pos)
end
end
P.opt = function(parser)
return function(input, pos)
local result = parser(input, pos)
return {
parsed = true,
value = result.value,
pos = result.pos,
}
end
end
P.seq = function(...)
local parsers = { ... }
return function(input, pos)
local values = {}
local new_pos = pos
for i, parser in ipairs(parsers) do
local result = parser(input, new_pos)
if result.parsed then
values[i] = result.value
new_pos = result.pos
else
return P.unmatch(pos)
end
end
return {
parsed = true,
value = values,
pos = new_pos,
}
end
end
local Node = {}
Node.Type = {
SNIPPET = 0,
TABSTOP = 1,
PLACEHOLDER = 2,
VARIABLE = 3,
CHOICE = 4,
TRANSFORM = 5,
FORMAT = 6,
TEXT = 7,
}
function Node:__tostring()
local insert_text = {}
if self.type == Node.Type.SNIPPET then
for _, c in ipairs(self.children) do
table.insert(insert_text, tostring(c))
end
elseif self.type == Node.Type.CHOICE then
table.insert(insert_text, self.items[1])
elseif self.type == Node.Type.PLACEHOLDER then
for _, c in ipairs(self.children or {}) do
table.insert(insert_text, tostring(c))
end
elseif self.type == Node.Type.TEXT then
table.insert(insert_text, self.esc)
end
return table.concat(insert_text, '')
end
--@see https://code.visualstudio.com/docs/editor/userdefinedsnippets#_grammar
local S = {}
S.dollar = P.token('$')
S.open = P.token('{')
S.close = P.token('}')
S.colon = P.token(':')
S.slash = P.token('/')
S.comma = P.token(',')
S.pipe = P.token('|')
S.plus = P.token('+')
S.minus = P.token('-')
S.question = P.token('?')
S.int = P.map(P.pattern('[0-9]+'), function(value)
return tonumber(value, 10)
end)
S.var = P.pattern('[%a_][%w_]+')
S.text = function(targets, specials)
return P.map(P.take_until(targets, specials), function(value)
return setmetatable({
type = Node.Type.TEXT,
raw = value.raw,
esc = value.esc,
}, Node)
end)
end
S.toplevel = P.lazy(function()
return P.any(S.placeholder, S.tabstop, S.variable, S.choice)
end)
S.format = P.any(
P.map(P.seq(S.dollar, S.int), function(values)
return setmetatable({
type = Node.Type.FORMAT,
capture_index = values[2],
}, Node)
end),
P.map(P.seq(S.dollar, S.open, S.int, S.close), function(values)
return setmetatable({
type = Node.Type.FORMAT,
capture_index = values[3],
}, Node)
end),
P.map(
P.seq(
S.dollar,
S.open,
S.int,
S.colon,
S.slash,
P.any(
P.token('upcase'),
P.token('downcase'),
P.token('capitalize'),
P.token('camelcase'),
P.token('pascalcase')
),
S.close
),
function(values)
return setmetatable({
type = Node.Type.FORMAT,
capture_index = values[3],
modifier = values[6],
}, Node)
end
),
P.map(
P.seq(
S.dollar,
S.open,
S.int,
S.colon,
P.seq(
S.question,
P.opt(P.take_until({ ':' }, { '\\' })),
S.colon,
P.opt(P.take_until({ '}' }, { '\\' }))
),
S.close
),
function(values)
return setmetatable({
type = Node.Type.FORMAT,
capture_index = values[3],
if_text = values[5][2] and values[5][2].esc or '',
else_text = values[5][4] and values[5][4].esc or '',
}, Node)
end
),
P.map(
P.seq(
S.dollar,
S.open,
S.int,
S.colon,
P.seq(S.plus, P.opt(P.take_until({ '}' }, { '\\' }))),
S.close
),
function(values)
return setmetatable({
type = Node.Type.FORMAT,
capture_index = values[3],
if_text = values[5][2] and values[5][2].esc or '',
else_text = '',
}, Node)
end
),
P.map(
P.seq(
S.dollar,
S.open,
S.int,
S.colon,
S.minus,
P.opt(P.take_until({ '}' }, { '\\' })),
S.close
),
function(values)
return setmetatable({
type = Node.Type.FORMAT,
capture_index = values[3],
if_text = '',
else_text = values[6] and values[6].esc or '',
}, Node)
end
),
P.map(
P.seq(S.dollar, S.open, S.int, S.colon, P.opt(P.take_until({ '}' }, { '\\' })), S.close),
function(values)
return setmetatable({
type = Node.Type.FORMAT,
capture_index = values[3],
if_text = '',
else_text = values[5] and values[5].esc or '',
}, Node)
end
)
)
S.transform = P.map(
P.seq(
S.slash,
P.take_until({ '/' }, { '\\' }),
S.slash,
P.many(P.any(S.format, S.text({ '$', '/' }, { '\\' }))),
S.slash,
P.opt(P.pattern('[ig]+'))
),
function(values)
return setmetatable({
type = Node.Type.TRANSFORM,
pattern = values[2].raw,
format = values[4],
option = values[6],
}, Node)
end
)
S.tabstop = P.any(
P.map(P.seq(S.dollar, S.int), function(values)
return setmetatable({
type = Node.Type.TABSTOP,
tabstop = values[2],
}, Node)
end),
P.map(P.seq(S.dollar, S.open, S.int, S.close), function(values)
return setmetatable({
type = Node.Type.TABSTOP,
tabstop = values[3],
}, Node)
end),
P.map(P.seq(S.dollar, S.open, S.int, S.transform, S.close), function(values)
return setmetatable({
type = Node.Type.TABSTOP,
tabstop = values[3],
transform = values[4],
}, Node)
end)
)
S.placeholder = P.any(
P.map(
P.seq(
S.dollar,
S.open,
S.int,
S.colon,
P.opt(P.many(P.any(S.toplevel, S.text({ '$', '}' }, { '\\' })))),
S.close
),
function(values)
return setmetatable({
type = Node.Type.PLACEHOLDER,
tabstop = values[3],
-- insert empty text if opt did not match.
children = values[5] or {
setmetatable({
type = Node.Type.TEXT,
raw = '',
esc = '',
}, Node),
},
}, Node)
end
)
)
S.choice = P.map(
P.seq(
S.dollar,
S.open,
S.int,
S.pipe,
P.many(P.map(P.seq(S.text({ ',', '|' }), P.opt(S.comma)), function(values)
return values[1].esc
end)),
S.pipe,
S.close
),
function(values)
return setmetatable({
type = Node.Type.CHOICE,
tabstop = values[3],
items = values[5],
}, Node)
end
)
S.variable = P.any(
P.map(P.seq(S.dollar, S.var), function(values)
return setmetatable({
type = Node.Type.VARIABLE,
name = values[2],
}, Node)
end),
P.map(P.seq(S.dollar, S.open, S.var, S.close), function(values)
return setmetatable({
type = Node.Type.VARIABLE,
name = values[3],
}, Node)
end),
P.map(P.seq(S.dollar, S.open, S.var, S.transform, S.close), function(values)
return setmetatable({
type = Node.Type.VARIABLE,
name = values[3],
transform = values[4],
}, Node)
end),
P.map(
P.seq(
S.dollar,
S.open,
S.var,
S.colon,
P.many(P.any(S.toplevel, S.text({ '$', '}' }, { '\\' }))),
S.close
),
function(values)
return setmetatable({
type = Node.Type.VARIABLE,
name = values[3],
children = values[5],
}, Node)
end
)
)
S.snippet = P.map(P.many(P.any(S.toplevel, S.text({ '$' }, { '}', '\\' }))), function(values)
return setmetatable({
type = Node.Type.SNIPPET,
children = values,
}, Node)
end)
local M = {}
---The snippet node type enum
---@types table<string, integer>
M.NodeType = Node.Type
---Parse snippet string and returns the AST
---@param input string
---@return table
function M.parse(input)
local result = S.snippet(input, 1)
if not result.parsed then
error('snippet parsing failed.')
end
return result.value
end
return M

View File

@ -1,5 +1,5 @@
local protocol = require('vim.lsp.protocol')
local snippet = require('vim.lsp._snippet')
local snippet = require('vim.lsp._snippet_grammar')
local validate = vim.validate
local api = vim.api
local list_extend = vim.list_extend
@ -610,12 +610,41 @@ end
---@return string parsed snippet
function M.parse_snippet(input)
local ok, parsed = pcall(function()
return tostring(snippet.parse(input))
return snippet.parse(input)
end)
if not ok then
return input
end
return parsed
--- @param node vim.snippet.Node<any>
--- @return string
local function node_to_string(node)
local insert_text = {}
if node.type == snippet.NodeType.Snippet then
for _, child in
ipairs((node.data --[[@as vim.snippet.SnippetData]]).children)
do
table.insert(insert_text, node_to_string(child))
end
elseif node.type == snippet.NodeType.Choice then
table.insert(insert_text, (node.data --[[@as vim.snippet.ChoiceData]]).values[1])
elseif node.type == snippet.NodeType.Placeholder then
table.insert(
insert_text,
node_to_string((node.data --[[@as vim.snippet.PlaceholderData]]).value)
)
elseif node.type == snippet.NodeType.Text then
table.insert(
insert_text,
node
.data --[[@as vim.snippet.TextData]]
.text
)
end
return table.concat(insert_text)
end
return node_to_string(parsed)
end
--- Sorts by CompletionItem.sortText.

View File

@ -1,130 +1,70 @@
local helpers = require('test.functional.helpers')(after_each)
local snippet = require('vim.lsp._snippet')
local snippet = require('vim.lsp._snippet_grammar')
local eq = helpers.eq
local exec_lua = helpers.exec_lua
describe('vim.lsp._snippet', function()
describe('vim.lsp._snippet_grammar', function()
before_each(helpers.clear)
after_each(helpers.clear)
local parse = function(...)
return exec_lua('return require("vim.lsp._snippet").parse(...)', ...)
local res = exec_lua('return require("vim.lsp._snippet_grammar").parse(...)', ...)
return res.data.children
end
it('should parse only text', function()
it('parses only text', function()
eq({
type = snippet.NodeType.SNIPPET,
children = {
{
type = snippet.NodeType.TEXT,
raw = 'TE\\$\\}XT',
esc = 'TE$}XT',
},
},
{ type = snippet.NodeType.Text, data = { text = 'TE$}XT' } },
}, parse('TE\\$\\}XT'))
end)
it('should parse tabstop', function()
it('parses tabstops', function()
eq({
type = snippet.NodeType.SNIPPET,
children = {
{
type = snippet.NodeType.TABSTOP,
tabstop = 1,
},
{
type = snippet.NodeType.TABSTOP,
tabstop = 2,
},
},
{ type = snippet.NodeType.Tabstop, data = { tabstop = 1 } },
{ type = snippet.NodeType.Tabstop, data = { tabstop = 2 } },
}, parse('$1${2}'))
end)
it('should parse placeholders', function()
it('parses nested placeholders', function()
eq({
type = snippet.NodeType.SNIPPET,
children = {
{
type = snippet.NodeType.PLACEHOLDER,
{
type = snippet.NodeType.Placeholder,
data = {
tabstop = 1,
children = {
{
type = snippet.NodeType.PLACEHOLDER,
value = {
type = snippet.NodeType.Placeholder,
data = {
tabstop = 2,
children = {
{
type = snippet.NodeType.TEXT,
raw = 'TE\\$\\}XT',
esc = 'TE$}XT',
},
{
type = snippet.NodeType.TABSTOP,
tabstop = 3,
},
{
type = snippet.NodeType.TABSTOP,
tabstop = 1,
transform = {
type = snippet.NodeType.TRANSFORM,
pattern = 'regex',
option = 'i',
format = {
{
type = snippet.NodeType.FORMAT,
capture_index = 1,
modifier = 'upcase',
},
},
},
},
{
type = snippet.NodeType.TEXT,
raw = 'TE\\$\\}XT',
esc = 'TE$}XT',
},
},
value = { type = snippet.NodeType.Tabstop, data = { tabstop = 3 } },
},
},
},
},
}, parse('${1:${2:TE\\$\\}XT$3${1/regex/${1:/upcase}/i}TE\\$\\}XT}}'))
}, parse('${1:${2:${3}}}'))
end)
it('should parse variables', function()
it('parses variables', function()
eq({
type = snippet.NodeType.SNIPPET,
children = {
{
type = snippet.NodeType.VARIABLE,
{ type = snippet.NodeType.Variable, data = { name = 'VAR' } },
{ type = snippet.NodeType.Variable, data = { name = 'VAR' } },
{
type = snippet.NodeType.Variable,
data = {
name = 'VAR',
default = { type = snippet.NodeType.Tabstop, data = { tabstop = 1 } },
},
{
type = snippet.NodeType.VARIABLE,
},
{
type = snippet.NodeType.Variable,
data = {
name = 'VAR',
},
{
type = snippet.NodeType.VARIABLE,
name = 'VAR',
children = {
regex = 'regex',
options = '',
format = {
{
type = snippet.NodeType.TABSTOP,
tabstop = 1,
},
},
},
{
type = snippet.NodeType.VARIABLE,
name = 'VAR',
transform = {
type = snippet.NodeType.TRANSFORM,
pattern = 'regex',
format = {
{
type = snippet.NodeType.FORMAT,
capture_index = 1,
modifier = 'upcase',
},
type = snippet.NodeType.Format,
data = { capture = 1, modifier = 'upcase' },
},
},
},
@ -132,105 +72,82 @@ describe('vim.lsp._snippet', function()
}, parse('$VAR${VAR}${VAR:$1}${VAR/regex/${1:/upcase}/}'))
end)
it('should parse choice', function()
it('parses choice', function()
eq({
type = snippet.NodeType.SNIPPET,
children = {
{
type = snippet.NodeType.CHOICE,
tabstop = 1,
items = {
',',
'|',
},
},
{
type = snippet.NodeType.Choice,
data = { tabstop = 1, values = { ',', '|' } },
},
}, parse('${1|\\,,\\||}'))
end)
it('should parse format', function()
eq({
type = snippet.NodeType.SNIPPET,
children = {
it('parses format', function()
eq(
{
{
type = snippet.NodeType.VARIABLE,
name = 'VAR',
transform = {
type = snippet.NodeType.TRANSFORM,
pattern = 'regex',
type = snippet.NodeType.Variable,
data = {
name = 'VAR',
regex = 'regex',
options = '',
format = {
{
type = snippet.NodeType.FORMAT,
capture_index = 1,
modifier = 'upcase',
type = snippet.NodeType.Format,
data = { capture = 1, modifier = 'upcase' },
},
{
type = snippet.NodeType.FORMAT,
capture_index = 1,
if_text = 'if_text',
else_text = '',
type = snippet.NodeType.Format,
data = { capture = 1, if_text = 'if_text' },
},
{
type = snippet.NodeType.FORMAT,
capture_index = 1,
if_text = '',
else_text = 'else_text',
type = snippet.NodeType.Format,
data = { capture = 1, else_text = 'else_text' },
},
{
type = snippet.NodeType.FORMAT,
capture_index = 1,
else_text = 'else_text',
if_text = 'if_text',
type = snippet.NodeType.Format,
data = { capture = 1, if_text = 'if_text', else_text = 'else_text' },
},
{
type = snippet.NodeType.FORMAT,
capture_index = 1,
if_text = '',
else_text = 'else_text',
type = snippet.NodeType.Format,
data = { capture = 1, else_text = 'else_text' },
},
},
},
},
},
}, parse('${VAR/regex/${1:/upcase}${1:+if_text}${1:-else_text}${1:?if_text:else_text}${1:else_text}/}'))
parse(
'${VAR/regex/${1:/upcase}${1:+if_text}${1:-else_text}${1:?if_text:else_text}${1:else_text}/}'
)
)
end)
it('should parse empty strings', function()
it('parses empty strings', function()
eq({
children = {
{
children = { {
esc = '',
raw = '',
type = 7,
} },
{
type = snippet.NodeType.Placeholder,
data = {
tabstop = 1,
type = 2,
},
{
esc = ' ',
raw = ' ',
type = 7,
},
{
name = 'VAR',
transform = {
format = {
{
capture_index = 1,
else_text = '',
if_text = '',
type = 6,
},
},
option = 'g',
pattern = 'erg',
type = 5,
},
type = 3,
value = { type = snippet.NodeType.Text, data = { text = '' } },
},
},
type = 0,
}, parse('${1:} ${VAR/erg/${1:?:}/g}'))
{
type = snippet.NodeType.Text,
data = { text = ' ' },
},
{
type = snippet.NodeType.Variable,
data = {
name = 'VAR',
regex = 'erg',
format = {
{
type = snippet.NodeType.Format,
data = { capture = 1, if_text = '' },
},
},
options = 'g',
},
},
}, parse('${1:} ${VAR/erg/${1:+}/g}'))
end)
end)

View File

@ -2301,7 +2301,7 @@ describe('LSP', function()
{ label='foocar', sortText="g", insertText='foodar', insertTextFormat=2, textEdit={newText='foobar(${1:place holder}, ${2:more ...holder{\\}})'} },
{ label='foocar', sortText="h", insertText='foodar(${1:var1} typ1, ${2:var2} *typ2) {$0\\}', insertTextFormat=2, textEdit={} },
-- nested snippet tokens
{ label='foocar', sortText="i", insertText='foodar(${1:var1 ${2|typ2,typ3|} ${3:tail}}) {$0\\}', insertTextFormat=2, textEdit={} },
{ label='foocar', sortText="i", insertText='foodar(${1:${2|typ1,typ2|}}) {$0\\}', insertTextFormat=2, textEdit={} },
-- braced tabstop
{ label='foocar', sortText="j", insertText='foodar()${0}', insertTextFormat=2, textEdit={} },
-- plain text
@ -2317,7 +2317,7 @@ describe('LSP', function()
{ abbr = 'foocar', dup = 1, empty = 1, icase = 1, kind = 'Unknown', menu = '', word = 'foobar', user_data = { nvim = { lsp = { completion_item = { label='foocar', sortText="f", textEdit={newText='foobar'} } } } } },
{ abbr = 'foocar', dup = 1, empty = 1, icase = 1, kind = 'Unknown', menu = '', word = 'foobar(place holder, more ...holder{})', user_data = { nvim = { lsp = { completion_item = { label='foocar', sortText="g", insertText='foodar', insertTextFormat=2, textEdit={newText='foobar(${1:place holder}, ${2:more ...holder{\\}})'} } } } } },
{ abbr = 'foocar', dup = 1, empty = 1, icase = 1, kind = 'Unknown', menu = '', word = 'foodar(var1 typ1, var2 *typ2) {}', user_data = { nvim = { lsp = { completion_item = { label='foocar', sortText="h", insertText='foodar(${1:var1} typ1, ${2:var2} *typ2) {$0\\}', insertTextFormat=2, textEdit={} } } } } },
{ abbr = 'foocar', dup = 1, empty = 1, icase = 1, kind = 'Unknown', menu = '', word = 'foodar(var1 typ2 tail) {}', user_data = { nvim = { lsp = { completion_item = { label='foocar', sortText="i", insertText='foodar(${1:var1 ${2|typ2,typ3|} ${3:tail}}) {$0\\}', insertTextFormat=2, textEdit={} } } } } },
{ abbr = 'foocar', dup = 1, empty = 1, icase = 1, kind = 'Unknown', menu = '', word = 'foodar(typ1) {}', user_data = { nvim = { lsp = { completion_item = { label='foocar', sortText="i", insertText='foodar(${1:${2|typ1,typ2|}}) {$0\\}', insertTextFormat=2, textEdit={} } } } } },
{ abbr = 'foocar', dup = 1, empty = 1, icase = 1, kind = 'Unknown', menu = '', word = 'foodar()', user_data = { nvim = { lsp = { completion_item = { label='foocar', sortText="j", insertText='foodar()${0}', insertTextFormat=2, textEdit={} } } } } },
{ abbr = 'foocar', dup = 1, empty = 1, icase = 1, kind = 'Unknown', menu = '', word = 'foodar(${1:var1})', user_data = { nvim = { lsp = { completion_item = { label='foocar', sortText="k", insertText='foodar(${1:var1})', insertTextFormat=1, textEdit={} } } } } },
}