neovim/test/unit/viml/expressions/lexer_spec.lua
Justin M. Keyes fd04877eb0 test: remove use of require('test.helpers')
The test.functional.helpers and test.unit.helpers modules now include
all of the public functions from test.helpers, so there is no need to
separately require('test.helpers').
2019-05-18 15:48:13 +02:00

427 lines
20 KiB
Lua

local helpers = require('test.unit.helpers')(after_each)
local itp = helpers.gen_itp(it)
local viml_helpers = require('test.unit.viml.helpers')
local child_call_once = helpers.child_call_once
local conv_enum = helpers.conv_enum
local cimport = helpers.cimport
local ffi = helpers.ffi
local eq = helpers.eq
local shallowcopy = helpers.shallowcopy
local intchar2lua = helpers.intchar2lua
local conv_ccs = viml_helpers.conv_ccs
local new_pstate = viml_helpers.new_pstate
local conv_cmp_type = viml_helpers.conv_cmp_type
local pstate_set_str = viml_helpers.pstate_set_str
local conv_expr_asgn_type = viml_helpers.conv_expr_asgn_type
local lib = cimport('./src/nvim/viml/parser/expressions.h')
local eltkn_type_tab, eltkn_mul_type_tab, eltkn_opt_scope_tab
child_call_once(function()
eltkn_type_tab = {
[tonumber(lib.kExprLexInvalid)] = 'Invalid',
[tonumber(lib.kExprLexMissing)] = 'Missing',
[tonumber(lib.kExprLexSpacing)] = 'Spacing',
[tonumber(lib.kExprLexEOC)] = 'EOC',
[tonumber(lib.kExprLexQuestion)] = 'Question',
[tonumber(lib.kExprLexColon)] = 'Colon',
[tonumber(lib.kExprLexOr)] = 'Or',
[tonumber(lib.kExprLexAnd)] = 'And',
[tonumber(lib.kExprLexComparison)] = 'Comparison',
[tonumber(lib.kExprLexPlus)] = 'Plus',
[tonumber(lib.kExprLexMinus)] = 'Minus',
[tonumber(lib.kExprLexDot)] = 'Dot',
[tonumber(lib.kExprLexMultiplication)] = 'Multiplication',
[tonumber(lib.kExprLexNot)] = 'Not',
[tonumber(lib.kExprLexNumber)] = 'Number',
[tonumber(lib.kExprLexSingleQuotedString)] = 'SingleQuotedString',
[tonumber(lib.kExprLexDoubleQuotedString)] = 'DoubleQuotedString',
[tonumber(lib.kExprLexOption)] = 'Option',
[tonumber(lib.kExprLexRegister)] = 'Register',
[tonumber(lib.kExprLexEnv)] = 'Env',
[tonumber(lib.kExprLexPlainIdentifier)] = 'PlainIdentifier',
[tonumber(lib.kExprLexBracket)] = 'Bracket',
[tonumber(lib.kExprLexFigureBrace)] = 'FigureBrace',
[tonumber(lib.kExprLexParenthesis)] = 'Parenthesis',
[tonumber(lib.kExprLexComma)] = 'Comma',
[tonumber(lib.kExprLexArrow)] = 'Arrow',
[tonumber(lib.kExprLexAssignment)] = 'Assignment',
}
eltkn_mul_type_tab = {
[tonumber(lib.kExprLexMulMul)] = 'Mul',
[tonumber(lib.kExprLexMulDiv)] = 'Div',
[tonumber(lib.kExprLexMulMod)] = 'Mod',
}
eltkn_opt_scope_tab = {
[tonumber(lib.kExprOptScopeUnspecified)] = 'Unspecified',
[tonumber(lib.kExprOptScopeGlobal)] = 'Global',
[tonumber(lib.kExprOptScopeLocal)] = 'Local',
}
end)
local function conv_eltkn_type(typ)
return conv_enum(eltkn_type_tab, typ)
end
local bracket_types = {
Bracket = true,
FigureBrace = true,
Parenthesis = true,
}
local function eltkn2lua(pstate, tkn)
local ret = {
type = conv_eltkn_type(tkn.type),
}
pstate_set_str(pstate, tkn.start, tkn.len, ret)
if not ret.error and (#(ret.str) ~= ret.len) then
ret.error = '#str /= len'
end
if ret.type == 'Comparison' then
ret.data = {
type = conv_cmp_type(tkn.data.cmp.type),
ccs = conv_ccs(tkn.data.cmp.ccs),
inv = (not not tkn.data.cmp.inv),
}
elseif ret.type == 'Multiplication' then
ret.data = { type = conv_enum(eltkn_mul_type_tab, tkn.data.mul.type) }
elseif bracket_types[ret.type] then
ret.data = { closing = (not not tkn.data.brc.closing) }
elseif ret.type == 'Register' then
ret.data = { name = intchar2lua(tkn.data.reg.name) }
elseif (ret.type == 'SingleQuotedString'
or ret.type == 'DoubleQuotedString') then
ret.data = { closed = (not not tkn.data.str.closed) }
elseif ret.type == 'Option' then
ret.data = {
scope = conv_enum(eltkn_opt_scope_tab, tkn.data.opt.scope),
name = ffi.string(tkn.data.opt.name, tkn.data.opt.len),
}
elseif ret.type == 'PlainIdentifier' then
ret.data = {
scope = intchar2lua(tkn.data.var.scope),
autoload = (not not tkn.data.var.autoload),
}
elseif ret.type == 'Number' then
ret.data = {
is_float = (not not tkn.data.num.is_float),
base = tonumber(tkn.data.num.base),
}
ret.data.val = tonumber(tkn.data.num.is_float
and tkn.data.num.val.floating
or tkn.data.num.val.integer)
elseif ret.type == 'Assignment' then
ret.data = { type = conv_expr_asgn_type(tkn.data.ass.type) }
elseif ret.type == 'Invalid' then
ret.data = { error = ffi.string(tkn.data.err.msg) }
end
return ret, tkn
end
local function next_eltkn(pstate, flags)
return eltkn2lua(pstate, lib.viml_pexpr_next_token(pstate, flags))
end
describe('Expressions lexer', function()
local flags = 0
local should_advance = true
local function check_advance(pstate, bytes_to_advance, initial_col)
local tgt = initial_col + bytes_to_advance
if should_advance then
if pstate.reader.lines.items[0].size == tgt then
eq(1, pstate.pos.line)
eq(0, pstate.pos.col)
else
eq(0, pstate.pos.line)
eq(tgt, pstate.pos.col)
end
else
eq(0, pstate.pos.line)
eq(initial_col, pstate.pos.col)
end
end
local function singl_eltkn_test(typ, str, data)
local pstate = new_pstate({str})
eq({data=data, len=#str, start={col=0, line=0}, str=str, type=typ},
next_eltkn(pstate, flags))
check_advance(pstate, #str, 0)
if not (
typ == 'Spacing'
or (typ == 'Register' and str == '@')
or ((typ == 'SingleQuotedString' or typ == 'DoubleQuotedString')
and not data.closed)
) then
pstate = new_pstate({str .. ' '})
eq({data=data, len=#str, start={col=0, line=0}, str=str, type=typ},
next_eltkn(pstate, flags))
check_advance(pstate, #str, 0)
end
pstate = new_pstate({'x' .. str})
pstate.pos.col = 1
eq({data=data, len=#str, start={col=1, line=0}, str=str, type=typ},
next_eltkn(pstate, flags))
check_advance(pstate, #str, 1)
end
local function scope_test(scope)
singl_eltkn_test('PlainIdentifier', scope .. ':test#var', {autoload=true, scope=scope})
singl_eltkn_test('PlainIdentifier', scope .. ':', {autoload=false, scope=scope})
end
local function comparison_test(op, inv_op, cmp_type)
singl_eltkn_test('Comparison', op, {type=cmp_type, inv=false, ccs='UseOption'})
singl_eltkn_test('Comparison', inv_op, {type=cmp_type, inv=true, ccs='UseOption'})
singl_eltkn_test('Comparison', op .. '#', {type=cmp_type, inv=false, ccs='MatchCase'})
singl_eltkn_test('Comparison', inv_op .. '#', {type=cmp_type, inv=true, ccs='MatchCase'})
singl_eltkn_test('Comparison', op .. '?', {type=cmp_type, inv=false, ccs='IgnoreCase'})
singl_eltkn_test('Comparison', inv_op .. '?', {type=cmp_type, inv=true, ccs='IgnoreCase'})
end
local function simple_test(pstate_arg, exp_type, exp_len, exp)
local pstate = new_pstate(pstate_arg)
exp = shallowcopy(exp)
exp.type = exp_type
exp.len = exp_len or #(pstate_arg[0])
exp.start = { col = 0, line = 0 }
eq(exp, next_eltkn(pstate, flags))
end
local function stable_tests()
singl_eltkn_test('Parenthesis', '(', {closing=false})
singl_eltkn_test('Parenthesis', ')', {closing=true})
singl_eltkn_test('Bracket', '[', {closing=false})
singl_eltkn_test('Bracket', ']', {closing=true})
singl_eltkn_test('FigureBrace', '{', {closing=false})
singl_eltkn_test('FigureBrace', '}', {closing=true})
singl_eltkn_test('Question', '?')
singl_eltkn_test('Colon', ':')
singl_eltkn_test('Dot', '.')
singl_eltkn_test('Assignment', '.=', {type='Concat'})
singl_eltkn_test('Plus', '+')
singl_eltkn_test('Assignment', '+=', {type='Add'})
singl_eltkn_test('Comma', ',')
singl_eltkn_test('Multiplication', '*', {type='Mul'})
singl_eltkn_test('Multiplication', '/', {type='Div'})
singl_eltkn_test('Multiplication', '%', {type='Mod'})
singl_eltkn_test('Spacing', ' \t\t \t\t')
singl_eltkn_test('Spacing', ' ')
singl_eltkn_test('Spacing', '\t')
singl_eltkn_test('Invalid', '\x01\x02\x03', {error='E15: Invalid control character present in input: %.*s'})
singl_eltkn_test('Number', '0123', {is_float=false, base=8, val=83})
singl_eltkn_test('Number', '01234567', {is_float=false, base=8, val=342391})
singl_eltkn_test('Number', '012345678', {is_float=false, base=10, val=12345678})
singl_eltkn_test('Number', '0x123', {is_float=false, base=16, val=291})
singl_eltkn_test('Number', '0x56FF', {is_float=false, base=16, val=22271})
singl_eltkn_test('Number', '0xabcdef', {is_float=false, base=16, val=11259375})
singl_eltkn_test('Number', '0xABCDEF', {is_float=false, base=16, val=11259375})
singl_eltkn_test('Number', '0x0', {is_float=false, base=16, val=0})
singl_eltkn_test('Number', '00', {is_float=false, base=8, val=0})
singl_eltkn_test('Number', '0b0', {is_float=false, base=2, val=0})
singl_eltkn_test('Number', '0b010111', {is_float=false, base=2, val=23})
singl_eltkn_test('Number', '0b100111', {is_float=false, base=2, val=39})
singl_eltkn_test('Number', '0', {is_float=false, base=10, val=0})
singl_eltkn_test('Number', '9', {is_float=false, base=10, val=9})
singl_eltkn_test('Env', '$abc')
singl_eltkn_test('Env', '$')
singl_eltkn_test('PlainIdentifier', 'test', {autoload=false, scope=0})
singl_eltkn_test('PlainIdentifier', '_test', {autoload=false, scope=0})
singl_eltkn_test('PlainIdentifier', '_test_foo', {autoload=false, scope=0})
singl_eltkn_test('PlainIdentifier', 't', {autoload=false, scope=0})
singl_eltkn_test('PlainIdentifier', 'test5', {autoload=false, scope=0})
singl_eltkn_test('PlainIdentifier', 't0', {autoload=false, scope=0})
singl_eltkn_test('PlainIdentifier', 'test#var', {autoload=true, scope=0})
singl_eltkn_test('PlainIdentifier', 'test#var#val###', {autoload=true, scope=0})
singl_eltkn_test('PlainIdentifier', 't#####', {autoload=true, scope=0})
singl_eltkn_test('And', '&&')
singl_eltkn_test('Or', '||')
singl_eltkn_test('Invalid', '&', {error='E112: Option name missing: %.*s'})
singl_eltkn_test('Option', '&opt', {scope='Unspecified', name='opt'})
singl_eltkn_test('Option', '&t_xx', {scope='Unspecified', name='t_xx'})
singl_eltkn_test('Option', '&t_\r\r', {scope='Unspecified', name='t_\r\r'})
singl_eltkn_test('Option', '&t_\t\t', {scope='Unspecified', name='t_\t\t'})
singl_eltkn_test('Option', '&t_ ', {scope='Unspecified', name='t_ '})
singl_eltkn_test('Option', '&g:opt', {scope='Global', name='opt'})
singl_eltkn_test('Option', '&l:opt', {scope='Local', name='opt'})
singl_eltkn_test('Invalid', '&l:', {error='E112: Option name missing: %.*s'})
singl_eltkn_test('Invalid', '&g:', {error='E112: Option name missing: %.*s'})
singl_eltkn_test('Register', '@', {name=-1})
singl_eltkn_test('Register', '@a', {name='a'})
singl_eltkn_test('Register', '@\r', {name=13})
singl_eltkn_test('Register', '@ ', {name=' '})
singl_eltkn_test('Register', '@\t', {name=9})
singl_eltkn_test('SingleQuotedString', '\'test', {closed=false})
singl_eltkn_test('SingleQuotedString', '\'test\'', {closed=true})
singl_eltkn_test('SingleQuotedString', '\'\'\'\'', {closed=true})
singl_eltkn_test('SingleQuotedString', '\'x\'\'\'', {closed=true})
singl_eltkn_test('SingleQuotedString', '\'\'\'x\'', {closed=true})
singl_eltkn_test('SingleQuotedString', '\'\'\'', {closed=false})
singl_eltkn_test('SingleQuotedString', '\'x\'\'', {closed=false})
singl_eltkn_test('SingleQuotedString', '\'\'\'x', {closed=false})
singl_eltkn_test('DoubleQuotedString', '"test', {closed=false})
singl_eltkn_test('DoubleQuotedString', '"test"', {closed=true})
singl_eltkn_test('DoubleQuotedString', '"\\""', {closed=true})
singl_eltkn_test('DoubleQuotedString', '"x\\""', {closed=true})
singl_eltkn_test('DoubleQuotedString', '"\\"x"', {closed=true})
singl_eltkn_test('DoubleQuotedString', '"\\"', {closed=false})
singl_eltkn_test('DoubleQuotedString', '"x\\"', {closed=false})
singl_eltkn_test('DoubleQuotedString', '"\\"x', {closed=false})
singl_eltkn_test('Not', '!')
singl_eltkn_test('Assignment', '=', {type='Plain'})
comparison_test('==', '!=', 'Equal')
comparison_test('=~', '!~', 'Matches')
comparison_test('>', '<=', 'Greater')
comparison_test('>=', '<', 'GreaterOrEqual')
singl_eltkn_test('Minus', '-')
singl_eltkn_test('Assignment', '-=', {type='Subtract'})
singl_eltkn_test('Arrow', '->')
singl_eltkn_test('Invalid', '~', {error='E15: Unidentified character: %.*s'})
simple_test({{data=nil, size=0}}, 'EOC', 0, {error='start.col >= #pstr'})
simple_test({''}, 'EOC', 0, {error='start.col >= #pstr'})
simple_test({'2.'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
simple_test({'2e5'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
simple_test({'2.x'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
simple_test({'2.2.'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
simple_test({'2.0x'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
simple_test({'2.0e'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
simple_test({'2.0e+'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
simple_test({'2.0e-'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
simple_test({'2.0e+x'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
simple_test({'2.0e-x'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
simple_test({'2.0e+1a'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
simple_test({'2.0e-1a'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
simple_test({'0b102'}, 'Number', 4, {data={is_float=false, base=2, val=2}, str='0b10'})
simple_test({'10F'}, 'Number', 2, {data={is_float=false, base=10, val=10}, str='10'})
simple_test({'0x0123456789ABCDEFG'}, 'Number', 18, {data={is_float=false, base=16, val=81985529216486895}, str='0x0123456789ABCDEF'})
simple_test({{data='00', size=2}}, 'Number', 2, {data={is_float=false, base=8, val=0}, str='00'})
simple_test({{data='009', size=2}}, 'Number', 2, {data={is_float=false, base=8, val=0}, str='00'})
simple_test({{data='01', size=1}}, 'Number', 1, {data={is_float=false, base=10, val=0}, str='0'})
end
local function regular_scope_tests()
scope_test('s')
scope_test('g')
scope_test('v')
scope_test('b')
scope_test('w')
scope_test('t')
scope_test('l')
scope_test('a')
simple_test({'g:'}, 'PlainIdentifier', 2, {data={scope='g', autoload=false}, str='g:'})
simple_test({'g:is#foo'}, 'PlainIdentifier', 8, {data={scope='g', autoload=true}, str='g:is#foo'})
simple_test({'g:isnot#foo'}, 'PlainIdentifier', 11, {data={scope='g', autoload=true}, str='g:isnot#foo'})
end
local function regular_is_tests()
comparison_test('is', 'isnot', 'Identical')
simple_test({'is'}, 'Comparison', 2, {data={type='Identical', inv=false, ccs='UseOption'}, str='is'})
simple_test({'isnot'}, 'Comparison', 5, {data={type='Identical', inv=true, ccs='UseOption'}, str='isnot'})
simple_test({'is?'}, 'Comparison', 3, {data={type='Identical', inv=false, ccs='IgnoreCase'}, str='is?'})
simple_test({'isnot?'}, 'Comparison', 6, {data={type='Identical', inv=true, ccs='IgnoreCase'}, str='isnot?'})
simple_test({'is#'}, 'Comparison', 3, {data={type='Identical', inv=false, ccs='MatchCase'}, str='is#'})
simple_test({'isnot#'}, 'Comparison', 6, {data={type='Identical', inv=true, ccs='MatchCase'}, str='isnot#'})
simple_test({'is#foo'}, 'Comparison', 3, {data={type='Identical', inv=false, ccs='MatchCase'}, str='is#'})
simple_test({'isnot#foo'}, 'Comparison', 6, {data={type='Identical', inv=true, ccs='MatchCase'}, str='isnot#'})
end
local function regular_number_tests()
simple_test({'2.0'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
simple_test({'2.0e5'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
simple_test({'2.0e+5'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
simple_test({'2.0e-5'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
end
local function regular_eoc_tests()
singl_eltkn_test('EOC', '|')
singl_eltkn_test('EOC', '\0')
singl_eltkn_test('EOC', '\n')
end
itp('works (single tokens, zero flags)', function()
stable_tests()
regular_eoc_tests()
regular_scope_tests()
regular_is_tests()
regular_number_tests()
end)
itp('peeks', function()
flags = tonumber(lib.kELFlagPeek)
should_advance = false
stable_tests()
regular_eoc_tests()
regular_scope_tests()
regular_is_tests()
regular_number_tests()
end)
itp('forbids scope', function()
flags = tonumber(lib.kELFlagForbidScope)
stable_tests()
regular_eoc_tests()
regular_is_tests()
regular_number_tests()
simple_test({'g:'}, 'PlainIdentifier', 1, {data={scope=0, autoload=false}, str='g'})
end)
itp('allows floats', function()
flags = tonumber(lib.kELFlagAllowFloat)
stable_tests()
regular_eoc_tests()
regular_scope_tests()
regular_is_tests()
simple_test({'2.2'}, 'Number', 3, {data={is_float=true, base=10, val=2.2}, str='2.2'})
simple_test({'2.0e5'}, 'Number', 5, {data={is_float=true, base=10, val=2e5}, str='2.0e5'})
simple_test({'2.0e+5'}, 'Number', 6, {data={is_float=true, base=10, val=2e5}, str='2.0e+5'})
simple_test({'2.0e-5'}, 'Number', 6, {data={is_float=true, base=10, val=2e-5}, str='2.0e-5'})
simple_test({'2.500000e-5'}, 'Number', 11, {data={is_float=true, base=10, val=2.5e-5}, str='2.500000e-5'})
simple_test({'2.5555e2'}, 'Number', 8, {data={is_float=true, base=10, val=2.5555e2}, str='2.5555e2'})
simple_test({'2.5555e+2'}, 'Number', 9, {data={is_float=true, base=10, val=2.5555e2}, str='2.5555e+2'})
simple_test({'2.5555e-2'}, 'Number', 9, {data={is_float=true, base=10, val=2.5555e-2}, str='2.5555e-2'})
simple_test({{data='2.5e-5', size=3}},
'Number', 3, {data={is_float=true, base=10, val=2.5}, str='2.5'})
simple_test({{data='2.5e5', size=4}},
'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
simple_test({{data='2.5e-50', size=6}},
'Number', 6, {data={is_float=true, base=10, val=2.5e-5}, str='2.5e-5'})
end)
itp('treats `is` as an identifier', function()
flags = tonumber(lib.kELFlagIsNotCmp)
stable_tests()
regular_eoc_tests()
regular_scope_tests()
regular_number_tests()
simple_test({'is'}, 'PlainIdentifier', 2, {data={scope=0, autoload=false}, str='is'})
simple_test({'isnot'}, 'PlainIdentifier', 5, {data={scope=0, autoload=false}, str='isnot'})
simple_test({'is?'}, 'PlainIdentifier', 2, {data={scope=0, autoload=false}, str='is'})
simple_test({'isnot?'}, 'PlainIdentifier', 5, {data={scope=0, autoload=false}, str='isnot'})
simple_test({'is#'}, 'PlainIdentifier', 3, {data={scope=0, autoload=true}, str='is#'})
simple_test({'isnot#'}, 'PlainIdentifier', 6, {data={scope=0, autoload=true}, str='isnot#'})
simple_test({'is#foo'}, 'PlainIdentifier', 6, {data={scope=0, autoload=true}, str='is#foo'})
simple_test({'isnot#foo'}, 'PlainIdentifier', 9, {data={scope=0, autoload=true}, str='isnot#foo'})
end)
itp('forbids EOC', function()
flags = tonumber(lib.kELFlagForbidEOC)
stable_tests()
regular_scope_tests()
regular_is_tests()
regular_number_tests()
singl_eltkn_test('Invalid', '|', {error='E15: Unexpected EOC character: %.*s'})
singl_eltkn_test('Invalid', '\0', {error='E15: Unexpected EOC character: %.*s'})
singl_eltkn_test('Invalid', '\n', {error='E15: Unexpected EOC character: %.*s'})
end)
end)