kevin/neovim
1
Fork 0
mirror of https://github.com/neovim/neovim.git synced 2024-12-23 20:55:18 -07:00
Code Issues Packages Projects Releases Wiki Activity
3e4a058b01
neovim/test/unit/mbyte_spec.lua
Jan Edmund Lazo 3e4a058b01 test: port kword_test to Lua for utf_char2bytes() 
Use LuaJIT FFI to create char pointer.
Validate output with utf_ptr2char(), vim_iswordc() and vim_iswordp().
Use const for LuaJIT string-to-char conversion.
2018-07-01 08:45:19 -04:00

288 lines
9.2 KiB
Lua
Raw Blame History

local helpers = require("test.unit.helpers")(after_each)
local itp = helpers.gen_itp(it)
local ffi = helpers.ffi
local eq = helpers.eq
local mbyte = helpers.cimport("./src/nvim/mbyte.h")
local charset = helpers.cimport('./src/nvim/charset.h')
describe('mbyte', function()
-- Array for composing characters
local intp = ffi.typeof('int[?]')
local function to_intp()
-- how to get MAX_MCO from globals.h?
return intp(7, 1)
end
-- Convert from bytes to string
local function to_string(bytes)
local s = {}
for i = 1, #bytes do
s[i] = string.char(bytes[i])
end
return table.concat(s)
end
before_each(function()
end)
itp('utf_ptr2char', function()
-- For strings with length 1 the first byte is returned.
for c = 0, 255 do
eq(c, mbyte.utf_ptr2char(to_string({c, 0})))
end
-- Some ill formed byte sequences that should not be recognized as UTF-8
-- First byte: 0xc0 or 0xc1
-- Second byte: 0x80 .. 0xbf
--eq(0x00c0, mbyte.utf_ptr2char(to_string({0xc0, 0x80})))
--eq(0x00c1, mbyte.utf_ptr2char(to_string({0xc1, 0xbf})))
--
-- Sequences with more than four bytes
end)
itp('utf_char2bytes', function()
local char_p = ffi.typeof('char[?]')
for c = 0, 0xFFFF do
local p = char_p(4, 0)
mbyte.utf_char2bytes(c, p)
eq(c, mbyte.utf_ptr2char(p))
eq(charset.vim_iswordc(c), charset.vim_iswordp(p))
end
end)
describe('utfc_ptr2char_len', function()
itp('1-byte sequences', function()
local pcc = to_intp()
for c = 0, 255 do
eq(c, mbyte.utfc_ptr2char_len(to_string({c}), pcc, 1))
eq(0, pcc[0])
end
end)
itp('2-byte sequences', function()
local pcc = to_intp()
-- No combining characters
eq(0x007f, mbyte.utfc_ptr2char_len(to_string({0x7f, 0x7f}), pcc, 2))
eq(0, pcc[0])
-- No combining characters
pcc = to_intp()
eq(0x007f, mbyte.utfc_ptr2char_len(to_string({0x7f, 0x80}), pcc, 2))
eq(0, pcc[0])
-- No UTF-8 sequence
pcc = to_intp()
eq(0x00c2, mbyte.utfc_ptr2char_len(to_string({0xc2, 0x7f}), pcc, 2))
eq(0, pcc[0])
-- One UTF-8 character
pcc = to_intp()
eq(0x0080, mbyte.utfc_ptr2char_len(to_string({0xc2, 0x80}), pcc, 2))
eq(0, pcc[0])
-- No UTF-8 sequence
pcc = to_intp()
eq(0x00c2, mbyte.utfc_ptr2char_len(to_string({0xc2, 0xc0}), pcc, 2))
eq(0, pcc[0])
end)
itp('3-byte sequences', function()
local pcc = to_intp()
-- No second UTF-8 character
eq(0x007f, mbyte.utfc_ptr2char_len(to_string({0x7f, 0x80, 0x80}), pcc, 3))
eq(0, pcc[0])
-- No combining character
pcc = to_intp()
eq(0x007f, mbyte.utfc_ptr2char_len(to_string({0x7f, 0xc2, 0x80}), pcc, 3))
eq(0, pcc[0])
-- Combining character is U+0300
pcc = to_intp()
eq(0x007f, mbyte.utfc_ptr2char_len(to_string({0x7f, 0xcc, 0x80}), pcc, 3))
eq(0x0300, pcc[0])
eq(0x0000, pcc[1])
-- No UTF-8 sequence
pcc = to_intp()
eq(0x00c2, mbyte.utfc_ptr2char_len(to_string({0xc2, 0x7f, 0xcc}), pcc, 3))
eq(0, pcc[0])
-- Incomplete combining character
pcc = to_intp()
eq(0x0080, mbyte.utfc_ptr2char_len(to_string({0xc2, 0x80, 0xcc}), pcc, 3))
eq(0, pcc[0])
-- One UTF-8 character
pcc = to_intp()
eq(0x20d0, mbyte.utfc_ptr2char_len(to_string({0xe2, 0x83, 0x90}), pcc, 3))
eq(0, pcc[0])
end)
itp('4-byte sequences', function()
local pcc = to_intp()
-- No following combining character
eq(0x007f, mbyte.utfc_ptr2char_len(to_string({0x7f, 0x7f, 0xcc, 0x80}), pcc, 4))
eq(0, pcc[0])
-- No second UTF-8 character
pcc = to_intp()
eq(0x007f, mbyte.utfc_ptr2char_len(to_string({0x7f, 0xc2, 0xcc, 0x80}), pcc, 4))
eq(0, pcc[0])
-- Combining character U+0300
pcc = to_intp()
eq(0x007f, mbyte.utfc_ptr2char_len(to_string({0x7f, 0xcc, 0x80, 0xcc}), pcc, 4))
eq(0x0300, pcc[0])
eq(0x0000, pcc[1])
-- No UTF-8 sequence
pcc = to_intp()
eq(0x00c2, mbyte.utfc_ptr2char_len(to_string({0xc2, 0x7f, 0xcc, 0x80}), pcc, 4))
eq(0, pcc[0])
-- No following UTF-8 character
pcc = to_intp()
eq(0x0080, mbyte.utfc_ptr2char_len(to_string({0xc2, 0x80, 0xcc, 0xcc}), pcc, 4))
eq(0, pcc[0])
-- Combining character U+0301
pcc = to_intp()
eq(0x0080, mbyte.utfc_ptr2char_len(to_string({0xc2, 0x80, 0xcc, 0x81}), pcc, 4))
eq(0x0301, pcc[0])
eq(0x0000, pcc[1])
-- One UTF-8 character
pcc = to_intp()
eq(0x100000, mbyte.utfc_ptr2char_len(to_string({0xf4, 0x80, 0x80, 0x80}), pcc, 4))
eq(0, pcc[0])
end)
itp('5+-byte sequences', function()
local pcc = to_intp()
-- No following combining character
eq(0x007f, mbyte.utfc_ptr2char_len(to_string({0x7f, 0x7f, 0xcc, 0x80, 0x80}), pcc, 5))
eq(0, pcc[0])
-- No second UTF-8 character
pcc = to_intp()
eq(0x007f, mbyte.utfc_ptr2char_len(to_string({0x7f, 0xc2, 0xcc, 0x80, 0x80}), pcc, 5))
eq(0, pcc[0])
-- Combining character U+0300
pcc = to_intp()
eq(0x007f, mbyte.utfc_ptr2char_len(to_string({0x7f, 0xcc, 0x80, 0xcc}), pcc, 5))
eq(0x0300, pcc[0])
eq(0x0000, pcc[1])
-- Combining characters U+0300 and U+0301
pcc = to_intp()
eq(0x007f, mbyte.utfc_ptr2char_len(to_string({0x7f, 0xcc, 0x80, 0xcc, 0x81}), pcc, 5))
eq(0x0300, pcc[0])
eq(0x0301, pcc[1])
eq(0x0000, pcc[2])
-- Combining characters U+0300, U+0301, U+0302
pcc = to_intp()
eq(0x007f, mbyte.utfc_ptr2char_len(to_string({0x7f, 0xcc, 0x80, 0xcc, 0x81, 0xcc, 0x82}), pcc, 7))
eq(0x0300, pcc[0])
eq(0x0301, pcc[1])
eq(0x0302, pcc[2])
eq(0x0000, pcc[3])
-- Combining characters U+0300, U+0301, U+0302, U+0303
pcc = to_intp()
eq(0x007f, mbyte.utfc_ptr2char_len(to_string({0x7f, 0xcc, 0x80, 0xcc, 0x81, 0xcc, 0x82, 0xcc, 0x83}), pcc, 9))
eq(0x0300, pcc[0])
eq(0x0301, pcc[1])
eq(0x0302, pcc[2])
eq(0x0303, pcc[3])
eq(0x0000, pcc[4])
-- Combining characters U+0300, U+0301, U+0302, U+0303, U+0304
pcc = to_intp()
eq(0x007f, mbyte.utfc_ptr2char_len(to_string(
{0x7f, 0xcc, 0x80, 0xcc, 0x81, 0xcc, 0x82, 0xcc, 0x83, 0xcc, 0x84}), pcc, 11))
eq(0x0300, pcc[0])
eq(0x0301, pcc[1])
eq(0x0302, pcc[2])
eq(0x0303, pcc[3])
eq(0x0304, pcc[4])
eq(0x0000, pcc[5])
-- Combining characters U+0300, U+0301, U+0302, U+0303, U+0304,
-- U+0305
pcc = to_intp()
eq(0x007f, mbyte.utfc_ptr2char_len(to_string(
{0x7f, 0xcc, 0x80, 0xcc, 0x81, 0xcc, 0x82, 0xcc, 0x83, 0xcc, 0x84, 0xcc, 0x85}), pcc, 13))
eq(0x0300, pcc[0])
eq(0x0301, pcc[1])
eq(0x0302, pcc[2])
eq(0x0303, pcc[3])
eq(0x0304, pcc[4])
eq(0x0305, pcc[5])
eq(1, pcc[6])
-- Combining characters U+0300, U+0301, U+0302, U+0303, U+0304,
-- U+0305, U+0306, but only save six (= MAX_MCO).
pcc = to_intp()
eq(0x007f, mbyte.utfc_ptr2char_len(to_string(
{0x7f, 0xcc, 0x80, 0xcc, 0x81, 0xcc, 0x82, 0xcc, 0x83, 0xcc, 0x84, 0xcc, 0x85, 0xcc, 0x86}), pcc, 15))
eq(0x0300, pcc[0])
eq(0x0301, pcc[1])
eq(0x0302, pcc[2])
eq(0x0303, pcc[3])
eq(0x0304, pcc[4])
eq(0x0305, pcc[5])
eq(0x0001, pcc[6])
-- Only three following combining characters U+0300, U+0301, U+0302
pcc = to_intp()
eq(0x007f, mbyte.utfc_ptr2char_len(to_string(
{0x7f, 0xcc, 0x80, 0xcc, 0x81, 0xcc, 0x82, 0xc2, 0x80, 0xcc, 0x84, 0xcc, 0x85}), pcc, 13))
eq(0x0300, pcc[0])
eq(0x0301, pcc[1])
eq(0x0302, pcc[2])
eq(0x0000, pcc[3])
-- No UTF-8 sequence
pcc = to_intp()
eq(0x00c2, mbyte.utfc_ptr2char_len(to_string({0xc2, 0x7f, 0xcc, 0x80, 0x80}), pcc, 5))
eq(0, pcc[0])
-- No following UTF-8 character
pcc = to_intp()
eq(0x0080, mbyte.utfc_ptr2char_len(to_string({0xc2, 0x80, 0xcc, 0xcc, 0x80}), pcc, 5))
eq(0, pcc[0])
-- Combining character U+0301
pcc = to_intp()
eq(0x0080, mbyte.utfc_ptr2char_len(to_string({0xc2, 0x80, 0xcc, 0x81, 0x7f}), pcc, 5))
eq(0x0301, pcc[0])
eq(0x0000, pcc[1])
-- Combining character U+0301
pcc = to_intp()
eq(0x0080, mbyte.utfc_ptr2char_len(to_string({0xc2, 0x80, 0xcc, 0x81, 0xcc}), pcc, 5))
eq(0x0301, pcc[0])
eq(0x0000, pcc[1])
-- One UTF-8 character
pcc = to_intp()
eq(0x100000, mbyte.utfc_ptr2char_len(to_string({0xf4, 0x80, 0x80, 0x80, 0x7f}), pcc, 5))
eq(0, pcc[0])
-- One UTF-8 character
pcc = to_intp()
eq(0x100000, mbyte.utfc_ptr2char_len(to_string({0xf4, 0x80, 0x80, 0x80, 0x80}), pcc, 5))
eq(0, pcc[0])
-- One UTF-8 character
pcc = to_intp()
eq(0x100000, mbyte.utfc_ptr2char_len(to_string({0xf4, 0x80, 0x80, 0x80, 0xcc}), pcc, 5))
eq(0, pcc[0])
-- Combining characters U+1AB0 and U+0301
pcc = to_intp()
eq(0x100000, mbyte.utfc_ptr2char_len(to_string(
{0xf4, 0x80, 0x80, 0x80, 0xe1, 0xaa, 0xb0, 0xcc, 0x81}), pcc, 9))
eq(0x1ab0, pcc[0])
eq(0x0301, pcc[1])
eq(0x0000, pcc[2])
end)
end)
end)
Reference in New Issue View Git Blame Copy Permalink