neovim/scripts/genvimvim.lua
Arnout Engelen cb757f2663
build: make generated source files reproducible #21586
Problem:
Build is not reproducible, because generated source files (.c/.h/) are not
deterministic, mostly because Lua pairs() is unordered by design (for security).

https://github.com/LuaJIT/LuaJIT/issues/626#issuecomment-707005671
https://www.lua.org/manual/5.1/manual.html#pdf-next
> The order in which the indices are enumerated is not specified [...]
>
>> The hardening of the VM deliberately randomizes string hashes. This in
>> turn randomizes the iteration order of tables with string keys.

Solution:
- Update the code generation scripts to be deterministic.
    - That is only a partial solution: the exported function
      (funcs_metadata.generated.h) and ui event
      (ui_events_metadata.generated.h) metadata have some mpack'ed
      tables, which are not serialized deterministically.
    - As a workaround, introduce `PRG_GEN_LUA` cmake setting, so you can
      inject a modified build of luajit (with LUAJIT_SECURITY_PRN=0)
      that preserves table order.
    - Longer-term we should change the mpack'ed data structure so it no
      longer uses tables keyed by strings.

Closes #20124

Co-Authored-By: dundargoc <gocdundar@gmail.com>
Co-Authored-By: Arnout Engelen <arnout@bzzt.net>
2023-01-23 01:26:46 -08:00

142 lines
3.4 KiB
Lua

local mpack = require('mpack')
if arg[1] == '--help' then
print('Usage: lua genvimvim.lua src/nvim runtime/syntax/vim/generated.vim')
os.exit(0)
end
local nvimsrcdir = arg[1]
local syntax_file = arg[2]
local funcs_file = arg[3]
package.path = nvimsrcdir .. '/?.lua;' .. package.path
_G.vim = loadfile(nvimsrcdir..'/../../runtime/lua/vim/shared.lua')()
local lld = {}
local syn_fd = io.open(syntax_file, 'w')
lld.line_length = 0
local function w(s)
syn_fd:write(s)
if s:find('\n') then
lld.line_length = #(s:gsub('.*\n', ''))
else
lld.line_length = lld.line_length + #s
end
end
local options = require('options')
local auevents = require('auevents')
local ex_cmds = require('ex_cmds')
local function cmd_kw(prev_cmd, cmd)
if not prev_cmd then
return cmd:sub(1, 1) .. '[' .. cmd:sub(2) .. ']'
else
local shift = 1
while cmd:sub(shift, shift) == prev_cmd:sub(shift, shift) do
shift = shift + 1
end
if shift >= #cmd then
return cmd
else
return cmd:sub(1, shift) .. '[' .. cmd:sub(shift + 1) .. ']'
end
end
end
-- Exclude these from the vimCommand keyword list, they are handled specially
-- in syntax/vim.vim (vimAugroupKey, vimAutoCmd, vimGlobal, vimSubst). #9327
local function is_special_cased_cmd(cmd)
return (cmd == 'augroup'
or cmd == 'autocmd'
or cmd == 'doautocmd'
or cmd == 'doautoall'
or cmd == 'global'
or cmd == 'substitute')
end
local vimcmd_start = 'syn keyword vimCommand contained '
w(vimcmd_start)
local prev_cmd = nil
for _, cmd_desc in ipairs(ex_cmds.cmds) do
if lld.line_length > 850 then
w('\n' .. vimcmd_start)
end
local cmd = cmd_desc.command
if cmd:match('%w') and cmd ~= 'z' and not is_special_cased_cmd(cmd) then
w(' ' .. cmd_kw(prev_cmd, cmd))
end
prev_cmd = cmd
end
local vimopt_start = 'syn keyword vimOption contained '
w('\n\n' .. vimopt_start)
for _, opt_desc in ipairs(options.options) do
if not opt_desc.varname or opt_desc.varname:sub(1, 7) ~= 'p_force' then
if lld.line_length > 850 then
w('\n' .. vimopt_start)
end
w(' ' .. opt_desc.full_name)
if opt_desc.abbreviation then
w(' ' .. opt_desc.abbreviation)
end
if opt_desc.type == 'bool' then
w(' inv' .. opt_desc.full_name)
w(' no' .. opt_desc.full_name)
if opt_desc.abbreviation then
w(' inv' .. opt_desc.abbreviation)
w(' no' .. opt_desc.abbreviation)
end
end
end
end
w('\n\nsyn case ignore')
local vimau_start = 'syn keyword vimAutoEvent contained '
w('\n\n' .. vimau_start)
for _, au in ipairs(auevents.events) do
if not auevents.nvim_specific[au] then
if lld.line_length > 850 then
w('\n' .. vimau_start)
end
w(' ' .. au)
end
end
for au, _ in pairs(auevents.aliases) do
if not auevents.nvim_specific[au] then
if lld.line_length > 850 then
w('\n' .. vimau_start)
end
w(' ' .. au)
end
end
local nvimau_start = 'syn keyword nvimAutoEvent contained '
w('\n\n' .. nvimau_start)
for au, _ in vim.spairs(auevents.nvim_specific) do
if lld.line_length > 850 then
w('\n' .. nvimau_start)
end
w(' ' .. au)
end
w('\n\nsyn case match')
local vimfun_start = 'syn keyword vimFuncName contained '
w('\n\n' .. vimfun_start)
local funcs = mpack.unpack(io.open(funcs_file, 'rb'):read("*all"))
for _, name in ipairs(funcs) do
if name then
if lld.line_length > 850 then
w('\n' .. vimfun_start)
end
w(' ' .. name)
end
end
w('\n')
syn_fd:close()