build: make generated source files reproducible #21586

Problem:
Build is not reproducible, because generated source files (.c/.h/) are not
deterministic, mostly because Lua pairs() is unordered by design (for security).

https://github.com/LuaJIT/LuaJIT/issues/626#issuecomment-707005671
https://www.lua.org/manual/5.1/manual.html#pdf-next
> The order in which the indices are enumerated is not specified [...]
>
>> The hardening of the VM deliberately randomizes string hashes. This in
>> turn randomizes the iteration order of tables with string keys.

Solution:
- Update the code generation scripts to be deterministic.
    - That is only a partial solution: the exported function
      (funcs_metadata.generated.h) and ui event
      (ui_events_metadata.generated.h) metadata have some mpack'ed
      tables, which are not serialized deterministically.
    - As a workaround, introduce `PRG_GEN_LUA` cmake setting, so you can
      inject a modified build of luajit (with LUAJIT_SECURITY_PRN=0)
      that preserves table order.
    - Longer-term we should change the mpack'ed data structure so it no
      longer uses tables keyed by strings.

Closes #20124

Co-Authored-By: dundargoc <gocdundar@gmail.com>
Co-Authored-By: Arnout Engelen <arnout@bzzt.net>
This commit is contained in:
Arnout Engelen 2023-01-23 10:26:46 +01:00 committed by GitHub
parent da671b21cc
commit cb757f2663
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 161 additions and 76 deletions

View File

@ -242,6 +242,17 @@ endif()
message(STATUS "Using Lua interpreter: ${LUA_PRG}") message(STATUS "Using Lua interpreter: ${LUA_PRG}")
# Some of the code generation still relies on stable table ordering in order to
# produce reproducible output - specifically the msgpack'ed data in
# funcs_metadata.generated.h and ui_events_metadata.generated.h. This should
# ideally be fixed in the generators, but until then as a workaround you may provide
# a specific lua implementation that provides the needed stability by setting LUA_GEN_PRG:
if(NOT LUA_GEN_PRG)
set(LUA_GEN_PRG "${LUA_PRG}" CACHE FILEPATH "Path to the lua used for code generation.")
endif()
message(STATUS "Using Lua interpreter for code generation: ${LUA_GEN_PRG}")
option(COMPILE_LUA "Pre-compile Lua sources into bytecode (for sources that are included in the binary)" ON) option(COMPILE_LUA "Pre-compile Lua sources into bytecode (for sources that are included in the binary)" ON)
if(COMPILE_LUA AND NOT WIN32) if(COMPILE_LUA AND NOT WIN32)

View File

@ -1694,6 +1694,18 @@ pesc({s}) *vim.pesc()*
See also: ~ See also: ~
https://github.com/rxi/lume https://github.com/rxi/lume
spairs({t}) *vim.spairs()*
Enumerate a table sorted by its keys.
Parameters: ~
• {t} (table) List-like table
Return: ~
iterator over sorted keys and their values
See also: ~
Based on https://github.com/premake/premake-core/blob/master/src/base/table.lua
split({s}, {sep}, {kwargs}) *vim.split()* split({s}, {sep}, {kwargs}) *vim.split()*
Splits a string at each instance of a separator. Splits a string at each instance of a separator.

View File

@ -143,6 +143,10 @@ The following new APIs or features were added.
instance in the background and display its UI on demand, which previously instance in the background and display its UI on demand, which previously
only was possible using an external UI implementation. only was possible using an external UI implementation.
• Several improvements were made to make the code generation scripts more
deterministic, and a `LUA_GEN_PRG` build parameter has been introduced to
allow for a workaround for some remaining reproducibility problems.
============================================================================== ==============================================================================
CHANGED FEATURES *news-changes* CHANGED FEATURES *news-changes*

View File

@ -458,6 +458,33 @@ function vim.tbl_flatten(t)
return result return result
end end
--- Enumerate a table sorted by its keys.
---
---@see Based on https://github.com/premake/premake-core/blob/master/src/base/table.lua
---
---@param t table List-like table
---@return iterator over sorted keys and their values
function vim.spairs(t)
assert(type(t) == 'table', string.format('Expected table, got %s', type(t)))
-- collect the keys
local keys = {}
for k in pairs(t) do
table.insert(keys, k)
end
table.sort(keys)
-- Return the iterator function.
-- TODO(justinmk): Return "iterator function, table {t}, and nil", like pairs()?
local i = 0
return function()
i = i + 1
if keys[i] then
return keys[i], t[keys[i]]
end
end
end
--- Tests if a Lua table can be treated as an array. --- Tests if a Lua table can be treated as an array.
--- ---
--- Empty table `{}` is assumed to be an array, unless it was created by --- Empty table `{}` is assumed to be an array, unless it was created by

View File

@ -11,6 +11,8 @@ local funcs_file = arg[3]
package.path = nvimsrcdir .. '/?.lua;' .. package.path package.path = nvimsrcdir .. '/?.lua;' .. package.path
_G.vim = loadfile(nvimsrcdir..'/../../runtime/lua/vim/shared.lua')()
local lld = {} local lld = {}
local syn_fd = io.open(syntax_file, 'w') local syn_fd = io.open(syntax_file, 'w')
lld.line_length = 0 lld.line_length = 0
@ -115,7 +117,7 @@ end
local nvimau_start = 'syn keyword nvimAutoEvent contained ' local nvimau_start = 'syn keyword nvimAutoEvent contained '
w('\n\n' .. nvimau_start) w('\n\n' .. nvimau_start)
for au, _ in pairs(auevents.nvim_specific) do for au, _ in vim.spairs(auevents.nvim_specific) do
if lld.line_length > 850 then if lld.line_length > 850 then
w('\n' .. nvimau_start) w('\n' .. nvimau_start)
end end
@ -126,7 +128,7 @@ w('\n\nsyn case match')
local vimfun_start = 'syn keyword vimFuncName contained ' local vimfun_start = 'syn keyword vimFuncName contained '
w('\n\n' .. vimfun_start) w('\n\n' .. vimfun_start)
local funcs = mpack.unpack(io.open(funcs_file, 'rb'):read("*all")) local funcs = mpack.unpack(io.open(funcs_file, 'rb'):read("*all"))
for name, _ in pairs(funcs) do for _, name in ipairs(funcs) do
if name then if name then
if lld.line_length > 850 then if lld.line_length > 850 then
w('\n' .. vimfun_start) w('\n' .. vimfun_start)

View File

@ -570,11 +570,11 @@ add_custom_command(OUTPUT ${GENERATED_UNICODE_TABLES}
add_custom_command( add_custom_command(
OUTPUT ${GENERATED_API_DISPATCH} ${GENERATED_FUNCS_METADATA} OUTPUT ${GENERATED_API_DISPATCH} ${GENERATED_FUNCS_METADATA}
${API_METADATA} ${LUA_API_C_BINDINGS} ${API_METADATA} ${LUA_API_C_BINDINGS}
COMMAND ${LUA_PRG} ${API_DISPATCH_GENERATOR} ${CMAKE_CURRENT_LIST_DIR} COMMAND ${LUA_GEN_PRG} ${API_DISPATCH_GENERATOR} ${CMAKE_CURRENT_LIST_DIR}
${GENERATED_API_DISPATCH} ${GENERATED_API_DISPATCH}
${GENERATED_FUNCS_METADATA} ${API_METADATA} ${GENERATED_FUNCS_METADATA} ${API_METADATA}
${LUA_API_C_BINDINGS} ${LUA_API_C_BINDINGS}
${API_HEADERS} ${API_HEADERS}
DEPENDS DEPENDS
${API_HEADERS} ${API_HEADERS}
${MSGPACK_RPC_HEADERS} ${MSGPACK_RPC_HEADERS}
@ -619,12 +619,12 @@ add_custom_command(
${GENERATED_UI_EVENTS_REMOTE} ${GENERATED_UI_EVENTS_REMOTE}
${GENERATED_UI_EVENTS_METADATA} ${GENERATED_UI_EVENTS_METADATA}
${GENERATED_UI_EVENTS_CLIENT} ${GENERATED_UI_EVENTS_CLIENT}
COMMAND ${LUA_PRG} ${API_UI_EVENTS_GENERATOR} ${CMAKE_CURRENT_LIST_DIR} COMMAND ${LUA_GEN_PRG} ${API_UI_EVENTS_GENERATOR} ${CMAKE_CURRENT_LIST_DIR}
${CMAKE_CURRENT_LIST_DIR}/api/ui_events.in.h ${CMAKE_CURRENT_LIST_DIR}/api/ui_events.in.h
${GENERATED_UI_EVENTS_CALL} ${GENERATED_UI_EVENTS_CALL}
${GENERATED_UI_EVENTS_REMOTE} ${GENERATED_UI_EVENTS_REMOTE}
${GENERATED_UI_EVENTS_METADATA} ${GENERATED_UI_EVENTS_METADATA}
${GENERATED_UI_EVENTS_CLIENT} ${GENERATED_UI_EVENTS_CLIENT}
DEPENDS DEPENDS
${API_UI_EVENTS_GENERATOR} ${API_UI_EVENTS_GENERATOR}
${GENERATOR_C_GRAMMAR} ${GENERATOR_C_GRAMMAR}

View File

@ -1,8 +1,8 @@
return { return {
context = { { 'context', {
"types"; "types";
}; }};
set_decoration_provider = { { 'set_decoration_provider', {
"on_start"; "on_start";
"on_buf"; "on_buf";
"on_win"; "on_win";
@ -10,8 +10,8 @@ return {
"on_end"; "on_end";
"_on_hl_def"; "_on_hl_def";
"_on_spell_nav"; "_on_spell_nav";
}; }};
set_extmark = { { 'set_extmark', {
"id"; "id";
"end_line"; "end_line";
"end_row"; "end_row";
@ -39,8 +39,8 @@ return {
"conceal"; "conceal";
"spell"; "spell";
"ui_watched"; "ui_watched";
}; }};
keymap = { { 'keymap', {
"noremap"; "noremap";
"nowait"; "nowait";
"silent"; "silent";
@ -50,11 +50,11 @@ return {
"callback"; "callback";
"desc"; "desc";
"replace_keycodes"; "replace_keycodes";
}; }};
get_commands = { { 'get_commands', {
"builtin"; "builtin";
}; }};
user_command = { { 'user_command', {
"addr"; "addr";
"bang"; "bang";
"bar"; "bar";
@ -67,8 +67,8 @@ return {
"preview"; "preview";
"range"; "range";
"register"; "register";
}; }};
float_config = { { 'float_config', {
"row"; "row";
"col"; "col";
"width"; "width";
@ -85,25 +85,25 @@ return {
"title_pos"; "title_pos";
"style"; "style";
"noautocmd"; "noautocmd";
}; }};
runtime = { { 'runtime', {
"is_lua"; "is_lua";
"do_source"; "do_source";
}; }};
eval_statusline = { { 'eval_statusline', {
"winid"; "winid";
"maxwidth"; "maxwidth";
"fillchar"; "fillchar";
"highlights"; "highlights";
"use_winbar"; "use_winbar";
"use_tabline"; "use_tabline";
}; }};
option = { { 'option', {
"scope"; "scope";
"win"; "win";
"buf"; "buf";
}; }};
highlight = { { 'highlight', {
"bold"; "bold";
"standout"; "standout";
"strikethrough"; "strikethrough";
@ -128,8 +128,8 @@ return {
"blend"; "blend";
"fg_indexed"; "fg_indexed";
"bg_indexed"; "bg_indexed";
}; }};
highlight_cterm = { { 'highlight_cterm', {
"bold"; "bold";
"standout"; "standout";
"strikethrough"; "strikethrough";
@ -141,15 +141,15 @@ return {
"italic"; "italic";
"reverse"; "reverse";
"nocombine"; "nocombine";
}; }};
-- Autocmds -- Autocmds
clear_autocmds = { { 'clear_autocmds', {
"buffer"; "buffer";
"event"; "event";
"group"; "group";
"pattern"; "pattern";
}; }};
create_autocmd = { { 'create_autocmd', {
"buffer"; "buffer";
"callback"; "callback";
"command"; "command";
@ -158,24 +158,24 @@ return {
"nested"; "nested";
"once"; "once";
"pattern"; "pattern";
}; }};
exec_autocmds = { { 'exec_autocmds', {
"buffer"; "buffer";
"group"; "group";
"modeline"; "modeline";
"pattern"; "pattern";
"data"; "data";
}; }};
get_autocmds = { { 'get_autocmds', {
"event"; "event";
"group"; "group";
"pattern"; "pattern";
"buffer"; "buffer";
}; }};
create_augroup = { { 'create_augroup', {
"clear"; "clear";
}; }};
cmd = { { 'cmd', {
"cmd"; "cmd";
"range"; "range";
"count"; "count";
@ -187,12 +187,12 @@ return {
"nargs"; "nargs";
"addr"; "addr";
"nextcmd"; "nextcmd";
}; }};
cmd_magic = { { 'cmd_magic', {
"file"; "file";
"bar"; "bar";
}; }};
cmd_mods = { { 'cmd_mods', {
"silent"; "silent";
"emsg_silent"; "emsg_silent";
"unsilent"; "unsilent";
@ -213,16 +213,15 @@ return {
"verbose"; "verbose";
"vertical"; "vertical";
"split"; "split";
}; }};
cmd_mods_filter = { { 'cmd_mods_filter', {
"pattern"; "pattern";
"force"; "force";
}; }};
cmd_opts = { { 'cmd_opts', {
"output"; "output";
}; }};
echo_opts = { { 'echo_opts', {
"verbose"; "verbose";
}; }};
} }

View File

@ -127,10 +127,22 @@ return {
'WinScrolled', -- after a window was scrolled or resized 'WinScrolled', -- after a window was scrolled or resized
}, },
aliases = { aliases = {
BufCreate = 'BufAdd', {
BufRead = 'BufReadPost', 'BufCreate',
BufWrite = 'BufWritePre', 'BufAdd'
FileEncoding = 'EncodingChanged', },
{
'BufRead',
'BufReadPost'
},
{
'BufWrite',
'BufWritePre'
},
{
'FileEncoding',
'EncodingChanged'
},
}, },
-- List of nvim-specific events or aliases for the purpose of generating -- List of nvim-specific events or aliases for the purpose of generating
-- syntax file -- syntax file

View File

@ -426,7 +426,9 @@ for _,fn in ipairs(functions) do
end end
remote_fns.redraw = {impl_name="ui_client_redraw", fast=true} remote_fns.redraw = {impl_name="ui_client_redraw", fast=true}
local hashorder, hashfun = hashy.hashy_hash("msgpack_rpc_get_handler_for", vim.tbl_keys(remote_fns), function (idx) local names = vim.tbl_keys(remote_fns)
table.sort(names)
local hashorder, hashfun = hashy.hashy_hash("msgpack_rpc_get_handler_for", names, function (idx)
return "method_handlers["..idx.."].name" return "method_handlers["..idx.."].name"
end) end)

View File

@ -73,14 +73,13 @@ for _,fun in ipairs(metadata) do
end end
end end
local func_names = vim.tbl_keys(funcs)
table.sort(func_names)
local funcsdata = io.open(funcs_file, 'w') local funcsdata = io.open(funcs_file, 'w')
funcsdata:write(mpack.pack(funcs)) funcsdata:write(mpack.pack(func_names))
funcsdata:close() funcsdata:close()
local neworder, hashfun = hashy.hashy_hash("find_internal_func", func_names, function (idx)
local names = vim.tbl_keys(funcs)
local neworder, hashfun = hashy.hashy_hash("find_internal_func", names, function (idx)
return "functions["..idx.."].name" return "functions["..idx.."].name"
end) end)
hashpipe:write("static const EvalFuncDef functions[] = {\n") hashpipe:write("static const EvalFuncDef functions[] = {\n")

View File

@ -32,7 +32,9 @@ for i, event in ipairs(events) do
end end
end end
for alias, event in pairs(aliases) do for _, v in ipairs(aliases) do
local alias = v[1]
local event = v[2]
names_tgt:write(('\n {%u, "%s", EVENT_%s},'):format(#alias, alias, event:upper())) names_tgt:write(('\n {%u, "%s", EVENT_%s},'):format(#alias, alias, event:upper()))
end end

View File

@ -1,4 +1,3 @@
local nvimsrcdir = arg[1] local nvimsrcdir = arg[1]
local shared_file = arg[2] local shared_file = arg[2]
local funcs_file = arg[3] local funcs_file = arg[3]
@ -38,7 +37,9 @@ local function sanitize(key)
return key return key
end end
for name, keys in pairs(keysets) do for _, v in ipairs(keysets) do
local name = v[1]
local keys = v[2]
local neworder, hashfun = hashy.hashy_hash(name, keys, function (idx) local neworder, hashfun = hashy.hashy_hash(name, keys, function (idx)
return name.."_table["..idx.."].str" return name.."_table["..idx.."].str"
end) end)

View File

@ -159,7 +159,7 @@ local dump_option = function(i, o)
if #o.scope == 2 then if #o.scope == 2 then
pv_name = 'OPT_BOTH(' .. pv_name .. ')' pv_name = 'OPT_BOTH(' .. pv_name .. ')'
end end
defines['PV_' .. varname:sub(3):upper()] = pv_name table.insert(defines, { 'PV_' .. varname:sub(3):upper() , pv_name})
w(' .indir=' .. pv_name) w(' .indir=' .. pv_name)
end end
if o.enable_if then if o.enable_if then
@ -192,7 +192,7 @@ w(' [' .. ('%u'):format(#options.options) .. ']={.fullname=NULL}')
w('};') w('};')
w('') w('')
for k, v in pairs(defines) do for _, v in ipairs(defines) do
w('#define ' .. k .. ' ' .. v) w('#define ' .. v[1] .. ' ' .. v[2])
end end
opt_fd:close() opt_fd:close()

View File

@ -763,6 +763,20 @@ describe('lua stdlib', function()
pcall_err(exec_lua, code)) pcall_err(exec_lua, code))
end) end)
it('vim.spairs', function()
local res = ''
local table = {
ccc=1,
bbb=2,
ddd=3,
aaa=4
}
for key, _ in vim.spairs(table) do
res = res .. key
end
matches('aaabbbcccddd', res)
end)
it('vim.call, vim.fn', function() it('vim.call, vim.fn', function()
eq(true, exec_lua([[return vim.call('sin', 0.0) == 0.0 ]])) eq(true, exec_lua([[return vim.call('sin', 0.0) == 0.0 ]]))
eq(true, exec_lua([[return vim.fn.sin(0.0) == 0.0 ]])) eq(true, exec_lua([[return vim.fn.sin(0.0) == 0.0 ]]))