refactor: gen_declarations.lua

Problem:
gen_declarations.lua is complex and has duplicate logic with
c_grammar.lua

Solution:
Move all lpeg logic to c_grammar.lua and refactor gen_declarations.lua.

src/nvim/CMakeLists.txt

@@ -556,7 +556,7 @@ foreach(sfile ${NVIM_SOURCES}
     set(PREPROC_OUTPUT -w -E -o ${gf_i})
   endif()
 
-  set(depends "${HEADER_GENERATOR}" "${sfile}" "${LUA_GEN_DEPS}")
+  set(depends "${HEADER_GENERATOR}" "${sfile}" "${LUA_GEN_DEPS}" "${GENERATOR_C_GRAMMAR}")
   if("${f}" STREQUAL "version.c")
     # Ensure auto/versiondef_git.h exists after "make clean".
     list(APPEND depends update_version_stamp "${NVIM_VERSION_GIT_H}" "${NVIM_VERSION_DEF_H}")

src/nvim/generators/c_grammar.lua

@@ -1,10 +1,37 @@
-local lpeg = vim.lpeg
-
 -- lpeg grammar for building api metadata from a set of header files. It
 -- ignores comments and preprocessor commands and parses a very small subset
 -- of C prototypes with a limited set of types
+
+--- @class nvim.c_grammar.Proto
+--- @field [1] 'proto'
+--- @field pos integer
+--- @field endpos integer
+--- @field fast boolean
+--- @field name string
+--- @field return_type string
+--- @field parameters [string, string][]
+--- @field static true?
+--- @field inline true?
+
+--- @class nvim.c_grammar.Preproc
+--- @field [1] 'preproc'
+--- @field content string
+
+--- @class nvim.c_grammar.Empty
+--- @field [1] 'empty'
+
+--- @alias nvim.c_grammar.result
+--- | nvim.c_grammar.Proto
+--- | nvim.c_grammar.Preproc
+--- | nvim.c_grammar.Empty
+
+--- @class nvim.c_grammar
+--- @field match fun(self, input: string): nvim.c_grammar.result[]
+
+local lpeg = vim.lpeg
+
 local P, R, S, V = lpeg.P, lpeg.R, lpeg.S, lpeg.V
-local C, Ct, Cc, Cg = lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cg
+local C, Ct, Cc, Cg, Cp = lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cg, lpeg.Cp
 
 --- @param pat vim.lpeg.Pattern
 local function rep(pat)
@@ -21,125 +48,248 @@ local function opt(pat)
   return pat ^ -1
 end
 
-local any = P(1) -- (consume one character)
+local any = P(1)
 local letter = R('az', 'AZ') + S('_$')
 local num = R('09')
 local alpha = letter + num
 local nl = P('\r\n') + P('\n')
-local not_nl = any - nl
 local space = S(' \t')
+local str = P('"') * rep((P('\\') * any) + (1 - P('"'))) * P('"')
+local char = P("'") * (any - P("'")) * P("'")
 local ws = space + nl
-local fill = rep(ws)
+local wb = #-alpha -- word boundary
-local c_comment = P('//') * rep(not_nl)
+local id = letter * rep(alpha)
-local cdoc_comment = P('///') * opt(Ct(Cg(rep(space) * rep(not_nl), 'comment')))
+
-local c_preproc = P('#') * rep(not_nl)
+local comment_inline = P('/*') * rep(1 - P('*/')) * P('*/')
-local dllexport = P('DLLEXPORT') * rep1(ws)
+local comment = P('//') * rep(1 - nl) * nl
+local preproc = Ct(Cc('preproc') * P('#') * Cg(rep(1 - nl) * nl, 'content'))
+
+local fill = rep(ws + comment_inline + comment + preproc)
+
+--- @param s string
+--- @return vim.lpeg.Pattern
+local function word(s)
+  return fill * P(s) * wb * fill
+end
 
 --- @param x vim.lpeg.Pattern
 local function comma1(x)
   return x * rep(fill * P(',') * fill * x)
 end
 
--- Define the grammar
+--- @param v string
-
+local function Pf(v)
-local basic_id = letter * rep(alpha)
+  return fill * P(v) * fill
-
+end
-local str = P('"') * rep(any - P('"')) * P('"')
 
 --- @param x vim.lpeg.Pattern
 local function paren(x)
   return P('(') * fill * x * fill * P(')')
 end
 
+local cdoc_comment = P('///') * opt(Ct(Cg(rep(space) * rep(1 - nl), 'comment')))
+
+local braces = P({
+  'S',
+  A = comment_inline + comment + preproc + str + char + (any - S('{}')),
+  S = P('{') * rep(V('A')) * rep(V('S') + V('A')) * P('}'),
+})
+
 -- stylua: ignore start
 local typed_container = P({
   'S',
-  ID = V('S') + basic_id,
   S = (
     (P('Union') * paren(comma1(V('ID'))))
-    + (P('ArrayOf') * paren(basic_id * opt(P(',') * fill * rep1(num))))
+    + (P('ArrayOf') * paren(id * opt(P(',') * fill * rep1(num))))
-    + (P('DictOf') * paren(basic_id))
+    + (P('DictOf') * paren(id))
     + (P('LuaRefOf') * paren(
-      paren(comma1((V('ID') + str) * rep1(ws) * opt(P('*')) * basic_id))
+      paren(comma1((V('ID') + str) * rep1(ws) * opt(P('*')) * id))
       * opt(P(',') * fill * opt(P('*')) * V('ID'))
     ))
-    + (P('Dict') * paren(basic_id))),
+    + (P('Dict') * paren(id))),
+  ID = V('S') + id,
 })
 -- stylua: ignore end
 
-local c_id = typed_container + basic_id
+local ptr_mod = word('restrict') + word('__restrict') + word('const')
-local c_void = P('void')
+local opt_ptr = rep(Pf('*') * opt(ptr_mod))
 
-local c_param_type = (
+--- @param name string
-  ((P('Error') * fill * P('*') * fill) * Cc('error'))
+--- @param var string
-  + ((P('Arena') * fill * P('*') * fill) * Cc('arena'))
+--- @return vim.lpeg.Pattern
-  + ((P('lua_State') * fill * P('*') * fill) * Cc('lstate'))
+local function attr(name, var)
-  + C(opt(P('const ')) * c_id * rep1(ws) * rep1(P('*')))
+  return Cg((P(name) * Cc(true)), var)
-  + (C(c_id) * rep1(ws))
+end
+
+--- @param name string
+--- @param var string
+--- @return vim.lpeg.Pattern
+local function attr_num(name, var)
+  return Cg((P(name) * paren(C(rep1(num)))), var)
+end
+
+local fattr = (
+  attr_num('FUNC_API_SINCE', 'since')
+  + attr_num('FUNC_API_DEPRECATED_SINCE', 'deprecated_since')
+  + attr('FUNC_API_FAST', 'fast')
+  + attr('FUNC_API_RET_ALLOC', 'ret_alloc')
+  + attr('FUNC_API_NOEXPORT', 'noexport')
+  + attr('FUNC_API_REMOTE_ONLY', 'remote_only')
+  + attr('FUNC_API_LUA_ONLY', 'lua_only')
+  + attr('FUNC_API_TEXTLOCK_ALLOW_CMDWIN', 'textlock_allow_cmdwin')
+  + attr('FUNC_API_TEXTLOCK', 'textlock')
+  + attr('FUNC_API_REMOTE_IMPL', 'remote_impl')
+  + attr('FUNC_API_COMPOSITOR_IMPL', 'compositor_impl')
+  + attr('FUNC_API_CLIENT_IMPL', 'client_impl')
+  + attr('FUNC_API_CLIENT_IGNORE', 'client_ignore')
+  + (P('FUNC_') * rep(alpha) * opt(fill * paren(rep(1 - P(')') * any))))
 )
 
-local c_type = (C(c_void) * (ws ^ 1)) + c_param_type
+local void = P('void') * wb
-local c_param = Ct(c_param_type * C(c_id))
-local c_param_list = c_param * (fill * (P(',') * fill * c_param) ^ 0)
-local c_params = Ct(c_void + c_param_list)
 
-local impl_line = (any - P('}')) * opt(rep(not_nl)) * nl
+local api_param_type = (
+  (word('Error') * opt_ptr * Cc('error'))
+  + (word('Arena') * opt_ptr * Cc('arena'))
+  + (word('lua_State') * opt_ptr * Cc('lstate'))
+)
 
-local ignore_line = rep1(not_nl) * nl
+local ctype = C(
+  opt(word('const'))
+    * (
+      typed_container
+      -- 'unsigned' is a type modifier, and a type itself
+      + (word('unsigned char') + word('unsigned'))
+      + (word('struct') * fill * id)
+      + id
+    )
+    * opt(word('const'))
+    * opt_ptr
+)
 
+local return_type = (C(void) * fill) + ctype
+
+-- stylua: ignore start
+local params = Ct(
+  (void * #P(')'))
+  + comma1(Ct(
+      (api_param_type + ctype)
+      * fill
+      * C(id)
+      * rep(Pf('[') * rep(alpha) * Pf(']'))
+      * rep(fill * fattr)
+    ))
+    * opt(Pf(',') * P('...'))
+)
+-- stylua: ignore end
+
+local ignore_line = rep1(1 - nl) * nl
 local empty_line = Ct(Cc('empty') * nl * nl)
 
-local c_proto = Ct(
+local proto_name = opt_ptr * fill * id
-  Cc('proto')
+
-    * opt(dllexport)
+-- __inline is used in MSVC
-    * opt(Cg(P('static') * fill * Cc(true), 'static'))
+local decl_mod = (
-    * Cg(c_type, 'return_type')
+  Cg(word('static') * Cc(true), 'static')
-    * Cg(c_id, 'name')
+  + Cg((word('inline') + word('__inline')) * Cc(true), 'inline')
+)
+
+local proto = Ct(
+  Cg(Cp(), 'pos')
+    * Cc('proto')
+    * -#P('typedef')
+    * #alpha
+    * opt(P('DLLEXPORT') * rep1(ws))
+    * rep(decl_mod)
+    * Cg(return_type, 'return_type')
     * fill
-    * (P('(') * fill * Cg(c_params, 'parameters') * fill * P(')'))
+    * Cg(proto_name, 'name')
+    * fill
+    * paren(Cg(params, 'parameters'))
     * Cg(Cc(false), 'fast')
-    * (fill * Cg((P('FUNC_API_SINCE(') * C(rep1(num))) * P(')'), 'since') ^ -1)
+    * rep(fill * fattr)
-    * (fill * Cg((P('FUNC_API_DEPRECATED_SINCE(') * C(rep1(num))) * P(')'), 'deprecated_since') ^ -1)
+    * Cg(Cp(), 'endpos')
-    * (fill * Cg((P('FUNC_API_FAST') * Cc(true)), 'fast') ^ -1)
+    * (fill * (S(';') + braces))
-    * (fill * Cg((P('FUNC_API_RET_ALLOC') * Cc(true)), 'ret_alloc') ^ -1)
-    * (fill * Cg((P('FUNC_API_NOEXPORT') * Cc(true)), 'noexport') ^ -1)
-    * (fill * Cg((P('FUNC_API_REMOTE_ONLY') * Cc(true)), 'remote_only') ^ -1)
-    * (fill * Cg((P('FUNC_API_LUA_ONLY') * Cc(true)), 'lua_only') ^ -1)
-    * (fill * (Cg(P('FUNC_API_TEXTLOCK_ALLOW_CMDWIN') * Cc(true), 'textlock_allow_cmdwin') + Cg(
-      P('FUNC_API_TEXTLOCK') * Cc(true),
-      'textlock'
-    )) ^ -1)
-    * (fill * Cg((P('FUNC_API_REMOTE_IMPL') * Cc(true)), 'remote_impl') ^ -1)
-    * (fill * Cg((P('FUNC_API_COMPOSITOR_IMPL') * Cc(true)), 'compositor_impl') ^ -1)
-    * (fill * Cg((P('FUNC_API_CLIENT_IMPL') * Cc(true)), 'client_impl') ^ -1)
-    * (fill * Cg((P('FUNC_API_CLIENT_IGNORE') * Cc(true)), 'client_ignore') ^ -1)
-    * fill
-    * (P(';') + (P('{') * nl + (impl_line ^ 0) * P('}')))
 )
 
-local dict_key = P('DictKey(') * Cg(rep1(any - P(')')), 'dict_key') * P(')')
+local keyset_field = Ct(
-local keyset_field =
+  Cg(ctype, 'type')
-  Ct(Cg(c_id, 'type') * ws * Cg(c_id, 'name') * fill * (dict_key ^ -1) * fill * P(';') * fill)
+    * fill
-local c_keyset = Ct(
+    * Cg(id, 'name')
+    * fill
+    * opt(P('DictKey') * paren(Cg(rep1(1 - P(')')), 'dict_key')))
+    * Pf(';')
+)
+
+local keyset = Ct(
   P('typedef')
-    * ws
+    * word('struct')
-    * P('struct')
+    * Pf('{')
-    * fill
+    * Cg(Ct(rep1(keyset_field)), 'fields')
-    * P('{')
+    * Pf('}')
-    * fill
-    * Cg(Ct(keyset_field ^ 1), 'fields')
-    * P('}')
-    * fill
     * P('Dict')
-    * fill
+    * paren(Cg(id, 'keyset_name'))
-    * P('(')
+    * Pf(';')
-    * Cg(c_id, 'keyset_name')
-    * fill
-    * P(')')
-    * P(';')
 )
 
-local grammar = Ct(
+local grammar =
-  rep1(empty_line + c_proto + cdoc_comment + c_comment + c_preproc + ws + c_keyset + ignore_line)
+  Ct(rep1(empty_line + proto + cdoc_comment + comment + preproc + ws + keyset + ignore_line))
-)
+
-return { grammar = grammar, typed_container = typed_container }
+if arg[1] == '--test' then
+  for i, t in ipairs({
+    'void multiqueue_put_event(MultiQueue *self, Event event) {} ',
+    'void *xmalloc(size_t size) {} ',
+    {
+      'struct tm *os_localtime_r(const time_t *restrict clock,',
+      '                          struct tm *restrict result) FUNC_ATTR_NONNULL_ALL {}',
+    },
+    {
+      '_Bool',
+      '# 163 "src/nvim/event/multiqueue.c"',
+      '    multiqueue_empty(MultiQueue *self)',
+      '{}',
+    },
+    'const char *find_option_end(const char *arg, OptIndex *opt_idxp) {}',
+    'bool semsg(const char *const fmt, ...) {}',
+    'int32_t utf_ptr2CharInfo_impl(uint8_t const *p, uintptr_t const len) {}',
+    'void ex_argdedupe(exarg_T *eap FUNC_ATTR_UNUSED) {}',
+    'static TermKeySym register_c0(TermKey *tk, TermKeySym sym, unsigned char ctrl, const char *name) {}',
+    'unsigned get_bkc_flags(buf_T *buf) {}',
+    'char *xstpcpy(char *restrict dst, const char *restrict src) {}',
+    'bool try_leave(const TryState *const tstate, Error *const err) {}',
+    'void api_set_error(ErrorType errType) {}',
+
+    -- Do not consume leading preproc statements
+    {
+      '#line 1 "D:/a/neovim/neovim/src\\nvim/mark.h"',
+      'static __inline int mark_global_index(const char name)',
+      '  FUNC_ATTR_CONST',
+      '{}',
+    },
+    {
+      '',
+      '#line 1 "D:/a/neovim/neovim/src\\nvim/mark.h"',
+      'static __inline int mark_global_index(const char name)',
+      '{}',
+    },
+    {
+      'size_t xstrlcpy(char *__restrict dst, const char *__restrict src, size_t dsize)',
+      ' FUNC_ATTR_NONNULL_ALL',
+      ' {}',
+    },
+  }) do
+    if type(t) == 'table' then
+      t = table.concat(t, '\n') .. '\n'
+    end
+    t = t:gsub(' +', ' ')
+    local r = grammar:match(t)
+    if not r then
+      print('Test ' .. i .. ' failed')
+      print('    |' .. table.concat(vim.split(t, '\n'), '\n    |'))
+    end
+  end
+end
+
+return {
+  grammar = grammar --[[@as nvim.c_grammar]],
+  typed_container = typed_container,
+}

src/nvim/generators/gen_api_ui_events.lua

@@ -98,7 +98,7 @@ local function call_ui_event_method(output, ev)
 end
 
 events = vim.tbl_filter(function(ev)
-  return ev[1] ~= 'empty'
+  return ev[1] ~= 'empty' and ev[1] ~= 'preproc'
 end, events)
 
 for i = 1, #events do

src/nvim/generators/gen_declarations.lua

@@ -1,136 +1,105 @@
-local fname = arg[1]
+local grammar = require('generators.c_grammar').grammar
-local static_fname = arg[2]
-local non_static_fname = arg[3]
-local preproc_fname = arg[4]
-local static_basename = arg[5]
 
-local lpeg = vim.lpeg
+--- @param fname string
+--- @return string?
+local function read_file(fname)
+  local f = io.open(fname, 'r')
+  if not f then
+    return
+  end
+  local contents = f:read('*a')
+  f:close()
+  return contents
+end
 
-local fold = function(func, ...)
+--- @param fname string
-  local result = nil
+--- @param contents string[]
-  for _, v in ipairs({ ... }) do
+local function write_file(fname, contents)
-    if result == nil then
+  local contents_s = table.concat(contents, '\n') .. '\n'
-      result = v
+  local fcontents = read_file(fname)
+  if fcontents == contents_s then
+    return
+  end
+  local f = assert(io.open(fname, 'w'))
+  f:write(contents_s)
+  f:close()
+end
+
+--- @param fname string
+--- @param non_static_fname string
+--- @return string? non_static
+local function add_iwyu_non_static(fname, non_static_fname)
+  if fname:find('.*/src/nvim/.*%.c$') then
+    -- Add an IWYU pragma comment if the corresponding .h file exists.
+    local header_fname = fname:sub(1, -3) .. '.h'
+    local header_f = io.open(header_fname, 'r')
+    if header_f then
+      header_f:close()
+      return (header_fname:gsub('.*/src/nvim/', 'nvim/'))
+    end
+  elseif non_static_fname:find('/include/api/private/dispatch_wrappers%.h%.generated%.h$') then
+    return 'nvim/api/private/dispatch.h'
+  elseif non_static_fname:find('/include/ui_events_call%.h%.generated%.h$') then
+    return 'nvim/ui.h'
+  elseif non_static_fname:find('/include/ui_events_client%.h%.generated%.h$') then
+    return 'nvim/ui_client.h'
+  elseif non_static_fname:find('/include/ui_events_remote%.h%.generated%.h$') then
+    return 'nvim/api/ui.h'
+  end
+end
+
+--- @param d string
+local function process_decl(d)
+  -- Comments are really handled by preprocessor, so the following is not
+  -- needed
+  d = d:gsub('/%*.-%*/', '')
+  d = d:gsub('//.-\n', '\n')
+  d = d:gsub('# .-\n', '')
+  d = d:gsub('\n', ' ')
+  d = d:gsub('%s+', ' ')
+  d = d:gsub(' ?%( ?', '(')
+  d = d:gsub(' ?, ?', ', ')
+  d = d:gsub(' ?(%*+) ?', ' %1')
+  d = d:gsub(' ?(FUNC_ATTR_)', ' %1')
+  d = d:gsub(' $', '')
+  d = d:gsub('^ ', '')
+  return d .. ';'
+end
+
+--- @param fname string
+--- @param text string
+--- @return string[] static
+--- @return string[] non_static
+--- @return boolean any_static
+local function gen_declarations(fname, text)
+  local non_static = {} --- @type string[]
+  local static = {} --- @type string[]
+
+  local neededfile = fname:match('[^/]+$')
+  local curfile = nil
+  local any_static = false
+  for _, node in ipairs(grammar:match(text)) do
+    if node[1] == 'preproc' then
+      curfile = node.content:match('^%a* %d+ "[^"]-/?([^"/]+)"') or curfile
+    elseif node[1] == 'proto' and curfile == neededfile then
+      local node_text = text:sub(node.pos, node.endpos - 1)
+      local declaration = process_decl(node_text)
+
+      if node.static then
+        if not any_static and declaration:find('FUNC_ATTR_') then
+          any_static = true
+        end
+        static[#static + 1] = declaration
       else
-      result = func(result, v)
+        non_static[#non_static + 1] = 'DLLEXPORT ' .. declaration
       end
     end
-  return result
-end
-
-local folder = function(func)
-  return function(...)
-    return fold(func, ...)
   end
+
+  return static, non_static, any_static
 end
 
-local lit = lpeg.P
+local usage = [[
-local set = function(...)
-  return lpeg.S(fold(function(a, b)
-    return a .. b
-  end, ...))
-end
-local any_character = lpeg.P(1)
-local rng = function(s, e)
-  return lpeg.R(s .. e)
-end
-local concat = folder(function(a, b)
-  return a * b
-end)
-local branch = folder(function(a, b)
-  return a + b
-end)
-local one_or_more = function(v)
-  return v ^ 1
-end
-local two_or_more = function(v)
-  return v ^ 2
-end
-local any_amount = function(v)
-  return v ^ 0
-end
-local one_or_no = function(v)
-  return v ^ -1
-end
-local look_behind = lpeg.B
-local look_ahead = function(v)
-  return #v
-end
-local neg_look_ahead = function(v)
-  return -v
-end
-local neg_look_behind = function(v)
-  return -look_behind(v)
-end
-
-local w = branch(rng('a', 'z'), rng('A', 'Z'), lit('_'))
-local aw = branch(w, rng('0', '9'))
-local s = set(' ', '\n', '\t')
-local raw_word = concat(w, any_amount(aw))
-local right_word = concat(raw_word, neg_look_ahead(aw))
-local word = branch(
-  concat(
-    branch(lit('ArrayOf('), lit('DictOf('), lit('Dict(')), -- typed container macro
-    one_or_more(any_character - lit(')')),
-    lit(')')
-  ),
-  concat(neg_look_behind(aw), right_word)
-)
-local inline_comment =
-  concat(lit('/*'), any_amount(concat(neg_look_ahead(lit('*/')), any_character)), lit('*/'))
-local spaces = any_amount(branch(
-  s,
-  -- Comments are really handled by preprocessor, so the following is not needed
-  inline_comment,
-  concat(lit('//'), any_amount(concat(neg_look_ahead(lit('\n')), any_character)), lit('\n')),
-  -- Linemarker inserted by preprocessor
-  concat(lit('# '), any_amount(concat(neg_look_ahead(lit('\n')), any_character)), lit('\n'))
-))
-local typ_part = concat(word, any_amount(concat(spaces, lit('*'))), spaces)
-
-local typ_id = two_or_more(typ_part)
-local arg = typ_id -- argument name is swallowed by typ
-local pattern = concat(
-  any_amount(branch(set(' ', '\t'), inline_comment)),
-  typ_id, -- return type with function name
-  spaces,
-  lit('('),
-  spaces,
-  one_or_no(branch( -- function arguments
-    concat(
-      arg, -- first argument, does not require comma
-      any_amount(concat( -- following arguments, start with a comma
-        spaces,
-        lit(','),
-        spaces,
-        arg,
-        any_amount(concat(lit('['), spaces, any_amount(aw), spaces, lit(']')))
-      )),
-      one_or_no(concat(spaces, lit(','), spaces, lit('...')))
-    ),
-    lit('void') -- also accepts just void
-  )),
-  spaces,
-  lit(')'),
-  any_amount(concat( -- optional attributes
-    spaces,
-    lit('FUNC_'),
-    any_amount(aw),
-    one_or_no(concat( -- attribute argument
-      spaces,
-      lit('('),
-      any_amount(concat(neg_look_ahead(lit(')')), any_character)),
-      lit(')')
-    ))
-  )),
-  look_ahead(concat( -- definition must be followed by "{"
-    spaces,
-    lit('{')
-  ))
-)
-
-if fname == '--help' then
-  print([[
 Usage:
 
     gen_declarations.lua definitions.c static.h non-static.h definitions.i
@@ -141,204 +110,77 @@ non-static.h. File `definitions.i' should contain an already preprocessed
 version of definitions.c and it is the only one which is actually parsed,
 definitions.c is needed only to determine functions from which file out of all
 functions found in definitions.i are needed and to generate an IWYU comment.
+]]
 
-Additionally uses the following environment variables:
+local function main()
+  local fname = arg[1]
+  local static_fname = arg[2]
+  local non_static_fname = arg[3]
+  local preproc_fname = arg[4]
+  local static_basename = arg[5]
 
-    NVIM_GEN_DECLARATIONS_LINE_NUMBERS:
+  if fname == '--help' or #arg < 5 then
-        If set to 1 then all generated declarations receive a comment with file
+    print(usage)
-        name and line number after the declaration. This may be useful for
-        debugging gen_declarations script, but not much beyond that with
-        configured development environment (i.e. with with clang/etc).
-
-        WARNING: setting this to 1 will cause extensive rebuilds: declarations
-                 generator script will not regenerate non-static.h file if its
-                 contents did not change, but including line numbers will make
-                 contents actually change.
-
-                 With contents changed timestamp of the file is regenerated even
-                 when no real changes were made (e.g. a few lines were added to
-                 a function which is not at the bottom of the file).
-
-                 With changed timestamp build system will assume that header
-                 changed, triggering rebuilds of all C files which depend on the
-                 "changed" header.
-]])
     os.exit()
-end
-
-local preproc_f = io.open(preproc_fname)
-local text = preproc_f:read('*all')
-preproc_f:close()
-
-local non_static = [[
-#define DEFINE_FUNC_ATTRIBUTES
-#include "nvim/func_attr.h"
-#undef DEFINE_FUNC_ATTRIBUTES
-#ifndef DLLEXPORT
-#  ifdef MSWIN
-#    define DLLEXPORT __declspec(dllexport)
-#  else
-#    define DLLEXPORT
-#  endif
-#endif
-]]
-
-local static = [[
-#define DEFINE_FUNC_ATTRIBUTES
-#include "nvim/func_attr.h"
-#undef DEFINE_FUNC_ATTRIBUTES
-]]
-
-local non_static_footer = [[
-#include "nvim/func_attr.h"
-]]
-
-local static_footer = [[
-#define DEFINE_EMPTY_ATTRIBUTES
-#include "nvim/func_attr.h"  // IWYU pragma: export
-]]
-
-if fname:find('.*/src/nvim/.*%.c$') then
-  -- Add an IWYU pragma comment if the corresponding .h file exists.
-  local header_fname = fname:sub(1, -3) .. '.h'
-  local header_f = io.open(header_fname, 'r')
-  if header_f ~= nil then
-    header_f:close()
-    non_static = ([[
-// IWYU pragma: private, include "%s"
-]]):format(header_fname:gsub('.*/src/nvim/', 'nvim/')) .. non_static
   end
-elseif fname:find('.*/src/nvim/.*%.h$') then
-  static = ([[
-// IWYU pragma: private, include "%s"
-]]):format(fname:gsub('.*/src/nvim/', 'nvim/')) .. static
-elseif non_static_fname:find('/include/api/private/dispatch_wrappers%.h%.generated%.h$') then
-  non_static = [[
-// IWYU pragma: private, include "nvim/api/private/dispatch.h"
-]] .. non_static
-elseif non_static_fname:find('/include/ui_events_call%.h%.generated%.h$') then
-  non_static = [[
-// IWYU pragma: private, include "nvim/ui.h"
-]] .. non_static
-elseif non_static_fname:find('/include/ui_events_client%.h%.generated%.h$') then
-  non_static = [[
-// IWYU pragma: private, include "nvim/ui_client.h"
-]] .. non_static
-elseif non_static_fname:find('/include/ui_events_remote%.h%.generated%.h$') then
-  non_static = [[
-// IWYU pragma: private, include "nvim/api/ui.h"
-]] .. non_static
-end
 
-local filepattern = '^#%a* (%d+) "([^"]-)/?([^"/]+)"'
+  local text = assert(read_file(preproc_fname))
 
-local init = 1
+  local static_decls, non_static_decls, any_static = gen_declarations(fname, text)
-local curfile = nil
-local neededfile = fname:match('[^/]+$')
-local declline = 0
-local declendpos = 0
-local curdir = nil
-local is_needed_file = false
-local init_is_nl = true
-local any_static = false
-while init ~= nil do
-  if init_is_nl and text:sub(init, init) == '#' then
-    local line, dir, file = text:match(filepattern, init)
-    if file ~= nil then
-      curfile = file
-      is_needed_file = (curfile == neededfile)
-      declline = tonumber(line) - 1
-      curdir = dir:gsub('.*/src/nvim/', '')
-    else
-      declline = declline - 1
-    end
-  elseif init < declendpos then -- luacheck: ignore 542
-    -- Skipping over declaration
-  elseif is_needed_file then
-    s = init
-    local e = pattern:match(text, init)
-    if e ~= nil then
-      local declaration = text:sub(s, e - 1)
-      -- Comments are really handled by preprocessor, so the following is not
-      -- needed
-      declaration = declaration:gsub('/%*.-%*/', '')
-      declaration = declaration:gsub('//.-\n', '\n')
 
-      declaration = declaration:gsub('# .-\n', '')
+  local static = {} --- @type string[]
+  if fname:find('.*/src/nvim/.*%.h$') then
+    static[#static + 1] = ('// IWYU pragma: private, include "%s"'):format(
+      fname:gsub('.*/src/nvim/', 'nvim/')
+    )
+  end
+  vim.list_extend(static, {
+    '#define DEFINE_FUNC_ATTRIBUTES',
+    '#include "nvim/func_attr.h"',
+    '#undef DEFINE_FUNC_ATTRIBUTES',
+  })
+  vim.list_extend(static, static_decls)
+  vim.list_extend(static, {
+    '#define DEFINE_EMPTY_ATTRIBUTES',
+    '#include "nvim/func_attr.h"  // IWYU pragma: export',
+    '',
+  })
 
-      declaration = declaration:gsub('\n', ' ')
+  write_file(static_fname, static)
-      declaration = declaration:gsub('%s+', ' ')
-      declaration = declaration:gsub(' ?%( ?', '(')
-      -- declaration = declaration:gsub(' ?%) ?', ')')
-      declaration = declaration:gsub(' ?, ?', ', ')
-      declaration = declaration:gsub(' ?(%*+) ?', ' %1')
-      declaration = declaration:gsub(' ?(FUNC_ATTR_)', ' %1')
-      declaration = declaration:gsub(' $', '')
-      declaration = declaration:gsub('^ ', '')
-      declaration = declaration .. ';'
 
-      if os.getenv('NVIM_GEN_DECLARATIONS_LINE_NUMBERS') == '1' then
+  if any_static then
-        declaration = declaration .. ('  // %s/%s:%u'):format(curdir, curfile, declline)
+    local orig_text = assert(read_file(fname))
-      end
-      declaration = declaration .. '\n'
-      if declaration:sub(1, 6) == 'static' then
-        if declaration:find('FUNC_ATTR_') then
-          any_static = true
-        end
-        static = static .. declaration
-      else
-        declaration = 'DLLEXPORT ' .. declaration
-        non_static = non_static .. declaration
-      end
-      declendpos = e
-    end
-  end
-  init = text:find('[\n;}]', init)
-  if init == nil then
-    break
-  end
-  init_is_nl = text:sub(init, init) == '\n'
-  init = init + 1
-  if init_is_nl and is_needed_file then
-    declline = declline + 1
-  end
-end
-
-non_static = non_static .. non_static_footer
-static = static .. static_footer
-
-local F
-F = io.open(static_fname, 'w')
-F:write(static)
-F:close()
-
-if any_static then
-  F = io.open(fname, 'r')
-  local orig_text = F:read('*a')
     local pat = '\n#%s?include%s+"' .. static_basename .. '"\n'
     local pat_comment = '\n#%s?include%s+"' .. static_basename .. '"%s*//'
-  if not string.find(orig_text, pat) and not string.find(orig_text, pat_comment) then
+    if not orig_text:find(pat) and not orig_text:find(pat_comment) then
-    error('fail: missing include for ' .. static_basename .. ' in ' .. fname)
+      error(('fail: missing include for %s in %s'):format(static_basename, fname))
     end
-  F:close()
-end
-
-if non_static_fname == 'SKIP' then
-  return -- only want static declarations
-end
-
--- Before generating the non-static headers, check if the current file (if
--- exists) is different from the new one. If they are the same, we won't touch
--- the current version to avoid triggering an unnecessary rebuilds of modules
--- that depend on this one
-F = io.open(non_static_fname, 'r')
-if F ~= nil then
-  if F:read('*a') == non_static then
-    os.exit(0)
   end
-  F:close()
+
+  if non_static_fname ~= 'SKIP' then
+    local non_static = {} --- @type string[]
+    local iwyu_non_static = add_iwyu_non_static(fname, non_static_fname)
+    if iwyu_non_static then
+      non_static[#non_static + 1] = ('// IWYU pragma: private, include "%s"'):format(
+        iwyu_non_static
+      )
+    end
+    vim.list_extend(non_static, {
+      '#define DEFINE_FUNC_ATTRIBUTES',
+      '#include "nvim/func_attr.h"',
+      '#undef DEFINE_FUNC_ATTRIBUTES',
+      '#ifndef DLLEXPORT',
+      '#  ifdef MSWIN',
+      '#    define DLLEXPORT __declspec(dllexport)',
+      '#  else',
+      '#    define DLLEXPORT',
+      '#  endif',
+      '#endif',
+    })
+    vim.list_extend(non_static, non_static_decls)
+    non_static[#non_static + 1] = '#include "nvim/func_attr.h"'
+    write_file(non_static_fname, non_static)
+  end
 end
 
-F = io.open(non_static_fname, 'w')
+return main()
-F:write(non_static)
-F:close()