docs(generators): bake into cmake

.github/workflows/api-docs.yml

@@ -26,14 +26,10 @@ jobs:
           ./.github/scripts/install_deps.sh
           sudo env DEBIAN_FRONTEND=noninteractive apt-get install -y doxygen python3 python3-msgpack
 
-      - name: Build metadata
-        run: |
-          make api-metadata
-
       - name: Generate docs
         id: docs
         run: |
-          python3 scripts/gen_vimdoc.py
+          make doc
           printf 'UPDATED_DOCS=%s\n' $([ -z "$(git diff)" ]; echo $?) >> $GITHUB_OUTPUT
 
       - name: FAIL, PR has not committed doc changes

CMakeLists.txt

@@ -242,6 +242,7 @@ add_glob_target(
   FLAGS -q
   GLOB_DIRS runtime/ scripts/ src/ test/
   GLOB_PAT *.lua
+  EXCLUDE runtime/lua/vim/_meta/.*
   TOUCH_STRATEGY SINGLE)
 add_dependencies(lintlua-luacheck luacheck)
 
@@ -253,7 +254,7 @@ add_glob_target(
   GLOB_PAT *.lua
   EXCLUDE
     /runtime/lua/vim/re.lua
-    /runtime/lua/vim/_meta/options.lua
+    /runtime/lua/vim/_meta/.*
   TOUCH_STRATEGY SINGLE)
 
 add_custom_target(lintlua)

CONTRIBUTING.md

@@ -274,13 +274,24 @@ Read [:help dev-doc][dev-doc-guide] to understand the expected documentation sty
 
 ### Generating :help
 
-Many `:help` docs are autogenerated from (C or Lua) docstrings by the `./scripts/gen_vimdoc.py` script.
-For convenience you can filter the regeneration by target (api, lua, lsp) using the `-t` option, for example:
+Many `:help` docs are autogenerated from (C or Lua) docstrings. To generate the documentation run:
 
 ```bash
-./scripts/gen_vimdoc.py -t lua
+make doc
 ```
 
+If you need to modify or debug the documentation flow, these are the main files:
+- `./scripts/gen_vimdoc.py`:
+  Main doc generator. Drives doxygen to generate xml files, and scrapes those
+  xml files to render vimdoc files.
+- `./scripts/lua2dox.lua`:
+  Used by `gen_vimdoc.py` to transform Lua files into a format compatible with doxygen.
+- `./scripts/gen_eval_files.lua`:
+  Generates documentation and Lua type files from metadata files including:
+    - Eval functions (`src/nvim/eval.lua`)
+    - Options (`src/nvim/options.lua`)
+    - API
+
 ### Lua docstrings
 
 Use [LuaLS] annotations in Lua docstrings to annotate parameter types, return

Makefile

@@ -131,7 +131,7 @@ functionaltest-lua: | nvim
 FORMAT=formatc formatlua format
 LINT=lintlua lintsh lintc clang-tidy lintcommit lint
 TEST=functionaltest unittest
-generated-sources benchmark $(FORMAT) $(LINT) $(TEST) api-metadata: | build/.ran-cmake
+generated-sources benchmark $(FORMAT) $(LINT) $(TEST) doc: | build/.ran-cmake
 	$(CMAKE_PRG) --build build --target $@
 
 test: $(TEST)

scripts/gen_eval_files.lua

@@ -1,5 +1,8 @@
 #!/usr/bin/env -S nvim -l
--- Generator for src/nvim/eval.lua
+-- Generator for various vimdoc and Lua type files
+
+local DEP_API_METADATA = 'build/api_metadata.mpack'
+local DEP_API_DOC  = 'runtime/doc/api.mpack'
 
 --- @class vim.api.metadata
 --- @field name string
@@ -168,11 +171,11 @@ end
 
 --- @return table<string, vim.EvalFn>
 local function get_api_meta()
-  local mpack_f = assert(io.open('build/api_metadata.mpack', 'rb'))
+  local mpack_f = assert(io.open(DEP_API_METADATA, 'rb'))
   local metadata = vim.mpack.decode(mpack_f:read('*all')) --[[@as vim.api.metadata[] ]]
   local ret = {} --- @type table<string, vim.EvalFn>
 
-  local doc_mpack_f = assert(io.open('runtime/doc/api.mpack', 'rb'))
+  local doc_mpack_f = assert(io.open(DEP_API_DOC, 'rb'))
   local doc_metadata = vim.mpack.decode(doc_mpack_f:read('*all')) --[[@as table<string,vim.gen_vim_doc_fun>]]
 
   for _, fun in ipairs(metadata) do
@@ -282,7 +285,7 @@ end
 
 --- @return table<string, vim.EvalFn>
 local function get_api_keysets_meta()
-  local mpack_f = assert(io.open('build/api_metadata.mpack', 'rb'))
+  local mpack_f = assert(io.open(DEP_API_METADATA, 'rb'))
 
   --- @diagnostic disable-next-line:no-unknown
   local metadata = assert(vim.mpack.decode(mpack_f:read('*all')))

scripts/gen_vimdoc.py

@@ -1359,7 +1359,4 @@ if __name__ == "__main__":
     else:
         main(Doxyfile, args)
 
-        print('Running ./scripts/gen_eval_files.lua')
-        subprocess.call(['./scripts/gen_eval_files.lua'])
-
 # vim: set ft=python ts=4 sw=4 tw=79 et :

src/nvim/CMakeLists.txt

@@ -281,16 +281,17 @@ set(UNICODE_TABLES_GENERATOR ${GENERATOR_DIR}/gen_unicode_tables.lua)
 set(UNICODE_DIR ${PROJECT_SOURCE_DIR}/src/unicode)
 set(GENERATED_UNICODE_TABLES ${GENERATED_DIR}/unicode_tables.generated.h)
 set(VIM_MODULE_FILE ${GENERATED_DIR}/lua/vim_module.generated.h)
-set(LUA_EDITOR_MODULE_SOURCE ${PROJECT_SOURCE_DIR}/runtime/lua/vim/_editor.lua)
-set(LUA_SHARED_MODULE_SOURCE ${PROJECT_SOURCE_DIR}/runtime/lua/vim/shared.lua)
-set(LUA_LOADER_MODULE_SOURCE ${PROJECT_SOURCE_DIR}/runtime/lua/vim/loader.lua)
-set(LUA_INSPECT_MODULE_SOURCE ${PROJECT_SOURCE_DIR}/runtime/lua/vim/inspect.lua)
-set(LUA_FS_MODULE_SOURCE ${PROJECT_SOURCE_DIR}/runtime/lua/vim/fs.lua)
-set(LUA_F_MODULE_SOURCE ${PROJECT_SOURCE_DIR}/runtime/lua/vim/F.lua)
-set(LUA_OPTIONS_MODULE_SOURCE ${PROJECT_SOURCE_DIR}/runtime/lua/vim/_options.lua)
-set(LUA_FILETYPE_MODULE_SOURCE ${PROJECT_SOURCE_DIR}/runtime/lua/vim/filetype.lua)
-set(LUA_INIT_PACKAGES_MODULE_SOURCE ${PROJECT_SOURCE_DIR}/runtime/lua/vim/_init_packages.lua)
-set(LUA_KEYMAP_MODULE_SOURCE ${PROJECT_SOURCE_DIR}/runtime/lua/vim/keymap.lua)
+set(NVIM_RUNTIME_DIR ${PROJECT_SOURCE_DIR}/runtime)
+set(LUA_EDITOR_MODULE_SOURCE ${NVIM_RUNTIME_DIR}/lua/vim/_editor.lua)
+set(LUA_SHARED_MODULE_SOURCE ${NVIM_RUNTIME_DIR}/lua/vim/shared.lua)
+set(LUA_LOADER_MODULE_SOURCE ${NVIM_RUNTIME_DIR}/lua/vim/loader.lua)
+set(LUA_INSPECT_MODULE_SOURCE ${NVIM_RUNTIME_DIR}/lua/vim/inspect.lua)
+set(LUA_FS_MODULE_SOURCE ${NVIM_RUNTIME_DIR}/lua/vim/fs.lua)
+set(LUA_F_MODULE_SOURCE ${NVIM_RUNTIME_DIR}/lua/vim/F.lua)
+set(LUA_OPTIONS_MODULE_SOURCE ${NVIM_RUNTIME_DIR}/lua/vim/_options.lua)
+set(LUA_FILETYPE_MODULE_SOURCE ${NVIM_RUNTIME_DIR}/lua/vim/filetype.lua)
+set(LUA_INIT_PACKAGES_MODULE_SOURCE ${NVIM_RUNTIME_DIR}/lua/vim/_init_packages.lua)
+set(LUA_KEYMAP_MODULE_SOURCE ${NVIM_RUNTIME_DIR}/lua/vim/keymap.lua)
 set(CHAR_BLOB_GENERATOR ${GENERATOR_DIR}/gen_char_blob.lua)
 set(LUAJIT_RUNTIME_DIR ${DEPS_PREFIX}/share/luajit-2.1/jit)
 
@@ -857,6 +858,57 @@ add_custom_target(generated-sources DEPENDS
   ${NVIM_GENERATED_SOURCES}
 )
 
-add_custom_target(api-metadata DEPENDS ${API_METADATA})
-
 add_subdirectory(po)
+
+#-------------------------------------------------------------------------------
+# Docs
+#-------------------------------------------------------------------------------
+
+set(VIMDOC_FILES
+  ${NVIM_RUNTIME_DIR}/doc/api.mpack
+  ${NVIM_RUNTIME_DIR}/doc/api.txt
+  ${NVIM_RUNTIME_DIR}/doc/diagnostic.mpack
+  ${NVIM_RUNTIME_DIR}/doc/diagnostic.txt
+  ${NVIM_RUNTIME_DIR}/doc/lsp.mpack
+  ${NVIM_RUNTIME_DIR}/doc/lsp.txt
+  ${NVIM_RUNTIME_DIR}/doc/lua.mpack
+  ${NVIM_RUNTIME_DIR}/doc/lua.txt
+  ${NVIM_RUNTIME_DIR}/doc/treesitter.mpack
+  ${NVIM_RUNTIME_DIR}/doc/treesitter.txt
+)
+
+glob_wrapper(API_SOURCES ${PROJECT_SOURCE_DIR}/src/nvim/api/*.c)
+glob_wrapper(LUA_SOURCES ${NVIM_RUNTIME_DIR}/lua/vim/*.lua)
+
+add_custom_command(
+  OUTPUT ${VIMDOC_FILES}
+  COMMAND ${PROJECT_SOURCE_DIR}/scripts/gen_vimdoc.py
+  DEPENDS ${API_SOURCES} ${LUA_SOURCES}
+  WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+)
+
+set(GEN_EVAL_FILES
+  ${NVIM_RUNTIME_DIR}/lua/vim/_meta/vimfn.lua
+  ${NVIM_RUNTIME_DIR}/lua/vim/_meta/api.lua
+  ${NVIM_RUNTIME_DIR}/lua/vim/_meta/api_keysets.lua
+  ${NVIM_RUNTIME_DIR}/doc/builtin.txt
+  ${NVIM_RUNTIME_DIR}/lua/vim/_meta/options.lua
+  ${NVIM_RUNTIME_DIR}/doc/options.txt
+)
+
+add_custom_command(
+  OUTPUT ${GEN_EVAL_FILES}
+  COMMAND ${PROJECT_SOURCE_DIR}/scripts/gen_eval_files.lua
+  DEPENDS
+    ${API_METADATA}
+    ${PROJECT_SOURCE_DIR}/scripts/gen_eval_files.lua
+    ${PROJECT_SOURCE_DIR}/src/nvim/eval.lua
+    ${PROJECT_SOURCE_DIR}/src/nvim/options.lua
+    ${NVIM_RUNTIME_DIR}/doc/api.mpack
+  WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+)
+
+add_custom_target(doc
+  DEPENDS ${VIMDOC_FILES} ${GEN_EVAL_FILES}
+)
+