kevin/neovim
1
Fork 0
mirror of https://github.com/neovim/neovim.git synced 2024-12-20 19:25:11 -07:00
Code Issues Packages Projects Releases Wiki Activity
b70f160414
neovim/scripts/gen_vimdoc.py
bfredl 612944c586 refactor(api): update vimdoc
2022-06-12 19:45:39 +02:00

1216 lines
41 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python3
"""Generates Nvim :help docs from C/Lua docstrings, using Doxygen.
Also generates *.mpack files. To inspect the *.mpack structure:
:new | put=v:lua.vim.inspect(msgpackparse(readfile('runtime/doc/api.mpack')))
Flow:
main
extract_from_xml
fmt_node_as_vimhelp \
para_as_map } recursive
update_params_map /
render_node
This would be easier using lxml and XSLT, but:
1. This should avoid needing Python dependencies, especially ones that are
C modules that have library dependencies (lxml requires libxml and
libxslt).
2. I wouldn't know how to deal with nested indentation in <para> tags using
XSLT.
Each function :help block is formatted as follows:
- Max width of 78 columns (`text_width`).
- Indent with spaces (not tabs).
- Indent of 16 columns for body text.
- Function signature and helptag (right-aligned) on the same line.
- Signature and helptag must have a minimum of 8 spaces between them.
- If the signature is too long, it is placed on the line after the helptag.
Signature wraps at `text_width - 8` characters with subsequent
lines indented to the open parenthesis.
- Subsection bodies are indented an additional 4 spaces.
- Body consists of function description, parameters, return description, and
C declaration (`INCLUDE_C_DECL`).
- Parameters are omitted for the `void` and `Error *` types, or if the
parameter is marked as [out].
- Each function documentation is separated by a single line.
"""
import argparse
import os
import re
import sys
import shutil
import textwrap
import subprocess
import collections
import msgpack
import logging
from xml.dom import minidom
MIN_PYTHON_VERSION = (3, 6)
MIN_DOXYGEN_VERSION = (1, 9, 0)
if sys.version_info < MIN_PYTHON_VERSION:
print("requires Python {}.{}+".format(*MIN_PYTHON_VERSION))
sys.exit(1)
doxygen_version = tuple([int(i) for i in subprocess.check_output(["doxygen", "-v"],
universal_newlines=True).split()[0].split('.')])
if doxygen_version < MIN_DOXYGEN_VERSION:
print("\nRequires doxygen {}.{}.{}+".format(*MIN_DOXYGEN_VERSION))
print("Your doxygen version is {}.{}.{}\n".format(*doxygen_version))
sys.exit(1)
# DEBUG = ('DEBUG' in os.environ)
INCLUDE_C_DECL = ('INCLUDE_C_DECL' in os.environ)
INCLUDE_DEPRECATED = ('INCLUDE_DEPRECATED' in os.environ)
log = logging.getLogger(__name__)
LOG_LEVELS = {
logging.getLevelName(level): level for level in [
logging.DEBUG, logging.INFO, logging.ERROR
]
}
text_width = 78
script_path = os.path.abspath(__file__)
base_dir = os.path.dirname(os.path.dirname(script_path))
out_dir = os.path.join(base_dir, 'tmp-{target}-doc')
filter_cmd = '%s %s' % (sys.executable, script_path)
msgs = [] # Messages to show on exit.
lua2dox_filter = os.path.join(base_dir, 'scripts', 'lua2dox_filter')
CONFIG = {
'api': {
'mode': 'c',
'filename': 'api.txt',
# Section ordering.
'section_order': [
'vim.c',
'vimscript.c',
'command.c',
'options.c',
'buffer.c',
'extmark.c',
'window.c',
'win_config.c',
'tabpage.c',
'autocmd.c',
'ui.c',
],
# List of files/directories for doxygen to read, relative to `base_dir`
'files': ['src/nvim/api'],
# file patterns used by doxygen
'file_patterns': '*.h *.c',
# Only function with this prefix are considered
'fn_name_prefix': 'nvim_',
# Section name overrides.
'section_name': {
'vim.c': 'Global',
},
# For generated section names.
'section_fmt': lambda name: f'{name} Functions',
# Section helptag.
'helptag_fmt': lambda name: f'*api-{name.lower()}*',
# Per-function helptag.
'fn_helptag_fmt': lambda fstem, name: f'*{name}()*',
# Module name overrides (for Lua).
'module_override': {},
# Append the docs for these modules, do not start a new section.
'append_only': [],
},
'lua': {
'mode': 'lua',
'filename': 'lua.txt',
'section_order': [
'_editor.lua',
'shared.lua',
'uri.lua',
'ui.lua',
'filetype.lua',
'keymap.lua',
'fs.lua',
],
'files': [
'runtime/lua/vim/_editor.lua',
'runtime/lua/vim/shared.lua',
'runtime/lua/vim/uri.lua',
'runtime/lua/vim/ui.lua',
'runtime/lua/vim/filetype.lua',
'runtime/lua/vim/keymap.lua',
'runtime/lua/vim/fs.lua',
],
'file_patterns': '*.lua',
'fn_name_prefix': '',
'section_name': {
'lsp.lua': 'core',
},
'section_fmt': lambda name: (
'Lua module: vim'
if name.lower() == '_editor'
else f'Lua module: {name.lower()}'),
'helptag_fmt': lambda name: (
'*lua-vim*'
if name.lower() == '_editor'
else f'*lua-{name.lower()}*'),
'fn_helptag_fmt': lambda fstem, name: (
f'*vim.{name}()*'
if fstem.lower() == '_editor'
else f'*{fstem}.{name}()*'),
'module_override': {
# `shared` functions are exposed on the `vim` module.
'shared': 'vim',
'uri': 'vim',
'ui': 'vim.ui',
'filetype': 'vim.filetype',
'keymap': 'vim.keymap',
'fs': 'vim.fs',
},
'append_only': [
'shared.lua',
],
},
'lsp': {
'mode': 'lua',
'filename': 'lsp.txt',
'section_order': [
'lsp.lua',
'buf.lua',
'diagnostic.lua',
'codelens.lua',
'tagfunc.lua',
'handlers.lua',
'util.lua',
'log.lua',
'rpc.lua',
'sync.lua',
'protocol.lua',
],
'files': [
'runtime/lua/vim/lsp',
'runtime/lua/vim/lsp.lua',
],
'file_patterns': '*.lua',
'fn_name_prefix': '',
'section_name': {'lsp.lua': 'lsp'},
'section_fmt': lambda name: (
'Lua module: vim.lsp'
if name.lower() == 'lsp'
else f'Lua module: vim.lsp.{name.lower()}'),
'helptag_fmt': lambda name: (
'*lsp-core*'
if name.lower() == 'lsp'
else f'*lsp-{name.lower()}*'),
'fn_helptag_fmt': lambda fstem, name: (
f'*vim.lsp.{name}()*'
if fstem == 'lsp' and name != 'client'
else (
'*vim.lsp.client*'
# HACK. TODO(justinmk): class/structure support in lua2dox
if 'lsp.client' == f'{fstem}.{name}'
else f'*vim.lsp.{fstem}.{name}()*')),
'module_override': {},
'append_only': [],
},
'diagnostic': {
'mode': 'lua',
'filename': 'diagnostic.txt',
'section_order': [
'diagnostic.lua',
],
'files': ['runtime/lua/vim/diagnostic.lua'],
'file_patterns': '*.lua',
'fn_name_prefix': '',
'section_name': {'diagnostic.lua': 'diagnostic'},
'section_fmt': lambda _: 'Lua module: vim.diagnostic',
'helptag_fmt': lambda _: '*diagnostic-api*',
'fn_helptag_fmt': lambda fstem, name: f'*vim.{fstem}.{name}()*',
'module_override': {},
'append_only': [],
},
'treesitter': {
'mode': 'lua',
'filename': 'treesitter.txt',
'section_order': [
'treesitter.lua',
'language.lua',
'query.lua',
'highlighter.lua',
'languagetree.lua',
],
'files': [
'runtime/lua/vim/treesitter.lua',
'runtime/lua/vim/treesitter/',
],
'file_patterns': '*.lua',
'fn_name_prefix': '',
'section_name': {},
'section_fmt': lambda name: (
'Lua module: vim.treesitter'
if name.lower() == 'treesitter'
else f'Lua module: vim.treesitter.{name.lower()}'),
'helptag_fmt': lambda name: (
'*lua-treesitter-core*'
if name.lower() == 'treesitter'
else f'*treesitter-{name.lower()}*'),
'fn_helptag_fmt': lambda fstem, name: (
f'*{name}()*'
if name != 'new'
else f'*{fstem}.{name}()*'),
# 'fn_helptag_fmt': lambda fstem, name: (
# f'*vim.treesitter.{name}()*'
# if fstem == 'treesitter'
# else (
# '*vim.lsp.client*'
# # HACK. TODO(justinmk): class/structure support in lua2dox
# if 'lsp.client' == f'{fstem}.{name}'
# else f'*vim.lsp.{fstem}.{name}()*')),
'module_override': {},
'append_only': [],
}
}
param_exclude = (
'channel_id',
)
# Annotations are displayed as line items after API function descriptions.
annotation_map = {
'FUNC_API_FAST': '|api-fast|',
'FUNC_API_CHECK_TEXTLOCK': 'not allowed when |textlock| is active',
'FUNC_API_REMOTE_ONLY': '|RPC| only',
'FUNC_API_LUA_ONLY': '|vim.api| only',
}
# Raises an error with details about `o`, if `cond` is in object `o`,
# or if `cond()` is callable and returns True.
def debug_this(o, cond=True):
name = ''
if not isinstance(o, str):
try:
name = o.nodeName
o = o.toprettyxml(indent=' ', newl='\n')
except Exception:
pass
if ((callable(cond) and cond())
or (not callable(cond) and cond)
or (not callable(cond) and cond in o)):
raise RuntimeError('xxx: {}\n{}'.format(name, o))
# Appends a message to a list which will be printed on exit.
def msg(s):
msgs.append(s)
# Print all collected messages.
def msg_report():
for m in msgs:
print(f' {m}')
# Print collected messages, then throw an exception.
def fail(s):
msg_report()
raise RuntimeError(s)
def find_first(parent, name):
"""Finds the first matching node within parent."""
sub = parent.getElementsByTagName(name)
if not sub:
return None
return sub[0]
def iter_children(parent, name):
"""Yields matching child nodes within parent."""
for child in parent.childNodes:
if child.nodeType == child.ELEMENT_NODE and child.nodeName == name:
yield child
def get_child(parent, name):
"""Gets the first matching child node."""
for child in iter_children(parent, name):
return child
return None
def self_or_child(n):
"""Gets the first child node, or self."""
if len(n.childNodes) == 0:
return n
return n.childNodes[0]
def clean_lines(text):
"""Removes superfluous lines.
The beginning and end of the string is trimmed. Empty lines are collapsed.
"""
return re.sub(r'\A\n\s*\n*|\n\s*\n*\Z', '', re.sub(r'(\n\s*\n+)+', '\n\n', text))
def is_blank(text):
return '' == clean_lines(text)
def get_text(n, preformatted=False):
"""Recursively concatenates all text in a node tree."""
text = ''
if n.nodeType == n.TEXT_NODE:
return n.data
if n.nodeName == 'computeroutput':
for node in n.childNodes:
text += get_text(node)
return '`{}`'.format(text)
for node in n.childNodes:
if node.nodeType == node.TEXT_NODE:
text += node.data
elif node.nodeType == node.ELEMENT_NODE:
text += get_text(node, preformatted)
return text
# Gets the length of the last line in `text`, excluding newline ("\n") char.
def len_lastline(text):
lastnl = text.rfind('\n')
if -1 == lastnl:
return len(text)
if '\n' == text[-1]:
return lastnl - (1 + text.rfind('\n', 0, lastnl))
return len(text) - (1 + lastnl)
def len_lastline_withoutindent(text, indent):
n = len_lastline(text)
return (n - len(indent)) if n > len(indent) else 0
# Returns True if node `n` contains only inline (not block-level) elements.
def is_inline(n):
# if len(n.childNodes) == 0:
# return n.nodeType == n.TEXT_NODE or n.nodeName == 'computeroutput'
for c in n.childNodes:
if c.nodeType != c.TEXT_NODE and c.nodeName != 'computeroutput':
return False
if not is_inline(c):
return False
return True
def doc_wrap(text, prefix='', width=70, func=False, indent=None):
"""Wraps text to `width`.
First line is prefixed with `prefix`, subsequent lines are aligned.
If `func` is True, only wrap at commas.
"""
if not width:
# return prefix + text
return text
# Whitespace used to indent all lines except the first line.
indent = ' ' * len(prefix) if indent is None else indent
indent_only = (prefix == '' and indent is not None)
if func:
lines = [prefix]
for part in text.split(', '):
if part[-1] not in ');':
part += ', '
if len(lines[-1]) + len(part) > width:
lines.append(indent)
lines[-1] += part
return '\n'.join(x.rstrip() for x in lines).rstrip()
# XXX: Dummy prefix to force TextWrapper() to wrap the first line.
if indent_only:
prefix = indent
tw = textwrap.TextWrapper(break_long_words=False,
break_on_hyphens=False,
width=width,
initial_indent=prefix,
subsequent_indent=indent)
result = '\n'.join(tw.wrap(text.strip()))
# XXX: Remove the dummy prefix.
if indent_only:
result = result[len(indent):]
return result
def max_name(names):
if len(names) == 0:
return 0
return max(len(name) for name in names)
def update_params_map(parent, ret_map, width=62):
"""Updates `ret_map` with name:desc key-value pairs extracted
from Doxygen XML node `parent`.
"""
params = collections.OrderedDict()
for node in parent.childNodes:
if node.nodeType == node.TEXT_NODE:
continue
name_node = find_first(node, 'parametername')
if name_node.getAttribute('direction') == 'out':
continue
name = get_text(name_node)
if name in param_exclude:
continue
params[name.strip()] = node
max_name_len = max_name(params.keys()) + 8
# `ret_map` is a name:desc map.
for name, node in params.items():
desc = ''
desc_node = get_child(node, 'parameterdescription')
if desc_node:
desc = fmt_node_as_vimhelp(
desc_node, width=width, indent=(' ' * max_name_len))
ret_map[name] = desc
return ret_map
def render_node(n, text, prefix='', indent='', width=62, fmt_vimhelp=False):
"""Renders a node as Vim help text, recursively traversing all descendants."""
def ind(s):
return s if fmt_vimhelp else ''
text = ''
# space_preceding = (len(text) > 0 and ' ' == text[-1][-1])
# text += (int(not space_preceding) * ' ')
if n.nodeName == 'preformatted':
o = get_text(n, preformatted=True)
ensure_nl = '' if o[-1] == '\n' else '\n'
text += '>{}{}\n<'.format(ensure_nl, o)
elif is_inline(n):
text = doc_wrap(get_text(n), indent=indent, width=width)
elif n.nodeName == 'verbatim':
# TODO: currently we don't use this. The "[verbatim]" hint is there as
# a reminder that we must decide how to format this if we do use it.
text += ' [verbatim] {}'.format(get_text(n))
elif n.nodeName == 'listitem':
for c in n.childNodes:
result = render_node(
c,
text,
indent=indent + (' ' * len(prefix)),
width=width
)
if is_blank(result):
continue
text += indent + prefix + result
elif n.nodeName in ('para', 'heading'):
for c in n.childNodes:
if (is_inline(c)
and '' != get_text(c).strip()
and text
and ' ' != text[-1]):
text += ' '
text += render_node(c, text, indent=indent, width=width)
elif n.nodeName == 'itemizedlist':
for c in n.childNodes:
text += '{}\n'.format(render_node(c, text, prefix='• ',
indent=indent, width=width))
elif n.nodeName == 'orderedlist':
i = 1
for c in n.childNodes:
if is_blank(get_text(c)):
text += '\n'
continue
text += '{}\n'.format(render_node(c, text, prefix='{}. '.format(i),
indent=indent, width=width))
i = i + 1
elif n.nodeName == 'simplesect' and 'note' == n.getAttribute('kind'):
text += '\nNote:\n '
for c in n.childNodes:
text += render_node(c, text, indent=' ', width=width)
text += '\n'
elif n.nodeName == 'simplesect' and 'warning' == n.getAttribute('kind'):
text += 'Warning:\n '
for c in n.childNodes:
text += render_node(c, text, indent=' ', width=width)
text += '\n'
elif (n.nodeName == 'simplesect'
and n.getAttribute('kind') in ('return', 'see')):
text += ind(' ')
for c in n.childNodes:
text += render_node(c, text, indent=' ', width=width)
elif n.nodeName == 'computeroutput':
return get_text(n)
else:
raise RuntimeError('unhandled node type: {}\n{}'.format(
n.nodeName, n.toprettyxml(indent=' ', newl='\n')))
return text
def para_as_map(parent, indent='', width=62, fmt_vimhelp=False):
"""Extracts a Doxygen XML <para> node to a map.
Keys:
'text': Text from this <para> element
'params': <parameterlist> map
'return': List of @return strings
'seealso': List of @see strings
'xrefs': ?
"""
chunks = {
'text': '',
'params': collections.OrderedDict(),
'return': [],
'seealso': [],
'xrefs': []
}
# Ordered dict of ordered lists.
groups = collections.OrderedDict([
('params', []),
('return', []),
('seealso', []),
('xrefs', []),
])
# Gather nodes into groups. Mostly this is because we want "parameterlist"
# nodes to appear together.
text = ''
kind = ''
last = ''
if is_inline(parent):
# Flatten inline text from a tree of non-block nodes.
text = doc_wrap(render_node(parent, "", fmt_vimhelp=fmt_vimhelp),
indent=indent, width=width)
else:
prev = None # Previous node
for child in parent.childNodes:
if child.nodeName == 'parameterlist':
groups['params'].append(child)
elif child.nodeName == 'xrefsect':
groups['xrefs'].append(child)
elif child.nodeName == 'simplesect':
last = kind
kind = child.getAttribute('kind')
if kind == 'return' or (kind == 'note' and last == 'return'):
groups['return'].append(child)
elif kind == 'see':
groups['seealso'].append(child)
elif kind in ('note', 'warning'):
text += render_node(child, text, indent=indent,
width=width, fmt_vimhelp=fmt_vimhelp)
else:
raise RuntimeError('unhandled simplesect: {}\n{}'.format(
child.nodeName, child.toprettyxml(indent=' ', newl='\n')))
else:
if (prev is not None
and is_inline(self_or_child(prev))
and is_inline(self_or_child(child))
and '' != get_text(self_or_child(child)).strip()
and text
and ' ' != text[-1]):
text += ' '
text += render_node(child, text, indent=indent, width=width,
fmt_vimhelp=fmt_vimhelp)
prev = child
chunks['text'] += text
# Generate map from the gathered items.
if len(groups['params']) > 0:
for child in groups['params']:
update_params_map(child, ret_map=chunks['params'], width=width)
for child in groups['return']:
chunks['return'].append(render_node(
child, '', indent=indent, width=width, fmt_vimhelp=fmt_vimhelp))
for child in groups['seealso']:
chunks['seealso'].append(render_node(
child, '', indent=indent, width=width, fmt_vimhelp=fmt_vimhelp))
xrefs = set()
for child in groups['xrefs']:
# XXX: Add a space (or any char) to `title` here, otherwise xrefs
# ("Deprecated" section) acts very weird...
title = get_text(get_child(child, 'xreftitle')) + ' '
xrefs.add(title)
xrefdesc = get_text(get_child(child, 'xrefdescription'))
chunks['xrefs'].append(doc_wrap(xrefdesc, prefix='{}: '.format(title),
width=width) + '\n')
return chunks, xrefs
def fmt_node_as_vimhelp(parent, width=62, indent='', fmt_vimhelp=False):
"""Renders (nested) Doxygen <para> nodes as Vim :help text.
NB: Blank lines in a docstring manifest as <para> tags.
"""
rendered_blocks = []
def fmt_param_doc(m):
"""Renders a params map as Vim :help text."""
max_name_len = max_name(m.keys()) + 4
out = ''
for name, desc in m.items():
name = ' {}'.format('{{{}}}'.format(name).ljust(max_name_len))
out += '{}{}\n'.format(name, desc)
return out.rstrip()
def has_nonexcluded_params(m):
"""Returns true if any of the given params has at least
one non-excluded item."""
if fmt_param_doc(m) != '':
return True
for child in parent.childNodes:
para, _ = para_as_map(child, indent, width, fmt_vimhelp)
# Generate text from the gathered items.
chunks = [para['text']]
if len(para['params']) > 0 and has_nonexcluded_params(para['params']):
chunks.append('\nParameters: ~')
chunks.append(fmt_param_doc(para['params']))
if len(para['return']) > 0:
chunks.append('\nReturn: ~')
for s in para['return']:
chunks.append(s)
if len(para['seealso']) > 0:
chunks.append('\nSee also: ~')
for s in para['seealso']:
chunks.append(s)
for s in para['xrefs']:
chunks.append(s)
rendered_blocks.append(clean_lines('\n'.join(chunks).strip()))
rendered_blocks.append('')
return clean_lines('\n'.join(rendered_blocks).strip())
def extract_from_xml(filename, target, width, fmt_vimhelp):
"""Extracts Doxygen info as maps without formatting the text.
Returns two maps:
1. Functions
2. Deprecated functions
The `fmt_vimhelp` variable controls some special cases for use by
fmt_doxygen_xml_as_vimhelp(). (TODO: ugly :)
"""
fns = {} # Map of func_name:docstring.
deprecated_fns = {} # Map of func_name:docstring.
dom = minidom.parse(filename)
compoundname = get_text(dom.getElementsByTagName('compoundname')[0])
for member in dom.getElementsByTagName('memberdef'):
if member.getAttribute('static') == 'yes' or \
member.getAttribute('kind') != 'function' or \
member.getAttribute('prot') == 'private' or \
get_text(get_child(member, 'name')).startswith('_'):
continue
loc = find_first(member, 'location')
if 'private' in loc.getAttribute('file'):
continue
return_type = get_text(get_child(member, 'type'))
if return_type == '':
continue
if return_type.startswith(('ArrayOf', 'DictionaryOf')):
parts = return_type.strip('_').split('_')
return_type = '{}({})'.format(parts[0], ', '.join(parts[1:]))
name = get_text(get_child(member, 'name'))
annotations = get_text(get_child(member, 'argsstring'))
if annotations and ')' in annotations:
annotations = annotations.rsplit(')', 1)[-1].strip()
# XXX: (doxygen 1.8.11) 'argsstring' only includes attributes of
# non-void functions. Special-case void functions here.
if name == 'nvim_get_mode' and len(annotations) == 0:
annotations += 'FUNC_API_FAST'
annotations = filter(None, map(lambda x: annotation_map.get(x),
annotations.split()))
params = []
type_length = 0
for param in iter_children(member, 'param'):
param_type = get_text(get_child(param, 'type')).strip()
param_name = ''
declname = get_child(param, 'declname')
if declname:
param_name = get_text(declname).strip()
elif CONFIG[target]['mode'] == 'lua':
# XXX: this is what lua2dox gives us...
param_name = param_type
param_type = ''
if param_name in param_exclude:
continue
if fmt_vimhelp and param_type.endswith('*'):
param_type = param_type.strip('* ')
param_name = '*' + param_name
type_length = max(type_length, len(param_type))
params.append((param_type, param_name))
# Handle Object Oriented style functions here.
# We make sure they have "self" in the parameters,
# and a parent function
if return_type.startswith('function') \
and len(return_type.split(' ')) >= 2 \
and any(x[1] == 'self' for x in params):
split_return = return_type.split(' ')
name = f'{split_return[1]}:{name}'
c_args = []
for param_type, param_name in params:
c_args.append((' ' if fmt_vimhelp else '') + (
'%s %s' % (param_type.ljust(type_length), param_name)).strip())
if not fmt_vimhelp:
pass
else:
fstem = '?'
if '.' in compoundname:
fstem = compoundname.split('.')[0]
fstem = CONFIG[target]['module_override'].get(fstem, fstem)
vimtag = CONFIG[target]['fn_helptag_fmt'](fstem, name)
prefix = '%s(' % name
suffix = '%s)' % ', '.join('{%s}' % a[1] for a in params
if a[0] not in ('void', 'Error'))
if not fmt_vimhelp:
c_decl = '%s %s(%s);' % (return_type, name, ', '.join(c_args))
signature = prefix + suffix
else:
c_decl = textwrap.indent('%s %s(\n%s\n);' % (return_type, name,
',\n'.join(c_args)),
' ')
# Minimum 8 chars between signature and vimtag
lhs = (width - 8) - len(vimtag)
if len(prefix) + len(suffix) > lhs:
signature = vimtag.rjust(width) + '\n'
signature += doc_wrap(suffix, width=width, prefix=prefix,
func=True)
else:
signature = prefix + suffix
signature += vimtag.rjust(width - len(signature))
# Tracks `xrefsect` titles. As of this writing, used only for separating
# deprecated functions.
xrefs_all = set()
paras = []
brief_desc = find_first(member, 'briefdescription')
if brief_desc:
for child in brief_desc.childNodes:
para, xrefs = para_as_map(child)
xrefs_all.update(xrefs)
desc = find_first(member, 'detaileddescription')
if desc:
for child in desc.childNodes:
para, xrefs = para_as_map(child)
paras.append(para)
xrefs_all.update(xrefs)
log.debug(
textwrap.indent(
re.sub(r'\n\s*\n+', '\n',
desc.toprettyxml(indent=' ', newl='\n')), ' ' * 16))
fn = {
'annotations': list(annotations),
'signature': signature,
'parameters': params,
'parameters_doc': collections.OrderedDict(),
'doc': [],
'return': [],
'seealso': [],
}
if fmt_vimhelp:
fn['desc_node'] = desc
fn['brief_desc_node'] = brief_desc
for m in paras:
if 'text' in m:
if not m['text'] == '':
fn['doc'].append(m['text'])
if 'params' in m:
# Merge OrderedDicts.
fn['parameters_doc'].update(m['params'])
if 'return' in m and len(m['return']) > 0:
fn['return'] += m['return']
if 'seealso' in m and len(m['seealso']) > 0:
fn['seealso'] += m['seealso']
if INCLUDE_C_DECL:
fn['c_decl'] = c_decl
if 'Deprecated' in str(xrefs_all):
deprecated_fns[name] = fn
elif name.startswith(CONFIG[target]['fn_name_prefix']):
fns[name] = fn
fns = collections.OrderedDict(sorted(
fns.items(),
key=lambda key_item_tuple: key_item_tuple[0].lower()))
deprecated_fns = collections.OrderedDict(sorted(deprecated_fns.items()))
return fns, deprecated_fns
def fmt_doxygen_xml_as_vimhelp(filename, target):
"""Entrypoint for generating Vim :help from from Doxygen XML.
Returns 3 items:
1. Vim help text for functions found in `filename`.
2. Vim help text for deprecated functions.
"""
fns_txt = {} # Map of func_name:vim-help-text.
deprecated_fns_txt = {} # Map of func_name:vim-help-text.
fns, _ = extract_from_xml(filename, target, text_width, True)
for name, fn in fns.items():
# Generate Vim :help for parameters.
if fn['desc_node']:
doc = fmt_node_as_vimhelp(fn['desc_node'], fmt_vimhelp=True)
if not doc and fn['brief_desc_node']:
doc = fmt_node_as_vimhelp(fn['brief_desc_node'])
if not doc:
doc = 'TODO: Documentation'
annotations = '\n'.join(fn['annotations'])
if annotations:
annotations = ('\n\nAttributes: ~\n' +
textwrap.indent(annotations, ' '))
i = doc.rfind('Parameters: ~')
if i == -1:
doc += annotations
else:
doc = doc[:i] + annotations + '\n\n' + doc[i:]
if INCLUDE_C_DECL:
doc += '\n\nC Declaration: ~\n>\n'
doc += fn['c_decl']
doc += '\n<'
func_doc = fn['signature'] + '\n'
func_doc += textwrap.indent(clean_lines(doc), ' ' * 16)
# Verbatim handling.
func_doc = re.sub(r'^\s+([<>])$', r'\1', func_doc, flags=re.M)
split_lines = func_doc.split('\n')
start = 0
while True:
try:
start = split_lines.index('>', start)
except ValueError:
break
try:
end = split_lines.index('<', start)
except ValueError:
break
split_lines[start + 1:end] = [
(' ' + x).rstrip()
for x in textwrap.dedent(
"\n".join(
split_lines[start+1:end]
)
).split("\n")
]
start = end
func_doc = "\n".join(split_lines)
if name.startswith(CONFIG[target]['fn_name_prefix']):
fns_txt[name] = func_doc
return ('\n\n'.join(list(fns_txt.values())),
'\n\n'.join(list(deprecated_fns_txt.values())))
def delete_lines_below(filename, tokenstr):
"""Deletes all lines below the line containing `tokenstr`, the line itself,
and one line above it.
"""
lines = open(filename).readlines()
i = 0
found = False
for i, line in enumerate(lines, 1):
if tokenstr in line:
found = True
break
if not found:
raise RuntimeError(f'not found: "{tokenstr}"')
i = max(0, i - 2)
with open(filename, 'wt') as fp:
fp.writelines(lines[0:i])
def main(config, args):
"""Generates:
1. Vim :help docs
2. *.mpack files for use by API clients
Doxygen is called and configured through stdin.
"""
for target in CONFIG:
if args.target is not None and target != args.target:
continue
mpack_file = os.path.join(
base_dir, 'runtime', 'doc',
CONFIG[target]['filename'].replace('.txt', '.mpack'))
if os.path.exists(mpack_file):
os.remove(mpack_file)
output_dir = out_dir.format(target=target)
log.info("Generating documentation for %s in folder %s",
target, output_dir)
debug = args.log_level >= logging.DEBUG
p = subprocess.Popen(
['doxygen', '-'],
stdin=subprocess.PIPE,
# silence warnings
# runtime/lua/vim/lsp.lua:209: warning: argument 'foo' not found
stderr=(subprocess.STDOUT if debug else subprocess.DEVNULL))
p.communicate(
config.format(
input=' '.join(
[f'"{file}"' for file in CONFIG[target]['files']]),
output=output_dir,
filter=filter_cmd,
file_patterns=CONFIG[target]['file_patterns'])
.encode('utf8')
)
if p.returncode:
sys.exit(p.returncode)
fn_map_full = {} # Collects all functions as each module is processed.
sections = {}
intros = {}
sep = '=' * text_width
base = os.path.join(output_dir, 'xml')
dom = minidom.parse(os.path.join(base, 'index.xml'))
# generate docs for section intros
for compound in dom.getElementsByTagName('compound'):
if compound.getAttribute('kind') != 'group':
continue
groupname = get_text(find_first(compound, 'name'))
groupxml = os.path.join(base, '%s.xml' %
compound.getAttribute('refid'))
group_parsed = minidom.parse(groupxml)
doc_list = []
brief_desc = find_first(group_parsed, 'briefdescription')
if brief_desc:
for child in brief_desc.childNodes:
doc_list.append(fmt_node_as_vimhelp(child))
desc = find_first(group_parsed, 'detaileddescription')
if desc:
doc = fmt_node_as_vimhelp(desc)
if doc:
doc_list.append(doc)
intros[groupname] = "\n".join(doc_list)
for compound in dom.getElementsByTagName('compound'):
if compound.getAttribute('kind') != 'file':
continue
filename = get_text(find_first(compound, 'name'))
if filename.endswith('.c') or filename.endswith('.lua'):
xmlfile = os.path.join(base,
'{}.xml'.format(compound.getAttribute('refid')))
# Extract unformatted (*.mpack).
fn_map, _ = extract_from_xml(xmlfile, target, 9999, False)
# Extract formatted (:help).
functions_text, deprecated_text = fmt_doxygen_xml_as_vimhelp(
os.path.join(base, '{}.xml'.format(
compound.getAttribute('refid'))), target)
if not functions_text and not deprecated_text:
continue
else:
name = os.path.splitext(
os.path.basename(filename))[0].lower()
sectname = name.upper() if name == 'ui' else name.title()
doc = ''
intro = intros.get(f'api-{name}')
if intro:
doc += '\n\n' + intro
if functions_text:
doc += '\n\n' + functions_text
if INCLUDE_DEPRECATED and deprecated_text:
doc += f'\n\n\nDeprecated {sectname} Functions: ~\n\n'
doc += deprecated_text
if doc:
filename = os.path.basename(filename)
sectname = CONFIG[target]['section_name'].get(
filename, sectname)
title = CONFIG[target]['section_fmt'](sectname)
helptag = CONFIG[target]['helptag_fmt'](sectname)
sections[filename] = (title, helptag, doc)
fn_map_full.update(fn_map)
if len(sections) == 0:
fail(f'no sections for target: {target}')
if len(sections) > len(CONFIG[target]['section_order']):
raise RuntimeError(
'found new modules "{}"; update the "section_order" map'.format(
set(sections).difference(CONFIG[target]['section_order'])))
first_section_tag = sections[CONFIG[target]['section_order'][0]][1]
docs = ''
i = 0
for filename in CONFIG[target]['section_order']:
try:
title, helptag, section_doc = sections.pop(filename)
except KeyError:
msg(f'warning: empty docs, skipping (target={target}): {filename}')
msg(f' existing docs: {sections.keys()}')
continue
i += 1
if filename not in CONFIG[target]['append_only']:
docs += sep
docs += '\n%s%s' % (title,
helptag.rjust(text_width - len(title)))
docs += section_doc
docs += '\n\n\n'
docs = docs.rstrip() + '\n\n'
docs += ' vim:tw=78:ts=8:ft=help:norl:\n'
doc_file = os.path.join(base_dir, 'runtime', 'doc',
CONFIG[target]['filename'])
if os.path.exists(doc_file):
delete_lines_below(doc_file, first_section_tag)
with open(doc_file, 'ab') as fp:
fp.write(docs.encode('utf8'))
fn_map_full = collections.OrderedDict(sorted(fn_map_full.items()))
with open(mpack_file, 'wb') as fp:
fp.write(msgpack.packb(fn_map_full, use_bin_type=True))
if not args.keep_tmpfiles:
shutil.rmtree(output_dir)
msg_report()
def filter_source(filename):
name, extension = os.path.splitext(filename)
if extension == '.lua':
p = subprocess.run([lua2dox_filter, filename], stdout=subprocess.PIPE)
op = ('?' if 0 != p.returncode else p.stdout.decode('utf-8'))
print(op)
else:
"""Filters the source to fix macros that confuse Doxygen."""
with open(filename, 'rt') as fp:
print(re.sub(r'^(ArrayOf|DictionaryOf)(\(.*?\))',
lambda m: m.group(1)+'_'.join(
re.split(r'[^\w]+', m.group(2))),
fp.read(), flags=re.M))
def parse_args():
targets = ', '.join(CONFIG.keys())
ap = argparse.ArgumentParser(
description="Generate helpdoc from source code")
ap.add_argument(
"--log-level", "-l", choices=LOG_LEVELS.keys(),
default=logging.getLevelName(logging.ERROR), help="Set log verbosity"
)
ap.add_argument('source_filter', nargs='*',
help="Filter source file(s)")
ap.add_argument('-k', '--keep-tmpfiles', action='store_true',
help="Keep temporary files")
ap.add_argument('-t', '--target',
help=f'One of ({targets}), defaults to "all"')
return ap.parse_args()
Doxyfile = textwrap.dedent('''
OUTPUT_DIRECTORY = {output}
INPUT = {input}
INPUT_ENCODING = UTF-8
FILE_PATTERNS = {file_patterns}
RECURSIVE = YES
INPUT_FILTER = "{filter}"
EXCLUDE =
EXCLUDE_SYMLINKS = NO
EXCLUDE_PATTERNS = */private/* */health.lua */_*.lua
EXCLUDE_SYMBOLS =
EXTENSION_MAPPING = lua=C
EXTRACT_PRIVATE = NO
GENERATE_HTML = NO
GENERATE_DOCSET = NO
GENERATE_HTMLHELP = NO
GENERATE_QHP = NO
GENERATE_TREEVIEW = NO
GENERATE_LATEX = NO
GENERATE_RTF = NO
GENERATE_MAN = NO
GENERATE_DOCBOOK = NO
GENERATE_AUTOGEN_DEF = NO
GENERATE_XML = YES
XML_OUTPUT = xml
XML_PROGRAMLISTING = NO
ENABLE_PREPROCESSING = YES
MACRO_EXPANSION = YES
EXPAND_ONLY_PREDEF = NO
MARKDOWN_SUPPORT = YES
''')
if __name__ == "__main__":
args = parse_args()
print("Setting log level to %s" % args.log_level)
args.log_level = LOG_LEVELS[args.log_level]
log.setLevel(args.log_level)
log.addHandler(logging.StreamHandler())
if len(args.source_filter) > 0:
filter_source(args.source_filter[0])
else:
main(Doxyfile, args)
# vim: set ft=python ts=4 sw=4 tw=79 et :
Reference in New Issue View Git Blame Copy Permalink