neovim/scripts/gen_api_vimdoc.py
Tommy Allen fd9cc8b0b2 automation: Generate API documentation (#5798)
runtime: Add underscore to {} helpSpecial syntax pattern

docs: Added generated api-funcs.txt
2017-01-03 13:11:19 +01:00

515 lines
16 KiB
Python

#!/usr/bin/env python
"""Parses Doxygen XML output to generate Neovim's API documentation.
This would be easier using lxml and XSLT, but:
1. This should avoid needing Python dependencies, especially ones that are
C modules that have library dependencies (lxml requires libxml and
libxslt).
2. I wouldn't know how to deal with nested indentation in <para> tags using
XSLT.
Each function documentation is formatted with the following rules:
- Maximum width of 78 characters (`text_width`).
- Spaces for indentation.
- Function signature and helptag are on the same line.
- Helptag is right aligned.
- Signature and helptag must have a minimum of 8 spaces between them.
- If the signature is too long, it is placed on the line after the
helptag. The signature wraps at `text_width - 8` characters with
subsequent lines indented to the open parenthesis.
- Documentation body will be indented by 16 spaces.
- Subsection bodies are indented an additional 4 spaces.
- Documentation body consists of the function description, parameter details,
return description, and C declaration.
- Parameters are omitted for the `void` and `Error *` types, or if the
parameter is marked as [out].
- Each function documentation is separated by a single line.
The C declaration is added to the end to show actual argument types.
"""
import os
import re
import sys
import shutil
import textwrap
import subprocess
from xml.dom import minidom
# Text at the top of the doc file.
preamble = '''
Note: This documentation is generated from Neovim's API source code.
'''
doc_filename = 'api-funcs.txt'
# Section name overrides.
section_name = {
'vim.c': 'Global',
}
# Section ordering.
section_order = (
'vim.c',
'buffer.c',
'window.c',
'tabpage.c',
'ui.c',
)
param_exclude = (
'channel_id',
)
text_width = 78
script_path = os.path.abspath(__file__)
base_dir = os.path.dirname(os.path.dirname(script_path))
src_dir = os.path.join(base_dir, 'src/nvim/api')
out_dir = os.path.join(base_dir, 'tmp/api_doc')
filter_cmd = '%s %s' % (sys.executable, script_path)
seen_funcs = set()
# Tracks `xrefsect` titles. As of this writing, used only for separating
# deprecated functions.
xrefs = set()
# XML Parsing Utilities {{{
def find_first(parent, name):
"""Finds the first matching node within parent."""
sub = parent.getElementsByTagName(name)
if not sub:
return None
return sub[0]
def get_children(parent, name):
"""Yield matching child nodes within parent."""
for child in parent.childNodes:
if child.nodeType == child.ELEMENT_NODE and child.nodeName == name:
yield child
def get_child(parent, name):
"""Get the first matching child node."""
for child in get_children(parent, name):
return child
return None
def clean_text(text):
"""Cleans text.
Only cleans superfluous whitespace at the moment.
"""
return ' '.join(text.split()).strip()
def clean_lines(text):
"""Removes superfluous lines.
The beginning and end of the string is trimmed. Empty lines are collapsed.
"""
return re.sub(r'\A\n\s*\n*|\n\s*\n*\Z', '', re.sub(r'(\n\s*\n+)+', '\n\n', text))
def get_text(parent):
"""Combine all text in a node."""
if parent.nodeType == parent.TEXT_NODE:
return parent.data
out = ''
for node in parent.childNodes:
if node.nodeType == node.TEXT_NODE:
out += clean_text(node.data)
elif node.nodeType == node.ELEMENT_NODE:
out += ' ' + get_text(node)
return out
def doc_wrap(text, prefix='', width=70, func=False):
"""Wraps text to `width`.
The first line is prefixed with `prefix`, and subsequent lines are aligned.
If `func` is True, only wrap at commas.
"""
if not width:
return text
indent_space = ' ' * len(prefix)
if func:
lines = [prefix]
for part in text.split(', '):
if part[-1] not in ');':
part += ', '
if len(lines[-1]) + len(part) > width:
lines.append(indent_space)
lines[-1] += part
return '\n'.join(x.rstrip() for x in lines).rstrip()
return '\n'.join(textwrap.wrap(text.strip(), width=width,
initial_indent=prefix,
subsequent_indent=indent_space))
def parse_params(parent, width=62):
"""Parse Doxygen `parameterlist`."""
name_length = 0
items = []
for child in parent.childNodes:
if child.nodeType == child.TEXT_NODE:
continue
name_node = find_first(child, 'parametername')
if name_node.getAttribute('direction') == 'out':
continue
name = get_text(name_node)
if name in param_exclude:
continue
name = '{%s}' % name
name_length = max(name_length, len(name) + 2)
desc = ''
desc_node = get_child(child, 'parameterdescription')
if desc_node:
desc = parse_parblock(desc_node, width=None)
items.append((name.strip(), desc.strip()))
out = 'Parameters:~\n'
for name, desc in items:
name = ' %s' % name.ljust(name_length)
out += doc_wrap(desc, prefix=name, width=width) + '\n'
return out.strip()
def parse_para(parent, width=62):
"""Parse doxygen `para` tag.
I assume <para> is a paragraph block or "a block of text". It can contain
text nodes, or other tags.
"""
line = ''
lines = []
for child in parent.childNodes:
if child.nodeType == child.TEXT_NODE:
line += child.data
elif child.nodeName == 'computeroutput':
line += '`%s`' % get_text(child)
else:
if line:
lines.append(doc_wrap(line, width=width))
line = ''
if child.nodeName == 'parameterlist':
lines.append(parse_params(child, width=width))
elif child.nodeName == 'xrefsect':
title = get_text(get_child(child, 'xreftitle'))
xrefs.add(title)
xrefdesc = parse_para(get_child(child, 'xrefdescription'))
lines.append(doc_wrap(xrefdesc, prefix='%s: ' % title,
width=width) + '\n')
elif child.nodeName == 'simplesect':
kind = child.getAttribute('kind')
if kind == 'return':
lines.append('%s:~' % kind.title())
lines.append(doc_wrap(parse_para(child),
prefix=' ',
width=width))
else:
lines.append(get_text(child))
if line:
lines.append(doc_wrap(line, width=width))
return clean_lines('\n'.join(lines).strip())
def parse_parblock(parent, width=62):
"""Parses a nested block of `para` tags.
Named after the \parblock command, but not directly related.
"""
paragraphs = []
for child in parent.childNodes:
if child.nodeType == child.TEXT_NODE:
paragraphs.append(doc_wrap(child.data, width=width))
elif child.nodeName == 'para':
paragraphs.append(parse_para(child, width=width))
else:
paragraphs.append(doc_wrap(get_text(child), width=width))
paragraphs.append('')
return clean_lines('\n'.join(paragraphs).strip())
# }}}
def parse_source_xml(filename):
"""Collects API functions.
This returns two strings:
1. The API functions
2. The deprecated API functions
The caller decides what to do with the deprecated documentation.
"""
global xrefs
xrefs = set()
functions = []
deprecated_functions = []
dom = minidom.parse(filename)
for member in dom.getElementsByTagName('memberdef'):
if member.getAttribute('static') == 'yes' or \
member.getAttribute('kind') != 'function':
continue
loc = find_first(member, 'location')
if 'private' in loc.getAttribute('file'):
continue
return_type = get_text(get_child(member, 'type'))
if return_type == '':
continue
if return_type.startswith(('ArrayOf', 'DictionaryOf')):
parts = return_type.strip('_').split('_')
return_type = '%s(%s)' % (parts[0], ', '.join(parts[1:]))
name = get_text(get_child(member, 'name'))
vimtag = '*%s()*' % name
args = []
type_length = 0
for param in get_children(member, 'param'):
arg_type = get_text(get_child(param, 'type')).strip()
arg_name = ''
declname = get_child(param, 'declname')
if declname:
arg_name = get_text(declname).strip()
if arg_name in param_exclude:
continue
if arg_type.endswith('*'):
arg_type = arg_type.strip('* ')
arg_name = '*' + arg_name
type_length = max(type_length, len(arg_type))
args.append((arg_type, arg_name))
c_args = []
for arg_type, arg_name in args:
c_args.append(' ' + (
'%s %s' % (arg_type.ljust(type_length), arg_name)).strip())
c_decl = textwrap.indent('%s %s(\n%s\n);' % (return_type, name,
',\n'.join(c_args)),
' ')
prefix = '%s(' % name
suffix = '%s)' % ', '.join('{%s}' % a[1] for a in args
if a[0] not in ('void', 'Error'))
# Minimum 8 chars between signature and vimtag
lhs = (text_width - 8) - len(prefix)
if len(prefix) + len(suffix) > lhs:
signature = vimtag.rjust(text_width) + '\n'
signature += doc_wrap(suffix, width=text_width-8, prefix=prefix,
func=True)
else:
signature = prefix + suffix
signature += vimtag.rjust(text_width - len(signature))
doc = ''
desc = find_first(member, 'detaileddescription')
if desc:
doc = parse_parblock(desc)
if 'DEBUG' in os.environ:
print(textwrap.indent(
re.sub(r'\n\s*\n+', '\n',
desc.toprettyxml(indent=' ', newl='\n')), ' ' * 16))
if not doc:
doc = 'TODO: Documentation'
if 'INCLUDE_C_DECL' in os.environ:
doc += '\n\nC Declaration:~\n>\n'
doc += c_decl
doc += '\n<'
func_doc = signature + '\n'
func_doc += textwrap.indent(clean_lines(doc), ' ' * 16)
func_doc = re.sub(r'^\s+([<>])$', r'\1', func_doc, flags=re.M)
if 'Deprecated' in xrefs:
deprecated_functions.append(func_doc)
else:
functions.append(func_doc)
xrefs.clear()
return '\n\n'.join(functions), '\n\n'.join(deprecated_functions)
def gen_docs(config):
"""Generate documentation.
Doxygen is called and configured through stdin.
"""
p = subprocess.Popen(['doxygen', '-'], stdin=subprocess.PIPE)
p.communicate(config.format(input=src_dir, output=out_dir,
filter=filter_cmd).encode('utf8'))
if p.returncode:
sys.exit(p.returncode)
title_length = 0
sections = {}
sep = '=' * text_width
base = os.path.join(out_dir, 'xml')
dom = minidom.parse(os.path.join(base, 'index.xml'))
for compound in dom.getElementsByTagName('compound'):
if compound.getAttribute('kind') != 'file':
continue
filename = get_text(find_first(compound, 'name'))
if filename.endswith('.c'):
functions, deprecated = parse_source_xml(
os.path.join(base, '%s.xml' % compound.getAttribute('refid')))
if not functions and not deprecated:
continue
if functions or deprecated:
name = os.path.splitext(os.path.basename(filename))[0]
if name == 'ui':
name = name.upper()
else:
name = name.title()
doc = ''
if functions:
doc += '\n\n' + functions
if 'INCLUDE_DEPRECATED' in os.environ and deprecated:
doc += '\n\n\nDeprecated %s Functions:~\n\n' % name
doc += deprecated
if doc:
filename = os.path.basename(filename)
name = section_name.get(filename, name)
title = '%s Functions' % name
helptag = '*api-%s*' % name.lower()
title_length = max(title_length, len(title))
sections[filename] = (title, helptag, doc)
if not sections:
return
title_left = '*%s*' % doc_filename
title_center = 'Neovim API Function Reference'
title_right = '{Nvim}'
margin = max(len(title_left), len(title_right))
head = (title_left.ljust(margin) +
title_center.center(text_width - margin * 2) +
title_right.rjust(margin)) + '\n'
head += '\n%s\n\n' % doc_wrap(preamble, width=text_width)
head += 'Contents:\n\n'
docs = ''
title_length += len(str(len(section_order))) + 2
i = 0
for filename in section_order:
if filename not in sections:
continue
title, helptag, section_doc = sections.pop(filename)
i += 1
docs += sep
title = '%d. %s' % (i, title)
head += (title.ljust(title_length) + ' ' +
helptag.replace('*', '|') + '\n')
docs += '\n%s%s' % (title, helptag.rjust(text_width - len(title)))
docs += section_doc
docs += '\n\n\n'
if sections:
# In case new API sources are added without updating the order dict.
for title, helptag, section_doc in sections.values():
i += 1
docs += sep
title = '%d. %s' % (i, title)
head += (title.ljust(title_length) + ' ' +
helptag.replace('*', '|') + '\n')
docs += '\n%s%s' % (title, helptag.rjust(text_width - len(title)))
docs += section_doc
docs += '\n\n\n'
docs = '%s\n%s' % (head, docs)
docs = docs.rstrip() + '\n\n'
docs += ' vim:tw=78:ts=8:ft=help:norl:'
doc_file = os.path.join(base_dir, 'runtime/doc', doc_filename)
with open(doc_file, 'wb') as fp:
fp.write(docs.encode('utf8'))
shutil.rmtree(out_dir)
def filter_source(filename):
"""Filters the source to fix macros that confuse Doxygen."""
with open(filename, 'rt') as fp:
print(re.sub(r'^(ArrayOf|DictionaryOf)(\(.*?\))',
lambda m: m.group(1)+'_'.join(
re.split(r'[^\w]+', m.group(2))),
fp.read(), flags=re.M))
# Doxygen Config {{{
Doxyfile = '''
OUTPUT_DIRECTORY = {output}
INPUT = {input}
INPUT_ENCODING = UTF-8
FILE_PATTERNS = *.h *.c
RECURSIVE = YES
INPUT_FILTER = "{filter}"
EXCLUDE =
EXCLUDE_SYMLINKS = NO
EXCLUDE_PATTERNS = */private/*
EXCLUDE_SYMBOLS =
GENERATE_HTML = NO
GENERATE_DOCSET = NO
GENERATE_HTMLHELP = NO
GENERATE_QHP = NO
GENERATE_TREEVIEW = NO
GENERATE_LATEX = NO
GENERATE_RTF = NO
GENERATE_MAN = NO
GENERATE_DOCBOOK = NO
GENERATE_AUTOGEN_DEF = NO
GENERATE_XML = YES
XML_OUTPUT = xml
XML_PROGRAMLISTING = NO
ENABLE_PREPROCESSING = YES
MACRO_EXPANSION = YES
EXPAND_ONLY_PREDEF = NO
'''
# }}}
if __name__ == "__main__":
if len(sys.argv) > 1:
filter_source(sys.argv[1])
else:
gen_docs(Doxyfile)
# vim: set ft=python ts=4 sw=4 tw=79 et fdm=marker :