fix(docs-html): update parser

- Improve generated HTML by updating parser which includes fixes for
  single "'" and single "|":
  https://github.com/neovim/tree-sitter-vimdoc/pull/31
- Updated parser also fixes the conceal issue for "help" highlight
  queries https://github.com/neovim/tree-sitter-vimdoc/issues/23 by
  NOT including whitespace in nodes.
  - But this means we need to restore the getws() function which scrapes
    leading whitespace from the original input (buffer).

(cherry picked from commit a7a83bc4c2)

runtime/doc/develop.txt

@@ -164,6 +164,9 @@ Strict "vimdoc" subset:
   that you don't want auto-wrapped. Lists are always rendered with "flow"
   (soft-wrapped) layout instead of preformatted (hard-wrapped) layout common
   in legacy :help docs.
+  - Limitation: currently the parser https://github.com/neovim/tree-sitter-vimdoc
+    does not understand numbered listitems, so use a bullet symbol (- or •)
+    before numbered items, e.g. "- 1." instead of "1.".
 - Separate blocks (paragraphs) of content by a blank line(s).
 - Do not use indentation in random places—that prevents the page from using
   "flow" layout. If you need a preformatted section, put it in

runtime/doc/lua.txt

@@ -144,7 +144,7 @@ documentation at https://www.lua.org/manual/5.1/manual.html#pdf-require.
 
 For example, if 'runtimepath' is `foo,bar` and |package.cpath| was
 `./?.so;./?.dll` at startup, `require('mod')` searches these paths in order
-and loads the first module found ("first wins"):
+and loads the first module found ("first wins"): >
     foo/lua/mod.lua
     foo/lua/mod/init.lua
     bar/lua/mod.lua
@@ -153,7 +153,7 @@ and loads the first module found ("first wins"):
     foo/lua/mod.dll
     bar/lua/mod.so
     bar/lua/mod.dll
-
+<
                                                         *lua-package-path*
 Nvim automatically adjusts |package.path| and |package.cpath| according to the
 effective 'runtimepath' value. Adjustment happens whenever 'runtimepath' is
@@ -166,37 +166,33 @@ added to |package.cpath|. In this case, instead of appending `/lua/?.lua` and
 `/lua/?/init.lua` to each runtimepath, all unique `?`-containing suffixes of
 the existing |package.cpath| are used. Example:
 
-1. Given that
+- 1. Given that
    - 'runtimepath' contains `/foo/bar,/xxx;yyy/baz,/abc`;
-   - initial (defined at compile-time or derived from
+   - initial |package.cpath| (defined at compile-time or derived from
-     `$LUA_CPATH`/`$LUA_INIT`) |package.cpath| contains
+     `$LUA_CPATH` / `$LUA_INIT`) contains `./?.so;/def/ghi/a?d/j/g.elf;/def/?.so`.
-     `./?.so;/def/ghi/a?d/j/g.elf;/def/?.so`.
+- 2. It finds `?`-containing suffixes `/?.so`, `/a?d/j/g.elf` and `/?.so`, in
-2. It finds `?`-containing suffixes `/?.so`, `/a?d/j/g.elf` and `/?.so`, in
+     order: parts of the path starting from the first path component containing
-   order: parts of the path starting from the first path component containing
+     question mark and preceding path separator.
-   question mark and preceding path separator.
+- 3. The suffix of `/def/?.so`, namely `/?.so` is not unique, as it’s the same
-3. The suffix of `/def/?.so`, namely `/?.so` is not unique, as it’s the same
+     as the suffix of the first path from |package.path| (i.e. `./?.so`). Which
-   as the suffix of the first path from |package.path| (i.e. `./?.so`). Which
+     leaves `/?.so` and `/a?d/j/g.elf`, in this order.
-   leaves `/?.so` and `/a?d/j/g.elf`, in this order.
+- 4. 'runtimepath' has three paths: `/foo/bar`, `/xxx;yyy/baz` and `/abc`. The
-4. 'runtimepath' has three paths: `/foo/bar`, `/xxx;yyy/baz` and `/abc`. The
+     second one contains a semicolon which is a paths separator so it is out,
-   second one contains a semicolon which is a paths separator so it is out,
+     leaving only `/foo/bar` and `/abc`, in order.
-   leaving only `/foo/bar` and `/abc`, in order.
+- 5. The cartesian product of paths from 4. and suffixes from 3. is taken,
-5. The cartesian product of paths from 4. and suffixes from 3. is taken,
+     giving four variants. In each variant a `/lua` path segment is inserted
-   giving four variants. In each variant, a `/lua` path segment is inserted
+     between path and suffix, leaving:
-   between path and suffix, leaving:
+     - `/foo/bar/lua/?.so`
+     - `/foo/bar/lua/a?d/j/g.elf`
+     - `/abc/lua/?.so`
+     - `/abc/lua/a?d/j/g.elf`
+- 6. New paths are prepended to the original |package.cpath|.
 
-   - `/foo/bar/lua/?.so`
+The result will look like this: >
-   - `/foo/bar/lua/a?d/j/g.elf`
-   - `/abc/lua/?.so`
-   - `/abc/lua/a?d/j/g.elf`
 
-6. New paths are prepended to the original |package.cpath|.
+    /foo/bar,/xxx;yyy/baz,/abc ('runtimepath')
-
+    × ./?.so;/def/ghi/a?d/j/g.elf;/def/?.so (package.cpath)
-The result will look like this:
+    = /foo/bar/lua/?.so;/foo/bar/lua/a?d/j/g.elf;/abc/lua/?.so;/abc/lua/a?d/j/g.elf;./?.so;/def/ghi/a?d/j/g.elf;/def/?.so
-
-    `/foo/bar,/xxx;yyy/baz,/abc` ('runtimepath')
-    × `./?.so;/def/ghi/a?d/j/g.elf;/def/?.so` (`package.cpath`)
-
-    = `/foo/bar/lua/?.so;/foo/bar/lua/a?d/j/g.elf;/abc/lua/?.so;/abc/lua/a?d/j/g.elf;./?.so;/def/ghi/a?d/j/g.elf;/def/?.so`
 
 Note:
 
@@ -278,7 +274,7 @@ arguments separated by " " (space) instead of "\t" (tab).
         :lua require"lpeg"
         :lua -- balanced parenthesis grammar:
         :lua bp = lpeg.P{ "(" * ((1 - lpeg.S"()") + lpeg.V(1))^0 * ")" }
-        :luado if bp:match(line) then return "-->\t" .. line end
+        :luado if bp:match(line) then return "=>\t" .. line end
 <
                                                                     *:luafile*
 :luafile {file}
@@ -595,12 +591,12 @@ vim.highlight.range({bufnr}, {ns}, {hlgroup}, {start}, {finish}, {opts})
     Apply highlight group to range of text.
 
             Parameters: ~
-                {bufnr}   buffer number
+              • {bufnr}   buffer number
-                {ns}      namespace for highlights
+              • {ns}      namespace for highlights
-                {hlgroup} highlight group name
+              • {hlgroup} highlight group name
-                {start}   starting position (tuple {line,col})
+              • {start}   starting position (tuple {line,col})
-                {finish}  finish position (tuple {line,col})
+              • {finish}  finish position (tuple {line,col})
-                {opts}    optional parameters:
+              • {opts}    optional parameters:
                           • `regtype`: type of range (characterwise, linewise,
                             or blockwise, see |setreg()|), default `'v'`
                           • `inclusive`: range includes end position,
@@ -653,22 +649,22 @@ vim.diff({a}, {b}, {opts})                                        *vim.diff()*
     Examples: >
 
         vim.diff('a\n', 'b\nc\n')
-        -->
+        =>
         @@ -1 +1,2 @@
         -a
         +b
         +c
 
         vim.diff('a\n', 'b\nc\n', {result_type = 'indices'})
-        -->
+        =>
         {
             {1, 1, 1, 2}
         }
 <
     Parameters: ~
-        {a}      First string to compare
+      • {a}      First string to compare
-        {b}      Second string to compare
+      • {b}      Second string to compare
-        {opts}   Optional parameters:
+      • {opts}   Optional parameters:
                  • `on_hunk` (callback):
                    Invoked for each hunk in the diff. Return a negative number
                    to cancel the callback for any remaining hunks.
@@ -734,13 +730,13 @@ vim.spell.check({str})                                     *vim.spell.check()*
     Example: >
 
         vim.spell.check("the quik brown fox")
-        -->
+        =>
         {
             {'quik', 'bad', 4}
         }
 <
     Parameters: ~
-        {str}    String to spell check.
+      • {str}    String to spell check.
 
     Return: ~
       List of tuples with three items:
@@ -829,9 +825,9 @@ vim.iconv({str}, {from}, {to}[, {opts}])                        *vim.iconv()*
         can accept, see ":Man 3 iconv".
 
         Parameters: ~
-            {str}   (string) Text to convert
+          • {str}   (string) Text to convert
-            {from}  (string) Encoding of {str}
+          • {from}  (string) Encoding of {str}
-            {to}    (string) Target encoding
+          • {to}    (string) Target encoding
 
         Returns: ~
             Converted string if conversion succeeds, `nil` otherwise.
@@ -849,8 +845,8 @@ vim.defer_fn({fn}, {timeout})                                   *vim.defer_fn*
     safe to call.
 
     Parameters: ~
-        {fn}        Callback to call once {timeout} expires
+      • {fn}        Callback to call once {timeout} expires
-        {timeout}   Time in ms to wait before calling {fn}
+      • {timeout}   Time in ms to wait before calling {fn}
 
     Returns: ~
         |vim.loop|.new_timer() object
@@ -863,10 +859,10 @@ vim.wait({time} [, {callback}, {interval}, {fast_only}])          *vim.wait()*
     this time.
 
     Parameters: ~
-        {time}      Number of milliseconds to wait
+      • {time}      Number of milliseconds to wait
-        {callback}  Optional callback. Waits until {callback} returns true
+      • {callback}  Optional callback. Waits until {callback} returns true
-        {interval}  (Approximate) number of milliseconds to wait between polls
+      • {interval}  (Approximate) number of milliseconds to wait between polls
-        {fast_only} If true, only |api-fast| events will be processed.
+      • {fast_only} If true, only |api-fast| events will be processed.
                         If called from while in an |api-fast| event, will
                         automatically be set to `true`.
 
@@ -1254,7 +1250,7 @@ Option:get()
         --  { space = "_", tab = ">~", }
 
         for char, representation in pairs(vim.opt.listchars:get()) do
-            print(char, "->", representation)
+            print(char, "=>", representation)
         end
 <
     For values that are lists of flags, a set will be returned with the flags
@@ -1634,10 +1630,10 @@ split({s}, {sep}, {kwargs})                                      *vim.split()*
 
     Examples: >
 
-      split(":aa::b:", ":")     --> {'','aa','','b',''}
+      split(":aa::b:", ":")     => {'','aa','','b',''}
-      split("axaby", "ab?")     --> {'','x','y'}
+      split("axaby", "ab?")     => {'','x','y'}
-      split("x*yz*o", "*", {plain=true})  --> {'x','yz','o'}
+      split("x*yz*o", "*", {plain=true})  => {'x','yz','o'}
-      split("|x|y|z|", "|", {trimempty=true}) --> {'x', 'y', 'z'}
+      split("|x|y|z|", "|", {trimempty=true}) => {'x', 'y', 'z'}
 <
 
     Parameters: ~

runtime/lua/vim/shared.lua

@@ -99,10 +99,10 @@ end
 ---
 --- Examples:
 --- <pre>
----  split(":aa::b:", ":")     --> {'','aa','','b',''}
+---  split(":aa::b:", ":")     => {'','aa','','b',''}
----  split("axaby", "ab?")     --> {'','x','y'}
+---  split("axaby", "ab?")     => {'','x','y'}
----  split("x*yz*o", "*", {plain=true})  --> {'x','yz','o'}
+---  split("x*yz*o", "*", {plain=true})  => {'x','yz','o'}
----  split("|x|y|z|", "|", {trimempty=true}) --> {'x', 'y', 'z'}
+---  split("|x|y|z|", "|", {trimempty=true}) => {'x', 'y', 'z'}
 --- </pre>
 ---
 ---@see |vim.gsplit()|

scripts/gen_help_html.lua

@@ -248,6 +248,16 @@ local function getbuflinestr(node, bufnr, offset)
   return table.concat(lines, '\n')
 end
 
+-- Gets the whitespace just before `node` from the raw buffer text.
+-- Needed for preformatted `old` lines.
+local function getws(node, bufnr)
+  local line1, c1, line2, _ = node:range()
+  local raw = vim.fn.getbufline(bufnr, line1 + 1, line2 + 1)[1]
+  local text_before = raw:sub(1, c1)
+  local leading_ws = text_before:match('%s+$') or ''
+  return leading_ws
+end
+
 local function get_tagname(node, bufnr)
   local text = vim.treesitter.get_node_text(node, bufnr)
   local tag = (node:type() == 'optionlink' or node:parent():type() == 'optionlink') and ("'%s'"):format(text) or text
@@ -353,12 +363,21 @@ local function visit_node(root, level, lang_tree, headings, opt, stats)
   local parent = root:parent() and root:parent():type() or nil
   local text = ''
   local trimmed
-  local function node_text(node)
+  -- Gets leading whitespace of `node`.
-    return vim.treesitter.get_node_text(node or root, opt.buf)
+  local function ws(node)
+    node = node or root
+    local ws_ = getws(node, opt.buf)
+    -- XXX: first node of a (line) includes whitespace, even after
+    -- https://github.com/neovim/tree-sitter-vimdoc/pull/31 ?
+    if ws_ == '' then
+      ws_ = vim.treesitter.get_node_text(node, opt.buf):match('^%s+') or ''
+    end
+    return ws_
   end
-  -- Gets leading whitespace of the current node.
+  local function node_text(node, ws_)
-  local function ws()
+    node = node or root
-    return node_text():match('^%s+') or ''
+    ws_ = (ws_ == nil or ws_ == true) and getws(node, opt.buf) or ''
+    return string.format('%s%s', ws_, vim.treesitter.get_node_text(node, opt.buf))
   end
 
   if root:child_count() == 0 or node_name == 'ERROR' then
@@ -400,7 +419,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats)
     if root:has_error() then
       return text
     end
-    return ('<div class="help-column_heading">%s%s</div>'):format(ws(), trimmed)
+    return ('<div class="help-column_heading">%s</div>'):format(text)
   elseif node_name == 'block' then
     if is_blank(text) then
       return ''
@@ -461,9 +480,9 @@ local function visit_node(root, level, lang_tree, headings, opt, stats)
     end
     local in_heading = vim.tbl_contains({'h1', 'h2', 'h3'}, parent)
     local cssclass = (not in_heading and get_indent(node_text()) > 8) and 'help-tag-right' or 'help-tag'
-    local tagname = node_text(root:child(1))
+    local tagname = node_text(root:child(1), false)
     if vim.tbl_count(stats.first_tags) < 2 then
-      -- First 2 tags in the doc will be anchored at the main heading.
+      -- Force the first 2 tags in the doc to be anchored at the main heading.
       table.insert(stats.first_tags, tagname)
       return ''
     end