feat(treesitter): vertical conceal for conceal nodes

TSHighlighter now places marks for conceal_lines metadata. A new
internal decor provider callback _on_conceal_line was added that
instructs the highlighter to place conceal_lines marks whenever the
editor needs to know whether a line is concealed. The bundled markdown
queries use `conceal_lines` metadata to conceal code block fence lines.

runtime/doc/news.txt

@@ -291,6 +291,8 @@ TREESITTER
 • |treesitter-directive-trim!| can trim all whitespace (not just empty lines)
   from both sides of a node.
 • |vim.treesitter.get_captures_at_pos()| now returns the `id` of each capture
+• Bundled markdown highlight queries use `conceal_lines` metadata to conceal
+  code block fence lines vertically.
 
 TUI
 

runtime/doc/treesitter.txt

@@ -493,10 +493,10 @@ capture marks comments as to be checked: >query
 There is also `@nospell` which disables spellchecking regions with `@spell`.
 
                                                 *treesitter-highlight-conceal*
-Treesitter highlighting supports |conceal| via the `conceal` metadata. By
+Treesitter highlighting supports |conceal| via the `conceal` and `conceal_lines`
-convention, nodes to be concealed are captured as `@conceal`, but any capture
+metadata. By convention, nodes to be concealed are captured as `@conceal`, but
-can be used. For example, the following query can be used to hide code block
+any capture can be used. For example, the following query can be used to hide
-delimiters in Markdown: >query
+code block delimiters in Markdown: >query
 
     (fenced_code_block_delimiter @conceal (#set! conceal ""))
 <
@@ -507,7 +507,13 @@ still highlighted the same as other operators: >query
 
     "!=" @operator (#set! conceal "≠")
 <
-Conceals specified in this way respect 'conceallevel'.
+To conceal an entire line (do not draw it at all), a query with `conceal_lines`
+metadata can be used: >query
+
+    ((comment) @comment @spell
+      (#set! conceal_lines ""))
+<
+Conceals specified in this way respect 'conceallevel' and 'concealcursor'.
 
                                                *treesitter-highlight-priority*
 Treesitter uses |nvim_buf_set_extmark()| to set highlights with a default

runtime/lua/vim/_meta/api_keysets.lua

@@ -234,6 +234,7 @@ error('Cannot require a meta file')
 --- @field on_end? fun(_: "end", tick: integer)
 --- @field _on_hl_def? fun(_: "hl_def")
 --- @field _on_spell_nav? fun(_: "spell_nav")
+--- @field _on_conceal_line? fun(_: "conceal_line")
 
 --- @class vim.api.keyset.set_extmark
 --- @field id? integer

runtime/lua/vim/treesitter/highlighter.lua

@@ -67,6 +67,8 @@ end
 --- This state is kept during rendering across each line update.
 ---@field private _highlight_states vim.treesitter.highlighter.State[]
 ---@field private _queries table<string,vim.treesitter.highlighter.Query>
+---@field  _conceal_line boolean?
+---@field  _conceal_checked table<integer, boolean>
 ---@field tree vim.treesitter.LanguageTree
 ---@field private redraw_count integer
 local TSHighlighter = {
@@ -114,7 +116,7 @@ function TSHighlighter.new(tree, opts)
 
   self.bufnr = source
   self.redraw_count = 0
-  self._highlight_states = {}
+  self._conceal_checked = {}
   self._queries = {}
 
   -- Queries for a specific language can be overridden by a custom
@@ -122,6 +124,7 @@ function TSHighlighter.new(tree, opts)
   if opts.queries then
     for lang, query_string in pairs(opts.queries) do
       self._queries[lang] = TSHighlighterQuery.new(lang, query_string)
+      self._conceal_line = self._conceal_line or self._queries[lang]:query().conceal_line
     end
   end
 
@@ -140,7 +143,7 @@ function TSHighlighter.new(tree, opts)
   -- immediately afterwards will not error.
   if vim.g.syntax_on ~= 1 then
     vim.cmd.runtime({ 'syntax/synload.vim', bang = true })
-    vim.api.nvim_create_augroup('syntaxset', { clear = false })
+    api.nvim_create_augroup('syntaxset', { clear = false })
   end
 
   vim._with({ buf = self.bufnr }, function()
@@ -148,6 +151,7 @@ function TSHighlighter.new(tree, opts)
   end)
 
   self.tree:parse()
+  self:set_decoration_provider()
 
   return self
 end
@@ -160,6 +164,7 @@ function TSHighlighter:destroy()
   if api.nvim_buf_is_loaded(self.bufnr) then
     vim.bo[self.bufnr].spelloptions = self.orig_spelloptions
     vim.b[self.bufnr].ts_highlight = nil
+    api.nvim_buf_clear_namespace(self.bufnr, ns, 0, -1)
     if vim.g.syntax_on == 1 then
       api.nvim_exec_autocmds('FileType', { group = 'syntaxset', buffer = self.bufnr })
     end
@@ -185,10 +190,14 @@ function TSHighlighter:prepare_highlight_states(srow, erow)
       return
     end
 
-    local highlighter_query = self:get_query(tree:lang())
+    local hl_query = self:get_query(tree:lang())
+    if hl_query:query().conceal_line and not self._conceal_line then
+      self._conceal_line = true
+      self:set_decoration_provider()
+    end
 
     -- Some injected languages may not have highlight queries.
-    if not highlighter_query:query() then
+    if not hl_query:query() then
       return
     end
 
@@ -198,7 +207,7 @@ function TSHighlighter:prepare_highlight_states(srow, erow)
       tstree = tstree,
       next_row = 0,
       iter = nil,
-      highlighter_query = highlighter_query,
+      highlighter_query = hl_query,
     })
   end)
 end
@@ -220,7 +229,10 @@ end
 ---@param changes Range6[]
 function TSHighlighter:on_changedtree(changes)
   for _, ch in ipairs(changes) do
-    api.nvim__redraw({ buf = self.bufnr, range = { ch[1], ch[4] + 1 }, flush = false })
+    api.nvim__redraw({ buf = self.bufnr, range = { ch[1], ch[4] }, flush = false })
+    for i = ch[1], self._conceal_line and ch[4] or 0 do
+      self._conceal_checked[i] = false
+    end
   end
 end
 
@@ -279,8 +291,10 @@ end
 ---@param self vim.treesitter.highlighter
 ---@param buf integer
 ---@param line integer
----@param is_spell_nav boolean
+---@param on_spell boolean
-local function on_line_impl(self, buf, line, is_spell_nav)
+---@param on_conceal boolean
+local function on_line_impl(self, buf, line, on_spell, on_conceal)
+  self._conceal_checked[line] = true
   self:for_each_highlight_state(function(state)
     local root_node = state.tstree:root()
     local root_start_row, _, root_end_row, _ = root_node:range()
@@ -326,7 +340,7 @@ local function on_line_impl(self, buf, line, is_spell_nav)
 
         local url = get_url(match, buf, capture, metadata)
 
-        if hl and end_row >= line and (not is_spell_nav or spell ~= nil) then
+        if hl and end_row >= line and not on_conceal and (not on_spell or spell ~= nil) then
           api.nvim_buf_set_extmark(buf, ns, start_row, start_col, {
             end_line = end_row,
             end_col = end_col,
@@ -338,6 +352,16 @@ local function on_line_impl(self, buf, line, is_spell_nav)
             url = url,
           })
         end
+
+        if
+          (metadata.conceal_lines or metadata[capture] and metadata[capture].conceal_lines)
+          and #api.nvim_buf_get_extmarks(buf, ns, { start_row, 0 }, { start_row, 0 }, {}) == 0
+        then
+          api.nvim_buf_set_extmark(buf, ns, start_row, 0, {
+            end_line = end_row,
+            conceal_lines = '',
+          })
+        end
       end
 
       if start_row > line then
@@ -357,7 +381,7 @@ function TSHighlighter._on_line(_, _win, buf, line, _)
     return
   end
 
-  on_line_impl(self, buf, line, false)
+  on_line_impl(self, buf, line, false, false)
 end
 
 ---@private
@@ -376,17 +400,45 @@ function TSHighlighter._on_spell_nav(_, _, buf, srow, _, erow, _)
   self:prepare_highlight_states(srow, erow)
 
   for row = srow, erow do
-    on_line_impl(self, buf, row, true)
+    on_line_impl(self, buf, row, true, false)
   end
   self._highlight_states = highlight_states
 end
 
 ---@private
----@param _win integer
+---@param buf integer
+---@param row integer
+function TSHighlighter._on_conceal_line(_, _, buf, row)
+  local self = TSHighlighter.active[buf]
+  if not self or self._conceal_checked[row] then
+    return
+  end
+
+  -- Do not affect potentially populated highlight state.
+  local highlight_states = self._highlight_states
+  self:prepare_highlight_states(row, row)
+  on_line_impl(self, buf, row, false, true)
+  self._highlight_states = highlight_states
+end
+
+---@private
+--- Clear conceal_lines marks whenever we redraw for a buffer change. Marks are
+--- added back as either the _conceal_line or on_win callback comes across them.
+function TSHighlighter._on_buf(_, buf)
+  local self = TSHighlighter.active[buf]
+  if not self then
+    return
+  end
+
+  api.nvim_buf_clear_namespace(buf, ns, 0, -1)
+  self._conceal_checked = {}
+end
+
+---@private
 ---@param buf integer
 ---@param topline integer
 ---@param botline integer
-function TSHighlighter._on_win(_, _win, buf, topline, botline)
+function TSHighlighter._on_win(_, _, buf, topline, botline)
   local self = TSHighlighter.active[buf]
   if not self then
     return false
@@ -397,10 +449,15 @@ function TSHighlighter._on_win(_, _win, buf, topline, botline)
   return true
 end
 
-api.nvim_set_decoration_provider(ns, {
+---@private
-  on_win = TSHighlighter._on_win,
+function TSHighlighter:set_decoration_provider()
-  on_line = TSHighlighter._on_line,
+  api.nvim_set_decoration_provider(ns, {
-  _on_spell_nav = TSHighlighter._on_spell_nav,
+    on_win = TSHighlighter._on_win,
-})
+    on_line = TSHighlighter._on_line,
+    _on_spell_nav = TSHighlighter._on_spell_nav,
+    _on_conceal_line = self._conceal_line and TSHighlighter._on_conceal_line or nil,
+    on_buf = self._conceal_line and TSHighlighter._on_buf or nil,
+  })
+end
 
 return TSHighlighter

runtime/lua/vim/treesitter/query.lua

@@ -11,6 +11,7 @@ local M = {}
 ---@field lang string name of the language for this parser
 ---@field captures string[] list of (unique) capture names defined in query
 ---@field info vim.treesitter.QueryInfo contains information used in the query (e.g. captures, predicates, directives)
+---@field conceal_line boolean whether this query sets conceal_lines metadata.
 ---@field query TSQuery userdata query object
 local Query = {}
 Query.__index = Query
@@ -30,6 +31,17 @@ function Query.new(lang, ts_query)
     patterns = query_info.patterns,
   }
   self.captures = self.info.captures
+
+  -- Instruct highlighter to place marks when query contains conceal_lines metadata.
+  for _, preds in pairs(self.info.patterns) do
+    for _, pred in ipairs(preds) do
+      if vim.deep_equal(pred, { 'set!', 'conceal_lines', '' }) then
+        self.conceal_line = true
+        break
+      end
+    end
+  end
+
   return self
 end
 

runtime/queries/markdown/highlights.scm

@@ -49,12 +49,12 @@
 
 (fenced_code_block
   (fenced_code_block_delimiter) @markup.raw.block
-  (#set! conceal ""))
+  (#set! conceal_lines ""))
 
 (fenced_code_block
   (info_string
     (language) @label
-    (#set! conceal "")))
+    (#set! conceal_lines "")))
 
 (link_destination) @markup.link.url
 

src/nvim/api/extmark.c

@@ -1064,6 +1064,7 @@ void nvim_set_decoration_provider(Integer ns_id, Dict(set_decoration_provider) *
     { "on_end", &opts->on_end, &p->redraw_end },
     { "_on_hl_def", &opts->_on_hl_def, &p->hl_def },
     { "_on_spell_nav", &opts->_on_spell_nav, &p->spell_nav },
+    { "_on_conceal_line", &opts->_on_conceal_line, &p->conceal_line },
     { NULL, NULL, NULL },
   };
 

src/nvim/api/keysets_defs.h

@@ -20,6 +20,7 @@ typedef struct {
   LuaRefOf(("end" _, Integer tick)) on_end;
   LuaRefOf(("hl_def" _)) _on_hl_def;
   LuaRefOf(("spell_nav" _)) _on_spell_nav;
+  LuaRefOf(("conceal_line" _)) _on_conceal_line;
 } Dict(set_decoration_provider);
 
 typedef struct {

src/nvim/decoration.c

@@ -12,6 +12,7 @@
 #include "nvim/buffer_defs.h"
 #include "nvim/change.h"
 #include "nvim/decoration.h"
+#include "nvim/decoration_provider.h"
 #include "nvim/drawscreen.h"
 #include "nvim/extmark.h"
 #include "nvim/fold.h"
@@ -844,11 +845,17 @@ static const uint32_t conceal_filter[kMTMetaCount] = {[kMTMetaConcealLines] = kM
 
 bool decor_conceal_line(win_T *wp, int row, bool check_cursor)
 {
-  if (wp->w_p_cole < 2 || !buf_meta_total(wp->w_buffer, kMTMetaConcealLines)
+  if (wp->w_p_cole < 2
       || (!check_cursor && ((row + 1 == wp->w_cursor.lnum) && !conceal_cursor_line(wp)))) {
     return false;
   }
 
+  size_t keys = wp->w_buffer->b_marktree->n_keys;
+  if (!buf_meta_total(wp->w_buffer, kMTMetaConcealLines)) {
+    decor_providers_invoke_conceal_line(wp, row);
+    return wp->w_buffer->b_marktree->n_keys > keys;
+  }
+
   MTPair pair;
   MarkTreeIter itr[1];
   marktree_itr_get_overlap(wp->w_buffer->b_marktree, row, 0, itr);
@@ -870,13 +877,16 @@ bool decor_conceal_line(win_T *wp, int row, bool check_cursor)
     }
     marktree_itr_next_filter(wp->w_buffer->b_marktree, itr, row + 1, 0, conceal_filter);
   }
-  return false;
+
+  decor_providers_invoke_conceal_line(wp, row);
+  return wp->w_buffer->b_marktree->n_keys > keys;
 }
 
 bool win_lines_concealed(win_T *wp)
 {
   return hasAnyFolding(wp)
-         || (wp->w_p_cole >= 2 && buf_meta_total(wp->w_buffer, kMTMetaConcealLines));
+         || (wp->w_p_cole >= 2
+             && (conceal_provider || buf_meta_total(wp->w_buffer, kMTMetaConcealLines)));
 }
 
 static const uint32_t sign_filter[kMTMetaCount] = {[kMTMetaSignText] = kMTFilterSelect,

src/nvim/decoration_defs.h

@@ -144,6 +144,7 @@ typedef struct {
   LuaRef redraw_end;
   LuaRef hl_def;
   LuaRef spell_nav;
+  LuaRef conceal_line;
   int hl_valid;
   bool hl_cached;
 

src/nvim/decoration_provider.c

@@ -30,7 +30,7 @@ static kvec_t(DecorProvider) decor_providers = KV_INITIAL_VALUE;
 #define DECORATION_PROVIDER_INIT(ns_id) (DecorProvider) \
   { ns_id, kDecorProviderDisabled, LUA_NOREF, LUA_NOREF, \
     LUA_NOREF, LUA_NOREF, LUA_NOREF, \
-    LUA_NOREF, -1, false, false, 0 }
+    LUA_NOREF, LUA_NOREF, -1, false, false, 0 }
 
 static void decor_provider_error(DecorProvider *provider, const char *name, const char *msg)
 {
@@ -92,6 +92,21 @@ void decor_providers_invoke_spell(win_T *wp, int start_row, int start_col, int e
   }
 }
 
+void decor_providers_invoke_conceal_line(win_T *wp, int row)
+{
+  for (size_t i = 0; i < kv_size(decor_providers); i++) {
+    DecorProvider *p = &kv_A(decor_providers, i);
+    if (p->state != kDecorProviderDisabled && p->conceal_line != LUA_NOREF) {
+      conceal_provider = true;
+      MAXSIZE_TEMP_ARRAY(args, 4);
+      ADD_C(args, INTEGER_OBJ(wp->handle));
+      ADD_C(args, INTEGER_OBJ(wp->w_buffer->handle));
+      ADD_C(args, INTEGER_OBJ(row));
+      decor_provider_invoke((int)i, "conceal_line", p->conceal_line, args, true);
+    }
+  }
+}
+
 /// For each provider invoke the 'start' callback
 ///
 /// @param[out] providers Decoration providers
@@ -264,6 +279,7 @@ void decor_provider_clear(DecorProvider *p)
   NLUA_CLEAR_REF(p->redraw_line);
   NLUA_CLEAR_REF(p->redraw_end);
   NLUA_CLEAR_REF(p->spell_nav);
+  NLUA_CLEAR_REF(p->conceal_line);
   p->state = kDecorProviderDisabled;
 }
 

src/nvim/decoration_provider.h

@@ -7,6 +7,7 @@
 #include "nvim/types_defs.h"  // IWYU pragma: keep
 
 EXTERN bool provider_active INIT( = false);
+EXTERN bool conceal_provider INIT( = false);
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "decoration_provider.h.generated.h"

test/functional/treesitter/highlight_spec.lua

@@ -1119,14 +1119,16 @@ describe('treesitter highlighting (markdown)', function()
     })
   end)
 
-  it('works with spellchecked and smoothscrolled topline', function()
+  local code_block = [[
-    insert([[
 - $f(0)=\sum_{k=1}^{\infty}\frac{2}{\pi^{2}k^{2}}+\lim_{w \to 0}x$.
 
 ```c
 printf('Hello World!');
 ```
-    ]])
+    ]]
+
+  it('works with spellchecked and smoothscrolled topline', function()
+    insert(code_block)
     command('set spell smoothscroll')
     feed('gg<C-E>')
     screen:add_extra_attr_ids({ [100] = { undercurl = true, special = Screen.colors.Red } })
@@ -1141,6 +1143,105 @@ printf('Hello World!');
       ]],
     })
   end)
+
+  it('works with concealed lines', function()
+    insert(code_block)
+    screen:expect({
+      grid = [[
+                                                |
+        {18:```}{15:c}                                    |
+        {25:printf}{16:(}{26:'Hello World!'}{16:);}                 |
+        {18:```}                                     |
+           ^                                     |
+                                                |
+      ]],
+    })
+    feed('ggj')
+    command('set number conceallevel=3')
+    screen:expect({
+      grid = [[
+        {8:  1 }{16:- }$f(0)=\sum_{k=1}^{\infty}\frac{2}{|
+        {8:    }\pi^{2}k^{2}}+\lim_{w \to 0}x$.     |
+        {8:  2 }^                                    |
+        {8:  4 }{25:printf}{16:(}{26:'Hello World!'}{16:);}             |
+        {8:  6 }                                    |
+                                                |
+      ]],
+    })
+    feed('j')
+    screen:expect({
+      grid = [[
+        {8:  1 }{16:- }$f(0)=\sum_{k=1}^{\infty}\frac{2}{|
+        {8:    }\pi^{2}k^{2}}+\lim_{w \to 0}x$.     |
+        {8:  2 }                                    |
+        {8:  3 }{18:^```}{15:c}                                |
+        {8:  4 }{25:printf}{16:(}{26:'Hello World!'}{16:);}             |
+                                                |
+      ]],
+    })
+    feed('j')
+    screen:expect({
+      grid = [[
+        {8:  1 }{16:- }$f(0)=\sum_{k=1}^{\infty}\frac{2}{|
+        {8:    }\pi^{2}k^{2}}+\lim_{w \to 0}x$.     |
+        {8:  2 }                                    |
+        {8:  4 }{25:^printf}{16:(}{26:'Hello World!'}{16:);}             |
+        {8:  6 }                                    |
+                                                |
+      ]],
+    })
+    feed('j')
+    screen:expect({
+      grid = [[
+        {8:  1 }{16:- }$f(0)=\sum_{k=1}^{\infty}\frac{2}{|
+        {8:    }\pi^{2}k^{2}}+\lim_{w \to 0}x$.     |
+        {8:  2 }                                    |
+        {8:  4 }{25:printf}{16:(}{26:'Hello World!'}{16:);}             |
+        {8:  5 }{18:^```}                                 |
+                                                |
+      ]],
+    })
+    -- Concealed lines highlight until changed botline
+    screen:try_resize(screen._width, 16)
+    feed('y3k30P:<Esc><C-F><C-B>')
+    screen:expect([[
+      {8:  1 }{16:- }$f(0)=\sum_{k=1}^{\infty}\frac{2}{|
+      {8:    }\pi^{2}k^{2}}+\lim_{w \to 0}x$.     |
+      {8:  2 }                                    |
+      {8:  4 }{25:printf}{16:(}{26:'Hello World!'}{16:);}             |
+      {8:  6 }                                    |
+      {8:  8 }{25:printf}{16:(}{26:'Hello World!'}{16:);}             |
+      {8: 10 }                                    |
+      {8: 12 }{25:printf}{16:(}{26:'Hello World!'}{16:);}             |
+      {8: 14 }                                    |
+      {8: 16 }{25:printf}{16:(}{26:'Hello World!'}{16:);}             |
+      {8: 18 }                                    |
+      {8: 20 }{25:printf}{16:(}{26:'Hello World!'}{16:);}             |
+      {8: 22 }                                    |
+      {8: 24 }{25:printf}{16:(}{26:'Hello World!'}{16:);}             |
+      {8: 25 }{18:^```}                                 |
+                                              |
+    ]])
+    feed('G')
+    screen:expect([[
+      {8: 98 }                                    |
+      {8:100 }{25:printf}{16:(}{26:'Hello World!'}{16:);}             |
+      {8:102 }                                    |
+      {8:104 }{25:printf}{16:(}{26:'Hello World!'}{16:);}             |
+      {8:106 }                                    |
+      {8:108 }{25:printf}{16:(}{26:'Hello World!'}{16:);}             |
+      {8:110 }                                    |
+      {8:112 }{25:printf}{16:(}{26:'Hello World!'}{16:);}             |
+      {8:114 }                                    |
+      {8:116 }{25:printf}{16:(}{26:'Hello World!'}{16:);}             |
+      {8:118 }                                    |
+      {8:120 }{25:printf}{16:(}{26:'Hello World!'}{16:);}             |
+      {8:122 }                                    |
+      {8:124 }{25:printf}{16:(}{26:'Hello World!'}{16:);}             |
+      {8:126 }   ^                                 |
+                                              |
+    ]])
+  end)
 end)
 
 it('starting and stopping treesitter highlight in init.lua works #29541', function()