feat(decorations): support virtual lines (for now: only one block at a time)

src/nvim/api/buffer.c

@@ -1415,6 +1415,10 @@ Array nvim_buf_get_extmarks(Buffer buffer, Integer ns_id, Object start, Object e
 ///               - end_col : ending col of the mark, 0-based exclusive.
 ///               - hl_group : name of the highlight group used to highlight
 ///                   this mark.
+///               - hl_eol : when true, for a multiline highlight covering the
+///                          EOL of a line, continue the highlight for the rest
+///                          of the screen line (just like for diff and
+///                          cursorline highlight).
 ///               - virt_text : virtual text to link to this mark.
 ///                   A list of [text, highlight] tuples, each representing a
 ///                   text chunk with specified highlight. `highlight` element
@@ -1442,10 +1446,28 @@ Array nvim_buf_get_extmarks(Buffer buffer, Integer ns_id, Object start, Object e
 ///                              default
 ///                 - "combine": combine with background text color
 ///                 - "blend": blend with background text color.
-///               - hl_eol : when true, for a multiline highlight covering the
+///
-///                          EOL of a line, continue the highlight for the rest
+///               - virt_lines : virtual lines to add next to this mark
-///                          of the screen line (just like for diff and
+///                   This should be an array over lines, where each line in
-///                          cursorline highlight).
+///                   turn is an array over [text, highlight] tuples. In
+///                   general, buffer and window options do not affect the
+///                   display of the text. In particular 'wrap'
+///                   and 'linebreak' options do not take effect, so
+///                   the number of extra screen lines will always match
+///                   the size of the array. However the 'tabstop' buffer
+///                   option is still used for hard tabs. By default lines are
+///                   placed below the buffer line containing the mark.
+///
+///                   Note: currently virtual lines are limited to one block
+///                   per buffer. Thus setting a new mark disables any previous
+///                   `virt_lines` decoration. However plugins should not rely
+///                   on this behaviour, as this limitation is planned to be
+///                   removed.
+///
+///               - virt_lines_above: place virtual lines above instead.
+///               - virt_lines_leftcol: Place extmarks in the leftmost
+///                                     column of the window, bypassing
+///                                     sign and number columns.
 ///
 ///               - ephemeral : for use with |nvim_set_decoration_provider|
 ///                   callbacks. The mark will only be used for the current
@@ -1487,6 +1509,10 @@ Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id, Integer line, Integer
   bool end_right_gravity = false;
   bool end_gravity_set = false;
 
+  VirtLines virt_lines = KV_INITIAL_VALUE;
+  bool virt_lines_above = false;
+  bool virt_lines_leftcol = false;
+
   for (size_t i = 0; i < opts.size; i++) {
     String k = opts.items[i].key;
     Object *v = &opts.items[i].value;
@@ -1584,6 +1610,36 @@ Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id, Integer line, Integer
       if (ERROR_SET(err)) {
         goto error;
       }
+    } else if (strequal("virt_lines", k.data)) {
+      if (v->type != kObjectTypeArray) {
+        api_set_error(err, kErrorTypeValidation,
+                      "virt_lines is not an Array");
+        goto error;
+      }
+      Array a = v->data.array;
+      for (size_t j = 0; j < a.size; j++) {
+        if (a.items[j].type != kObjectTypeArray) {
+          api_set_error(err, kErrorTypeValidation,
+                        "virt_text_line item is not an Array");
+          goto error;
+        }
+        int dummig;
+        VirtText jtem = parse_virt_text(a.items[j].data.array, err, &dummig);
+        kv_push(virt_lines, jtem);
+        if (ERROR_SET(err)) {
+          goto error;
+        }
+      }
+    } else if (strequal("virt_lines_above", k.data)) {
+      virt_lines_above = api_object_to_bool(*v, "virt_lines_above", false, err);
+      if (ERROR_SET(err)) {
+        goto error;
+      }
+    } else if (strequal("virt_lines_leftcol", k.data)) {
+      virt_lines_leftcol = api_object_to_bool(*v, "virt_lines_leftcol", false, err);
+      if (ERROR_SET(err)) {
+        goto error;
+      }
     } else if (strequal("hl_eol", k.data)) {
       decor.hl_eol = api_object_to_bool(*v, "hl_eol", false, err);
       if (ERROR_SET(err)) {
@@ -1721,9 +1777,23 @@ Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id, Integer line, Integer
       goto error;
     }
 
-    id = extmark_set(buf, (uint64_t)ns_id, id, (int)line, (colnr_T)col,
+    if (kv_size(virt_lines) && buf->b_virt_line_mark) {
-                     line2, col2, d, right_gravity,
+      mtpos_t pos = marktree_lookup(buf->b_marktree, buf->b_virt_line_mark, NULL);
-                     end_right_gravity, kExtmarkNoUndo);
+      clear_virt_lines(buf, pos.row);  // handles pos.row == -1
+    }
+
+    uint64_t mark = extmark_set(buf, (uint64_t)ns_id, &id, (int)line, (colnr_T)col,
+                                line2, col2, d, right_gravity,
+                                end_right_gravity, kExtmarkNoUndo);
+
+    if (kv_size(virt_lines)) {
+      buf->b_virt_lines = virt_lines;
+      buf->b_virt_line_mark = mark;
+      buf->b_virt_line_pos = -1;
+      buf->b_virt_line_above = virt_lines_above;
+      buf->b_virt_line_leftcol = virt_lines_leftcol;
+      redraw_buf_line_later(buf, MIN(buf->b_ml.ml_line_count, line+1+(virt_lines_above?0:1)));
+    }
   }
 
   return (Integer)id;
@@ -1827,7 +1897,7 @@ Integer nvim_buf_add_highlight(Buffer buffer, Integer ns_id, String hl_group, In
     end_line++;
   }
 
-  extmark_set(buf, ns, 0,
+  extmark_set(buf, ns, NULL,
               (int)line, (colnr_T)col_start,
               end_line, (colnr_T)col_end,
               decor_hl(hl_id), true, false, kExtmarkNoUndo);

src/nvim/api/deprecated.c

@@ -150,7 +150,7 @@ Integer nvim_buf_set_virtual_text(Buffer buffer, Integer src_id, Integer line, A
   decor->virt_text = virt_text;
   decor->virt_text_width = width;
 
-  extmark_set(buf, ns_id, 0, (int)line, 0, -1, -1, decor, true,
+  extmark_set(buf, ns_id, NULL, (int)line, 0, -1, -1, decor, true,
               false, kExtmarkNoUndo);
   return src_id;
 }

src/nvim/api/private/helpers.c

@@ -1627,7 +1627,7 @@ VirtText parse_virt_text(Array chunks, Error *err, int *width)
       }
     }
 
-    char *text = transstr(str.size > 0 ? str.data : "");  // allocates
+    char *text = transstr(str.size > 0 ? str.data : "", false);  // allocates
     w += (int)mb_string2cells((char_u *)text);
 
     kv_push(virt_text, ((VirtTextChunk){ .text = text, .hl_id = hl_id }));

src/nvim/buffer.c

@@ -65,6 +65,7 @@
 #include "nvim/os/time.h"
 #include "nvim/os_unix.h"
 #include "nvim/path.h"
+#include "nvim/plines.h"
 #include "nvim/quickfix.h"
 #include "nvim/regexp.h"
 #include "nvim/screen.h"
@@ -820,6 +821,7 @@ static void free_buffer_stuff(buf_T *buf, int free_flags)
   uc_clear(&buf->b_ucmds);               // clear local user commands
   buf_delete_signs(buf, (char_u *)"*");  // delete any signs
   extmark_free_all(buf);                 // delete any extmarks
+  clear_virt_lines(buf, -1);
   map_clear_int(buf, MAP_ALL_MODES, true, false);    // clear local mappings
   map_clear_int(buf, MAP_ALL_MODES, true, true);     // clear local abbrevs
   XFREE_CLEAR(buf->b_start_fenc);
@@ -3262,7 +3264,7 @@ void maketitle(void)
         buf_p += MIN(size, SPACE_FOR_FNAME);
       } else {
         buf_p += transstr_buf((const char *)path_tail(curbuf->b_fname),
-                              buf_p, SPACE_FOR_FNAME + 1);
+                              buf_p, SPACE_FOR_FNAME + 1, true);
       }
 
       switch (bufIsChanged(curbuf)
@@ -3311,7 +3313,7 @@ void maketitle(void)
         // room for the server name.  When there is no room (very long
         // file name) use (...).
         if ((size_t)(buf_p - buf) < SPACE_FOR_DIR) {
-          char *const tbuf = transstr(buf_p);
+          char *const tbuf = transstr(buf_p, true);
           const size_t free_space = SPACE_FOR_DIR - (size_t)(buf_p - buf) + 1;
           const size_t dir_len = xstrlcpy(buf_p, tbuf, free_space);
           buf_p += MIN(dir_len, free_space - 1);
@@ -4657,7 +4659,7 @@ void get_rel_pos(win_T *wp, char_u *buf, int buflen)
   long below;          // number of lines below window
 
   above = wp->w_topline - 1;
-  above += diff_check_fill(wp, wp->w_topline) - wp->w_topfill;
+  above += win_get_fill(wp, wp->w_topline) - wp->w_topfill;
   if (wp->w_topline == 1 && wp->w_topfill >= 1) {
     // All buffer lines are displayed and there is an indication
     // of filler lines, that can be considered seeing all lines.

src/nvim/buffer_defs.h

@@ -868,6 +868,12 @@ struct file_buffer {
   Map(uint64_t, ExtmarkItem) b_extmark_index[1];
   Map(uint64_t, ExtmarkNs) b_extmark_ns[1];         // extmark namespaces
 
+  VirtLines b_virt_lines;
+  uint64_t b_virt_line_mark;
+  int b_virt_line_pos;
+  bool b_virt_line_above;
+  bool b_virt_line_leftcol;
+
   // array of channel_id:s which have asked to receive updates for this
   // buffer.
   kvec_t(uint64_t) update_channels;

src/nvim/charset.c

@@ -310,7 +310,7 @@ void trans_characters(char_u *buf, int bufsize)
 ///
 /// @return number of bytes needed to hold a translation of `s`, NUL byte not
 ///         included.
-size_t transstr_len(const char *const s)
+size_t transstr_len(const char *const s, bool untab)
   FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_PURE
 {
   const char *p = s;
@@ -331,6 +331,9 @@ size_t transstr_len(const char *const s)
         }
       }
       p += l;
+    } else if (*p == TAB && !untab) {
+      len += 1;
+      p++;
     } else {
       const int b2c_l = byte2cells((uint8_t)(*p++));
       // Illegal byte sequence may occupy up to 4 characters.
@@ -346,9 +349,10 @@ size_t transstr_len(const char *const s)
 /// @param[out]  buf  Buffer to which result should be saved.
 /// @param[in]  len  Buffer length. Resulting string may not occupy more then
 ///                  len - 1 bytes (one for trailing NUL byte).
+/// @param[in]  untab  remove tab characters
 ///
 /// @return length of the resulting string, without the NUL byte.
-size_t transstr_buf(const char *const s, char *const buf, const size_t len)
+size_t transstr_buf(const char *const s, char *const buf, const size_t len, bool untab)
   FUNC_ATTR_NONNULL_ALL
 {
   const char *p = s;
@@ -379,6 +383,8 @@ size_t transstr_buf(const char *const s, char *const buf, const size_t len)
         }
       }
       p += l;
+    } else if (*p == TAB && !untab) {
+      *buf_p++ = *p++;
     } else {
       const char *const tb = (const char *)transchar_byte((uint8_t)(*p++));
       const size_t tb_len = strlen(tb);
@@ -401,14 +407,14 @@ size_t transstr_buf(const char *const s, char *const buf, const size_t len)
 /// @param[in]  s  String to replace characters from.
 ///
 /// @return [allocated] translated string
-char *transstr(const char *const s)
+char *transstr(const char *const s, bool untab)
   FUNC_ATTR_NONNULL_RET
 {
   // Compute the length of the result, taking account of unprintable
   // multi-byte characters.
-  const size_t len = transstr_len((const char *)s) + 1;
+  const size_t len = transstr_len((const char *)s, untab) + 1;
   char *const buf = xmalloc(len);
-  transstr_buf(s, buf, len);
+  transstr_buf(s, buf, len, untab);
   return buf;
 }
 

src/nvim/decoration.c

@@ -59,7 +59,7 @@ void bufhl_add_hl_pos_offset(buf_T *buf, int src_id, int hl_id, lpos_T pos_start
       hl_start = pos_start.col + offset;
       hl_end = pos_end.col + offset;
     }
-    (void)extmark_set(buf, (uint64_t)src_id, 0,
+    (void)extmark_set(buf, (uint64_t)src_id, NULL,
                       (int)lnum-1, hl_start, (int)lnum-1+end_off, hl_end,
                       decor, true, false, kExtmarkNoUndo);
   }
@@ -412,3 +412,35 @@ void decor_free_all_mem(void)
   }
   kv_destroy(decor_providers);
 }
+
+
+int decor_virtual_lines(win_T *wp, linenr_T lnum)
+{
+  buf_T *buf = wp->w_buffer;
+  if (!buf->b_virt_line_mark) {
+    return 0;
+  }
+  if (buf->b_virt_line_pos < 0) {
+      mtpos_t pos = marktree_lookup(buf->b_marktree, buf->b_virt_line_mark, NULL);
+      if (pos.row < 0) {
+        buf->b_virt_line_mark = 0;
+      }
+      buf->b_virt_line_pos = pos.row + (buf->b_virt_line_above ? 0 : 1);
+  }
+
+  return (lnum-1 == buf->b_virt_line_pos) ? (int)kv_size(buf->b_virt_lines) : 0;
+}
+
+void clear_virt_lines(buf_T *buf, int row)
+{
+  if (row > -1) {
+    redraw_buf_line_later(buf, MIN(buf->b_ml.ml_line_count,
+                                   row+1+(buf->b_virt_line_above?0:1)));
+  }
+  for (size_t i = 0; i < kv_size(buf->b_virt_lines); i++) {
+    clear_virttext(&kv_A(buf->b_virt_lines, i));
+  }
+  kv_destroy(buf->b_virt_lines);  // re-initializes
+  buf->b_virt_line_pos = -1;
+  buf->b_virt_line_mark = 0;
+}

src/nvim/decoration.h

@@ -7,14 +7,6 @@
 
 // actual Decoration data is in extmark_defs.h
 
-typedef struct {
-  char *text;
-  int hl_id;
-} VirtTextChunk;
-
-typedef kvec_t(VirtTextChunk) VirtText;
-#define VIRTTEXT_EMPTY ((VirtText)KV_INITIAL_VALUE)
-
 typedef uint16_t DecorPriority;
 #define DECOR_PRIORITY_BASE 0x1000
 

src/nvim/diff.c

@@ -1985,26 +1985,6 @@ static int diff_cmp(char_u *s1, char_u *s2)
   return 0;
 }
 
-/// Return the number of filler lines above "lnum".
-///
-/// @param wp
-/// @param lnum
-///
-/// @return Number of filler lines above lnum
-int diff_check_fill(win_T *wp, linenr_T lnum)
-{
-  // be quick when there are no filler lines
-  if (!(diff_flags & DIFF_FILLER)) {
-    return 0;
-  }
-  int n = diff_check(wp, lnum);
-
-  if (n <= 0) {
-    return 0;
-  }
-  return n;
-}
-
 /// Set the topline of "towin" to match the position in "fromwin", so that they
 /// show the same diff'ed lines.
 ///
@@ -2030,6 +2010,7 @@ void diff_set_topline(win_T *fromwin, win_T *towin)
   }
   towin->w_topfill = 0;
 
+
   // search for a change that includes "lnum" in the list of diffblocks.
   for (dp = curtab->tp_first_diff; dp != NULL; dp = dp->df_next) {
     if (lnum <= dp->df_lnum[fromidx] + dp->df_count[fromidx]) {
@@ -2255,6 +2236,13 @@ bool diffopt_closeoff(void)
   return (diff_flags & DIFF_CLOSE_OFF) != 0;
 }
 
+// Return true if 'diffopt' contains "filler".
+bool diffopt_filler(void)
+  FUNC_ATTR_PURE FUNC_ATTR_WARN_UNUSED_RESULT
+{
+  return (diff_flags & DIFF_FILLER) != 0;
+}
+
 /// Find the difference within a changed line.
 ///
 /// @param  wp      window whose current buffer to check

src/nvim/edit.c

@@ -3344,12 +3344,10 @@ static char_u *ins_compl_mode(void)
  */
 static int ins_compl_bs(void)
 {
-  char_u *line;
+  char_u *line = get_cursor_line_ptr();
-  char_u *p;
+  char_u *p = line + curwin->w_cursor.col;
-
-  line = get_cursor_line_ptr();
-  p = line + curwin->w_cursor.col;
   MB_PTR_BACK(line, p);
+  ptrdiff_t p_off = p - line;
 
   // Stop completion when the whole word was deleted.  For Omni completion
   // allow the word to be deleted, we won't match everything.
@@ -3369,8 +3367,12 @@ static int ins_compl_bs(void)
     ins_compl_restart();
   }
 
+  // ins_compl_restart() calls update_screen(0) which may invalidate the pointer
+  // TODO(bfredl): get rid of random update_screen() calls deep inside completion logic
+  line = get_cursor_line_ptr();
+
   xfree(compl_leader);
-  compl_leader = vim_strnsave(line + compl_col, (int)(p - line) - compl_col);
+  compl_leader = vim_strnsave(line + compl_col, (int)p_off - compl_col);
   ins_compl_new_leader();
   if (compl_shown_match != NULL) {
     // Make sure current match is not a hidden item.

src/nvim/eval/funcs.c

@@ -1802,7 +1802,7 @@ static void f_did_filetype(typval_T *argvars, typval_T *rettv, FunPtr fptr)
  */
 static void f_diff_filler(typval_T *argvars, typval_T *rettv, FunPtr fptr)
 {
-  rettv->vval.v_number = diff_check_fill(curwin, tv_get_lnum(argvars));
+  rettv->vval.v_number = MAX(0, diff_check(curwin, tv_get_lnum(argvars)));
 }
 
 /*
@@ -10801,7 +10801,7 @@ static void f_strridx(typval_T *argvars, typval_T *rettv, FunPtr fptr)
 static void f_strtrans(typval_T *argvars, typval_T *rettv, FunPtr fptr)
 {
   rettv->v_type = VAR_STRING;
-  rettv->vval.v_string = (char_u *)transstr(tv_get_string(&argvars[0]));
+  rettv->vval.v_string = (char_u *)transstr(tv_get_string(&argvars[0]), true);
 }
 
 /*

src/nvim/extmark.c

@@ -56,8 +56,8 @@ static ExtmarkNs *buf_ns_ref(buf_T *buf, uint64_t ns_id, bool put) {
 /// Create or update an extmark
 ///
 /// must not be used during iteration!
-/// @returns the mark id
+/// @returns the internal mark id
-uint64_t extmark_set(buf_T *buf, uint64_t ns_id, uint64_t id, int row, colnr_T col, int end_row,
+uint64_t extmark_set(buf_T *buf, uint64_t ns_id, uint64_t *idp, int row, colnr_T col, int end_row,
                      colnr_T end_col, Decoration *decor, bool right_gravity, bool end_right_gravity,
                      ExtmarkOp op)
 {
@@ -65,6 +65,7 @@ uint64_t extmark_set(buf_T *buf, uint64_t ns_id, uint64_t id, int row, colnr_T c
   assert(ns != NULL);
   mtpos_t old_pos;
   uint64_t mark = 0;
+  uint64_t id = idp ? *idp : 0;
 
   if (id == 0) {
     id = ns->free_id++;
@@ -118,7 +119,11 @@ revised:
   if (decor) {
     decor_redraw(buf, row, end_row > -1 ? end_row : row, decor);
   }
-  return id;
+
+  if (idp) {
+    *idp = id;
+  }
+  return mark;
 }
 
 static bool extmark_setraw(buf_T *buf, uint64_t mark, int row, colnr_T col)
@@ -169,6 +174,10 @@ bool extmark_del(buf_T *buf, uint64_t ns_id, uint64_t id)
     decor_free(item.decor);
   }
 
+  if (mark == buf->b_virt_line_mark) {
+    clear_virt_lines(buf, pos.row);
+  }
+
   map_del(uint64_t, uint64_t)(ns->map, id);
   map_del(uint64_t, ExtmarkItem)(buf->b_extmark_index, mark);
 
@@ -227,6 +236,9 @@ bool extmark_clear(buf_T *buf, uint64_t ns_id, int l_row, colnr_T l_col, int u_r
     }
 
     uint64_t start_id = mark.id & ~MARKTREE_END_FLAG;
+    if (start_id == buf->b_virt_line_mark) {
+      clear_virt_lines(buf, mark.row);
+    }
     ExtmarkItem item = map_get(uint64_t, ExtmarkItem)(buf->b_extmark_index,
                                                       start_id);
 
@@ -496,6 +508,7 @@ void extmark_apply_undo(ExtmarkUndoObject undo_info, bool undo)
                           kExtmarkNoUndo);
     }
   }
+  curbuf->b_virt_line_pos = -1;
 }
 
 
@@ -574,7 +587,8 @@ void extmark_splice_impl(buf_T *buf, int start_row, colnr_T start_col, bcount_t
                          int old_row, colnr_T old_col, bcount_t old_byte, int new_row,
                          colnr_T new_col, bcount_t new_byte, ExtmarkOp undo)
 {
-  curbuf->deleted_bytes2 = 0;
+  buf->deleted_bytes2 = 0;
+  buf->b_virt_line_pos = -1;
   buf_updates_send_splice(buf, start_row, start_col, start_byte,
                           old_row, old_col, old_byte,
                           new_row, new_col, new_byte);
@@ -665,7 +679,8 @@ void extmark_move_region(buf_T *buf, int start_row, colnr_T start_col, bcount_t
                          int extent_row, colnr_T extent_col, bcount_t extent_byte, int new_row,
                          colnr_T new_col, bcount_t new_byte, ExtmarkOp undo)
 {
-  curbuf->deleted_bytes2 = 0;
+  buf->deleted_bytes2 = 0;
+  buf->b_virt_line_pos = -1;
   // TODO(bfredl): this is not synced to the buffer state inside the callback.
   // But unless we make the undo implementation smarter, this is not ensured
   // anyway.

src/nvim/extmark_defs.h

@@ -6,6 +6,16 @@
 
 typedef struct Decoration Decoration;
 
+typedef struct {
+  char *text;
+  int hl_id;
+} VirtTextChunk;
+
+typedef kvec_t(VirtTextChunk) VirtText;
+#define VIRTTEXT_EMPTY ((VirtText)KV_INITIAL_VALUE)
+typedef kvec_t(VirtText) VirtLines;
+
+
 typedef struct
 {
   uint64_t ns_id;

src/nvim/fold.c

@@ -1875,7 +1875,7 @@ char_u *get_foldtext(win_T *wp, linenr_T lnum, linenr_T lnume, foldinfo_T foldin
         }
       }
       if (*p != NUL) {
-        p = (char_u *)transstr((const char *)text);
+        p = (char_u *)transstr((const char *)text, true);
         xfree(text);
         text = p;
       }

src/nvim/mouse.c

@@ -241,7 +241,7 @@ retnomove:
     if (row < 0) {
       count = 0;
       for (first = true; curwin->w_topline > 1; ) {
-        if (curwin->w_topfill < diff_check(curwin, curwin->w_topline)) {
+        if (curwin->w_topfill < win_get_fill(curwin, curwin->w_topline)) {
           count++;
         } else {
           count += plines_win(curwin, curwin->w_topline - 1, true);
@@ -251,8 +251,8 @@ retnomove:
         }
         first = false;
         (void)hasFolding(curwin->w_topline, &curwin->w_topline, NULL);
-        if (curwin->w_topfill < diff_check(curwin, curwin->w_topline)) {
+        if (curwin->w_topfill < win_get_fill(curwin, curwin->w_topline)) {
-          ++curwin->w_topfill;
+          curwin->w_topfill++;
         } else {
           --curwin->w_topline;
           curwin->w_topfill = 0;
@@ -283,11 +283,10 @@ retnomove:
         }
 
         if (curwin->w_topfill > 0) {
-          --curwin->w_topfill;
+          curwin->w_topfill--;
         } else {
-          ++curwin->w_topline;
+          curwin->w_topline++;
-          curwin->w_topfill =
+          curwin->w_topfill = win_get_fill(curwin, curwin->w_topline);
-            diff_check_fill(curwin, curwin->w_topline);
         }
       }
       check_topfill(curwin, false);
@@ -373,12 +372,12 @@ bool mouse_comp_pos(win_T *win, int *rowp, int *colp, linenr_T *lnump)
 
   while (row > 0) {
     // Don't include filler lines in "count"
-    if (win->w_p_diff
+    if (win_may_fill(win)
         && !hasFoldingWin(win, lnum, NULL, NULL, true, NULL)) {
       if (lnum == win->w_topline) {
         row -= win->w_topfill;
       } else {
-        row -= diff_check_fill(win, lnum);
+        row -= win_get_fill(win, lnum);
       }
       count = plines_win_nofill(win, lnum, true);
     } else {

src/nvim/move.c

@@ -197,7 +197,7 @@ void update_topline(win_T *wp)
       }
     }
     // Check if there are more filler lines than allowed.
-    if (!check_topline && wp->w_topfill > diff_check_fill(wp, wp->w_topline)) {
+    if (!check_topline && wp->w_topfill > win_get_fill(wp, wp->w_topline)) {
       check_topline = true;
     }
 
@@ -582,8 +582,7 @@ static void curs_rows(win_T *wp)
         --i;                            // hold at inserted lines
       }
     }
-    if (valid
+    if (valid && (lnum != wp->w_topline || !win_may_fill(wp))) {
-        && (lnum != wp->w_topline || !wp->w_p_diff)) {
       lnum = wp->w_lines[i].wl_lastlnum + 1;
       // Cursor inside folded lines, don't count this row
       if (lnum > wp->w_cursor.lnum) {
@@ -854,7 +853,7 @@ void curs_columns(win_T *wp, int may_scroll)
   if (wp->w_cursor.lnum == wp->w_topline) {
     wp->w_wrow += wp->w_topfill;
   } else {
-    wp->w_wrow += diff_check_fill(wp, wp->w_cursor.lnum);
+    wp->w_wrow += win_get_fill(wp, wp->w_cursor.lnum);
   }
 
   prev_skipcol = wp->w_skipcol;
@@ -1041,7 +1040,7 @@ bool scrolldown(long line_count, int byfold)
   (void)hasFolding(curwin->w_topline, &curwin->w_topline, NULL);
   validate_cursor();            // w_wrow needs to be valid
   while (line_count-- > 0) {
-    if (curwin->w_topfill < diff_check(curwin, curwin->w_topline)
+    if (curwin->w_topfill < win_get_fill(curwin, curwin->w_topline)
         && curwin->w_topfill < curwin->w_height_inner - 1) {
       curwin->w_topfill++;
       done++;
@@ -1122,7 +1121,7 @@ bool scrollup(long line_count, int byfold)
   linenr_T botline = curwin->w_botline;
 
   if ((byfold && hasAnyFolding(curwin))
-      || curwin->w_p_diff) {
+      || win_may_fill(curwin)) {
     // count each sequence of folded lines as one logical line
     linenr_T lnum = curwin->w_topline;
     while (line_count--) {
@@ -1135,8 +1134,8 @@ bool scrollup(long line_count, int byfold)
         if (lnum >= curbuf->b_ml.ml_line_count) {
           break;
         }
-        ++lnum;
+        lnum++;
-        curwin->w_topfill = diff_check_fill(curwin, lnum);
+        curwin->w_topfill = win_get_fill(curwin, lnum);
       }
     }
     // approximate w_botline
@@ -1207,7 +1206,7 @@ static void max_topfill(void)
   if (n >= curwin->w_height_inner) {
     curwin->w_topfill = 0;
   } else {
-    curwin->w_topfill = diff_check_fill(curwin, curwin->w_topline);
+    curwin->w_topfill = win_get_fill(curwin, curwin->w_topline);
     if (curwin->w_topfill + n > curwin->w_height_inner) {
       curwin->w_topfill = curwin->w_height_inner - n;
     }
@@ -1220,8 +1219,7 @@ static void max_topfill(void)
  */
 void scrolldown_clamp(void)
 {
-  int can_fill = (curwin->w_topfill
+  int can_fill = (curwin->w_topfill < win_get_fill(curwin, curwin->w_topline));
-                  < diff_check_fill(curwin, curwin->w_topline));
 
   if (curwin->w_topline <= 1
       && !can_fill) {
@@ -1302,7 +1300,7 @@ void scrollup_clamp(void)
  */
 static void topline_back(win_T *wp, lineoff_T *lp)
 {
-  if (lp->fill < diff_check_fill(wp, lp->lnum)) {
+  if (lp->fill < win_get_fill(wp, lp->lnum)) {
     // Add a filler line
     lp->fill++;
     lp->height = 1;
@@ -1328,7 +1326,7 @@ static void topline_back(win_T *wp, lineoff_T *lp)
  */
 static void botline_forw(win_T *wp, lineoff_T *lp)
 {
-  if (lp->fill < diff_check_fill(wp, lp->lnum + 1)) {
+  if (lp->fill < win_get_fill(wp, lp->lnum + 1)) {
     // Add a filler line.
     lp->fill++;
     lp->height = 1;
@@ -1355,8 +1353,8 @@ static void botline_forw(win_T *wp, lineoff_T *lp)
 static void botline_topline(lineoff_T *lp)
 {
   if (lp->fill > 0) {
-    ++lp->lnum;
+    lp->lnum++;
-    lp->fill = diff_check_fill(curwin, lp->lnum) - lp->fill + 1;
+    lp->fill = win_get_fill(curwin, lp->lnum) - lp->fill + 1;
   }
 }
 
@@ -1368,8 +1366,8 @@ static void botline_topline(lineoff_T *lp)
 static void topline_botline(lineoff_T *lp)
 {
   if (lp->fill > 0) {
-    lp->fill = diff_check_fill(curwin, lp->lnum) - lp->fill + 1;
+    lp->fill = win_get_fill(curwin, lp->lnum) - lp->fill + 1;
-    --lp->lnum;
+    lp->lnum--;
   }
 }
 
@@ -1417,7 +1415,7 @@ void scroll_cursor_top(int min_scroll, int always)
   // "used" already contains the number of filler lines above, don't add it
   // again.
   // Hide filler lines above cursor line by adding them to "extra".
-  int extra = diff_check_fill(curwin, curwin->w_cursor.lnum);
+  int extra = win_get_fill(curwin, curwin->w_cursor.lnum);
 
   /*
    * Check if the lines from "top" to "bot" fit in the window.  If they do,
@@ -1475,7 +1473,7 @@ void scroll_cursor_top(int min_scroll, int always)
     if (curwin->w_topline > curwin->w_cursor.lnum) {
       curwin->w_topline = curwin->w_cursor.lnum;
     }
-    curwin->w_topfill = diff_check_fill(curwin, curwin->w_topline);
+    curwin->w_topfill = win_get_fill(curwin, curwin->w_topline);
     if (curwin->w_topfill > 0 && extra > off) {
       curwin->w_topfill -= extra - off;
       if (curwin->w_topfill < 0) {
@@ -1505,7 +1503,7 @@ void set_empty_rows(win_T *wp, int used)
   } else {
     wp->w_empty_rows = wp->w_height_inner - used;
     if (wp->w_botline <= wp->w_buffer->b_ml.ml_line_count) {
-      wp->w_filler_rows = diff_check_fill(wp, wp->w_botline);
+      wp->w_filler_rows = win_get_fill(wp, wp->w_botline);
       if (wp->w_empty_rows > wp->w_filler_rows) {
         wp->w_empty_rows -= wp->w_filler_rows;
       } else {
@@ -1590,7 +1588,7 @@ void scroll_cursor_bot(int min_scroll, int set_topbot)
   }
   loff.fill = 0;
   boff.fill = 0;
-  fill_below_window = diff_check_fill(curwin, curwin->w_botline)
+  fill_below_window = win_get_fill(curwin, curwin->w_botline)
                       - curwin->w_filler_rows;
 
   while (loff.lnum > 1) {
@@ -1835,7 +1833,7 @@ void cursor_correct(void)
 
       // Count filler lines below this line as context.
       if (topline < botline) {
-        above += diff_check_fill(curwin, topline + 1);
+        above += win_get_fill(curwin, topline + 1);
       }
       ++topline;
     }
@@ -1889,9 +1887,7 @@ int onepage(Direction dir, long count)
         ? ((curwin->w_topline >= curbuf->b_ml.ml_line_count - so)
            && curwin->w_botline > curbuf->b_ml.ml_line_count)
         : (curwin->w_topline == 1
-           && curwin->w_topfill ==
+           && curwin->w_topfill == win_get_fill(curwin, curwin->w_topline))) {
-           diff_check_fill(curwin, curwin->w_topline)
-           )) {
       beep_flush();
       retval = FAIL;
       break;
@@ -1919,7 +1915,7 @@ int onepage(Direction dir, long count)
         /* For the overlap, start with the line just below the window
          * and go upwards. */
         loff.lnum = curwin->w_botline;
-        loff.fill = diff_check_fill(curwin, loff.lnum)
+        loff.fill = win_get_fill(curwin, loff.lnum)
                     - curwin->w_filler_rows;
         get_scroll_overlap(&loff, -1);
         curwin->w_topline = loff.lnum;
@@ -1956,8 +1952,7 @@ int onepage(Direction dir, long count)
        * line at the bottom of the window.  Make sure this results in
        * the same line as before doing CTRL-F. */
       loff.lnum = curwin->w_topline - 1;
-      loff.fill = diff_check_fill(curwin, loff.lnum + 1)
+      loff.fill = win_get_fill(curwin, loff.lnum + 1) - curwin->w_topfill;
-                  - curwin->w_topfill;
       get_scroll_overlap(&loff, 1);
 
       if (loff.lnum >= curbuf->b_ml.ml_line_count) {
@@ -2000,8 +1995,7 @@ int onepage(Direction dir, long count)
           /* First try using the maximum number of filler lines.  If
            * that's not enough, backup one line. */
           loff.fill = curwin->w_topfill;
-          if (curwin->w_topfill < diff_check_fill(curwin,
+          if (curwin->w_topfill < win_get_fill(curwin, curwin->w_topline)) {
-                                                  curwin->w_topline)) {
             max_topfill();
           }
           if (curwin->w_topfill == loff.fill) {
@@ -2146,8 +2140,8 @@ void halfpage(bool flag, linenr_T Prenum)
           break;
         }
         (void)hasFolding(curwin->w_topline, NULL, &curwin->w_topline);
-        ++curwin->w_topline;
+        curwin->w_topline++;
-        curwin->w_topfill = diff_check_fill(curwin, curwin->w_topline);
+        curwin->w_topfill = win_get_fill(curwin, curwin->w_topline);
 
         if (curwin->w_cursor.lnum < curbuf->b_ml.ml_line_count) {
           ++curwin->w_cursor.lnum;
@@ -2158,11 +2152,9 @@ void halfpage(bool flag, linenr_T Prenum)
       curwin->w_valid &= ~(VALID_CROW|VALID_WROW);
       scrolled += i;
 
-      /*
+      // Correct w_botline for changed w_topline.
-       * Correct w_botline for changed w_topline.
+      // Won't work when there are filler lines.
-       * Won't work when there are filler lines.
+      if (win_may_fill(curwin)) {
-       */
-      if (curwin->w_p_diff) {
         curwin->w_valid &= ~(VALID_BOTLINE|VALID_BOTLINE_AP);
       } else {
         room += i;
@@ -2197,7 +2189,7 @@ void halfpage(bool flag, linenr_T Prenum)
      * scroll the text down
      */
     while (n > 0 && curwin->w_topline > 1) {
-      if (curwin->w_topfill < diff_check_fill(curwin, curwin->w_topline)) {
+      if (curwin->w_topfill < win_get_fill(curwin, curwin->w_topline)) {
         i = 1;
         n--;
         curwin->w_topfill++;

src/nvim/normal.c

@@ -5261,16 +5261,15 @@ static void nv_scroll(cmdarg_T *cap)
   } else {
     if (cap->cmdchar == 'M') {
       // Don't count filler lines above the window.
-      used -= diff_check_fill(curwin, curwin->w_topline)
+      used -= win_get_fill(curwin, curwin->w_topline)
               - curwin->w_topfill;
       validate_botline(curwin);  // make sure w_empty_rows is valid
       half = (curwin->w_height_inner - curwin->w_empty_rows + 1) / 2;
       for (n = 0; curwin->w_topline + n < curbuf->b_ml.ml_line_count; n++) {
         // Count half he number of filler lines to be "below this
         // line" and half to be "above the next line".
-        if (n > 0 && used + diff_check_fill(curwin, curwin->w_topline
+        if (n > 0 && used + win_get_fill(curwin, curwin->w_topline + n) / 2 >= half) {
-                                            + n) / 2 >= half) {
+          n--;
-          --n;
           break;
         }
         used += plines_win(curwin, curwin->w_topline + n, true);

src/nvim/plines.c

@@ -13,6 +13,7 @@
 #include "nvim/buffer.h"
 #include "nvim/charset.h"
 #include "nvim/cursor.h"
+#include "nvim/decoration.h"
 #include "nvim/diff.h"
 #include "nvim/fold.h"
 #include "nvim/func_attr.h"
@@ -41,7 +42,34 @@ int plines_win(win_T *wp, linenr_T lnum, bool winheight)
 {
   // Check for filler lines above this buffer line.  When folded the result
   // is one line anyway.
-  return plines_win_nofill(wp, lnum, winheight) + diff_check_fill(wp, lnum);
+  return plines_win_nofill(wp, lnum, winheight) + win_get_fill(wp, lnum);
+}
+
+
+/// Return the number of filler lines above "lnum".
+///
+/// @param wp
+/// @param lnum
+///
+/// @return Number of filler lines above lnum
+int win_get_fill(win_T *wp, linenr_T lnum)
+{
+  int virt_lines = decor_virtual_lines(wp, lnum);
+
+  // be quick when there are no filler lines
+  if (diffopt_filler()) {
+    int n = diff_check(wp, lnum);
+
+    if (n > 0) {
+      return virt_lines+n;
+    }
+  }
+  return virt_lines;
+}
+
+bool win_may_fill(win_T *wp)
+{
+  return (wp->w_p_diff && diffopt_filler()) || wp->w_buffer->b_virt_line_mark;
 }
 
 /// @param winheight when true limit to window height
@@ -107,7 +135,7 @@ int plines_win_col(win_T *wp, linenr_T lnum, long column)
 {
   // Check for filler lines above this buffer line.  When folded the result
   // is one line anyway.
-  int lines = diff_check_fill(wp, lnum);
+  int lines = win_get_fill(wp, lnum);
 
   if (!wp->w_p_wrap) {
     return lines + 1;

src/nvim/popupmnu.c

@@ -514,7 +514,7 @@ void pum_redraw(void)
             char_u saved = *p;
 
             *p = NUL;
-            st = (char_u *)transstr((const char *)s);
+            st = (char_u *)transstr((const char *)s, true);
             *p = saved;
 
             if (pum_rl) {

src/nvim/screen.c

@@ -1003,8 +1003,7 @@ static void win_update(win_T *wp, Providers *providers)
         i = plines_m_win(wp, wp->w_topline, wp->w_lines[0].wl_lnum - 1);
         // insert extra lines for previously invisible filler lines
         if (wp->w_lines[0].wl_lnum != wp->w_topline) {
-          i += diff_check_fill(wp, wp->w_lines[0].wl_lnum)
+          i += win_get_fill(wp, wp->w_lines[0].wl_lnum) - wp->w_old_topfill;
-               - wp->w_old_topfill;
         }
         if (i != 0 && i < wp->w_grid.Rows - 2) {  // less than a screen off
           // Try to insert the correct number of lines.
@@ -1067,7 +1066,7 @@ static void win_update(win_T *wp, Providers *providers)
         if (wp->w_lines[0].wl_lnum == wp->w_topline) {
           row += wp->w_old_topfill;
         } else {
-          row += diff_check_fill(wp, wp->w_topline);
+          row += win_get_fill(wp, wp->w_topline);
         }
         // ... but don't delete new filler lines.
         row -= wp->w_topfill;
@@ -1101,12 +1100,12 @@ static void win_update(win_T *wp, Providers *providers)
               break;
             }
           }
-          /* Correct the first entry for filler lines at the top
+
-           * when it won't get updated below. */
+          // Correct the first entry for filler lines at the top
-          if (wp->w_p_diff && bot_start > 0) {
+          // when it won't get updated below.
-            wp->w_lines[0].wl_size =
+          if (win_may_fill(wp) && bot_start > 0) {
-              plines_win_nofill(wp, wp->w_topline, true)
+            wp->w_lines[0].wl_size = (plines_win_nofill(wp, wp->w_topline, true)
-              + wp->w_topfill;
+                                      + wp->w_topfill);
           }
         }
       }
@@ -1564,7 +1563,7 @@ static void win_update(win_T *wp, Providers *providers)
           && lnum > wp->w_topline
           && !(dy_flags & (DY_LASTLINE | DY_TRUNCATE))
           && srow + wp->w_lines[idx].wl_size > wp->w_grid.Rows
-          && diff_check_fill(wp, lnum) == 0) {
+          && win_get_fill(wp, lnum) == 0) {
         // This line is not going to fit.  Don't draw anything here,
         // will draw "@  " lines below.
         row = wp->w_grid.Rows + 1;
@@ -1664,7 +1663,7 @@ static void win_update(win_T *wp, Providers *providers)
        * Don't overwrite it, it can be edited.
        */
       wp->w_botline = lnum + 1;
-    } else if (diff_check_fill(wp, lnum) >= wp->w_grid.Rows - srow) {
+    } else if (win_get_fill(wp, lnum) >= wp->w_grid.Rows - srow) {
       // Window ends in filler lines.
       wp->w_botline = lnum;
       wp->w_filler_rows = wp->w_grid.Rows - srow;
@@ -1691,7 +1690,7 @@ static void win_update(win_T *wp, Providers *providers)
   } else {
     if (eof) {  // we hit the end of the file
       wp->w_botline = buf->b_ml.ml_line_count + 1;
-      j = diff_check_fill(wp, wp->w_botline);
+      j = win_get_fill(wp, wp->w_botline);
       if (j > 0 && !wp->w_botfill) {
         // Display filler text below last line. win_line() will check
         // for ml_line_count+1 and only draw filler lines
@@ -1866,7 +1865,7 @@ static int compute_foldcolumn(win_T *wp, int col)
 /// Put a single char from an UTF-8 buffer into a line buffer.
 ///
 /// Handles composing chars and arabic shaping state.
-static int line_putchar(LineState *s, schar_T *dest, int maxcells, bool rl)
+static int line_putchar(buf_T *buf, LineState *s, schar_T *dest, int maxcells, bool rl, int vcol)
 {
   const char_u *p = (char_u *)s->p;
   int cells = utf_ptr2cells(p);
@@ -1876,7 +1875,13 @@ static int line_putchar(LineState *s, schar_T *dest, int maxcells, bool rl)
     return -1;
   }
   u8c = utfc_ptr2char(p, u8cc);
-  if (*p < 0x80 && u8cc[0] == 0) {
+  if (*p == TAB) {
+    cells = MIN(tabstop_padding(vcol, buf->b_p_ts, buf->b_p_vts_array), maxcells);
+    for (int c = 0; c < cells; c++) {
+      schar_from_ascii(dest[c], ' ');
+    }
+    goto done;
+  } else if (*p < 0x80 && u8cc[0] == 0) {
     schar_from_ascii(dest[0], *p);
     s->prev_c = u8c;
   } else {
@@ -1909,6 +1914,7 @@ static int line_putchar(LineState *s, schar_T *dest, int maxcells, bool rl)
   if (cells > 1) {
     dest[1][0] = 0;
   }
+done:
   s->p += c_len;
   return cells;
 }
@@ -2366,8 +2372,11 @@ static int win_line(win_T *wp, linenr_T lnum, int startrow, int endrow, bool noc
     filler_lines = 0;
     area_highlighting = true;
   }
+  int virtual_lines = decor_virtual_lines(wp, lnum);
+  filler_lines += virtual_lines;
   if (lnum == wp->w_topline) {
     filler_lines = wp->w_topfill;
+    virtual_lines = MIN(virtual_lines, filler_lines);
   }
   filler_todo = filler_lines;
 
@@ -2895,7 +2904,18 @@ static int win_line(win_T *wp, linenr_T lnum, int startrow, int endrow, bool noc
 
       if (draw_state == WL_SBR - 1 && n_extra == 0) {
         draw_state = WL_SBR;
-        if (filler_todo > 0) {
+        if (filler_todo > filler_lines - virtual_lines) {
+          // TODO(bfredl): check this doesn't inhibit TUI-style
+          //               clear-to-end-of-line.
+          c_extra = ' ';
+          c_final = NUL;
+          if (wp->w_p_rl) {
+            n_extra = col + 1;
+          } else {
+            n_extra = grid->Columns - col;
+          }
+          char_attr = 0;
+        } else if (filler_todo > 0) {
           // draw "deleted" diff line(s)
           if (char2cells(wp->w_p_fcs_chars.diff) > 1) {
             c_extra = '-';
@@ -4402,7 +4422,19 @@ static int win_line(win_T *wp, linenr_T lnum, int startrow, int endrow, bool noc
                   && !wp->w_p_rl;              // Not right-to-left.
 
       int draw_col = col - boguscols;
-      draw_virt_text(buf, win_col_offset, &draw_col, grid->Columns);
+      if (filler_todo > 0) {
+        int index = filler_todo - (filler_lines - virtual_lines);
+        if (index > 0) {
+          int fpos = kv_size(buf->b_virt_lines) - index;
+          assert(fpos >= 0);
+          int offset = buf->b_virt_line_leftcol ? 0 : win_col_offset;
+          draw_virt_text_item(buf, offset, kv_A(buf->b_virt_lines, fpos),
+                              kHlModeReplace, grid->Columns, offset);
+        }
+      } else {
+        draw_virt_text(buf, win_col_offset, &draw_col, grid->Columns);
+      }
+
       grid_put_linebuf(grid, row, 0, draw_col, grid->Columns, wp->w_p_rl,
                        wp, wp->w_hl_attr_normal, wrap);
       if (wrap) {
@@ -4485,69 +4517,82 @@ void draw_virt_text(buf_T *buf, int col_off, int *end_col, int max_col)
   bool do_eol = state->eol_col > -1;
   for (size_t i = 0; i < kv_size(state->active); i++) {
     DecorRange *item = &kv_A(state->active, i);
-    if (item->start_row == state->row && kv_size(item->decor.virt_text)) {
+    if (!(item->start_row == state->row && kv_size(item->decor.virt_text))) {
-      if (item->win_col == -1) {
+      continue;
-        if (item->decor.virt_text_pos == kVTRightAlign) {
-          right_pos -= item->decor.virt_text_width;
-          item->win_col = right_pos;
-        } else if (item->decor.virt_text_pos == kVTEndOfLine && do_eol) {
-          item->win_col = state->eol_col;
-          state->eol_col += item->decor.virt_text_width;
-        } else if (item->decor.virt_text_pos == kVTWinCol) {
-          item->win_col = MAX(item->decor.col+col_off, 0);
-        }
-      }
-      if (item->win_col < 0) {
-        continue;
-      }
-      VirtText vt = item->decor.virt_text;
-      HlMode hl_mode = item->decor.hl_mode;
-      LineState s = LINE_STATE("");
-      int virt_attr = 0;
-      int col = item->win_col;
-      size_t virt_pos = 0;
-      item->win_col = -2;  // deactivate
-
-      while (col < max_col) {
-        if (!*s.p) {
-          if (virt_pos >= kv_size(vt)) {
-            break;
-          }
-          virt_attr = 0;
-          do {
-            s.p = kv_A(vt, virt_pos).text;
-            int hl_id = kv_A(vt, virt_pos).hl_id;
-            virt_attr = hl_combine_attr(virt_attr,
-                                        hl_id > 0 ? syn_id2attr(hl_id) : 0);
-            virt_pos++;
-          } while (!s.p && virt_pos < kv_size(vt));
-          if (!s.p) {
-            break;
-          }
-        }
-        int attr;
-        bool through = false;
-        if (hl_mode == kHlModeCombine) {
-          attr = hl_combine_attr(linebuf_attr[col], virt_attr);
-        } else if (hl_mode == kHlModeBlend) {
-          through = (*s.p == ' ');
-          attr = hl_blend_attrs(linebuf_attr[col], virt_attr, &through);
-        } else {
-          attr = virt_attr;
-        }
-        schar_T dummy[2];
-        int cells = line_putchar(&s, through ? dummy : &linebuf_char[col],
-                                 max_col-col, false);
-        linebuf_attr[col++] = attr;
-        if (cells > 1) {
-          linebuf_attr[col++] = attr;
-        }
-      }
-      *end_col = MAX(*end_col, col);
     }
+    if (item->win_col == -1) {
+      if (item->decor.virt_text_pos == kVTRightAlign) {
+        right_pos -= item->decor.virt_text_width;
+        item->win_col = right_pos;
+      } else if (item->decor.virt_text_pos == kVTEndOfLine && do_eol) {
+        item->win_col = state->eol_col;
+      } else if (item->decor.virt_text_pos == kVTWinCol) {
+        item->win_col = MAX(item->decor.col+col_off, 0);
+      }
+    }
+    if (item->win_col < 0) {
+      continue;
+    }
+
+    int col = draw_virt_text_item(buf, item->win_col, item->decor.virt_text,
+                                  item->decor.hl_mode, max_col, item->win_col-col_off);
+    item->win_col = -2;  // deactivate
+    if (item->decor.virt_text_pos == kVTEndOfLine && do_eol) {
+      state->eol_col = col+1;
+    }
+
+    *end_col = MAX(*end_col, col);
   }
 }
 
+static int draw_virt_text_item(buf_T *buf, int col, VirtText vt, HlMode hl_mode,
+                               int max_col, int vcol)
+{
+  LineState s = LINE_STATE("");
+  int virt_attr = 0;
+  size_t virt_pos = 0;
+
+  while (col < max_col) {
+    if (!*s.p) {
+      if (virt_pos >= kv_size(vt)) {
+        break;
+      }
+      virt_attr = 0;
+      do {
+        s.p = kv_A(vt, virt_pos).text;
+        int hl_id = kv_A(vt, virt_pos).hl_id;
+        virt_attr = hl_combine_attr(virt_attr,
+                                    hl_id > 0 ? syn_id2attr(hl_id) : 0);
+        virt_pos++;
+      } while (!s.p && virt_pos < kv_size(vt));
+      if (!s.p) {
+        break;
+      }
+    }
+    int attr;
+    bool through = false;
+    if (hl_mode == kHlModeCombine) {
+      attr = hl_combine_attr(linebuf_attr[col], virt_attr);
+    } else if (hl_mode == kHlModeBlend) {
+      through = (*s.p == ' ');
+      attr = hl_blend_attrs(linebuf_attr[col], virt_attr, &through);
+    } else {
+      attr = virt_attr;
+    }
+    schar_T dummy[2];
+    int cells = line_putchar(buf, &s, through ? dummy : &linebuf_char[col],
+                             max_col-col, false, vcol);
+    // if we failed to emit a char, we still need to advance
+    cells = MAX(cells, 1);
+
+    for (int c = 0; c < cells; c++) {
+      linebuf_attr[col++] = attr;
+    }
+    vcol += cells;
+  }
+  return col;
+}
+
 /// Determine if dedicated window grid should be used or the default_grid
 ///
 /// If UI did not request multigrid support, draw all windows on the
@@ -5489,7 +5534,7 @@ static void win_redr_custom(win_T *wp, bool draw_ruler)
   ewp->w_p_crb = p_crb_save;
 
   // Make all characters printable.
-  p = (char_u *)transstr((const char *)buf);
+  p = (char_u *)transstr((const char *)buf, true);
   len = STRLCPY(buf, p, sizeof(buf));
   len = (size_t)len < sizeof(buf) ? len : (int)sizeof(buf) - 1;
   xfree(p);

test/functional/ui/decorations_spec.lua

@@ -790,3 +790,511 @@ end]]
     helpers.assert_alive()
   end)
 end)
+
+describe('decorations: virtual lines', function()
+  local screen, ns
+  before_each(function()
+    clear()
+    screen = Screen.new(50, 12)
+    screen:attach()
+    screen:set_default_attr_ids {
+      [1] = {bold=true, foreground=Screen.colors.Blue};
+      [2] = {foreground = Screen.colors.Cyan4};
+      [3] = {background = Screen.colors.Yellow1};
+      [4] = {bold = true};
+      [5] = {background = Screen.colors.Yellow, foreground = Screen.colors.Blue};
+      [6] = {foreground = Screen.colors.Blue};
+      [7] = {foreground = Screen.colors.SlateBlue};
+      [8] = {background = Screen.colors.WebGray, foreground = Screen.colors.DarkBlue};
+      [9] = {foreground = Screen.colors.Brown};
+    }
+
+    ns = meths.create_namespace 'test'
+  end)
+
+  local example_text = [[
+if (h->n_buckets < new_n_buckets) { // expand
+  khkey_t *new_keys = (khkey_t *)krealloc((void *)h->keys, new_n_buckets * sizeof(khkey_t));
+  h->keys = new_keys;
+  if (kh_is_map && val_size) {
+    char *new_vals = krealloc( h->vals_buf, new_n_buckets * val_size);
+    h->vals_buf = new_vals;
+  }
+}]]
+
+  it('works with one line', function()
+    insert(example_text)
+    feed 'gg'
+    meths.buf_set_extmark(0, ns, 1, 33, {
+      virt_lines={ {{">> ", "NonText"}, {"krealloc", "Identifier"}, {": change the size of an allocation"}}};
+      virt_lines_above=true;
+    })
+
+    screen:expect{grid=[[
+      ^if (h->n_buckets < new_n_buckets) { // expand     |
+      {1:>> }{2:krealloc}: change the size of an allocation     |
+        khkey_t *new_keys = (khkey_t *)krealloc((void *)|
+      h->keys, new_n_buckets * sizeof(khkey_t));        |
+        h->keys = new_keys;                             |
+        if (kh_is_map && val_size) {                    |
+          char *new_vals = krealloc( h->vals_buf, new_n_|
+      buckets * val_size);                              |
+          h->vals_buf = new_vals;                       |
+        }                                               |
+      }                                                 |
+                                                        |
+    ]]}
+
+    feed '/krealloc<cr>'
+    screen:expect{grid=[[
+      if (h->n_buckets < new_n_buckets) { // expand     |
+      {1:>> }{2:krealloc}: change the size of an allocation     |
+        khkey_t *new_keys = (khkey_t *){3:^krealloc}((void *)|
+      h->keys, new_n_buckets * sizeof(khkey_t));        |
+        h->keys = new_keys;                             |
+        if (kh_is_map && val_size) {                    |
+          char *new_vals = {3:krealloc}( h->vals_buf, new_n_|
+      buckets * val_size);                              |
+          h->vals_buf = new_vals;                       |
+        }                                               |
+      }                                                 |
+      /krealloc                                         |
+    ]]}
+
+    -- virtual line remains anchored to the extmark
+    feed 'i<cr>'
+    screen:expect{grid=[[
+      if (h->n_buckets < new_n_buckets) { // expand     |
+        khkey_t *new_keys = (khkey_t *)                 |
+      {1:>> }{2:krealloc}: change the size of an allocation     |
+      {3:^krealloc}((void *)h->keys, new_n_buckets * sizeof(k|
+      hkey_t));                                         |
+        h->keys = new_keys;                             |
+        if (kh_is_map && val_size) {                    |
+          char *new_vals = {3:krealloc}( h->vals_buf, new_n_|
+      buckets * val_size);                              |
+          h->vals_buf = new_vals;                       |
+        }                                               |
+      {4:-- INSERT --}                                      |
+    ]]}
+
+    feed '<esc>3+'
+    screen:expect{grid=[[
+      if (h->n_buckets < new_n_buckets) { // expand     |
+        khkey_t *new_keys = (khkey_t *)                 |
+      {1:>> }{2:krealloc}: change the size of an allocation     |
+      {3:krealloc}((void *)h->keys, new_n_buckets * sizeof(k|
+      hkey_t));                                         |
+        h->keys = new_keys;                             |
+        if (kh_is_map && val_size) {                    |
+          ^char *new_vals = {3:krealloc}( h->vals_buf, new_n_|
+      buckets * val_size);                              |
+          h->vals_buf = new_vals;                       |
+        }                                               |
+                                                        |
+    ]]}
+
+    meths.buf_set_extmark(0, ns, 5, 0, {
+      virt_lines = { {{"^^ REVIEW:", "Todo"}, {" new_vals variable seems unneccesary?", "Comment"}} };
+    })
+    -- TODO: what about the cursor??
+    screen:expect{grid=[[
+      if (h->n_buckets < new_n_buckets) { // expand     |
+        khkey_t *new_keys = (khkey_t *)                 |
+      {3:krealloc}((void *)h->keys, new_n_buckets * sizeof(k|
+      hkey_t));                                         |
+        h->keys = new_keys;                             |
+        if (kh_is_map && val_size) {                    |
+          char *new_vals = {3:krealloc}( h->vals_buf, new_n_|
+      buck^ets * val_size);                              |
+      {5:^^ REVIEW:}{6: new_vals variable seems unneccesary?}   |
+          h->vals_buf = new_vals;                       |
+        }                                               |
+                                                        |
+    ]]}
+
+    meths.buf_clear_namespace(0, ns, 0, -1)
+    screen:expect{grid=[[
+      if (h->n_buckets < new_n_buckets) { // expand     |
+        khkey_t *new_keys = (khkey_t *)                 |
+      {3:krealloc}((void *)h->keys, new_n_buckets * sizeof(k|
+      hkey_t));                                         |
+        h->keys = new_keys;                             |
+        if (kh_is_map && val_size) {                    |
+          char *new_vals = {3:krealloc}( h->vals_buf, new_n_|
+      buck^ets * val_size);                              |
+          h->vals_buf = new_vals;                       |
+        }                                               |
+      }                                                 |
+                                                        |
+    ]]}
+  end)
+
+
+  it('works with text at the beginning of the buffer', function()
+    insert(example_text)
+    feed 'gg'
+
+    screen:expect{grid=[[
+      ^if (h->n_buckets < new_n_buckets) { // expand     |
+        khkey_t *new_keys = (khkey_t *)krealloc((void *)|
+      h->keys, new_n_buckets * sizeof(khkey_t));        |
+        h->keys = new_keys;                             |
+        if (kh_is_map && val_size) {                    |
+          char *new_vals = krealloc( h->vals_buf, new_n_|
+      buckets * val_size);                              |
+          h->vals_buf = new_vals;                       |
+        }                                               |
+      }                                                 |
+      {1:~                                                 }|
+                                                        |
+    ]]}
+
+    meths.buf_set_extmark(0, ns, 0, 0, {
+      virt_lines={
+        {{"refactor(khash): ", "Special"}, {"take size of values as parameter"}};
+        {{"Author: Dev Devsson, "}, {"Tue Aug 31 10:13:37 2021", "Comment"}};
+      };
+      virt_lines_above=true;
+      right_gravity=false;
+    })
+
+    -- placing virt_text on topline does not automatically cause a scroll
+    screen:expect{grid=[[
+      ^if (h->n_buckets < new_n_buckets) { // expand     |
+        khkey_t *new_keys = (khkey_t *)krealloc((void *)|
+      h->keys, new_n_buckets * sizeof(khkey_t));        |
+        h->keys = new_keys;                             |
+        if (kh_is_map && val_size) {                    |
+          char *new_vals = krealloc( h->vals_buf, new_n_|
+      buckets * val_size);                              |
+          h->vals_buf = new_vals;                       |
+        }                                               |
+      }                                                 |
+      {1:~                                                 }|
+                                                        |
+    ]], unchanged=true}
+
+    feed '<c-b>'
+    screen:expect{grid=[[
+      {7:refactor(khash): }take size of values as parameter |
+      Author: Dev Devsson, {6:Tue Aug 31 10:13:37 2021}     |
+      ^if (h->n_buckets < new_n_buckets) { // expand     |
+        khkey_t *new_keys = (khkey_t *)krealloc((void *)|
+      h->keys, new_n_buckets * sizeof(khkey_t));        |
+        h->keys = new_keys;                             |
+        if (kh_is_map && val_size) {                    |
+          char *new_vals = krealloc( h->vals_buf, new_n_|
+      buckets * val_size);                              |
+          h->vals_buf = new_vals;                       |
+        }                                               |
+                                                        |
+    ]]}
+  end)
+
+  it('works with text et the end of the buffer', function()
+    insert(example_text)
+    feed 'G'
+
+    screen:expect{grid=[[
+      if (h->n_buckets < new_n_buckets) { // expand     |
+        khkey_t *new_keys = (khkey_t *)krealloc((void *)|
+      h->keys, new_n_buckets * sizeof(khkey_t));        |
+        h->keys = new_keys;                             |
+        if (kh_is_map && val_size) {                    |
+          char *new_vals = krealloc( h->vals_buf, new_n_|
+      buckets * val_size);                              |
+          h->vals_buf = new_vals;                       |
+        }                                               |
+      ^}                                                 |
+      {1:~                                                 }|
+                                                        |
+    ]]}
+
+    local id = meths.buf_set_extmark(0, ns, 7, 0, {
+      virt_lines={{{"Grugg"}}};
+      right_gravity=false;
+    })
+
+    screen:expect{grid=[[
+      if (h->n_buckets < new_n_buckets) { // expand     |
+        khkey_t *new_keys = (khkey_t *)krealloc((void *)|
+      h->keys, new_n_buckets * sizeof(khkey_t));        |
+        h->keys = new_keys;                             |
+        if (kh_is_map && val_size) {                    |
+          char *new_vals = krealloc( h->vals_buf, new_n_|
+      buckets * val_size);                              |
+          h->vals_buf = new_vals;                       |
+        }                                               |
+      ^}                                                 |
+      Grugg                                             |
+                                                        |
+    ]]}
+
+    meths.buf_del_extmark(0, ns, id)
+    screen:expect{grid=[[
+      if (h->n_buckets < new_n_buckets) { // expand     |
+        khkey_t *new_keys = (khkey_t *)krealloc((void *)|
+      h->keys, new_n_buckets * sizeof(khkey_t));        |
+        h->keys = new_keys;                             |
+        if (kh_is_map && val_size) {                    |
+          char *new_vals = krealloc( h->vals_buf, new_n_|
+      buckets * val_size);                              |
+          h->vals_buf = new_vals;                       |
+        }                                               |
+      ^}                                                 |
+      {1:~                                                 }|
+                                                        |
+    ]]}
+  end)
+
+  it('works with a block scrolling up', function()
+    screen:try_resize(30, 7)
+    insert("aa\nbb\ncc\ndd\nee\nff\ngg\nhh")
+    feed 'gg'
+
+    meths.buf_set_extmark(0, ns, 6, 0, {
+      virt_lines={
+        {{"they see me"}};
+        {{"scrolling", "Special"}};
+        {{"they"}};
+        {{"hatin'", "Special"}};
+      };
+    })
+
+    screen:expect{grid=[[
+      ^aa                            |
+      bb                            |
+      cc                            |
+      dd                            |
+      ee                            |
+      ff                            |
+                                    |
+    ]]}
+
+    feed '<c-e>'
+    screen:expect{grid=[[
+      ^bb                            |
+      cc                            |
+      dd                            |
+      ee                            |
+      ff                            |
+      gg                            |
+                                    |
+    ]]}
+
+    feed '<c-e>'
+    screen:expect{grid=[[
+      ^cc                            |
+      dd                            |
+      ee                            |
+      ff                            |
+      gg                            |
+      they see me                   |
+                                    |
+    ]]}
+
+    feed '<c-e>'
+    screen:expect{grid=[[
+      ^dd                            |
+      ee                            |
+      ff                            |
+      gg                            |
+      they see me                   |
+      {7:scrolling}                     |
+                                    |
+    ]]}
+
+    feed '<c-e>'
+    screen:expect{grid=[[
+      ^ee                            |
+      ff                            |
+      gg                            |
+      they see me                   |
+      {7:scrolling}                     |
+      they                          |
+                                    |
+    ]]}
+
+    feed '<c-e>'
+    screen:expect{grid=[[
+      ^ff                            |
+      gg                            |
+      they see me                   |
+      {7:scrolling}                     |
+      they                          |
+      {7:hatin'}                        |
+                                    |
+    ]]}
+
+    feed '<c-e>'
+    screen:expect{grid=[[
+      ^gg                            |
+      they see me                   |
+      {7:scrolling}                     |
+      they                          |
+      {7:hatin'}                        |
+      hh                            |
+                                    |
+    ]]}
+
+    feed '<c-e>'
+    screen:expect{grid=[[
+      they see me                   |
+      {7:scrolling}                     |
+      they                          |
+      {7:hatin'}                        |
+      ^hh                            |
+      {1:~                             }|
+                                    |
+    ]]}
+
+    feed '<c-e>'
+    screen:expect{grid=[[
+      {7:scrolling}                     |
+      they                          |
+      {7:hatin'}                        |
+      ^hh                            |
+      {1:~                             }|
+      {1:~                             }|
+                                    |
+    ]]}
+
+    feed '<c-e>'
+    screen:expect{grid=[[
+      they                          |
+      {7:hatin'}                        |
+      ^hh                            |
+      {1:~                             }|
+      {1:~                             }|
+      {1:~                             }|
+                                    |
+    ]]}
+
+    feed '<c-e>'
+    screen:expect{grid=[[
+      {7:hatin'}                        |
+      ^hh                            |
+      {1:~                             }|
+      {1:~                             }|
+      {1:~                             }|
+      {1:~                             }|
+                                    |
+    ]]}
+
+    feed '<c-e>'
+    screen:expect{grid=[[
+      ^hh                            |
+      {1:~                             }|
+      {1:~                             }|
+      {1:~                             }|
+      {1:~                             }|
+      {1:~                             }|
+                                    |
+    ]]}
+  end)
+
+  it('works with sign and numbercolumns', function()
+    insert(example_text)
+    feed 'gg'
+    command 'set number signcolumn=yes'
+    screen:expect{grid=[[
+      {8:  }{9:  1 }^if (h->n_buckets < new_n_buckets) { // expan|
+      {8:  }{9:    }d                                           |
+      {8:  }{9:  2 }  khkey_t *new_keys = (khkey_t *)krealloc((v|
+      {8:  }{9:    }oid *)h->keys, new_n_buckets * sizeof(khkey_|
+      {8:  }{9:    }t));                                        |
+      {8:  }{9:  3 }  h->keys = new_keys;                       |
+      {8:  }{9:  4 }  if (kh_is_map && val_size) {              |
+      {8:  }{9:  5 }    char *new_vals = krealloc( h->vals_buf, |
+      {8:  }{9:    }new_n_buckets * val_size);                  |
+      {8:  }{9:  6 }    h->vals_buf = new_vals;                 |
+      {8:  }{9:  7 }  }                                         |
+                                                        |
+    ]]}
+
+    meths.buf_set_extmark(0, ns, 2, 0, {
+      virt_lines={
+        {{"Some special", "Special"}};
+        {{"remark about codes", "Comment"}};
+      };
+    })
+
+    screen:expect{grid=[[
+      {8:  }{9:  1 }^if (h->n_buckets < new_n_buckets) { // expan|
+      {8:  }{9:    }d                                           |
+      {8:  }{9:  2 }  khkey_t *new_keys = (khkey_t *)krealloc((v|
+      {8:  }{9:    }oid *)h->keys, new_n_buckets * sizeof(khkey_|
+      {8:  }{9:    }t));                                        |
+      {8:  }{9:  3 }  h->keys = new_keys;                       |
+      {8:  }{9:    }{7:Some special}                                |
+      {8:  }{9:    }{6:remark about codes}                          |
+      {8:  }{9:  4 }  if (kh_is_map && val_size) {              |
+      {8:  }{9:  5 }    char *new_vals = krealloc( h->vals_buf, |
+      {8:  }{9:    }new_n_buckets * val_size);                  |
+                                                        |
+    ]]}
+
+    meths.buf_set_extmark(0, ns, 2, 0, {
+      virt_lines={
+        {{"Some special", "Special"}};
+        {{"remark about codes", "Comment"}};
+      };
+      virt_lines_leftcol=true;
+    })
+    screen:expect{grid=[[
+      {8:  }{9:  1 }^if (h->n_buckets < new_n_buckets) { // expan|
+      {8:  }{9:    }d                                           |
+      {8:  }{9:  2 }  khkey_t *new_keys = (khkey_t *)krealloc((v|
+      {8:  }{9:    }oid *)h->keys, new_n_buckets * sizeof(khkey_|
+      {8:  }{9:    }t));                                        |
+      {8:  }{9:  3 }  h->keys = new_keys;                       |
+      {7:Some special}                                      |
+      {6:remark about codes}                                |
+      {8:  }{9:  4 }  if (kh_is_map && val_size) {              |
+      {8:  }{9:  5 }    char *new_vals = krealloc( h->vals_buf, |
+      {8:  }{9:    }new_n_buckets * val_size);                  |
+                                                        |
+    ]]}
+  end)
+
+
+  it('works with hard tabs', function()
+    insert(example_text)
+    feed 'gg'
+    meths.buf_set_extmark(0, ns, 1, 0, {
+      virt_lines={ {{">>", "NonText"}, {"\tvery\ttabby", "Identifier"}, {"text\twith\ttabs"}}};
+    })
+    screen:expect{grid=[[
+      ^if (h->n_buckets < new_n_buckets) { // expand     |
+        khkey_t *new_keys = (khkey_t *)krealloc((void *)|
+      h->keys, new_n_buckets * sizeof(khkey_t));        |
+      {1:>>}{2:      very    tabby}text       with    tabs      |
+        h->keys = new_keys;                             |
+        if (kh_is_map && val_size) {                    |
+          char *new_vals = krealloc( h->vals_buf, new_n_|
+      buckets * val_size);                              |
+          h->vals_buf = new_vals;                       |
+        }                                               |
+      }                                                 |
+                                                        |
+    ]]}
+
+    command 'set tabstop=4'
+    screen:expect{grid=[[
+      ^if (h->n_buckets < new_n_buckets) { // expand     |
+        khkey_t *new_keys = (khkey_t *)krealloc((void *)|
+      h->keys, new_n_buckets * sizeof(khkey_t));        |
+      {1:>>}{2:  very    tabby}text   with    tabs              |
+        h->keys = new_keys;                             |
+        if (kh_is_map && val_size) {                    |
+          char *new_vals = krealloc( h->vals_buf, new_n_|
+      buckets * val_size);                              |
+          h->vals_buf = new_vals;                       |
+        }                                               |
+      }                                                 |
+                                                        |
+    ]]}
+  end)
+
+end)