fix(:source, nvim_exec): handle Vimscript line continuations #14809

Problem:
Anonymous :source (no args) and nvim_exec() don't support Vimscript line continuations.

Solution:
Factor out the concat logic into concat_continued_line() and a
CONCAT_CONTINUED_LINES macro for simple concatenations where lines are
fetched individually.

Closes #14807

src/nvim/charset.c

@@ -1150,15 +1150,29 @@ void getvcols(win_T *wp, pos_T *pos1, pos_T *pos2, colnr_T *left,
 
 /// skipwhite: skip over ' ' and '\t'.
 ///
-/// @param[in]  q  String to skip in.
+/// @param[in]  p  String to skip in.
 ///
 /// @return Pointer to character after the skipped whitespace.
-char_u *skipwhite(const char_u *q)
+char_u *skipwhite(const char_u *const p)
   FUNC_ATTR_PURE FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL
   FUNC_ATTR_NONNULL_RET
 {
-  const char_u *p = q;
+  return skipwhite_len(p, STRLEN(p));
-  while (ascii_iswhite(*p)) {
+}
+
+/// Like `skipwhite`, but skip up to `len` characters.
+/// @see skipwhite
+///
+/// @param[in]  p    String to skip in.
+/// @param[in]  len  Max length to skip.
+///
+/// @return Pointer to character after the skipped whitespace, or the `len`-th
+///         character in the string.
+char_u *skipwhite_len(const char_u *p, size_t len)
+  FUNC_ATTR_PURE FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL
+  FUNC_ATTR_NONNULL_RET
+{
+  for (; len > 0 && ascii_iswhite(*p); len--) {
     p++;
   }
   return (char_u *)p;
@@ -1304,6 +1318,18 @@ char_u* skiptowhite_esc(char_u *p) {
   return p;
 }
 
+/// Skip over text until '\n' or NUL.
+///
+/// @param[in]  p  Text to skip over.
+///
+/// @return Pointer to the next '\n' or NUL character.
+char_u *skip_to_newline(const char_u *const p)
+  FUNC_ATTR_PURE FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL
+  FUNC_ATTR_NONNULL_RET
+{
+  return (char_u *)xstrchrnul((const char *)p, NL);
+}
+
 /// Gets a number from a string and skips over it, signalling overflow.
 ///
 /// @param[out]  pp  A pointer to a pointer to char_u.

src/nvim/ex_cmds2.c

@@ -1829,6 +1829,59 @@ static void cmd_source(char_u *fname, exarg_T *eap)
   }
 }
 
+/// Concatenate VimL line if it starts with a line continuation into a growarray
+/// (excluding the continuation chars and leading whitespace)
+///
+/// @note Growsize of the growarray may be changed to speed up concatenations!
+///
+/// @param ga  the growarray to append to
+/// @param init_growsize  the starting growsize value of the growarray
+/// @param p  pointer to the beginning of the line to consider
+/// @param len  the length of this line
+///
+/// @return true if this line did begin with a continuation (the next line
+///         should also be considered, if it exists); false otherwise
+static bool concat_continued_line(garray_T *const ga, const int init_growsize,
+                                  const char_u *const p, size_t len)
+  FUNC_ATTR_NONNULL_ALL
+{
+  const char_u *const line = skipwhite_len(p, len);
+  len -= (size_t)(line - p);
+  // Skip lines starting with '\" ', concat lines starting with '\'
+  if (len >= 3 && STRNCMP(line, "\"\\ ", 3) == 0) {
+    return true;
+  } else if (len == 0 || line[0] != '\\') {
+    return false;
+  }
+  if (ga->ga_len > init_growsize) {
+    ga_set_growsize(ga, MAX(ga->ga_len, 8000));
+  }
+  ga_concat_len(ga, (const char *)line + 1, len - 1);
+  return true;
+}
+
+/// Concatenate (possibly many) VimL lines starting with line continuations into
+/// a growarray. @see concat_continued_line
+///
+/// @note All parameters, excluding `ga`, accept expressions that are evaluated
+///       once for each line; side-effects may be triggered many times!
+///
+/// @param ga  the growarray to append to
+/// @param cond  should evaluate to true if a next line exists
+/// @param line  should evaluate to the current line
+/// @param next  should handle fetching the next line when evaluated
+#define CONCAT_CONTINUED_LINES(ga, cond, line, next) \
+  do { \
+    garray_T *const ga_ = (ga); \
+    const int init_growsize_ = ga_->ga_growsize; \
+    for (; (cond); (next)) { \
+      const char_u *const line_ = (line); \
+      if (!concat_continued_line(ga_, init_growsize_, line_, STRLEN(line_))) { \
+        break; \
+      } \
+    } \
+  } while (false)
+
 typedef struct {
   linenr_T curr_lnum;
   const linenr_T final_lnum;
@@ -1845,9 +1898,15 @@ static char_u *get_buffer_line(int c, void *cookie, int indent, bool do_concat)
   if (p->curr_lnum > p->final_lnum) {
     return NULL;
   }
-  char_u *curr_line = ml_get(p->curr_lnum);
+  garray_T ga;
-  p->curr_lnum++;
+  ga_init(&ga, sizeof(char_u), 400);
-  return (char_u *)xstrdup((const char *)curr_line);
+  ga_concat(&ga, ml_get(p->curr_lnum++));
+  if (do_concat && vim_strchr(p_cpo, CPO_CONCAT) == NULL) {
+    CONCAT_CONTINUED_LINES(&ga, p->curr_lnum <= p->final_lnum,
+                           ml_get(p->curr_lnum), p->curr_lnum++);
+  }
+  ga_append(&ga, NUL);
+  return ga.ga_data;
 }
 
 static void cmd_source_buffer(const exarg_T *eap)
@@ -1919,17 +1978,27 @@ typedef struct {
 static char_u *get_str_line(int c, void *cookie, int indent, bool do_concat)
 {
   GetStrLineCookie *p = cookie;
-  size_t i = p->offset;
+  if (STRLEN(p->buf) <= p->offset) {
-  if (strlen((char *)p->buf) <= p->offset) {
     return NULL;
   }
-  while (!(p->buf[i] == '\n' || p->buf[i] == '\0')) {
+  const char_u *line = p->buf + p->offset;
-    i++;
+  const char_u *eol = skip_to_newline(line);
+  garray_T ga;
+  ga_init(&ga, sizeof(char_u), 400);
+  ga_concat_len(&ga, (const char *)line, (size_t)(eol - line));
+  if (do_concat && vim_strchr(p_cpo, CPO_CONCAT) == NULL) {
+    while (eol[0] != NUL) {
+      line = eol + 1;
+      const char_u *const next_eol = skip_to_newline(line);
+      if (!concat_continued_line(&ga, 400, line, (size_t)(next_eol - line))) {
+        break;
+      }
+      eol = next_eol;
+    }
   }
-  size_t line_length = i - p->offset;
+  ga_append(&ga, NUL);
-  char_u *buf = xmemdupz(p->buf + p->offset, line_length);
+  p->offset = (size_t)(eol - p->buf) + 1;
-  p->offset = i + 1;
+  return ga.ga_data;
-  return buf;
 }
 
 static int source_using_linegetter(void *cookie,
@@ -2421,27 +2490,9 @@ char_u *getsourceline(int c, void *cookie, int indent, bool do_concat)
 
       ga_init(&ga, (int)sizeof(char_u), 400);
       ga_concat(&ga, line);
-      if (*p == '\\') {
+      CONCAT_CONTINUED_LINES(&ga, sp->nextline != NULL, sp->nextline,
-        ga_concat(&ga, p + 1);
+                             (xfree(sp->nextline),
-      }
+                              sp->nextline = get_one_sourceline(sp)));
-      for (;; ) {
-        xfree(sp->nextline);
-        sp->nextline = get_one_sourceline(sp);
-        if (sp->nextline == NULL) {
-          break;
-        }
-        p = skipwhite(sp->nextline);
-        if (*p == '\\') {
-          // Adjust the growsize to the current length to speed up
-          // concatenating many lines.
-          if (ga.ga_len > 400) {
-            ga_set_growsize(&ga, (ga.ga_len > 8000) ? 8000 : ga.ga_len);
-          }
-          ga_concat(&ga, p + 1);
-        } else if (p[0] != '"' || p[1] != '\\' || p[2] != ' ') {
-          break;
-        }
-      }
       ga_append(&ga, NUL);
       xfree(line);
       line = ga.ga_data;

test/functional/api/vim_spec.lua

@@ -117,6 +117,19 @@ describe('API', function()
       nvim('exec','autocmd BufAdd * :let x1 = "Hello"', false)
       nvim('command', 'new foo')
       eq('Hello', request('nvim_eval', 'g:x1'))
+
+      -- Line continuations
+      nvim('exec', [[
+        let abc = #{
+          \ a: 1,
+         "\ b: 2,
+          \ c: 3
+          \ }]], false)
+      eq({a = 1, c = 3}, request('nvim_eval', 'g:abc'))
+
+      -- try no spaces before continuations to catch off-by-one error
+      nvim('exec', 'let ab = #{\n\\a: 98,\n"\\ b: 2\n\\}', false)
+      eq({a = 98}, request('nvim_eval', 'g:ab'))
     end)
 
     it('non-ASCII input', function()

test/functional/ex_cmds/source_spec.lua

@@ -8,6 +8,8 @@ local feed = helpers.feed
 local feed_command = helpers.feed_command
 local write_file = helpers.write_file
 local exec = helpers.exec
+local exc_exec = helpers.exc_exec
+local exec_lua = helpers.exec_lua
 local eval = helpers.eval
 local exec_capture = helpers.exec_capture
 local neq = helpers.neq
@@ -18,16 +20,30 @@ describe(':source', function()
   end)
 
   it('current buffer', function()
-    insert('let a = 2')
+    insert([[
+      let a = 2
+      let b = #{
+        \ k: "v"
+       "\ (o_o)
+        \ }]])
+
     command('source')
     eq('2', meths.exec('echo a', true))
+    eq("{'k': 'v'}", meths.exec('echo b', true))
+
+    exec('set cpoptions+=C')
+    eq('Vim(let):E15: Invalid expression: #{', exc_exec('source'))
   end)
 
   it('selection in current buffer', function()
-    insert(
+    insert([[
-      'let a = 2\n'..
+      let a = 2
-      'let a = 3\n'..
+      let a = 3
-      'let a = 4\n')
+      let a = 4
+      let b = #{
+       "\ (>_<)
+        \ K: "V"
+        \ }]])
 
     -- Source the 2nd line only
     feed('ggjV')
@@ -38,13 +54,17 @@ describe(':source', function()
     feed('ggjVG')
     feed_command(':source')
     eq('4', meths.exec('echo a', true))
+    eq("{'K': 'V'}", meths.exec('echo b', true))
+
+    exec('set cpoptions+=C')
+    eq('Vim(let):E15: Invalid expression: #{', exc_exec("'<,'>source"))
   end)
 
   it('multiline heredoc command', function()
-    insert(
+    insert([[
-      'lua << EOF\n'..
+      lua << EOF
-      'y = 4\n'..
+      y = 4
-      'EOF\n')
+      EOF]])
 
     command('source')
     eq('4', meths.exec('echo luaeval("y")', true))
@@ -67,13 +87,21 @@ describe(':source', function()
       vim.g.b = 5
       vim.g.b = 6
       vim.g.b = 7
+      a = [=[
+       "\ a
+        \ b]=]
     ]])
 
     command('edit '..test_file)
+
     feed('ggjV')
     feed_command(':source')
-
     eq(6, eval('g:b'))
+
+    feed('GVkk')
+    feed_command(':source')
+    eq('   "\\ a\n    \\ b', exec_lua('return _G.a'))
+
     os.remove(test_file)
   end)
 
@@ -84,12 +112,16 @@ describe(':source', function()
       vim.g.c = 10
       vim.g.c = 11
       vim.g.c = 12
+      a = [=[
+        \ 1
+       "\ 2]=]
     ]])
 
     command('edit '..test_file)
     feed_command(':source')
 
     eq(12, eval('g:c'))
+    eq('    \\ 1\n   "\\ 2', exec_lua('return _G.a'))
     os.remove(test_file)
   end)