viml/parser: Handle encoding conversions

src/nvim/viml/parser/expressions.c

@@ -25,8 +25,13 @@
 #define AUTOLOAD_CHAR '#'
 
 /// Get next token for the VimL expression input
-LexExprToken viml_pexpr_next_token(ParserState *const pstate)
-  FUNC_ATTR_WARN_UNUSED_RESULT
+///
+/// @param  pstate  Parser state.
+/// @param[in]  peek  If true, do not advance pstate cursor.
+///
+/// @return Next token.
+LexExprToken viml_pexpr_next_token(ParserState *const pstate, const bool peek)
+  FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL
 {
   LexExprToken ret = {
     .type = kExprLexInvalid,
@@ -362,6 +367,8 @@ viml_pexpr_next_token_invalid_comparison:
   }
 #undef GET_CCS
 viml_pexpr_next_token_adv_return:
-  viml_parser_advance(pstate, ret.len);
+  if (!peek) {
+    viml_parser_advance(pstate, ret.len);
+  }
   return ret;
 }

src/nvim/viml/parser/parser.h

@@ -7,11 +7,13 @@
 
 #include "nvim/lib/kvec.h"
 #include "nvim/func_attr.h"
+#include "nvim/mbyte.h"
 
 /// One parsed line
 typedef struct {
   const char *data;  ///< Parsed line pointer
   size_t size;  ///< Parsed line size
+  bool allocated;  ///< True if line may be freed.
 } ParserLine;
 
 /// Line getter type for parser
@@ -48,6 +50,8 @@ typedef struct {
   void *cookie;
   /// All lines obtained by get_line.
   kvec_withinit_t(ParserLine, 4) lines;
+  /// Conversion, for :scriptencoding.
+  vimconv_T conv;
 } ParserInputReader;
 
 /// Highlighted region definition
@@ -76,6 +80,33 @@ typedef struct {
   bool can_continuate;
 } ParserState;
 
+static inline void viml_preader_get_line(ParserInputReader *const preader,
+                                         ParserLine *const ret_pline)
+  REAL_FATTR_NONNULL_ALL;
+
+/// Get one line from ParserInputReader
+static inline void viml_preader_get_line(ParserInputReader *const preader,
+                                         ParserLine *const ret_pline)
+{
+  ParserLine pline;
+  preader->get_line(preader->cookie, &pline);
+  if (preader->conv.vc_type != CONV_NONE && pline.size) {
+    ParserLine cpline = {
+      .allocated = true,
+      .size = pline.size,
+    };
+    cpline.data = (char *)string_convert(&preader->conv,
+                                         (char_u *)pline.data,
+                                         &cpline.size);
+    if (pline.allocated) {
+      xfree((void *)pline.data);
+    }
+    pline = cpline;
+  }
+  kvi_push(preader->lines, pline);
+  *ret_pline = pline;
+}
+
 static inline bool viml_parser_get_remaining_line(ParserState *const pstate,
                                                   ParserLine *const ret_pline)
   REAL_FATTR_ALWAYS_INLINE REAL_FATTR_WARN_UNUSED_RESULT REAL_FATTR_NONNULL_ALL;
@@ -90,8 +121,7 @@ static inline bool viml_parser_get_remaining_line(ParserState *const pstate,
 {
   const size_t num_lines = kv_size(pstate->reader.lines);
   if (pstate->pos.line == num_lines) {
-    pstate->reader.get_line(pstate->reader.cookie, ret_pline);
-    kvi_push(pstate->reader.lines, *ret_pline);
+    viml_preader_get_line(&pstate->reader, ret_pline);
   } else {
     *ret_pline = kv_last(pstate->reader.lines);
   }

test/unit/viml/expressions/lexer_spec.lua

@@ -110,15 +110,22 @@ local function new_pstate(strings)
     end
     ret_pline.data = data
     ret_pline.size = size
+    ret_pline.allocated = false
   end
   local pline_init = {
     data = nil,
     size = 0,
+    allocated = false,
   }
   local state = {
     reader = {
       get_line = get_line,
       cookie = nil,
+      conv = {
+        vc_type = 0,
+        vc_factor = 1,
+        vc_fail = false,
+      },
     },
     pos = { line = 0, col = 0 },
     colors = kvi_new('ParserHighlight'),