os/input.c: parse keycodes in non-string context #7411

cb02137dfa had two mistakes, of the same nature: trans_special() must
be invoked with in_string=false unless the parsing context is a VimL
string. replace_termcodes() and input_enqueue() are low-level
mechanisms where VimL strings do not exist.

keymap.c: adjust double-quote case to satisfy keymap_spec.lua

closes #7410

src/nvim/keymap.c

@@ -481,7 +481,7 @@ char_u *get_special_key_name(int c, int modifiers)
   return string;
 }
 
-/// Try translating a <> name
+/// Try translating a <> name ("keycode").
 ///
 /// @param[in,out]  srcp  Source from which <> are translated. Is advanced to
 ///                       after the <> name if there is a match.
@@ -619,9 +619,11 @@ int find_special_key(const char_u **srcp, const size_t src_len, int *const modp,
 
         // Modifier with single letter, or special key name.
         if (in_string && last_dash[1] == '\\' && last_dash[2] == '"') {
-          off = 2;
+          // Special case for a double-quoted string
+          off = l = 2;
+        } else {
+          l = mb_ptr2len(last_dash + 1);
         }
-        l = mb_ptr2len(last_dash + 1);
         if (modifiers != 0 && last_dash[l + 1] == '>') {
           key = PTR2CHAR(last_dash + off);
         } else {
@@ -834,7 +836,7 @@ char_u *replace_termcodes(const char_u *from, const size_t from_len,
       }
 
       slen = trans_special(&src, (size_t)(end - src) + 1, result + dlen, true,
-                           true);
+                           false);
       if (slen) {
         dlen += slen;
         continue;

src/nvim/os/input.c

@@ -188,7 +188,7 @@ size_t input_enqueue(String keys)
     uint8_t buf[6] = { 0 };
     unsigned int new_size
         = trans_special((const uint8_t **)&ptr, (size_t)(end - ptr), buf, true,
-                        true);
+                        false);
 
     if (new_size) {
       new_size = handle_mouse_event(&ptr, buf, new_size);

test/functional/ex_cmds/map_spec.lua

@@ -5,6 +5,7 @@ local feed = helpers.feed
 local meths = helpers.meths
 local clear = helpers.clear
 local command = helpers.command
+local expect = helpers.expect
 
 describe(':*map', function()
   before_each(clear)
@@ -18,4 +19,10 @@ describe(':*map', function()
     feed('\24\25<C-x><C-y>')
     eq(4, meths.get_var('counter'))
   end)
+
+  it(':imap <M-">', function()
+    command('imap <M-"> foo')
+    feed('i-<M-">-')
+    expect('-foo-')
+  end)
 end)

test/unit/keymap_spec.lua

@@ -0,0 +1,71 @@
+local helpers = require("test.unit.helpers")(after_each)
+local itp = helpers.gen_itp(it)
+
+local ffi     = helpers.ffi
+local eq      = helpers.eq
+local neq     = helpers.neq
+
+local keymap = helpers.cimport("./src/nvim/keymap.h")
+
+describe('keymap.c', function()
+
+  describe('find_special_key()', function()
+    local srcp = ffi.new('const unsigned char *[1]')
+    local modp = ffi.new('int[1]')
+
+    itp('no keycode', function()
+      srcp[0] = 'abc'
+      eq(0, keymap.find_special_key(srcp, 3, modp, false, false, false))
+    end)
+
+    itp('keycode with multiple modifiers', function()
+      srcp[0] = '<C-M-S-A>'
+      neq(0, keymap.find_special_key(srcp, 9, modp, false, false, false))
+      neq(0, modp[0])
+    end)
+
+    itp('case-insensitive', function()
+      -- Compare other capitalizations to this.
+      srcp[0] = '<C-A>'
+      local all_caps_key =
+          keymap.find_special_key(srcp, 5, modp, false, false, false)
+      local all_caps_mod = modp[0]
+
+      srcp[0] = '<C-a>'
+      eq(all_caps_key,
+         keymap.find_special_key(srcp, 5, modp, false, false, false))
+      eq(all_caps_mod, modp[0])
+
+      srcp[0] = '<c-A>'
+      eq(all_caps_key,
+         keymap.find_special_key(srcp, 5, modp, false, false, false))
+      eq(all_caps_mod, modp[0])
+
+      srcp[0] = '<c-a>'
+      eq(all_caps_key,
+         keymap.find_special_key(srcp, 5, modp, false, false, false))
+      eq(all_caps_mod, modp[0])
+    end)
+
+    itp('double-quote in keycode #7411', function()
+      -- Unescaped with in_string=false
+      srcp[0] = '<C-">'
+      eq(string.byte('"'),
+         keymap.find_special_key(srcp, 5, modp, false, false, false))
+
+      -- Unescaped with in_string=true
+      eq(0, keymap.find_special_key(srcp, 5, modp, false, false, true))
+
+      -- Escaped with in_string=false
+      srcp[0] = '<C-\\">'
+      -- Should fail because the key is invalid
+      -- (more than 1 non-modifier character).
+      eq(0, keymap.find_special_key(srcp, 6, modp, false, false, false))
+
+      -- Escaped with in_string=true
+      eq(string.byte('"'),
+         keymap.find_special_key(srcp, 6, modp, false, false, true))
+    end)
+  end)
+
+end)