Merge pull request #15721 from bfredl/electricboogalo

fix(runtime): some ordering issues (and start work on cache mechanism)

src/nvim/option.c

@@ -2397,6 +2397,8 @@ static char_u *did_set_string_option(int opt_idx, char_u **varp, bool new_value_
       os_setenv("VIMRUNTIME", "", 1);
       didset_vimruntime = false;
     }
+  } else if (varp == &p_rtp || varp == &p_pp) {  // 'runtimepath' 'packpath'
+    invalidate_search_path();
   } else if (varp == &curwin->w_p_culopt
              || gvarp == &curwin->w_allbuf_opt.wo_culopt) {  // 'cursorlineopt'
     if (**varp == NUL || fill_culopt_flags(*varp, curwin) != OK) {

src/nvim/runtime.c

@@ -6,6 +6,7 @@
 /// Management of runtime files (including packages)
 
 #include "nvim/ascii.h"
+#include "nvim/api/private/helpers.h"
 #include "nvim/charset.h"
 #include "nvim/eval.h"
 #include "nvim/ex_cmds.h"
@@ -20,6 +21,8 @@
 # include "runtime.c.generated.h"
 #endif
 
+static bool search_path_valid = false;
+static RuntimeSearchPath runtime_search_path;
 
 /// ":runtime [what] {name}"
 void ex_runtime(exarg_T *eap)
@@ -89,8 +92,7 @@ int do_in_path(char_u *path, char_u *name, int flags, DoInRuntimepathCB callback
 
       // Skip after or non-after directories.
       if (flags & (DIP_NOAFTER | DIP_AFTER)) {
-        bool is_after = buflen >= 5
+        bool is_after = path_is_after(buf, buflen);
-                        && STRCMP(buf + buflen - 5, "after") == 0;
 
         if ((is_after && (flags & DIP_NOAFTER))
             || (!is_after && (flags & DIP_AFTER))) {
@@ -155,6 +157,98 @@ int do_in_path(char_u *path, char_u *name, int flags, DoInRuntimepathCB callback
   return did_one ? OK : FAIL;
 }
 
+/// Find the file "name" in all directories in "path" and invoke
+/// "callback(fname, cookie)".
+/// "name" can contain wildcards.
+/// When "flags" has DIP_ALL: source all files, otherwise only the first one.
+/// When "flags" has DIP_DIR: find directories instead of files.
+/// When "flags" has DIP_ERR: give an error message if there is no match.
+///
+/// return FAIL when no file could be sourced, OK otherwise.
+int do_in_cached_path(char_u *name, int flags, DoInRuntimepathCB callback, void *cookie)
+{
+  validate_search_path();
+  char_u      *tail;
+  int num_files;
+  char_u      **files;
+  int i;
+  bool did_one = false;
+
+  char_u buf[MAXPATHL];
+
+  if (p_verbose > 10 && name != NULL) {
+    verbose_enter();
+    smsg(_("Searching for \"%s\" in runtime path"), (char *)name);
+    verbose_leave();
+  }
+
+  // Loop over all entries in 'runtimepath'.
+  for (size_t j = 0; j < kv_size(runtime_search_path); j++) {
+    SearchPathItem item = kv_A(runtime_search_path, j);
+    size_t buflen = strlen(item.path);
+
+    // Skip after or non-after directories.
+    if (flags & (DIP_NOAFTER | DIP_AFTER)) {
+      if ((item.after && (flags & DIP_NOAFTER))
+          || (!item.after && (flags & DIP_AFTER))) {
+        continue;
+      }
+    }
+
+    if (name == NULL) {
+      (*callback)((char_u *)item.path, cookie);
+      did_one = true;
+    } else if (buflen + STRLEN(name) + 2 < MAXPATHL) {
+      STRCPY(buf, item.path);
+      add_pathsep((char *)buf);
+      tail = buf + STRLEN(buf);
+
+      // Loop over all patterns in "name"
+      char_u *np = name;
+      while (*np != NUL && ((flags & DIP_ALL) || !did_one)) {
+        // Append the pattern from "name" to buf[].
+        assert(MAXPATHL >= (tail - buf));
+        copy_option_part(&np, tail, (size_t)(MAXPATHL - (tail - buf)),
+                         "\t ");
+
+        if (p_verbose > 10) {
+          verbose_enter();
+          smsg(_("Searching for \"%s\""), buf);
+          verbose_leave();
+        }
+
+        int ew_flags = ((flags & DIP_DIR) ? EW_DIR : EW_FILE)
+                       | (flags & DIP_DIRFILE) ? (EW_DIR|EW_FILE) : 0;
+
+        // Expand wildcards, invoke the callback for each match.
+        char_u *(pat[]) = { buf };
+        if (gen_expand_wildcards(1, pat, &num_files, &files, ew_flags) == OK) {
+          for (i = 0; i < num_files; i++) {
+            (*callback)(files[i], cookie);
+            did_one = true;
+            if (!(flags & DIP_ALL)) {
+              break;
+            }
+          }
+          FreeWild(num_files, files);
+        }
+      }
+    }
+  }
+
+  if (!did_one && name != NULL) {
+    if (flags & DIP_ERR) {
+      EMSG3(_(e_dirnotf), "runtime path", name);
+    } else if (p_verbose > 0) {
+      verbose_enter();
+      smsg(_("not found in runtime path: \"%s\""), name);
+      verbose_leave();
+    }
+  }
+
+
+  return did_one ? OK : FAIL;
+}
 /// Find "name" in "path".  When found, invoke the callback function for
 /// it: callback(fname, "cookie")
 /// When "flags" has DIP_ALL repeat for all matches, otherwise only the first
@@ -167,13 +261,6 @@ int do_in_path_and_pp(char_u *path, char_u *name, int flags, DoInRuntimepathCB c
                       void *cookie)
 {
   int done = FAIL;
-  if (!(flags & (DIP_NOAFTER | DIP_AFTER))) {
-    done = do_in_path_and_pp(path, name, flags | DIP_NOAFTER, callback, cookie);
-    if (done == OK && !(flags & DIP_ALL)) {
-      return done;
-    }
-    flags |= DIP_AFTER;
-  }
 
   if ((flags & DIP_NORTP) == 0) {
     done |= do_in_path(path, (name && !*name) ? NULL : name, flags, callback, cookie);
@@ -227,10 +314,152 @@ int do_in_path_and_pp(char_u *path, char_u *name, int flags, DoInRuntimepathCB c
   return done;
 }
 
+static void push_path(RuntimeSearchPath *search_path, char *entry, bool after)
+{
+  kv_push(*search_path, ((SearchPathItem){ entry, after }));
+}
+
+static void expand_pack_entry(RuntimeSearchPath *search_path, CharVec *after_path,
+                              char_u *pack_entry)
+{
+  static char_u buf[MAXPATHL], buf2[MAXPATHL];
+  char *start_dir = "/pack/*/start/*/";  // NOLINT
+  if (STRLEN(pack_entry) + STRLEN(start_dir) + 1 < MAXPATHL) {
+    xstrlcpy((char *)buf, (char *)pack_entry, MAXPATHL);
+    xstrlcpy((char *)buf2, (char *)pack_entry, MAXPATHL);
+    xstrlcat((char *)buf, start_dir, sizeof buf);
+    xstrlcat((char *)buf2, "/start/*/", sizeof buf);  // NOLINT
+    int num_files;
+    char_u **files;
+
+    char_u *(pat[]) = { buf, buf2 };
+    if (gen_expand_wildcards(2, pat, &num_files, &files, EW_DIR) == OK) {
+      for (int i = 0; i < num_files; i++) {
+        push_path(search_path, xstrdup((char *)files[i]), false);
+        size_t after_size = STRLEN(files[i])+6;
+        char *after = xmallocz(after_size);
+        xstrlcpy(after, (char *)files[i], after_size);
+        xstrlcat(after, "after/", after_size);
+        if (os_isdir((char_u *)after)) {
+          push_path(search_path, after, true);
+        } else {
+          xfree(after);
+        }
+      }
+      FreeWild(num_files, files);
+    }
+  }
+}
+
+static bool path_is_after(char_u *buf, size_t buflen)
+{
+  // NOTE: we only consider dirs exactly matching "after" to be an AFTER dir.
+  // vim8 considers all dirs like "foo/bar_after", "Xafter" etc, as an
+  // "after" dir in SOME codepaths not not in ALL codepaths.
+  return buflen >= 5
+         && (!(buflen >= 6) || vim_ispathsep(buf[buflen-6]))
+         && STRCMP(buf + buflen - 5, "after") == 0;
+}
+
+RuntimeSearchPath build_runtime_search_path(void)
+{
+  kvec_t(String) pack_entries = KV_INITIAL_VALUE;
+  Map(String, handle_T) pack_used = MAP_INIT;
+  // TODO(bfredl): add a set of existing rtp entries to not duplicate those
+  RuntimeSearchPath search_path = KV_INITIAL_VALUE;
+  CharVec after_path = KV_INITIAL_VALUE;
+
+  static char_u buf[MAXPATHL];
+  for (char *entry = (char *)p_pp; *entry != NUL; ) {
+    char *cur_entry = entry;
+    copy_option_part((char_u **)&entry, buf, MAXPATHL, ",");
+
+    String the_entry = { .data = cur_entry, .size = STRLEN(buf) };
+
+    kv_push(pack_entries, the_entry);
+    map_put(String, handle_T)(&pack_used, the_entry, 0);
+  }
+
+
+  char *rtp_entry;
+  for (rtp_entry = (char *)p_rtp; *rtp_entry != NUL; ) {
+    char *cur_entry = rtp_entry;
+    copy_option_part((char_u **)&rtp_entry, buf, MAXPATHL, ",");
+    size_t buflen = STRLEN(buf);
+
+    if (path_is_after(buf, buflen)) {
+      rtp_entry = cur_entry;
+      break;
+    }
+
+    push_path(&search_path, xstrdup((char *)buf), false);
+
+    handle_T *h = map_ref(String, handle_T)(&pack_used, cstr_as_string((char *)buf), false);
+    if (h) {
+      (*h)++;
+      expand_pack_entry(&search_path, &after_path, buf);
+    }
+  }
+
+  for (size_t i = 0; i < kv_size(pack_entries); i++) {
+    handle_T h = map_get(String, handle_T)(&pack_used, cstr_as_string((char *)buf));
+    if (h == 0) {
+      expand_pack_entry(&search_path, &after_path, (char_u *)kv_A(pack_entries, i).data);
+    }
+  }
+
+  // "after" packages
+  for (size_t i = 0; i < kv_size(after_path); i++) {
+    push_path(&search_path, kv_A(after_path, i), true);
+  }
+
+  // "after" dirs in rtp
+  for (; *rtp_entry != NUL;) {
+    copy_option_part((char_u **)&rtp_entry, buf, MAXPATHL, ",");
+    push_path(&search_path, xstrdup((char *)buf), path_is_after(buf, STRLEN(buf)));
+  }
+
+  // strings are not owned
+  kv_destroy(pack_entries);
+  map_destroy(String, handle_T)(&pack_used);
+
+  return search_path;
+}
+
+void invalidate_search_path(void)
+{
+  search_path_valid = false;
+}
+
+void validate_search_path(void)
+{
+  if (!search_path_valid) {
+    for (size_t j = 0; j < kv_size(runtime_search_path); j++) {
+      SearchPathItem item = kv_A(runtime_search_path, j);
+      xfree(item.path);
+    }
+    kv_destroy(runtime_search_path);
+    runtime_search_path = build_runtime_search_path();
+    search_path_valid = true;
+  }
+}
+
+
+
 /// Just like do_in_path_and_pp(), using 'runtimepath' for "path".
 int do_in_runtimepath(char_u *name, int flags, DoInRuntimepathCB callback, void *cookie)
 {
-  return do_in_path_and_pp(p_rtp, name, flags | DIP_START, callback, cookie);
+  int success = FAIL;
+  if (!(flags & DIP_NORTP)) {
+    success |= do_in_cached_path((name && !*name) ? NULL : name, flags, callback, cookie);
+    flags = (flags & ~DIP_START) | DIP_NORTP;
+  }
+  // TODO(bfredl): we could integrate disabled OPT dirs into the cached path
+  // which would effectivize ":packadd myoptpack" as well
+  if ((flags & (DIP_START|DIP_OPT)) && (success == FAIL || (flags & DIP_ALL))) {
+    success |= do_in_path_and_pp(p_rtp, name, flags, callback, cookie);
+  }
+  return success;
 }
 
 /// Source the file "name" from all directories in 'runtimepath'.
@@ -240,8 +469,7 @@ int do_in_runtimepath(char_u *name, int flags, DoInRuntimepathCB callback, void
 /// return FAIL when no file could be sourced, OK otherwise.
 int source_runtime(char_u *name, int flags)
 {
-  flags |= (flags & DIP_NORTP) ? 0 : DIP_START;
+  return do_in_runtimepath(name, flags, source_callback, NULL);
-  return source_in_path(p_rtp, name, flags);
 }
 
 /// Just like source_runtime(), but use "path" instead of 'runtimepath'.

src/nvim/runtime.h

@@ -7,10 +7,30 @@
 
 typedef void (*DoInRuntimepathCB)(char_u *, void *);
 
+typedef struct {
+  char *path;
+  bool after;
+} SearchPathItem;
+
+typedef kvec_t(SearchPathItem) RuntimeSearchPath;
+typedef kvec_t(char *) CharVec;
+
 // last argument for do_source()
 #define DOSO_NONE       0
 #define DOSO_VIMRC      1       // loading vimrc file
 
+// Used for flags in do_in_path()
+#define DIP_ALL 0x01    // all matches, not just the first one
+#define DIP_DIR 0x02    // find directories instead of files
+#define DIP_ERR 0x04    // give an error message when none found
+#define DIP_START 0x08  // also use "start" directory in 'packpath'
+#define DIP_OPT 0x10    // also use "opt" directory in 'packpath'
+#define DIP_NORTP 0x20  // do not use 'runtimepath'
+#define DIP_NOAFTER 0x40  // skip "after" directories
+#define DIP_AFTER   0x80  // only use "after" directories
+#define DIP_LUA  0x100    // also use ".lua" files
+#define DIP_DIRFILE 0x200  // find both files and directories
+
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "runtime.h.generated.h"

src/nvim/testdir/test_startup.vim

@@ -27,8 +27,8 @@ func Test_after_comes_later()
     set guioptions+=M
     let $HOME = "/does/not/exist"
     set loadplugins
-    set rtp=Xhere,Xafter,Xanother
+    set rtp=Xhere,Xdir/after,Xanother
-    set packpath=Xhere,Xafter
+    set packpath=Xhere,Xdir/after
     set nomore
     let g:sequence = ""
   [CODE]
@@ -50,8 +50,8 @@ func Test_after_comes_later()
   call mkdir('Xhere/pack/foo/start/foobar/plugin', 'p')
   call writefile(['let g:sequence .= "pack "'], 'Xhere/pack/foo/start/foobar/plugin/foo.vim')
 
-  call mkdir('Xafter/plugin', 'p')
+  call mkdir('Xdir/after/plugin', 'p')
-  call writefile(['let g:sequence .= "after "'], 'Xafter/plugin/later.vim')
+  call writefile(['let g:sequence .= "after "'], 'Xdir/after/plugin/later.vim')
 
   if RunVim(before, after, '')
 
@@ -74,7 +74,7 @@ func Test_after_comes_later()
   call delete('Xsequence')
   call delete('Xhere', 'rf')
   call delete('Xanother', 'rf')
-  call delete('Xafter', 'rf')
+  call delete('Xdir', 'rf')
 endfunc
 
 func Test_pack_in_rtp_when_plugins_run()

src/nvim/vim.h

@@ -305,18 +305,6 @@ enum { FOLD_TEXT_LEN = 51 };  //!< buffer size for get_foldtext()
 #include "nvim/buffer_defs.h"    // buffer and windows
 #include "nvim/ex_cmds_defs.h"   // Ex command defines
 
-// Used for flags in do_in_path()
-#define DIP_ALL 0x01    // all matches, not just the first one
-#define DIP_DIR 0x02    // find directories instead of files
-#define DIP_ERR 0x04    // give an error message when none found
-#define DIP_START 0x08  // also use "start" directory in 'packpath'
-#define DIP_OPT 0x10    // also use "opt" directory in 'packpath'
-#define DIP_NORTP 0x20  // do not use 'runtimepath'
-#define DIP_NOAFTER 0x40  // skip "after" directories
-#define DIP_AFTER   0x80  // only use "after" directories
-#define DIP_LUA  0x100    // also use ".lua" files
-#define DIP_DIRFILE 0x200  // find both files and directories
-
 // Lowest number used for window ID. Cannot have this many windows per tab.
 #define LOWEST_WIN_ID 1000
 

test/functional/core/startup_spec.lua

@@ -351,12 +351,12 @@ describe('startup', function()
 
   it("handles the correct order with start packages and after/", function()
     pack_clear [[ lua _G.test_loadorder = {} vim.cmd "runtime! filen.lua" ]]
-    eq({'ordinary', 'FANCY', 'ordinary after', 'FANCY after'}, exec_lua [[ return _G.test_loadorder ]])
+    eq({'ordinary', 'FANCY', 'FANCY after', 'ordinary after'}, exec_lua [[ return _G.test_loadorder ]])
   end)
 
   it("handles the correct order with opt packages and after/", function()
     pack_clear [[ lua _G.test_loadorder = {} vim.cmd "packadd! superspecial\nruntime! filen.lua" ]]
-    eq({'ordinary', 'SuperSpecial', 'FANCY', 'SuperSpecial after', 'ordinary after', 'FANCY after'}, exec_lua [[ return _G.test_loadorder ]])
+    eq({'ordinary', 'SuperSpecial', 'FANCY', 'FANCY after', 'SuperSpecial after', 'ordinary after'}, exec_lua [[ return _G.test_loadorder ]])
   end)
 end)