os: implement VimL libcall with {mch,os}_libcall

The old mch_libcall was removed from neovim. This is a partial
reimplementation on top of libuv. It doesn't catch exceptions (windows) nor
signals (unix) though, so it's quite a bit more prone to crashing if the
loadable library throws an exception or crashes. Still, it should be fine
for well-behaved libraries. Requested by @Shougo.

src/nvim/eval.c

@@ -72,6 +72,7 @@
 #include "nvim/os/channel.h"
 #include "nvim/api/private/helpers.h"
 #include "nvim/os/msgpack_rpc_helpers.h"
+#include "nvim/os/dl.h"
 
 #define DICT_MAXNEST 100        /* maximum nesting of lists and dicts */
 
@@ -10623,42 +10624,49 @@ static void f_len(typval_T *argvars, typval_T *rettv)
   }
 }
 
-
-static void libcall_common(typval_T *argvars, typval_T *rettv, int type)
+static void libcall_common(typval_T *argvars, typval_T *rettv, int out_type)
 {
-#ifdef FEAT_LIBCALL
-  char_u              *string_in;
-  char_u              **string_result;
-  int nr_result;
-#endif
-
-  rettv->v_type = type;
-  if (type != VAR_NUMBER)
+  rettv->v_type = out_type;
+  if (out_type != VAR_NUMBER) {
     rettv->vval.v_string = NULL;
-
-  if (check_restricted() || check_secure())
-    return;
-
-#ifdef FEAT_LIBCALL
-  /* The first two args must be strings, otherwise its meaningless */
-  if (argvars[0].v_type == VAR_STRING && argvars[1].v_type == VAR_STRING) {
-    string_in = NULL;
-    if (argvars[2].v_type == VAR_STRING)
-      string_in = argvars[2].vval.v_string;
-    if (type == VAR_NUMBER)
-      string_result = NULL;
-    else
-      string_result = &rettv->vval.v_string;
-    if (mch_libcall(argvars[0].vval.v_string,
-            argvars[1].vval.v_string,
-            string_in,
-            argvars[2].vval.v_number,
-            string_result,
-            &nr_result) == OK
-        && type == VAR_NUMBER)
-      rettv->vval.v_number = nr_result;
   }
-#endif
+
+  if (check_restricted() || check_secure()) {
+    return;
+  }
+
+  // The first two args (libname and funcname) must be strings
+  if (argvars[0].v_type != VAR_STRING || argvars[1].v_type != VAR_STRING) {
+    return;
+  }
+
+  const char *libname = (char *) argvars[0].vval.v_string;
+  const char *funcname = (char *) argvars[1].vval.v_string;
+
+  int in_type = argvars[2].v_type;
+
+  // input variables
+  char *str_in = (in_type == VAR_STRING)
+      ? (char *) argvars[2].vval.v_string : NULL;
+  int64_t int_in = argvars[2].vval.v_number;
+
+  // output variables
+  char **str_out = (out_type == VAR_STRING)
+      ? (char **) &rettv->vval.v_string : NULL;
+  int64_t int_out = 0;
+
+  bool success = os_libcall(libname, funcname,
+                            str_in, int_in,
+                            str_out, &int_out);
+
+  if (!success) {
+    EMSG2(_(e_libcall), funcname);
+    return;
+  }
+
+  if (out_type == VAR_NUMBER) {
+     rettv->vval.v_number = (int) int_out;
+  }
 }
 
 /*

src/nvim/os/dl.c

@@ -0,0 +1,86 @@
+/// Functions for using external native libraries
+
+#include <stdint.h>
+#include <uv.h>
+
+#include "nvim/os/os.h"
+#include "nvim/memory.h"
+#include "nvim/message.h"
+
+/// possible function prototypes that can be called by os_libcall()
+/// int -> int
+/// int -> string
+/// string -> string
+/// string -> int
+typedef void (*gen_fn)();
+typedef const char *(*str_str_fn)(const char *str);
+typedef int64_t (*str_int_fn)(const char *str);
+typedef const char *(*int_str_fn)(int64_t i);
+typedef int64_t (*int_int_fn)(int64_t i);
+
+/// os_libcall - call a function in a dynamic loadable library
+///
+/// an example of calling a function that takes a string and returns an int:
+///
+///   int64_t int_out = 0;
+///   os_libcall("mylib.so", "somefn", "string-argument", 0, NULL, &int_out);
+///
+/// @param libname the name of the library to load (e.g.: libsomething.so)
+/// @param funcname the name of the library function (e.g.: myfunc)
+/// @param argv the input string, NULL when using `argi`
+/// @param argi the input integer, not used when using `argv` != NULL
+/// @param[out] str_out an allocated output string, caller must free if
+///             not NULL. NULL when using `int_out`.
+/// @param[out] int_out the output integer param
+/// @return true on success, false on failure
+bool os_libcall(const char *libname,
+                const char *funcname,
+                const char *argv,
+                int64_t argi,
+                char **str_out,
+                int64_t *int_out)
+{
+  if (!libname || !funcname) {
+    return false;
+  }
+
+  uv_lib_t lib;
+
+  // open the dynamic loadable library
+  if (uv_dlopen(libname, &lib)) {
+      EMSG2(_("dlerror = \"%s\""), uv_dlerror(&lib));
+      return false;
+  }
+
+  // find and load the requested function in the library
+  gen_fn fn;
+  if (uv_dlsym(&lib, funcname, (void **) &fn)) {
+      EMSG2(_("dlerror = \"%s\""), uv_dlerror(&lib));
+      uv_dlclose(&lib);
+      return false;
+  }
+
+  // call the library and save the result
+  // TODO(aktau): catch signals and use jmp (if available) to handle
+  // exceptions. jmp's on UNIX seem to interact trickily with signals as
+  // well. So for now we only support those libraries that are well-behaved.
+  if (str_out) {
+    str_str_fn sfn = (str_str_fn) fn;
+    int_str_fn ifn = (int_str_fn) fn;
+
+    const char *res = argv ? sfn(argv) : ifn(argi);
+
+    // assume that ptr values of NULL, 1 or -1 are illegal
+    *str_out = (res && (intptr_t) res != 1 && (intptr_t) res != -1)
+        ? xstrdup(res) : NULL;
+  } else {
+    str_int_fn sfn = (str_int_fn) fn;
+    int_int_fn ifn = (int_int_fn) fn;
+    *int_out = argv ? sfn(argv) : ifn(argi);
+  }
+
+  // free the library
+  uv_dlclose(&lib);
+
+  return true;
+}

src/nvim/os/dl.h

@@ -0,0 +1,10 @@
+#ifndef NVIM_OS_DL_H
+#define NVIM_OS_DL_H
+
+#include <stdbool.h>
+#include <stdint.h>
+
+#ifdef INCLUDE_GENERATED_DECLARATIONS
+# include "os/dl.h.generated.h"
+#endif
+#endif  // NVIM_OS_DL_H