nvim_get_proc_children: fallback to shell

/proc/…/children may be unavailable because of an unset kernel option.
Fallback to `pgrep` invoked in a shell.

src/nvim/api/vim.c

@@ -1482,7 +1482,7 @@ Array nvim_list_uis(void)
 
 /// Gets the immediate children of process `pid`.
 ///
-/// @return Array of child process ids, or empty array if process not found.
+/// @return Array of child process ids. Empty array if process not found.
 Array nvim_get_proc_children(Integer pid, Error *err)
   FUNC_API_SINCE(4)
 {
@@ -1490,17 +1490,27 @@ Array nvim_get_proc_children(Integer pid, Error *err)
   int *proc_list = NULL;
 
   if (pid <= 0 || pid > INT_MAX) {
-    api_set_error(err, kErrorTypeException, "Invalid pid: %d", pid);
+    api_set_error(err, kErrorTypeException, "Invalid pid: %" PRId64, pid);
     goto end;
   }
 
   size_t proc_count;
   int rv = os_proc_children((int)pid, &proc_list, &proc_count);
-  if (rv == 1) {
+  if (rv != 0) {
-    goto end;  // Process not found; return empty list.
+    // syscall failed (possibly because of kernel options), try shelling out.
-  } else if (rv != 0) {
+    Array a = ARRAY_DICT_INIT;
+    ADD(a, INTEGER_OBJ(pid));
+    String s = cstr_to_string("return vim._os_proc_children(select(1, ...))");
+    Object o = nvim_execute_lua(s, a, err);
+    api_free_string(s);
+    api_free_array(a);
+    if (o.type == kObjectTypeArray) {
+      proc_array = o.data.array;
+    } else if (!ERROR_SET(err)) {
       api_set_error(err, kErrorTypeException,
-                  "Failed to get process children. pid=%d error=%d", pid, rv);
+                    "Failed to get process children. pid=%" PRId64 " error=%d",
+                    pid, rv);
+    }
     goto end;
   }
 

src/nvim/lua/vim.lua

@@ -1,3 +1,26 @@
+-- Gets the children of process `ppid` via the shell.
+-- Used by nvim_get_proc_children() as a fallback.
+local function _os_proc_children(ppid)
+  if ppid == nil or ppid <= 0 or type(ppid) ~= 'number' then
+    error('invalid ppid')
+  end
+  local out = vim.api.nvim_call_function('system', { 'pgrep -P '..ppid })
+  local err = vim.api.nvim_get_vvar('shell_error')
+  if 1 == err and out == '' then
+    return {}  -- Process not found.
+  elseif 0 ~= err then
+    error('pgrep failed')
+  end
+  local children = {}
+  for s in string.gmatch(out, '%S+') do
+    local i = tonumber(s)
+    if i ~= nil then
+      table.insert(children, i)
+    end
+  end
+  return children
+end
+
 -- TODO(ZyX-I): Create compatibility layer.
 --{{{1 package.path updater function
 -- Last inserted paths. Used to clear out items from package.[c]path when they
@@ -61,4 +84,5 @@ end
 --{{{1 Module definition
 return {
   _update_package_paths = _update_package_paths,
+  _os_proc_children = _os_proc_children,
 }

src/nvim/os/process.c

@@ -2,6 +2,7 @@
 // it. PVS-Studio Static Code Analyzer for C, C++ and C#: http://www.viva64.com
 
 #include <uv.h>  // for HANDLE (win32)
+
 #ifdef WIN32
 # include <tlhelp32.h>  // for CreateToolhelp32Snapshot
 #endif
@@ -165,7 +166,7 @@ int os_proc_children(int ppid, int **proc_list, size_t *proc_count)
   snprintf(proc_p, sizeof(proc_p), "/proc/%d/task/%d/children", ppid, ppid);
   FILE *fp = fopen(proc_p, "r");
   if (fp == NULL) {
-    return 1;  // Process not found.
+    return 2;  // Process not found, or /proc/…/children not supported.
   }
   int match_pid;
   while (fscanf(fp, "%d", &match_pid) > 0) {