fix: add forkpty for SunOS variants

forkpty is missing on Solaris < 11 and Illumos, provide fallback implementation
for non Solaris 11 users.

config/CMakeLists.txt

@@ -50,6 +50,19 @@ check_function_exists(strcasecmp HAVE_STRCASECMP)
 check_function_exists(strncasecmp HAVE_STRNCASECMP)
 check_function_exists(strptime HAVE_STRPTIME)
 
+if(CMAKE_SYSTEM_NAME STREQUAL "SunOS")
+  check_c_source_compiles("
+#include <termios.h>
+int
+main(void)
+{
+  return forkpty(0, NULL, NULL, NULL);
+}
+" HAVE_FORKPTY)
+else()
+  set(HAVE_FORKPTY 1)
+endif()
+
 # Symbols
 check_symbol_exists(FD_CLOEXEC "fcntl.h" HAVE_FD_CLOEXEC)
 if(HAVE_LANGINFO_H)

config/config.h.in

@@ -50,6 +50,7 @@
 #  undef HAVE_SYS_UIO_H
 # endif
 #endif
+#cmakedefine HAVE_FORKPTY
 
 #cmakedefine FEAT_TUI
 

src/nvim/os/pty_process_unix.c

@@ -15,7 +15,13 @@
 # include <libutil.h>
 #elif defined(__OpenBSD__) || defined(__NetBSD__) || defined(__APPLE__)
 # include <util.h>
-#elif !defined(__sun)
+#elif defined(__sun)
+#  include <sys/stream.h>
+#  include <sys/syscall.h>
+#  include <fcntl.h>
+#  include <unistd.h>
+#  include <signal.h>
+#else
 # include <pty.h>
 #endif
 
@@ -38,6 +44,118 @@
 # include "os/pty_process_unix.c.generated.h"
 #endif
 
+#if defined(__sun) && !defined(HAVE_FORKPTY)
+
+// this header defines STR, just as nvim.h, but it is defined as ('S'<<8),
+// to avoid #undef STR, #undef STR, #define STR ('S'<<8) just delay the
+// inclusion of the header even though it gets include out of order.
+#include <sys/stropts.h>
+
+static int openpty(int *amaster, int *aslave, char *name,
+                   struct termios *termp, struct winsize *winp)
+{
+  int slave = -1;
+  int master = open("/dev/ptmx", O_RDWR);
+  if (master == -1) {
+    goto error;
+  }
+
+  // grantpt will invoke a setuid program to change permissions
+  // and might fail if SIGCHLD handler is set, temporarily reset
+  // while running
+  void(*sig_saved)(int) = signal(SIGCHLD, SIG_DFL);
+  int res = grantpt(master);
+  signal(SIGCHLD, sig_saved);
+
+  if (res == -1 || unlockpt(master) == -1) {
+    goto error;
+  }
+
+  char *slave_name = ptsname(master);
+  if (slave_name == NULL) {
+    goto error;
+  }
+
+  slave = open(slave_name, O_RDWR|O_NOCTTY);
+  if (slave == -1) {
+    goto error;
+  }
+
+  // ptem emulates a terminal when used on a pseudo terminal driver,
+  // must be pushed before ldterm
+  ioctl(slave, I_PUSH, "ptem");
+  // ldterm provides most of the termio terminal interface
+  ioctl(slave, I_PUSH, "ldterm");
+  // ttcompat compatability with older terminal ioctls
+  ioctl(slave, I_PUSH, "ttcompat");
+
+  if (termp) {
+    tcsetattr(slave, TCSAFLUSH, termp);
+  }
+  if (winp) {
+    ioctl(slave, TIOCSWINSZ, winp);
+  }
+
+  *amaster = master;
+  *aslave = slave;
+  // ignoring name, not passed and size is unknown in the API
+
+  return 0;
+
+error:
+  if (slave != -1) {
+    close(slave);
+  }
+  if (master != -1) {
+    close(master);
+  }
+  return -1;
+}
+
+static int login_tty(int fd)
+{
+  setsid();
+  if (ioctl(fd, TIOCSCTTY, NULL) == -1) {
+    return -1;
+  }
+
+  dup2(fd, STDIN_FILENO);
+  dup2(fd, STDOUT_FILENO);
+  dup2(fd, STDERR_FILENO);
+  if (fd > STDERR_FILENO) {
+    close(fd);
+  }
+
+  return 0;
+}
+
+static pid_t forkpty(int *amaster, char *name,
+                     struct termios *termp, struct winsize *winp)
+{
+  int master, slave;
+  if (openpty(&master, &slave, name, termp, winp) == -1) {
+    return -1;
+  }
+
+  pid_t pid = fork();
+  switch (pid) {
+  case -1:
+    close(master);
+    close(slave);
+    return -1;
+  case 0:
+    close(master);
+    login_tty(slave);
+    return 0;
+  default:
+    close(slave);
+    *amaster = master;
+    return pid;
+  }
+}
+
+#endif
+
 /// termios saved at startup (for TUI) or initialized by pty_process_spawn().
 static struct termios termios_default;