api: Implement '--embedded-mode' command-line option

This option makes nvim run in "embedded mode", which creates an API channel via
stdin/stdout and disables all terminal-related code

src/nvim/ex_getln.c

@@ -2011,6 +2011,10 @@ void free_cmdline_buf(void)
  */
 static void draw_cmdline(int start, int len)
 {
+  if (embedded_mode) {
+    return;
+  }
+
   int i;
 
   if (cmdline_star > 0)

src/nvim/globals.h

@@ -1233,6 +1233,9 @@ EXTERN char *ignoredp;
  * os_unix.c */
 EXTERN int curr_tmode INIT(= TMODE_COOK); /* contains current terminal mode */
 
+// If a msgpack-rpc channel should be started over stdin/stdout
+EXTERN bool embedded_mode INIT(= false);
+
 /// Used to track the status of external functions.
 /// Currently only used for iconv().
 typedef enum {

src/nvim/main.c

@@ -256,10 +256,10 @@ int main(int argc, char **argv)
   TIME_MSG("shell init");
 
 
-  /*
-   * Print a warning if stdout is not a terminal.
-   */
-  check_tty(&params);
+  if (!embedded_mode) {
+    // Print a warning if stdout is not a terminal.
+    check_tty(&params);
+  }
 
   /* This message comes before term inits, but after setting "silent_mode"
    * when the input is not a tty. */
@@ -267,8 +267,16 @@ int main(int argc, char **argv)
     printf(_("%d files to edit\n"), GARGCOUNT);
 
   if (params.want_full_screen && !silent_mode) {
-    termcapinit(params.term);           /* set terminal name and get terminal
-                                           capabilities (will set full_screen) */
+    if (embedded_mode) {
+      // In embedded mode don't do terminal-related initializations, assume an
+      // initial screen size of 80x20
+      full_screen = true;
+      set_shellsize(80, 20, false);
+    } else { 
+      // set terminal name and get terminal capabilities (will set full_screen)
+      // Do some initialization of the screen
+      termcapinit(params.term);
+    }
     screen_start();             /* don't know where cursor is now */
     TIME_MSG("Termcap init");
   }
@@ -405,14 +413,18 @@ int main(int argc, char **argv)
     TIME_MSG("waiting for return");
   }
 
-  starttermcap();             /* start termcap if not done by wait_return() */
-  TIME_MSG("start termcap");
-  may_req_ambiguous_char_width();
+  if (!embedded_mode) {
+    starttermcap(); // start termcap if not done by wait_return()
+    TIME_MSG("start termcap");
+    may_req_ambiguous_char_width();
+    setmouse();  // may start using the mouse
 
-  setmouse();                             /* may start using the mouse */
-  if (scroll_region)
-    scroll_region_reset();                /* In case Rows changed */
-  scroll_start();         /* may scroll the screen to the right position */
+    if (scroll_region) {
+      scroll_region_reset(); // In case Rows changed
+    }
+
+    scroll_start(); // may scroll the screen to the right position
+  }
 
   /*
    * Don't clear the screen when starting in Ex mode, unless using the GUI.
@@ -1015,11 +1027,13 @@ static void command_line_scan(mparm_T *parmp)
             msg_putchar('\n');
             msg_didout = FALSE;
             mch_exit(0);
-	  } else if (STRICMP(argv[0] + argv_idx, "api-msgpack-metadata") == 0) {
+          } else if (STRICMP(argv[0] + argv_idx, "api-msgpack-metadata") == 0) {
             for (unsigned int i = 0; i<msgpack_metadata_size; i++) {
               putchar(msgpack_metadata[i]);
             }
             mch_exit(0);
+          } else if (STRICMP(argv[0] + argv_idx, "embedded-mode") == 0) {
+            embedded_mode = true;
           } else if (STRNICMP(argv[0] + argv_idx, "literal", 7) == 0) {
 #if !defined(UNIX)
             parmp->literal = TRUE;
@@ -2200,6 +2214,8 @@ static void usage(void)
   main_msg(_("--startuptime <file>\tWrite startup timing messages to <file>"));
   main_msg(_("-i <viminfo>\t\tUse <viminfo> instead of .viminfo"));
   main_msg(_("--api-msgpack-metadata\tDump API metadata information and exit"));
+  main_msg(_("--embedded-mode\tUse stdin/stdout as a msgpack-rpc channel. "
+             "This can be used for embedding Neovim into other programs"));
   main_msg(_("-h  or  --help\tPrint Help (this message) and exit"));
   main_msg(_("--version\t\tPrint version information and exit"));
 

src/nvim/os/channel.c

@@ -67,6 +67,10 @@ void channel_init(void)
   channels = pmap_new(uint64_t)();
   event_strings = pmap_new(cstr_t)();
   msgpack_sbuffer_init(&out_buffer);
+
+  if (embedded_mode) {
+    channel_from_stdio();
+  }
 }
 
 /// Teardown the module

src/nvim/os/input.c

@@ -6,6 +6,7 @@
 
 #include "nvim/os/input.h"
 #include "nvim/os/event.h"
+#include "nvim/os/signal.h"
 #include "nvim/os/rstream_defs.h"
 #include "nvim/os/rstream.h"
 #include "nvim/ascii.h"
@@ -34,6 +35,10 @@ static bool eof = false, started_reading = false;
 
 void input_init(void)
 {
+  if (embedded_mode) {
+    return;
+  }
+
   read_stream = rstream_new(read_cb, READ_BUFFER_SIZE, NULL, NULL);
   rstream_set_file(read_stream, read_cmd_fd);
 }
@@ -41,18 +46,30 @@ void input_init(void)
 // Listen for input
 void input_start(void)
 {
+  if (embedded_mode) {
+    return;
+  }
+
   rstream_start(read_stream);
 }
 
 // Stop listening for input
 void input_stop(void)
 {
+  if (embedded_mode) {
+    return;
+  }
+
   rstream_stop(read_stream);
 }
 
 // Copies (at most `count`) of was read from `read_cmd_fd` into `buf`
 uint32_t input_read(char *buf, uint32_t count)
 {
+  if (embedded_mode) {
+    return 0;
+  }
+
   return rstream_read(read_stream, buf, count);
 }
 
@@ -129,6 +146,11 @@ bool os_isatty(int fd)
 
 static bool input_poll(int32_t ms)
 {
+  if (embedded_mode) {
+    EventSource input_sources[] = { signal_event_source(), NULL };
+    return event_poll(ms, input_sources);
+  }
+
   EventSource input_sources[] = {
     rstream_event_source(read_stream),
     NULL

src/nvim/os/signal.c

@@ -39,7 +39,10 @@ void signal_init(void)
   uv_signal_start(&shup, signal_cb, SIGHUP);
   uv_signal_start(&squit, signal_cb, SIGQUIT);
   uv_signal_start(&sterm, signal_cb, SIGTERM);
-  uv_signal_start(&swinch, signal_cb, SIGWINCH);
+  if (!embedded_mode) {
+    // TODO(tarruda): There must be an API function for resizing window
+    uv_signal_start(&swinch, signal_cb, SIGWINCH);
+  }
 #ifdef SIGPWR
   uv_signal_init(uv_default_loop(), &spwr);
   uv_signal_start(&spwr, signal_cb, SIGPWR);

src/nvim/os_unix.c

@@ -88,6 +88,18 @@ static int did_set_icon = FALSE;
  */
 void mch_write(char_u *s, int len)
 {
+  if (embedded_mode) {
+    // TODO(tarruda): This is a temporary hack to stop Neovim from writing
+    // messages to stdout in embedded mode. In the future, embedded mode will
+    // be the only possibility(GUIs will always start neovim with a msgpack-rpc
+    // over stdio) and this function won't exist.
+    //
+    // The reason for this is because before Neovim fully migrates to a
+    // msgpack-rpc-driven architecture, we must have a fully functional
+    // UI working
+    return;
+  }
+
   ignored = (int)write(1, (char *)s, len);
   if (p_wd)             /* Unix is too fast, slow down a bit more */
     os_microdelay(p_wd, false);

src/nvim/term.c

@@ -2360,6 +2360,9 @@ void set_shellsize(int width, int height, int mustset)
  */
 void settmode(int tmode)
 {
+  if (embedded_mode) {
+    return;
+  }
 
   if (full_screen) {
     /*