patch 8.1.0824: SunOS/Solaris has a problem with ttysv8.1.0824

Problem:    SunOS/Solaris has a problem with ttys.
Solution:   Add mch_isatty() with extra handling for SunOS. (Ozaki Kiichi,
            closes #3865)

9 files changed, 147 insertions, 68 deletions

diff --git a/src/auto/configure b/src/auto/configure
index a3302d2857..8052cebf6e 100755
--- a/src/auto/configure
+++ b/src/auto/configure
@@ -11131,7 +11131,7 @@ for ac_header in stdint.h stdlib.h string.h \
 	unistd.h stropts.h errno.h sys/resource.h \
 	sys/systeminfo.h locale.h sys/stream.h termios.h \
 	libc.h sys/statfs.h poll.h sys/poll.h pwd.h \
-	utime.h sys/param.h libintl.h libgen.h \
+	utime.h sys/param.h sys/ptms.h libintl.h libgen.h \
 	util/debug.h util/msg18n.h frame.h sys/acl.h \
 	sys/access.h sys/sysinfo.h wchar.h wctype.h
 do :
diff --git a/src/channel.c b/src/channel.c
index b013a8558d..7c649b1714 100644
--- a/src/channel.c
+++ b/src/channel.c
@@ -1048,7 +1048,7 @@ channel_set_pipes(channel_T *channel, sock_T in, sock_T out, sock_T err)
 # if defined(UNIX)
 	/* Do not end the job when all output channels are closed, wait until
 	 * the job ended. */
-	if (isatty(in))
+	if (mch_isatty(in))
 	    channel->ch_to_be_closed |= (1U << PART_IN);
 # endif
     }
diff --git a/src/config.h.in b/src/config.h.in
index e13cc635f6..d1aaf708ea 100644
--- a/src/config.h.in
+++ b/src/config.h.in
@@ -258,6 +258,7 @@
 #undef HAVE_SYS_PARAM_H
 #undef HAVE_SYS_POLL_H
 #undef HAVE_SYS_PTEM_H
+#undef HAVE_SYS_PTMS_H
 #undef HAVE_SYS_RESOURCE_H
 #undef HAVE_SYS_SELECT_H
 #undef HAVE_SYS_STATFS_H
diff --git a/src/configure.ac b/src/configure.ac
index e5525eaa63..2b7725b0a7 100644
--- a/src/configure.ac
+++ b/src/configure.ac
@@ -3265,7 +3265,7 @@ AC_CHECK_HEADERS(stdint.h stdlib.h string.h \
 	unistd.h stropts.h errno.h sys/resource.h \
 	sys/systeminfo.h locale.h sys/stream.h termios.h \
 	libc.h sys/statfs.h poll.h sys/poll.h pwd.h \
-	utime.h sys/param.h libintl.h libgen.h \
+	utime.h sys/param.h sys/ptms.h libintl.h libgen.h \
 	util/debug.h util/msg18n.h frame.h sys/acl.h \
 	sys/access.h sys/sysinfo.h wchar.h wctype.h)
 
diff --git a/src/os_unix.c b/src/os_unix.c
index 5ab3bd3cf3..765cd57e7a 100644
--- a/src/os_unix.c
+++ b/src/os_unix.c
@@ -338,7 +338,9 @@ mch_chdir(char *path)
 }
 
 /* Why is NeXT excluded here (and not in os_unixx.h)? */
-#if defined(ECHOE) && defined(ICANON) && (defined(HAVE_TERMIO_H) || defined(HAVE_TERMIOS_H)) && !defined(__NeXT__)
+#if defined(ECHOE) && defined(ICANON) \
+    && (defined(HAVE_TERMIO_H) || defined(HAVE_TERMIOS_H)) \
+    && !defined(__NeXT__)
 # define NEW_TTY_SYSTEM
 #endif
 
@@ -3448,6 +3450,58 @@ may_core_dump(void)
 
 #ifndef VMS
 
+/*
+ * Get the file descriptor to use for tty operations.
+ */
+    static int
+get_tty_fd(int fd)
+{
+    int		tty_fd = fd;
+
+#if defined(HAVE_SVR4_PTYS) && defined(SUN_SYSTEM)
+    // On SunOS: Get the terminal parameters from "fd", or the slave device of
+    // "fd" when it is a master device.
+    if (mch_isatty(fd) > 1)
+    {
+	char *name;
+
+	name = ptsname(fd);
+	if (name == NULL)
+	    return -1;
+
+	tty_fd = open(name, O_RDONLY | O_NOCTTY | O_EXTRA, 0);
+	if (tty_fd < 0)
+	    return -1;
+    }
+#endif
+    return tty_fd;
+}
+
+    static int
+mch_tcgetattr(int fd, void *term)
+{
+    int		tty_fd;
+    int		retval = -1;
+
+    tty_fd = get_tty_fd(fd);
+    if (tty_fd >= 0)
+    {
+#ifdef NEW_TTY_SYSTEM
+# ifdef HAVE_TERMIOS_H
+	retval = tcgetattr(tty_fd, (struct termios *)term);
+# else
+	retval = ioctl(tty_fd, TCGETA, (struct termio *)term);
+# endif
+#else
+	// for "old" tty systems
+	retval = ioctl(tty_fd, TIOCGETP, (struct sgttyb *)term);
+#endif
+	if (tty_fd != fd)
+	    close(tty_fd);
+    }
+    return retval;
+}
+
     void
 mch_settmode(int tmode)
 {
@@ -3465,11 +3519,7 @@ mch_settmode(int tmode)
     if (first)
     {
 	first = FALSE;
-# if defined(HAVE_TERMIOS_H)
-	tcgetattr(read_cmd_fd, &told);
-# else
-	ioctl(read_cmd_fd, TCGETA, &told);
-# endif
+	mch_tcgetattr(read_cmd_fd, &told);
     }
 
     tnew = told;
@@ -3527,7 +3577,7 @@ mch_settmode(int tmode)
     if (first)
     {
 	first = FALSE;
-	ioctl(read_cmd_fd, TIOCGETP, &ttybold);
+	mch_tcgetattr(read_cmd_fd, &ttybold);
     }
 
     ttybnew = ttybold;
@@ -3587,13 +3637,7 @@ get_tty_info(int fd, ttyinfo_T *info)
     struct termio keys;
 # endif
 
-    if (
-# if defined(HAVE_TERMIOS_H)
-	    tcgetattr(fd, &keys) != -1
-# else
-	    ioctl(fd, TCGETA, &keys) != -1
-# endif
-       )
+    if (mch_tcgetattr(fd, &keys) != -1)
     {
 	info->backspace = keys.c_cc[VERASE];
 	info->interrupt = keys.c_cc[VINTR];
@@ -3611,7 +3655,7 @@ get_tty_info(int fd, ttyinfo_T *info)
     /* for "old" tty systems */
     struct sgttyb keys;
 
-    if (ioctl(fd, TIOCGETP, &keys) != -1)
+    if (mch_tcgetattr(fd, &keys) != -1)
     {
 	info->backspace = keys.sg_erase;
 	info->interrupt = keys.sg_kill;
@@ -4070,34 +4114,35 @@ mch_get_shellsize(void)
     int
 mch_report_winsize(int fd, int rows, int cols)
 {
-# ifdef TIOCSWINSZ
-    struct winsize	ws;
+    int		tty_fd;
+    int		retval = -1;
 
-    ws.ws_col = cols;
-    ws.ws_row = rows;
-    ws.ws_xpixel = cols * 5;
-    ws.ws_ypixel = rows * 10;
-    if (ioctl(fd, TIOCSWINSZ, &ws) == 0)
+    tty_fd = get_tty_fd(fd);
+    if (tty_fd >= 0)
     {
-	ch_log(NULL, "ioctl(TIOCSWINSZ) success");
-	return OK;
-    }
-    ch_log(NULL, "ioctl(TIOCSWINSZ) failed");
-# else
-#  ifdef TIOCSSIZE
-    struct ttysize	ts;
+# if defined(TIOCSWINSZ)
+	struct winsize ws;
 
-    ts.ts_cols = cols;
-    ts.ts_lines = rows;
-    if (ioctl(fd, TIOCSSIZE, &ws) == 0)
-    {
-	ch_log(NULL, "ioctl(TIOCSSIZE) success");
-	return OK;
-    }
-    ch_log(NULL, "ioctl(TIOCSSIZE) failed");
-#  endif
+	ws.ws_col = cols;
+	ws.ws_row = rows;
+	ws.ws_xpixel = cols * 5;
+	ws.ws_ypixel = rows * 10;
+	retval = ioctl(tty_fd, TIOCSWINSZ, &ws);
+	ch_log(NULL, "ioctl(TIOCSWINSZ) %s",
+					  retval == 0 ? "success" : "failed");
+# elif defined(TIOCSSIZE)
+	struct ttysize ts;
+
+	ts.ts_cols = cols;
+	ts.ts_lines = rows;
+	retval = ioctl(tty_fd, TIOCSSIZE, &ts);
+	ch_log(NULL, "ioctl(TIOCSSIZE) %s",
+					  retval == 0 ? "success" : "failed");
 # endif
-    return FAIL;
+	if (tty_fd != fd)
+	    close(tty_fd);
+    }
+    return retval == 0 ? OK : FAIL;
 }
 #endif
 
@@ -4273,7 +4318,7 @@ open_pty(int *pty_master_fd, int *pty_slave_fd, char_u **namep)
 {
     char	*tty_name;
 
-    *pty_master_fd = OpenPTY(&tty_name);	    /* open pty */
+    *pty_master_fd = mch_openpty(&tty_name);	    // open pty
     if (*pty_master_fd >= 0)
     {
 	/* Leaving out O_NOCTTY may lead to waitpid() always returning
@@ -4721,7 +4766,7 @@ mch_call_shell_fork(
 		{
 		    /* push stream discipline modules */
 		    if (options & SHELL_COOKED)
-			SetupSlavePTY(pty_slave_fd);
+			setup_slavepty(pty_slave_fd);
 #  ifdef TIOCSCTTY
 		    /* Try to become controlling tty (probably doesn't work,
 		     * unless run by root) */
@@ -5579,7 +5624,7 @@ mch_job_start(char **argv, job_T *job, jobopt_T *options, int is_terminal)
 	if (pty_slave_fd >= 0)
 	{
 	    /* push stream discipline modules */
-	    SetupSlavePTY(pty_slave_fd);
+	    setup_slavepty(pty_slave_fd);
 #  ifdef TIOCSCTTY
 	    /* Try to become controlling tty (probably doesn't work,
 	     * unless run by root) */
diff --git a/src/proto/pty.pro b/src/proto/pty.pro
index 35e5c2b7ac..52e20ef674 100644
--- a/src/proto/pty.pro
+++ b/src/proto/pty.pro
@@ -1,4 +1,5 @@
 /* pty.c */
-int SetupSlavePTY(int fd);
-int OpenPTY(char **ttyn);
+int setup_slavepty(int fd);
+int mch_openpty(char **ttyn);
+int mch_isatty(int fd);
 /* vim: set ft=c : */
diff --git a/src/pty.c b/src/pty.c
index 7171734758..23ea0c06c5 100644
--- a/src/pty.c
+++ b/src/pty.c
@@ -56,16 +56,19 @@
 #endif
 
 #if HAVE_STROPTS_H
-#include <sys/types.h>
-#ifdef sinix
-#define buf_T __system_buf_t__
-#endif
-#include <stropts.h>
-#ifdef sinix
-#undef buf_T
-#endif
+# include <sys/types.h>
+# ifdef sinix
+#  define buf_T __system_buf_t__
+# endif
+# include <stropts.h>
+# ifdef sinix
+#  undef buf_T
+# endif
 # ifdef SUN_SYSTEM
 #  include <sys/conf.h>
+#  if defined(HAVE_SYS_PTMS_H) && defined(HAVE_SVR4_PTYS)
+#   include <sys/ptms.h>
+#  endif
 # endif
 #endif
 
@@ -155,11 +158,12 @@ initmaster(int f UNUSED)
  * pty on others.  Needs to be tuned...
  */
     int
-SetupSlavePTY(int fd)
+setup_slavepty(int fd)
 {
     if (fd < 0)
 	return 0;
-#if defined(I_PUSH) && defined(HAVE_SVR4_PTYS) && !defined(sgi) && !defined(linux) && !defined(__osf__) && !defined(M_UNIX)
+#if defined(I_PUSH) && defined(HAVE_SVR4_PTYS) && !defined(sgi) \
+	&& !defined(linux) && !defined(__osf__) && !defined(M_UNIX)
 # if defined(HAVE_SYS_PTEM_H) || defined(hpux)
     if (ioctl(fd, I_PUSH, "ptem") != 0)
 	return -1;
@@ -178,7 +182,7 @@ SetupSlavePTY(int fd)
 #if defined(OSX) && !defined(PTY_DONE)
 #define PTY_DONE
     int
-OpenPTY(char **ttyn)
+mch_openpty(char **ttyn)
 {
     int		f;
     static char TtyName[32];
@@ -195,7 +199,7 @@ OpenPTY(char **ttyn)
 	&& !defined(PTY_DONE)
 #define PTY_DONE
     int
-OpenPTY(char **ttyn)
+mch_openpty(char **ttyn)
 {
     char	*m, *s;
     int		f;
@@ -219,7 +223,7 @@ OpenPTY(char **ttyn)
 #if defined(__sgi) && !defined(PTY_DONE)
 #define PTY_DONE
     int
-OpenPTY(char **ttyn)
+mch_openpty(char **ttyn)
 {
     int f;
     char *name;
@@ -244,7 +248,7 @@ OpenPTY(char **ttyn)
 #if defined(MIPS) && defined(HAVE_DEV_PTC) && !defined(PTY_DONE)
 #define PTY_DONE
     int
-OpenPTY(char **ttyn)
+mch_openpty(char **ttyn)
 {
     int		f;
     stat_T	buf;
@@ -272,7 +276,7 @@ OpenPTY(char **ttyn)
  * Same for Mac OS X Leopard (10.5). */
 #define PTY_DONE
     int
-OpenPTY(char **ttyn)
+mch_openpty(char **ttyn)
 {
     int		f;
     char	*m;
@@ -313,7 +317,7 @@ int aixhack = -1;
 #endif
 
     int
-OpenPTY(char **ttyn)
+mch_openpty(char **ttyn)
 {
     int		f;
     /* used for opening a new pty-pair: */
@@ -359,7 +363,7 @@ static char TtyProto[] = "/dev/ttyXY";
 # endif
 
     int
-OpenPTY(char **ttyn)
+mch_openpty(char **ttyn)
 {
     char	*p, *q, *l, *d;
     int		f;
@@ -410,4 +414,30 @@ OpenPTY(char **ttyn)
 }
 #endif
 
-#endif /* FEAT_GUI || FEAT_TERMINAL */
+/*
+ * Call isatty(fd), except for SunOS where it's done differently.
+ */
+    int
+mch_isatty(int fd)
+{
+# if defined(I_STR) && defined(HAVE_SYS_PTMS_H) && defined(HAVE_SVR4_PTYS) \
+	&& defined(SUN_SYSTEM)
+    // On SunOS, isatty() for /dev/ptmx returns false or sometimes can hang up
+    // in the inner ioctl(), and therefore first determine whether "fd" is a
+    // master device.
+    struct strioctl istr;
+
+    istr.ic_cmd = ISPTM;
+    istr.ic_timout = 0;
+    istr.ic_dp = NULL;
+    istr.ic_len = 0;
+
+    if (ioctl(fd, I_STR, &istr) == 0)
+	// Trick: return 2 in order to advice the caller that "fd" is a master
+	// device. cf. src/os_unix.c:get_tty_fd()
+	return 2;
+# endif
+    return isatty(fd);
+}
+
+#endif /* FEAT_GUI || FEAT_JOB_CHANNEL */
diff --git a/src/terminal.c b/src/terminal.c
index d485b745f3..f33521a9f3 100644
--- a/src/terminal.c
+++ b/src/terminal.c
@@ -873,7 +873,7 @@ get_tty_part(term_T *term)
     {
 	int fd = term->tl_job->jv_channel->ch_part[parts[i]].ch_fd;
 
-	if (isatty(fd))
+	if (mch_isatty(fd))
 	    return parts[i];
     }
 #endif
@@ -2182,7 +2182,7 @@ terminal_loop(int blocking)
 	 * them for every typed character is a bit of overhead, but it's needed
 	 * for the first character typed, e.g. when Vim starts in a shell.
 	 */
-	if (isatty(tty_fd))
+	if (mch_isatty(tty_fd))
 	{
 	    ttyinfo_T info;
 
@@ -5882,7 +5882,7 @@ term_report_winsize(term_T *term, int rows, int cols)
 	for (part = PART_OUT; part < PART_COUNT; ++part)
 	{
 	    fd = term->tl_job->jv_channel->ch_part[part].ch_fd;
-	    if (isatty(fd))
+	    if (mch_isatty(fd))
 		break;
 	}
 	if (part < PART_COUNT && mch_report_winsize(fd, rows, cols) == OK)
diff --git a/src/version.c b/src/version.c
index b9522911a4..ddf9b1496c 100644
--- a/src/version.c
+++ b/src/version.c
@@ -788,6 +788,8 @@ static char *(features[]) =
 static int included_patches[] =
 {   /* Add new patch number below this line */
 /**/
+    824,
+/**/
     823,
 /**/
     822,