From bfad290773ea79bbf861c12d33933ea2f67254ca Mon Sep 17 00:00:00 2001
From: Darren Tucker <dtucker@zip.com.au>
Date: Thu, 21 Aug 2003 18:23:13 +1000
Subject:  - (dtucker) [configure.ac] Bug #536: Test for and work around
 openpty/    controlling tty problem on Linux (fixes "could not set
 controlling tty"    errors).

---
 ChangeLog    |  5 ++++-
 configure.ac | 49 ++++++++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 52 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index d2b1b2a2..ed8387d5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -16,6 +16,9 @@
    invalid cmsg_type on Linux 1.x and 2.0 kernels, privsep should now work.
  - (dtucker) [acconfig.h configure.ac sshd.c] STREAMS_PUSH_ACQUIRES_CTTY ->
    SSHD_ACQUIRES_CTTY.
+ - (dtucker) [configure.ac] Bug #536: Test for and work around openpty/
+   controlling tty problem on Linux (fixes "could not set controlling tty"
+   errors).
 
 20030429
  - (djm) Add back radix.o (used by AFS support), after it went missing from
@@ -1321,4 +1324,4 @@
      save auth method before monitor_reset_key_state(); bugzilla bug #284;
      ok provos@
 
-$Id: ChangeLog,v 1.2648.2.9 2003/08/21 07:56:29 dtucker Exp $
+$Id: ChangeLog,v 1.2648.2.10 2003/08/21 08:23:13 dtucker Exp $
diff --git a/configure.ac b/configure.ac
index 72e75cd9..ad7a778e 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,4 +1,4 @@
-# $Id: configure.ac,v 1.113.2.4 2003/08/21 07:42:34 dtucker Exp $
+# $Id: configure.ac,v 1.113.2.5 2003/08/21 08:23:13 dtucker Exp $
 
 AC_INIT
 AC_CONFIG_SRCDIR([ssh.c])
@@ -183,6 +183,7 @@ main() { if (NSVersionOfRunTimeLibrary("System") >= (60 << 16))
 *-*-linux*)
 	no_dev_ptmx=1
 	check_for_libcrypt_later=1
+	check_for_openpty_ctty_bug=1
 	AC_DEFINE(DONT_TRY_OTHER_AF)
 	AC_DEFINE(PAM_TTY_KLUDGE)
 	AC_DEFINE(SETPROCTITLE_STRATEGY,PS_USE_CLOBBER_ARGV)
@@ -733,6 +734,52 @@ unlink(template); exit(0);
 )
 fi
 
+if test ! -z "$check_for_openpty_ctty_bug"; then
+	AC_MSG_CHECKING(if openpty correctly handles controlling tty)
+	AC_TRY_RUN(
+		[
+#include <stdio.h>
+#include <sys/fcntl.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+
+int
+main()
+{
+	pid_t pid;
+	int fd, ptyfd, ttyfd, status;
+
+	pid = fork();
+	if (pid < 0) {		/* failed */
+		exit(1);
+	} else if (pid > 0) {	/* parent */
+		waitpid(pid, &status, 0);
+		if (WIFEXITED(status))
+			exit(WEXITSTATUS(status));
+		else
+			exit(2);
+	} else {		/* child */
+		close(0); close(1); close(2);
+		setsid();
+		openpty(&ptyfd, &ttyfd, NULL, NULL, NULL);
+		fd = open("/dev/tty", O_RDWR | O_NOCTTY);
+		if (fd >= 0)
+			exit(3);	/* Acquired ctty: broken */
+		else
+			exit(0);	/* Did not acquire ctty: OK */
+	}
+}
+		],
+		[
+			AC_MSG_RESULT(yes)
+		],
+		[
+			AC_MSG_RESULT(no)
+			AC_DEFINE(SSHD_ACQUIRES_CTTY)
+		]
+	)
+fi
+
 AC_FUNC_GETPGRP
 
 # Check for PAM libs
-- 
cgit v1.2.3