- djm@cvs.openbsd.org 2010/01/26 01:28:35

     [channels.c channels.h clientloop.c clientloop.h mux.c nchan.c ssh.c]
     rewrite ssh(1) multiplexing code to a more sensible protocol.

     The new multiplexing code uses channels for the listener and
     accepted control sockets to make the mux master non-blocking, so
     no stalls when processing messages from a slave.

     avoid use of fatal() in mux master protocol parsing so an errant slave
     process cannot take down a running master.

     implement requesting of port-forwards over multiplexed sessions. Any
     port forwards requested by the slave are added to those the master has
     established.

     add support for stdio forwarding ("ssh -W host:port ...") in mux slaves.

     document master/slave mux protocol so that other tools can use it to
     control a running ssh(1). Note: there are no guarantees that this
     protocol won't be incompatibly changed (though it is versioned).

     feedback Salvador Fandino, dtucker@
     channel changes ok markus@

8 files changed, 1676 insertions, 509 deletions

diff --git a/ChangeLog b/ChangeLog
index 4334f676..b8535045 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -8,6 +8,29 @@
      [roaming_client.c]
      s/long long unsigned/unsigned long long/, from tim via portable
      (Id sync only, change already in portable)
+   - djm@cvs.openbsd.org 2010/01/26 01:28:35
+     [channels.c channels.h clientloop.c clientloop.h mux.c nchan.c ssh.c]
+     rewrite ssh(1) multiplexing code to a more sensible protocol.
+     
+     The new multiplexing code uses channels for the listener and
+     accepted control sockets to make the mux master non-blocking, so
+     no stalls when processing messages from a slave.
+     
+     avoid use of fatal() in mux master protocol parsing so an errant slave
+     process cannot take down a running master.
+     
+     implement requesting of port-forwards over multiplexed sessions. Any
+     port forwards requested by the slave are added to those the master has
+     established.
+     
+     add support for stdio forwarding ("ssh -W host:port ...") in mux slaves.
+     
+     document master/slave mux protocol so that other tools can use it to
+     control a running ssh(1). Note: there are no guarantees that this
+     protocol won't be incompatibly changed (though it is versioned).
+     
+     feedback Salvador Fandino, dtucker@
+     channel changes ok markus@
 
 20100122
  - (tim) [configure.ac] Due to constraints in Windows Sockets in terms of
diff --git a/channels.c b/channels.c
index e8589d8c..81261679 100644
--- a/channels.c
+++ b/channels.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: channels.c,v 1.301 2010/01/11 01:39:46 dtucker Exp $ */
+/* $OpenBSD: channels.c,v 1.302 2010/01/26 01:28:35 djm Exp $ */
 /*
  * Author: Tatu Ylonen <ylo@cs.hut.fi>
  * Copyright (c) 1995 Tatu Ylonen <ylo@cs.hut.fi>, Espoo, Finland
@@ -239,7 +239,6 @@ channel_register_fds(Channel *c, int rfd, int wfd, int efd,
 	c->rfd = rfd;
 	c->wfd = wfd;
 	c->sock = (rfd == wfd) ? rfd : -1;
-	c->ctl_fd = -1; /* XXX: set elsewhere */
 	c->efd = efd;
 	c->extended_usage = extusage;
 
@@ -328,6 +327,9 @@ channel_new(char *ctype, int type, int rfd, int wfd, int efd,
 	c->output_filter = NULL;
 	c->filter_ctx = NULL;
 	c->filter_cleanup = NULL;
+	c->ctl_chan = -1;
+	c->mux_rcb = NULL;
+	c->mux_ctx = NULL;
 	c->delayed = 1;		/* prevent call to channel_post handler */
 	TAILQ_INIT(&c->status_confirms);
 	debug("channel %d: new [%s]", found, remote_name);
@@ -370,11 +372,10 @@ channel_close_fd(int *fdp)
 static void
 channel_close_fds(Channel *c)
 {
-	debug3("channel %d: close_fds r %d w %d e %d c %d",
-	    c->self, c->rfd, c->wfd, c->efd, c->ctl_fd);
+	debug3("channel %d: close_fds r %d w %d e %d",
+	    c->self, c->rfd, c->wfd, c->efd);
 
 	channel_close_fd(&c->sock);
-	channel_close_fd(&c->ctl_fd);
 	channel_close_fd(&c->rfd);
 	channel_close_fd(&c->wfd);
 	channel_close_fd(&c->efd);
@@ -400,8 +401,6 @@ channel_free(Channel *c)
 
 	if (c->sock != -1)
 		shutdown(c->sock, SHUT_RDWR);
-	if (c->ctl_fd != -1)
-		shutdown(c->ctl_fd, SHUT_RDWR);
 	channel_close_fds(c);
 	buffer_free(&c->input);
 	buffer_free(&c->output);
@@ -523,6 +522,7 @@ channel_still_open(void)
 		case SSH_CHANNEL_X11_LISTENER:
 		case SSH_CHANNEL_PORT_LISTENER:
 		case SSH_CHANNEL_RPORT_LISTENER:
+		case SSH_CHANNEL_MUX_LISTENER:
 		case SSH_CHANNEL_CLOSED:
 		case SSH_CHANNEL_AUTH_SOCKET:
 		case SSH_CHANNEL_DYNAMIC:
@@ -536,6 +536,7 @@ channel_still_open(void)
 		case SSH_CHANNEL_OPENING:
 		case SSH_CHANNEL_OPEN:
 		case SSH_CHANNEL_X11_OPEN:
+		case SSH_CHANNEL_MUX_CLIENT:
 			return 1;
 		case SSH_CHANNEL_INPUT_DRAINING:
 		case SSH_CHANNEL_OUTPUT_DRAINING:
@@ -567,6 +568,8 @@ channel_find_open(void)
 		case SSH_CHANNEL_X11_LISTENER:
 		case SSH_CHANNEL_PORT_LISTENER:
 		case SSH_CHANNEL_RPORT_LISTENER:
+		case SSH_CHANNEL_MUX_LISTENER:
+		case SSH_CHANNEL_MUX_CLIENT:
 		case SSH_CHANNEL_OPENING:
 		case SSH_CHANNEL_CONNECTING:
 		case SSH_CHANNEL_ZOMBIE:
@@ -617,6 +620,8 @@ channel_open_message(void)
 		case SSH_CHANNEL_CLOSED:
 		case SSH_CHANNEL_AUTH_SOCKET:
 		case SSH_CHANNEL_ZOMBIE:
+		case SSH_CHANNEL_MUX_CLIENT:
+		case SSH_CHANNEL_MUX_LISTENER:
 			continue;
 		case SSH_CHANNEL_LARVAL:
 		case SSH_CHANNEL_OPENING:
@@ -627,12 +632,12 @@ channel_open_message(void)
 		case SSH_CHANNEL_INPUT_DRAINING:
 		case SSH_CHANNEL_OUTPUT_DRAINING:
 			snprintf(buf, sizeof buf,
-			    "  #%d %.300s (t%d r%d i%d/%d o%d/%d fd %d/%d cfd %d)\r\n",
+			    "  #%d %.300s (t%d r%d i%d/%d o%d/%d fd %d/%d cc %d)\r\n",
 			    c->self, c->remote_name,
 			    c->type, c->remote_id,
 			    c->istate, buffer_len(&c->input),
 			    c->ostate, buffer_len(&c->output),
-			    c->rfd, c->wfd, c->ctl_fd);
+			    c->rfd, c->wfd, c->ctl_chan);
 			buffer_append(&buffer, buf, strlen(buf));
 			continue;
 		default:
@@ -839,9 +844,6 @@ channel_pre_open(Channel *c, fd_set *readset, fd_set *writeset)
 			FD_SET(c->efd, readset);
 	}
 	/* XXX: What about efd? races? */
-	if (compat20 && c->ctl_fd != -1 &&
-	    c->istate == CHAN_INPUT_OPEN && c->ostate == CHAN_OUTPUT_OPEN)
-		FD_SET(c->ctl_fd, readset);
 }
 
 /* ARGSUSED */
@@ -986,6 +988,28 @@ channel_pre_x11_open(Channel *c, fd_set *readset, fd_set *writeset)
 	}
 }
 
+static void
+channel_pre_mux_client(Channel *c, fd_set *readset, fd_set *writeset)
+{
+	if (c->istate == CHAN_INPUT_OPEN &&
+	    buffer_check_alloc(&c->input, CHAN_RBUF))
+		FD_SET(c->rfd, readset);
+	if (c->istate == CHAN_INPUT_WAIT_DRAIN) {
+		/* clear buffer immediately (discard any partial packet) */
+		buffer_clear(&c->input);
+		chan_ibuf_empty(c);
+		/* Start output drain. XXX just kill chan? */
+		chan_rcvd_oclose(c);
+	}
+	if (c->ostate == CHAN_OUTPUT_OPEN ||
+	    c->ostate == CHAN_OUTPUT_WAIT_DRAIN) {
+		if (buffer_len(&c->output) > 0)
+			FD_SET(c->wfd, writeset);
+		else if (c->ostate == CHAN_OUTPUT_WAIT_DRAIN)
+			chan_obuf_empty(c);
+	}
+}
+
 /* try to decode a socks4 header */
 /* ARGSUSED */
 static int
@@ -1218,19 +1242,14 @@ channel_decode_socks5(Channel *c, fd_set *readset, fd_set *writeset)
 }
 
 Channel *
-channel_connect_stdio_fwd(const char *host_to_connect, u_short port_to_connect)
+channel_connect_stdio_fwd(const char *host_to_connect, u_short port_to_connect,
+    int in, int out)
 {
 	Channel *c;
-	int in, out;
 
 	debug("channel_connect_stdio_fwd %s:%d", host_to_connect,
 	    port_to_connect);
 
-	in = dup(STDIN_FILENO);
-	out = dup(STDOUT_FILENO);
-	if (in < 0 || out < 0)
-		fatal("channel_connect_stdio_fwd: dup() in/out failed");
-
 	c = channel_new("stdio-forward", SSH_CHANNEL_OPENING, in, out,
 	    -1, CHAN_TCP_WINDOW_DEFAULT, CHAN_TCP_PACKET_DEFAULT,
 	    0, "stdio-forward", /*nonblock*/0);
@@ -1749,36 +1768,6 @@ channel_handle_efd(Channel *c, fd_set *readset, fd_set *writeset)
 	return 1;
 }
 
-/* ARGSUSED */
-static int
-channel_handle_ctl(Channel *c, fd_set *readset, fd_set *writeset)
-{
-	char buf[16];
-	int len;
-
-	/* Monitor control fd to detect if the slave client exits */
-	if (c->ctl_fd != -1 && FD_ISSET(c->ctl_fd, readset)) {
-		len = read(c->ctl_fd, buf, sizeof(buf));
-		if (len < 0 &&
-		    (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK))
-			return 1;
-		if (len <= 0) {
-			debug2("channel %d: ctl read<=0", c->self);
-			if (c->type != SSH_CHANNEL_OPEN) {
-				debug2("channel %d: not open", c->self);
-				chan_mark_dead(c);
-				return -1;
-			} else {
-				chan_read_failed(c);
-				chan_write_failed(c);
-			}
-			return -1;
-		} else
-			fatal("%s: unexpected data on ctl fd", __func__);
-	}
-	return 1;
-}
-
 static int
 channel_check_window(Channel *c)
 {
@@ -1809,10 +1798,131 @@ channel_post_open(Channel *c, fd_set *readset, fd_set *writeset)
 	if (!compat20)
 		return;
 	channel_handle_efd(c, readset, writeset);
-	channel_handle_ctl(c, readset, writeset);
 	channel_check_window(c);
 }
 
+static u_int
+read_mux(Channel *c, u_int need)
+{
+	char buf[CHAN_RBUF];
+	int len;
+	u_int rlen;
+
+	if (buffer_len(&c->input) < need) {
+		rlen = need - buffer_len(&c->input);
+		len = read(c->rfd, buf, MIN(rlen, CHAN_RBUF));
+		if (len <= 0) {
+			if (errno != EINTR && errno != EAGAIN) {
+				debug2("channel %d: ctl read<=0 rfd %d len %d",
+				    c->self, c->rfd, len);
+				chan_read_failed(c);
+				return 0;
+			}
+		} else
+			buffer_append(&c->input, buf, len);
+	}
+	return buffer_len(&c->input);
+}
+
+static void
+channel_post_mux_client(Channel *c, fd_set *readset, fd_set *writeset)
+{
+	u_int need;
+	ssize_t len;
+
+	if (!compat20)
+		fatal("%s: entered with !compat20", __func__);
+
+	if (c->rfd != -1 && FD_ISSET(c->rfd, readset) &&
+	    (c->istate == CHAN_INPUT_OPEN ||
+	    c->istate == CHAN_INPUT_WAIT_DRAIN)) {
+		/*
+		 * Don't not read past the precise end of packets to
+		 * avoid disrupting fd passing.
+		 */
+		if (read_mux(c, 4) < 4) /* read header */
+			return;
+		need = get_u32(buffer_ptr(&c->input));
+#define CHANNEL_MUX_MAX_PACKET	(256 * 1024)
+		if (need > CHANNEL_MUX_MAX_PACKET) {
+			debug2("channel %d: packet too big %u > %u",
+			    c->self, CHANNEL_MUX_MAX_PACKET, need);
+			chan_rcvd_oclose(c);
+			return;
+		}
+		if (read_mux(c, need + 4) < need + 4) /* read body */
+			return;
+		if (c->mux_rcb(c) != 0) {
+			debug("channel %d: mux_rcb failed", c->self);
+			chan_mark_dead(c);
+			return;
+		}
+	}
+
+	if (c->wfd != -1 && FD_ISSET(c->wfd, writeset) &&
+	    buffer_len(&c->output) > 0) {
+		len = write(c->wfd, buffer_ptr(&c->output),
+		    buffer_len(&c->output));
+		if (len < 0 && (errno == EINTR || errno == EAGAIN))
+			return;
+		if (len <= 0) {
+			chan_mark_dead(c);
+			return;
+		}
+		buffer_consume(&c->output, len);
+	}
+}
+
+static void
+channel_post_mux_listener(Channel *c, fd_set *readset, fd_set *writeset)
+{
+	Channel *nc;
+	struct sockaddr_storage addr;
+	socklen_t addrlen;
+	int newsock;
+	uid_t euid;
+	gid_t egid;
+
+	if (!FD_ISSET(c->sock, readset))
+		return;
+
+	debug("multiplexing control connection");
+
+	/*
+	 * Accept connection on control socket
+	 */
+	memset(&addr, 0, sizeof(addr));
+	addrlen = sizeof(addr);
+	if ((newsock = accept(c->sock, (struct sockaddr*)&addr,
+	    &addrlen)) == -1) {
+		error("%s accept: %s", __func__, strerror(errno));
+		return;
+	}
+
+	if (getpeereid(newsock, &euid, &egid) < 0) {
+		error("%s getpeereid failed: %s", __func__,
+		    strerror(errno));
+		close(newsock);
+		return;
+	}
+	if ((euid != 0) && (getuid() != euid)) {
+		error("multiplex uid mismatch: peer euid %u != uid %u",
+		    (u_int)euid, (u_int)getuid());
+		close(newsock);
+		return;
+	}
+	nc = channel_new("multiplex client", SSH_CHANNEL_MUX_CLIENT,
+	    newsock, newsock, -1, c->local_window_max,
+	    c->local_maxpacket, 0, "mux-control", 1);
+	nc->mux_rcb = c->mux_rcb;
+	debug3("%s: new mux channel %d fd %d", __func__,
+	    nc->self, nc->sock);
+	/* establish state */
+	nc->mux_rcb(nc);
+	/* mux state transitions must not elicit protocol messages */
+	nc->flags |= CHAN_LOCAL;
+}
+
 /* ARGSUSED */
 static void
 channel_post_output_drain_13(Channel *c, fd_set *readset, fd_set *writeset)
@@ -1841,6 +1951,8 @@ channel_handler_init_20(void)
 	channel_pre[SSH_CHANNEL_AUTH_SOCKET] =		&channel_pre_listener;
 	channel_pre[SSH_CHANNEL_CONNECTING] =		&channel_pre_connecting;
 	channel_pre[SSH_CHANNEL_DYNAMIC] =		&channel_pre_dynamic;
+	channel_pre[SSH_CHANNEL_MUX_LISTENER] =		&channel_pre_listener;
+	channel_pre[SSH_CHANNEL_MUX_CLIENT] =		&channel_pre_mux_client;
 
 	channel_post[SSH_CHANNEL_OPEN] =		&channel_post_open;
 	channel_post[SSH_CHANNEL_PORT_LISTENER] =	&channel_post_port_listener;
@@ -1849,6 +1961,8 @@ channel_handler_init_20(void)
 	channel_post[SSH_CHANNEL_AUTH_SOCKET] =		&channel_post_auth_listener;
 	channel_post[SSH_CHANNEL_CONNECTING] =		&channel_post_connecting;
 	channel_post[SSH_CHANNEL_DYNAMIC] =		&channel_post_open;
+	channel_post[SSH_CHANNEL_MUX_LISTENER] =	&channel_post_mux_listener;
+	channel_post[SSH_CHANNEL_MUX_CLIENT] =		&channel_post_mux_client;
 }
 
 static void
diff --git a/channels.h b/channels.h
index 79ebe047..cc71885f 100644
--- a/channels.h
+++ b/channels.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: channels.h,v 1.102 2010/01/11 01:39:46 dtucker Exp $ */
+/* $OpenBSD: channels.h,v 1.103 2010/01/26 01:28:35 djm Exp $ */
 
 /*
  * Author: Tatu Ylonen <ylo@cs.hut.fi>
@@ -53,7 +53,9 @@
 #define SSH_CHANNEL_CONNECTING		12
 #define SSH_CHANNEL_DYNAMIC		13
 #define SSH_CHANNEL_ZOMBIE		14	/* Almost dead. */
-#define SSH_CHANNEL_MAX_TYPE		15
+#define SSH_CHANNEL_MUX_LISTENER	15	/* Listener for mux conn. */
+#define SSH_CHANNEL_MUX_CLIENT		16	/* Conn. to mux slave */
+#define SSH_CHANNEL_MAX_TYPE		17
 
 struct Channel;
 typedef struct Channel Channel;
@@ -81,6 +83,9 @@ struct channel_connect {
 	struct addrinfo *ai, *aitop;
 };
 
+/* Callbacks for mux channels back into client-specific code */
+typedef int mux_callback_fn(struct Channel *);
+
 struct Channel {
 	int     type;		/* channel type/state */
 	int     self;		/* my own channel identifier */
@@ -92,7 +97,7 @@ struct Channel {
 	int     wfd;		/* write fd */
 	int     efd;		/* extended fd */
 	int     sock;		/* sock fd */
-	int     ctl_fd;		/* control fd (client sharing) */
+	int     ctl_chan;	/* control channel (multiplexed connections) */
 	int     isatty;		/* rfd is a tty */
 	int     wfd_isatty;	/* wfd is a tty */
 	int	client_tty;	/* (client) TTY has been requested */
@@ -142,6 +147,10 @@ struct Channel {
 
 	/* non-blocking connect */
 	struct channel_connect	connect_ctx;
+
+	/* multiplexing protocol hook, called for each packet received */
+	mux_callback_fn		*mux_rcb;
+	void			*mux_ctx;
 };
 
 #define CHAN_EXTENDED_IGNORE		0
@@ -172,6 +181,7 @@ struct Channel {
 #define CHAN_CLOSE_RCVD			0x02
 #define CHAN_EOF_SENT			0x04
 #define CHAN_EOF_RCVD			0x08
+#define CHAN_LOCAL			0x10
 
 #define CHAN_RBUF	16*1024
 
@@ -243,7 +253,7 @@ void	 channel_clear_adm_permitted_opens(void);
 void 	 channel_print_adm_permitted_opens(void);
 int      channel_input_port_forward_request(int, int);
 Channel	*channel_connect_to(const char *, u_short, char *, char *);
-Channel	*channel_connect_stdio_fwd(const char*, u_short);
+Channel	*channel_connect_stdio_fwd(const char*, u_short, int, int);
 Channel	*channel_connect_by_listen_address(u_short, char *, char *);
 int	 channel_request_remote_forwarding(const char *, u_short,
 	     const char *, u_short);
diff --git a/clientloop.c b/clientloop.c
index 5793a6e9..05f4720a 100644
--- a/clientloop.c
+++ b/clientloop.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: clientloop.c,v 1.216 2010/01/09 05:04:24 djm Exp $ */
+/* $OpenBSD: clientloop.c,v 1.217 2010/01/26 01:28:35 djm Exp $ */
 /*
  * Author: Tatu Ylonen <ylo@cs.hut.fi>
  * Copyright (c) 1995 Tatu Ylonen <ylo@cs.hut.fi>, Espoo, Finland
@@ -121,7 +121,7 @@ extern int stdin_null_flag;
 extern int no_shell_flag;
 
 /* Control socket */
-extern int muxserver_sock;
+extern int muxserver_sock; /* XXX use mux_client_cleanup() instead */
 
 /*
  * Name of the host we are connecting to.  This is the name given on the
@@ -146,7 +146,7 @@ static volatile sig_atomic_t received_signal = 0;
 static int in_non_blocking_mode = 0;
 
 /* Common data for the client loop code. */
-static volatile sig_atomic_t quit_pending; /* Set non-zero to quit the loop. */
+volatile sig_atomic_t quit_pending; /* Set non-zero to quit the loop. */
 static int escape_char1;	/* Escape character. (proto1 only) */
 static int escape_pending1;	/* Last character was an escape (proto1 only) */
 static int last_was_cr;		/* Last character was a newline. */
@@ -564,9 +564,6 @@ client_wait_until_can_do_something(fd_set **readsetp, fd_set **writesetp,
 	if (packet_have_data_to_write())
 		FD_SET(connection_out, *writesetp);
 
-	if (muxserver_sock != -1)
-		FD_SET(muxserver_sock, *readsetp);
-
 	/*
 	 * Wait for something to happen.  This will suspend the process until
 	 * some selected descriptor can be read, written, or has some other
@@ -695,7 +692,7 @@ client_status_confirm(int type, Channel *c, void *ctx)
 
 	/* XXX supress on mux _client_ quietmode */
 	tochan = options.log_level >= SYSLOG_LEVEL_ERROR &&
-	    c->ctl_fd != -1 && c->extended_usage == CHAN_EXTENDED_WRITE;
+	    c->ctl_chan != -1 && c->extended_usage == CHAN_EXTENDED_WRITE;
 
 	if (type == SSH2_MSG_CHANNEL_SUCCESS) {
 		debug2("%s request accepted on channel %d",
@@ -839,6 +836,7 @@ process_cmdline(void)
 	while (isspace(*++s))
 		;
 
+	/* XXX update list of forwards in options */
 	if (delete) {
 		cancel_port = 0;
 		cancel_host = hpdelim(&s);	/* may be NULL */
@@ -936,7 +934,7 @@ process_escapes(Channel *c, Buffer *bin, Buffer *bout, Buffer *berr,
 				    escape_char);
 				buffer_append(berr, string, strlen(string));
 
-				if (c && c->ctl_fd != -1) {
+				if (c && c->ctl_chan != -1) {
 					chan_read_failed(c);
 					chan_write_failed(c);
 					return 0;
@@ -946,7 +944,7 @@ process_escapes(Channel *c, Buffer *bin, Buffer *bout, Buffer *berr,
 
 			case 'Z' - 64:
 				/* XXX support this for mux clients */
-				if (c && c->ctl_fd != -1) {
+				if (c && c->ctl_chan != -1) {
  noescape:
 					snprintf(string, sizeof string,
 					    "%c%c escape not available to "
@@ -991,7 +989,7 @@ process_escapes(Channel *c, Buffer *bin, Buffer *bout, Buffer *berr,
 				continue;
 
 			case '&':
-				if (c && c->ctl_fd != -1)
+				if (c && c->ctl_chan != -1)
 					goto noescape;
 				/*
 				 * Detach the program (continue to serve
@@ -1042,7 +1040,7 @@ process_escapes(Channel *c, Buffer *bin, Buffer *bout, Buffer *berr,
 				continue;
 
 			case '?':
-				if (c && c->ctl_fd != -1) {
+				if (c && c->ctl_chan != -1) {
 					snprintf(string, sizeof string,
 "%c?\r\n\
 Supported escape sequences:\r\n\
@@ -1091,7 +1089,7 @@ Supported escape sequences:\r\n\
 				continue;
 
 			case 'C':
-				if (c && c->ctl_fd != -1)
+				if (c && c->ctl_chan != -1)
 					goto noescape;
 				process_cmdline();
 				continue;
@@ -1327,8 +1325,6 @@ client_loop(int have_pty, int escape_char_arg, int ssh2_chan_id)
 	connection_in = packet_get_connection_in();
 	connection_out = packet_get_connection_out();
 	max_fd = MAX(connection_in, connection_out);
-	if (muxserver_sock != -1)
-		max_fd = MAX(max_fd, muxserver_sock);
 
 	if (!compat20) {
 		/* enable nonblocking unless tty */
@@ -1446,12 +1442,6 @@ client_loop(int have_pty, int escape_char_arg, int ssh2_chan_id)
 		/* Buffer input from the connection.  */
 		client_process_net_input(readset);
 
-		/* Accept control connections.  */
-		if (muxserver_sock != -1 &&FD_ISSET(muxserver_sock, readset)) {
-			if (muxserver_accept_control())
-				quit_pending = 1;
-		}
-
 		if (quit_pending)
 			break;
 
@@ -1859,9 +1849,8 @@ client_input_channel_req(int type, u_int32_t seq, void *ctxt)
 		chan_rcvd_eow(c);
 	} else if (strcmp(rtype, "exit-status") == 0) {
 		exitval = packet_get_int();
-		if (c->ctl_fd != -1) {
-			/* Dispatch to mux client */
-			atomicio(vwrite, c->ctl_fd, &exitval, sizeof(exitval));
+		if (c->ctl_chan != -1) {
+			mux_exit_message(c, exitval);
 			success = 1;
 		} else if (id == session_ident) {
 			/* Record exit value of local session */
diff --git a/clientloop.h b/clientloop.h
index 8bb874b3..0b8257b9 100644
--- a/clientloop.h
+++ b/clientloop.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: clientloop.h,v 1.22 2008/06/12 15:19:17 djm Exp $ */
+/* $OpenBSD: clientloop.h,v 1.23 2010/01/26 01:28:35 djm Exp $ */
 
 /*
  * Author: Tatu Ylonen <ylo@cs.hut.fi>
@@ -56,18 +56,14 @@ typedef void global_confirm_cb(int, u_int32_t seq, void *);
 void	 client_register_global_confirm(global_confirm_cb *, void *);
 
 /* Multiplexing protocol version */
-#define SSHMUX_VER			2
+#define SSHMUX_VER			4
 
 /* Multiplexing control protocol flags */
 #define SSHMUX_COMMAND_OPEN		1	/* Open new connection */
 #define SSHMUX_COMMAND_ALIVE_CHECK	2	/* Check master is alive */
 #define SSHMUX_COMMAND_TERMINATE	3	/* Ask master to exit */
-
-#define SSHMUX_FLAG_TTY			(1)	/* Request tty on open */
-#define SSHMUX_FLAG_SUBSYS		(1<<1)	/* Subsystem request on open */
-#define SSHMUX_FLAG_X11_FWD		(1<<2)	/* Request X11 forwarding */
-#define SSHMUX_FLAG_AGENT_FWD		(1<<3)	/* Request agent forwarding */
+#define SSHMUX_COMMAND_STDIO_FWD	4	/* Open stdio fwd (ssh -W) */
 
 void	muxserver_listen(void);
-int	muxserver_accept_control(void);
 void	muxclient(const char *);
+void	mux_exit_message(Channel *, int);
diff --git a/mux.c b/mux.c
index 239edd5f..dcd6dc02 100644
--- a/mux.c
+++ b/mux.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: mux.c,v 1.9 2010/01/09 05:04:24 djm Exp $ */
+/* $OpenBSD: mux.c,v 1.10 2010/01/26 01:28:35 djm Exp $ */
 /*
  * Copyright (c) 2002-2008 Damien Miller <djm@openbsd.org>
  *
@@ -17,25 +17,21 @@
 
 /* ssh session multiplexing support */
 
-#include "includes.h"
-
 /*
  * TODO:
- *   1. partial reads in muxserver_accept_control (maybe make channels
- *      from accepted connections)
- *   2. Better signalling from master to slave, especially passing of
+ *   - Better signalling from master to slave, especially passing of
  *      error messages
- *   3. Better fall-back from mux slave error to new connection.
- *   3. Add/delete forwardings via slave
- *   4. ExitOnForwardingFailure (after #3 obviously)
- *   5. Maybe extension mechanisms for multi-X11/multi-agent forwarding
- *   6. Document the mux mini-protocol somewhere.
- *   7. Support ~^Z in mux slaves.
- *   8. Inspect or control sessions in master.
- *   9. If we ever support the "signal" channel request, send signals on
- *      sessions in master.
+ *   - Better fall-back from mux slave error to new connection.
+ *   - ExitOnForwardingFailure
+ *   - Maybe extension mechanisms for multi-X11/multi-agent forwarding
+ *   - Support ~^Z in mux slaves.
+ *   - Inspect or control sessions in master.
+ *   - If we ever support the "signal" channel request, send signals on
+ *     sessions in master.
  */
 
+#include "includes.h"
+
 #include <sys/types.h>
 #include <sys/param.h>
 #include <sys/stat.h>
@@ -55,6 +51,14 @@
 #include <paths.h>
 #endif
 
+#ifdef HAVE_POLL_H
+#include <poll.h>
+#else
+# ifdef HAVE_SYS_POLL_H
+#  include <sys/poll.h>
+# endif
+#endif
+
 #ifdef HAVE_UTIL_H
 # include <util.h>
 #endif
@@ -88,13 +92,16 @@ extern int stdin_null_flag;
 extern char *host;
 extern int subsystem_flag;
 extern Buffer command;
+extern volatile sig_atomic_t quit_pending;
+extern char *stdio_forward_host;
+extern int stdio_forward_port;
 
 /* Context for session open confirmation callback */
 struct mux_session_confirm_ctx {
-	int want_tty;
-	int want_subsys;
-	int want_x_fwd;
-	int want_agent_fwd;
+	u_int want_tty;
+	u_int want_subsys;
+	u_int want_x_fwd;
+	u_int want_agent_fwd;
 	Buffer cmd;
 	char *term;
 	struct termios tio;
@@ -104,6 +111,9 @@ struct mux_session_confirm_ctx {
 /* fd to control socket */
 int muxserver_sock = -1;
 
+/* client request id */
+u_int muxclient_request_id = 0;
+
 /* Multiplexing control command */
 u_int muxclient_command = 0;
 
@@ -113,16 +123,823 @@ static volatile sig_atomic_t muxclient_terminate = 0;
 /* PID of multiplex server */
 static u_int muxserver_pid = 0;
 
+static Channel *mux_listener_channel = NULL;
+
+struct mux_master_state {
+	int hello_rcvd;
+};
+
+/* mux protocol messages */
+#define MUX_MSG_HELLO		0x00000001
+#define MUX_C_NEW_SESSION	0x10000002
+#define MUX_C_ALIVE_CHECK	0x10000004
+#define MUX_C_TERMINATE		0x10000005
+#define MUX_C_OPEN_FWD		0x10000006
+#define MUX_C_CLOSE_FWD		0x10000007
+#define MUX_C_NEW_STDIO_FWD	0x10000008
+#define MUX_S_OK		0x80000001
+#define MUX_S_PERMISSION_DENIED	0x80000002
+#define MUX_S_FAILURE		0x80000003
+#define MUX_S_EXIT_MESSAGE	0x80000004
+#define MUX_S_ALIVE		0x80000005
+#define MUX_S_SESSION_OPENED	0x80000006
+
+/* type codes for MUX_C_OPEN_FWD and MUX_C_CLOSE_FWD */
+#define MUX_FWD_LOCAL   1
+#define MUX_FWD_REMOTE  2
+#define MUX_FWD_DYNAMIC 3
+
+static void mux_session_confirm(int, void *);
+
+static int process_mux_master_hello(u_int, Channel *, Buffer *, Buffer *);
+static int process_mux_new_session(u_int, Channel *, Buffer *, Buffer *);
+static int process_mux_alive_check(u_int, Channel *, Buffer *, Buffer *);
+static int process_mux_terminate(u_int, Channel *, Buffer *, Buffer *);
+static int process_mux_open_fwd(u_int, Channel *, Buffer *, Buffer *);
+static int process_mux_close_fwd(u_int, Channel *, Buffer *, Buffer *);
+static int process_mux_stdio_fwd(u_int, Channel *, Buffer *, Buffer *);
+
+static const struct {
+	u_int type;
+	int (*handler)(u_int, Channel *, Buffer *, Buffer *);
+} mux_master_handlers[] = {
+	{ MUX_MSG_HELLO, process_mux_master_hello },
+	{ MUX_C_NEW_SESSION, process_mux_new_session },
+	{ MUX_C_ALIVE_CHECK, process_mux_alive_check },
+	{ MUX_C_TERMINATE, process_mux_terminate },
+	{ MUX_C_OPEN_FWD, process_mux_open_fwd },
+	{ MUX_C_CLOSE_FWD, process_mux_close_fwd },
+	{ MUX_C_NEW_STDIO_FWD, process_mux_stdio_fwd },
+	{ 0, NULL }
+};
+
+/* Cleanup callback fired on closure of mux slave _session_ channel */
+/* ARGSUSED */
+static void
+mux_master_session_cleanup_cb(int cid, void *unused)
+{
+	Channel *cc, *c = channel_by_id(cid);
+
+	debug3("%s: entering for channel %d", __func__, cid);
+	if (c == NULL)
+		fatal("%s: channel_by_id(%i) == NULL", __func__, cid);
+	if (c->ctl_chan != -1) {
+		if ((cc = channel_by_id(c->ctl_chan)) == NULL)
+			fatal("%s: channel %d missing control channel %d",
+			    __func__, c->self, c->ctl_chan);
+		c->ctl_chan = -1;
+		cc->remote_id = -1;
+		chan_rcvd_oclose(cc);
+	}
+	channel_cancel_cleanup(c->self);
+}
+
+/* Cleanup callback fired on closure of mux slave _control_ channel */
+/* ARGSUSED */
+static void
+mux_master_control_cleanup_cb(int cid, void *unused)
+{
+	Channel *sc, *c = channel_by_id(cid);
+
+	debug3("%s: entering for channel %d", __func__, cid);
+	if (c == NULL)
+		fatal("%s: channel_by_id(%i) == NULL", __func__, cid);
+	if (c->remote_id != -1) {
+		if ((sc = channel_by_id(c->remote_id)) == NULL)
+			debug2("%s: channel %d n session channel %d",
+			    __func__, c->self, c->remote_id);
+		c->remote_id = -1;
+		sc->ctl_chan = -1;
+		chan_mark_dead(sc);
+	}
+	channel_cancel_cleanup(c->self);
+}
+
+/* Check mux client environment variables before passing them to mux master. */
+static int
+env_permitted(char *env)
+{
+	int i, ret;
+	char name[1024], *cp;
+
+	if ((cp = strchr(env, '=')) == NULL || cp == env)
+		return 0;
+	ret = snprintf(name, sizeof(name), "%.*s", (int)(cp - env), env);
+	if (ret <= 0 || (size_t)ret >= sizeof(name)) {
+		error("env_permitted: name '%.100s...' too long", env);
+		return 0;
+	}
+
+	for (i = 0; i < options.num_send_env; i++)
+		if (match_pattern(name, options.send_env[i]))
+			return 1;
+
+	return 0;
+}
+
+/* Mux master protocol message handlers */
+
+static int
+process_mux_master_hello(u_int rid, Channel *c, Buffer *m, Buffer *r)
+{
+	u_int ver;
+	struct mux_master_state *state = (struct mux_master_state *)c->mux_ctx;
+
+	if (state == NULL)
+		fatal("%s: channel %d: c->mux_ctx == NULL", __func__, c->self);
+	if (state->hello_rcvd) {
+		error("%s: HELLO received twice", __func__);
+		return -1;
+	}
+	if (buffer_get_int_ret(&ver, m) != 0) {
+ malf:
+		error("%s: malformed message", __func__);
+		return -1;
+	}
+	if (ver != SSHMUX_VER) {
+		error("Unsupported multiplexing protocol version %d "
+		    "(expected %d)", ver, SSHMUX_VER);
+		return -1;
+	}
+	debug2("%s: channel %d slave version %u", __func__, c->self, ver);
+
+	/* No extensions are presently defined */
+	while (buffer_len(m) > 0) {
+		char *name = buffer_get_string_ret(m, NULL);
+		char *value = buffer_get_string_ret(m, NULL);
+
+		if (name == NULL || value == NULL) {
+			if (name != NULL)
+				xfree(name);
+			goto malf;
+		}
+		debug2("Unrecognised slave extension \"%s\"", name);
+		xfree(name);
+		xfree(value);
+	}
+	state->hello_rcvd = 1;
+	return 0;
+}
+
+static int
+process_mux_new_session(u_int rid, Channel *c, Buffer *m, Buffer *r)
+{
+	Channel *nc;
+	struct mux_session_confirm_ctx *cctx;
+	char *reserved, *cmd, *cp;
+	u_int i, j, len, env_len, escape_char, window, packetmax;
+	int new_fd[3];
+
+	/* Reply for SSHMUX_COMMAND_OPEN */
+	cctx = xcalloc(1, sizeof(*cctx));
+	cctx->term = NULL;
+	cmd = reserved = NULL;
+	if ((reserved = buffer_get_string_ret(m, NULL)) == NULL ||
+	    buffer_get_int_ret(&cctx->want_tty, m) != 0 ||
+	    buffer_get_int_ret(&cctx->want_x_fwd, m) != 0 ||
+	    buffer_get_int_ret(&cctx->want_agent_fwd, m) != 0 ||
+	    buffer_get_int_ret(&cctx->want_subsys, m) != 0 ||
+	    buffer_get_int_ret(&escape_char, m) != 0 ||
+	    (cctx->term = buffer_get_string_ret(m, &len)) == NULL ||
+	    (cmd = buffer_get_string_ret(m, &len)) == NULL) {
+ malf:
+		if (cmd != NULL)
+			xfree(cmd);
+		if (reserved != NULL)
+			xfree(reserved);
+		if (cctx->term != NULL)
+			xfree(cctx->term);
+		error("%s: malformed message", __func__);
+		return -1;
+	}
+	xfree(reserved);
+	reserved = NULL;
+
+	cctx->env = NULL;
+	env_len = 0;
+	while (buffer_len(m) > 0) {
+#define MUX_MAX_ENV_VARS	4096
+		if ((cp = buffer_get_string_ret(m, &len)) == NULL) {
+			xfree(cmd);
+			goto malf;
+		}
+		if (!env_permitted(cp)) {
+			xfree(cp);
+			continue;
+		}
+		cctx->env = xrealloc(cctx->env, env_len + 2,
+		    sizeof(*cctx->env));
+		cctx->env[env_len++] = cp;
+		cctx->env[env_len] = NULL;
+		if (env_len > MUX_MAX_ENV_VARS) {
+			error(">%d environment variables received, ignoring "
+			    "additional", MUX_MAX_ENV_VARS);
+			break;
+		}
+	}
+
+	debug2("%s: channel %d: request tty %d, X %d, agent %d, subsys %d, "
+	    "term \"%s\", cmd \"%s\", env %u", __func__, c->self,
+	    cctx->want_tty, cctx->want_x_fwd, cctx->want_agent_fwd,
+	    cctx->want_subsys, cctx->term, cmd, env_len);
+
+	buffer_init(&cctx->cmd);
+	buffer_append(&cctx->cmd, cmd, strlen(cmd));
+	xfree(cmd);
+	cmd = NULL;
+
+	/* Gather fds from client */
+	for(i = 0; i < 3; i++) {
+		if ((new_fd[i] = mm_receive_fd(c->sock)) == -1) {
+			error("%s: failed to receive fd %d from slave",
+			    __func__, i);
+			for (j = 0; j < i; j++)
+				close(new_fd[j]);
+			for (j = 0; j < env_len; j++)
+				xfree(cctx->env[j]);
+			if (env_len > 0)
+				xfree(cctx->env);
+			xfree(cctx->term);
+			buffer_free(&cctx->cmd);
+			xfree(cctx);
+
+			/* prepare reply */
+			buffer_put_int(r, MUX_S_FAILURE);
+			buffer_put_int(r, rid);
+			buffer_put_cstring(r,
+			    "did not receive file descriptors");
+			return -1;
+		}
+	}