Up to now, tmux sees \033\033[OA as M-Up and since we turned on

xterm-keys by default, generates \033[1;3A instead of
\033\033[OA. Unfortunately this confuses vi, which doesn't understand
xterm keys and now sees Escape+Up pressed within escape-time as Escape
followed by A.

The issue doesn't happen in xterm itself because it gets the keys from X
and can distinguish between a genuine M-Up and Escape+Up.

Because xterm can, tmux can too: xterm will give us \033[1;3A (that is,
kUP3) for a real M-Up and \033\033OA for Escape+Up - in fact, we can be
sure any \033 preceding an xterm key is a real Escape key press because
Meta would be part of the xterm key instead of a separate \033.

So change tmux to recognise both sequences as M-Up for its own purposes,
but generate the xterm version of M-Up only if it originally received
the xterm version from the terminal.

This means we will return to sending \033\033OA instead of the xterm key
for terminals that do not support xterm keys themselves, but there is no
practical way around this because they do not allow us to distinguish
between Escape+Up and M-Up. xterm style escape sequences are now the de
facto standard for these keys in any case.

Problem reported by jsing@ and subsequently by Cecile Tonglet in GitHub
issue 907.

6 files changed, 100 insertions, 74 deletions

diff --git a/input-keys.c b/input-keys.c
index eb2879a9..01b00b05 100644
--- a/input-keys.c
+++ b/input-keys.c
@@ -201,6 +201,7 @@ input_key(struct window_pane *wp, key_code key, struct mouse_event *m)
 			return;
 		}
 	}
+	key &= ~KEYC_XTERM;
 
 	/* Otherwise look the key up in the table. */
 	for (i = 0; i < nitems(input_keys); i++) {
diff --git a/key-bindings.c b/key-bindings.c
index 60dbe544..9e327655 100644
--- a/key-bindings.c
+++ b/key-bindings.c
@@ -92,7 +92,7 @@ key_bindings_add(const char *name, key_code key, int repeat,
 
 	table = key_bindings_get_table(name, 1);
 
-	bd_find.key = key;
+	bd_find.key = (key & ~KEYC_XTERM);
 	bd = RB_FIND(key_bindings, &table->key_bindings, &bd_find);
 	if (bd != NULL) {
 		RB_REMOVE(key_bindings, &table->key_bindings, bd);
@@ -119,7 +119,7 @@ key_bindings_remove(const char *name, key_code key)
 	if (table == NULL)
 		return;
 
-	bd_find.key = key;
+	bd_find.key = (key & ~KEYC_XTERM);
 	bd = RB_FIND(key_bindings, &table->key_bindings, &bd_find);
 	if (bd == NULL)
 		return;
diff --git a/server-client.c b/server-client.c
index e9e26bef..72b5d312 100644
--- a/server-client.c
+++ b/server-client.c
@@ -904,7 +904,7 @@ retry:
 		log_debug("currently repeating");
 
 	/* Try to see if there is a key binding in the current table. */
-	bd_find.key = key;
+	bd_find.key = (key & ~KEYC_XTERM);
 	bd = RB_FIND(key_bindings, &table->key_bindings, &bd_find);
 	if (bd != NULL) {
 		/*
diff --git a/tmux.h b/tmux.h
index ac2991e8..378c937a 100644
--- a/tmux.h
+++ b/tmux.h
@@ -91,9 +91,10 @@ struct tmuxproc;
 #define KEYC_ESCAPE 0x200000000000ULL
 #define KEYC_CTRL   0x400000000000ULL
 #define KEYC_SHIFT  0x800000000000ULL
+#define KEYC_XTERM 0x1000000000000ULL
 
 /* Mask to obtain key w/o modifiers. */
-#define KEYC_MASK_MOD (KEYC_ESCAPE|KEYC_CTRL|KEYC_SHIFT)
+#define KEYC_MASK_MOD (KEYC_ESCAPE|KEYC_CTRL|KEYC_SHIFT|KEYC_XTERM)
 #define KEYC_MASK_KEY (~KEYC_MASK_MOD)
 
 /* Is this a mouse key? */
diff --git a/tty-keys.c b/tty-keys.c
index a011fcab..d7797a9c 100644
--- a/tty-keys.c
+++ b/tty-keys.c
@@ -257,67 +257,70 @@ static const struct tty_default_key_code tty_default_code_keys[] = {
 	{ TTYC_KCUB1, KEYC_LEFT },
 	{ TTYC_KCUF1, KEYC_RIGHT },
 
-	/* Key and modifier capabilities. */
-	{ TTYC_KDC2, KEYC_DC|KEYC_SHIFT },
-	{ TTYC_KDC3, KEYC_DC|KEYC_ESCAPE },
-	{ TTYC_KDC4, KEYC_DC|KEYC_SHIFT|KEYC_ESCAPE },
-	{ TTYC_KDC5, KEYC_DC|KEYC_CTRL },
-	{ TTYC_KDC6, KEYC_DC|KEYC_SHIFT|KEYC_CTRL },
-	{ TTYC_KDC7, KEYC_DC|KEYC_ESCAPE|KEYC_CTRL },
-	{ TTYC_KDN2, KEYC_DOWN|KEYC_SHIFT },
-	{ TTYC_KDN3, KEYC_DOWN|KEYC_ESCAPE },
-	{ TTYC_KDN4, KEYC_DOWN|KEYC_SHIFT|KEYC_ESCAPE },
-	{ TTYC_KDN5, KEYC_DOWN|KEYC_CTRL },
-	{ TTYC_KDN6, KEYC_DOWN|KEYC_SHIFT|KEYC_CTRL },
-	{ TTYC_KDN7, KEYC_DOWN|KEYC_ESCAPE|KEYC_CTRL },
-	{ TTYC_KEND2, KEYC_END|KEYC_SHIFT },
-	{ TTYC_KEND3, KEYC_END|KEYC_ESCAPE },
-	{ TTYC_KEND4, KEYC_END|KEYC_SHIFT|KEYC_ESCAPE },
-	{ TTYC_KEND5, KEYC_END|KEYC_CTRL },
-	{ TTYC_KEND6, KEYC_END|KEYC_SHIFT|KEYC_CTRL },
-	{ TTYC_KEND7, KEYC_END|KEYC_ESCAPE|KEYC_CTRL },
-	{ TTYC_KHOM2, KEYC_HOME|KEYC_SHIFT },
-	{ TTYC_KHOM3, KEYC_HOME|KEYC_ESCAPE },
-	{ TTYC_KHOM4, KEYC_HOME|KEYC_SHIFT|KEYC_ESCAPE },
-	{ TTYC_KHOM5, KEYC_HOME|KEYC_CTRL },
-	{ TTYC_KHOM6, KEYC_HOME|KEYC_SHIFT|KEYC_CTRL },
-	{ TTYC_KHOM7, KEYC_HOME|KEYC_ESCAPE|KEYC_CTRL },
-	{ TTYC_KIC2, KEYC_IC|KEYC_SHIFT },
-	{ TTYC_KIC3, KEYC_IC|KEYC_ESCAPE },
-	{ TTYC_KIC4, KEYC_IC|KEYC_SHIFT|KEYC_ESCAPE },
-	{ TTYC_KIC5, KEYC_IC|KEYC_CTRL },
-	{ TTYC_KIC6, KEYC_IC|KEYC_SHIFT|KEYC_CTRL },
-	{ TTYC_KIC7, KEYC_IC|KEYC_ESCAPE|KEYC_CTRL },
-	{ TTYC_KLFT2, KEYC_LEFT|KEYC_SHIFT },
-	{ TTYC_KLFT3, KEYC_LEFT|KEYC_ESCAPE },
-	{ TTYC_KLFT4, KEYC_LEFT|KEYC_SHIFT|KEYC_ESCAPE },
-	{ TTYC_KLFT5, KEYC_LEFT|KEYC_CTRL },
-	{ TTYC_KLFT6, KEYC_LEFT|KEYC_SHIFT|KEYC_CTRL },
-	{ TTYC_KLFT7, KEYC_LEFT|KEYC_ESCAPE|KEYC_CTRL },
-	{ TTYC_KNXT2, KEYC_NPAGE|KEYC_SHIFT },
-	{ TTYC_KNXT3, KEYC_NPAGE|KEYC_ESCAPE },
-	{ TTYC_KNXT4, KEYC_NPAGE|KEYC_SHIFT|KEYC_ESCAPE },
-	{ TTYC_KNXT5, KEYC_NPAGE|KEYC_CTRL },
-	{ TTYC_KNXT6, KEYC_NPAGE|KEYC_SHIFT|KEYC_CTRL },
-	{ TTYC_KNXT7, KEYC_NPAGE|KEYC_ESCAPE|KEYC_CTRL },
-	{ TTYC_KPRV2, KEYC_PPAGE|KEYC_SHIFT },
-	{ TTYC_KPRV3, KEYC_PPAGE|KEYC_ESCAPE },
-	{ TTYC_KPRV4, KEYC_PPAGE|KEYC_SHIFT|KEYC_ESCAPE },
-	{ TTYC_KPRV5, KEYC_PPAGE|KEYC_CTRL },
-	{ TTYC_KPRV6, KEYC_PPAGE|KEYC_SHIFT|KEYC_CTRL },
-	{ TTYC_KPRV7, KEYC_PPAGE|KEYC_ESCAPE|KEYC_CTRL },
-	{ TTYC_KRIT2, KEYC_RIGHT|KEYC_SHIFT },
-	{ TTYC_KRIT3, KEYC_RIGHT|KEYC_ESCAPE },
-	{ TTYC_KRIT4, KEYC_RIGHT|KEYC_SHIFT|KEYC_ESCAPE },
-	{ TTYC_KRIT5, KEYC_RIGHT|KEYC_CTRL },
-	{ TTYC_KRIT6, KEYC_RIGHT|KEYC_SHIFT|KEYC_CTRL },
-	{ TTYC_KRIT7, KEYC_RIGHT|KEYC_ESCAPE|KEYC_CTRL },
-	{ TTYC_KUP2, KEYC_UP|KEYC_SHIFT },
-	{ TTYC_KUP3, KEYC_UP|KEYC_ESCAPE },
-	{ TTYC_KUP4, KEYC_UP|KEYC_SHIFT|KEYC_ESCAPE },
-	{ TTYC_KUP5, KEYC_UP|KEYC_CTRL },
-	{ TTYC_KUP6, KEYC_UP|KEYC_SHIFT|KEYC_CTRL },
-	{ TTYC_KUP7, KEYC_UP|KEYC_ESCAPE|KEYC_CTRL },
+	/*
+	 * Key and modifier capabilities. We set the xterm flag to mark that
+	 * any leading escape means an escape key press and not the modifier.
+	 */
+	{ TTYC_KDC2, KEYC_DC|KEYC_SHIFT|KEYC_XTERM },
+	{ TTYC_KDC3, KEYC_DC|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KDC4, KEYC_DC|KEYC_SHIFT|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KDC5, KEYC_DC|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KDC6, KEYC_DC|KEYC_SHIFT|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KDC7, KEYC_DC|KEYC_ESCAPE|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KDN2, KEYC_DOWN|KEYC_SHIFT|KEYC_XTERM },
+	{ TTYC_KDN3, KEYC_DOWN|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KDN4, KEYC_DOWN|KEYC_SHIFT|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KDN5, KEYC_DOWN|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KDN6, KEYC_DOWN|KEYC_SHIFT|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KDN7, KEYC_DOWN|KEYC_ESCAPE|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KEND2, KEYC_END|KEYC_SHIFT|KEYC_XTERM },
+	{ TTYC_KEND3, KEYC_END|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KEND4, KEYC_END|KEYC_SHIFT|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KEND5, KEYC_END|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KEND6, KEYC_END|KEYC_SHIFT|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KEND7, KEYC_END|KEYC_ESCAPE|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KHOM2, KEYC_HOME|KEYC_SHIFT|KEYC_XTERM },
+	{ TTYC_KHOM3, KEYC_HOME|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KHOM4, KEYC_HOME|KEYC_SHIFT|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KHOM5, KEYC_HOME|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KHOM6, KEYC_HOME|KEYC_SHIFT|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KHOM7, KEYC_HOME|KEYC_ESCAPE|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KIC2, KEYC_IC|KEYC_SHIFT|KEYC_XTERM },
+	{ TTYC_KIC3, KEYC_IC|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KIC4, KEYC_IC|KEYC_SHIFT|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KIC5, KEYC_IC|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KIC6, KEYC_IC|KEYC_SHIFT|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KIC7, KEYC_IC|KEYC_ESCAPE|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KLFT2, KEYC_LEFT|KEYC_SHIFT|KEYC_XTERM },
+	{ TTYC_KLFT3, KEYC_LEFT|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KLFT4, KEYC_LEFT|KEYC_SHIFT|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KLFT5, KEYC_LEFT|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KLFT6, KEYC_LEFT|KEYC_SHIFT|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KLFT7, KEYC_LEFT|KEYC_ESCAPE|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KNXT2, KEYC_NPAGE|KEYC_SHIFT|KEYC_XTERM },
+	{ TTYC_KNXT3, KEYC_NPAGE|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KNXT4, KEYC_NPAGE|KEYC_SHIFT|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KNXT5, KEYC_NPAGE|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KNXT6, KEYC_NPAGE|KEYC_SHIFT|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KNXT7, KEYC_NPAGE|KEYC_ESCAPE|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KPRV2, KEYC_PPAGE|KEYC_SHIFT|KEYC_XTERM },
+	{ TTYC_KPRV3, KEYC_PPAGE|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KPRV4, KEYC_PPAGE|KEYC_SHIFT|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KPRV5, KEYC_PPAGE|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KPRV6, KEYC_PPAGE|KEYC_SHIFT|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KPRV7, KEYC_PPAGE|KEYC_ESCAPE|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KRIT2, KEYC_RIGHT|KEYC_SHIFT|KEYC_XTERM },
+	{ TTYC_KRIT3, KEYC_RIGHT|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KRIT4, KEYC_RIGHT|KEYC_SHIFT|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KRIT5, KEYC_RIGHT|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KRIT6, KEYC_RIGHT|KEYC_SHIFT|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KRIT7, KEYC_RIGHT|KEYC_ESCAPE|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KUP2, KEYC_UP|KEYC_SHIFT|KEYC_XTERM },
+	{ TTYC_KUP3, KEYC_UP|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KUP4, KEYC_UP|KEYC_SHIFT|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KUP5, KEYC_UP|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KUP6, KEYC_UP|KEYC_SHIFT|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KUP7, KEYC_UP|KEYC_ESCAPE|KEYC_CTRL|KEYC_XTERM },
 };
 
 /* Add key to tree. */
@@ -476,6 +479,7 @@ tty_keys_next1(struct tty *tty, const char *buf, size_t len, key_code *key,
 	enum utf8_state		 more;
 	u_int			 i;
 	wchar_t			 wc;
+	int			 n;
 
 	log_debug("%s: next key is %zu (%.*s) (expired=%d)", c->name, len,
 	    (int)len, buf, expired);
@@ -493,6 +497,13 @@ tty_keys_next1(struct tty *tty, const char *buf, size_t len, key_code *key,
 		return (0);
 	}
 
+	/* Is this an an xterm(1) key? */
+	n = xterm_keys_find(buf, len, size, key);
+	if (n == 0)
+		return (0);
+	if (n == 1 && !expired)
+		return (1);
+
 	/* Is this valid UTF-8? */
 	more = utf8_open(&ud, (u_char)*buf);
 	if (more == UTF8_MORE) {
@@ -573,6 +584,18 @@ first_key:
 		/* Look for a key without the escape. */
 		n = tty_keys_next1(tty, buf + 1, len - 1, &key, &size, expired);
 		if (n == 0) {	/* found */
+			if (key & KEYC_XTERM) {
+				/*
+				 * We want the escape key as well as the xterm
+				 * key, because the xterm sequence implicitly
+				 * includes the escape (so if we see
+				 * \033\033[1;3D we know it is an Escape
+				 * followed by M-Left, not just M-Left).
+				 */
+				key = '\033';
+				size = 1;
+				goto complete_key;
+			}
 			key |= KEYC_ESCAPE;
 			size++;
 			goto complete_key;
@@ -588,13 +611,6 @@ first_key:
 	if (n == 1)
 		goto partial_key;
 
-	/* Is this an an xterm(1) key? */
-	n = xterm_keys_find(buf, len, &size, &key);
-	if (n == 0)
-		goto complete_key;
-	if (n == 1 && !expired)
-		goto partial_key;
-
 	/*
 	 * At this point, we know the key is not partial (with or without
 	 * escape). So pass it through even if the timer has not expired.
diff --git a/xterm-keys.c b/xterm-keys.c
index 5ffc845f..b10c10db 100644
--- a/xterm-keys.c
+++ b/xterm-keys.c
@@ -197,7 +197,7 @@ xterm_keys_find(const char *buf, size_t len, size_t *size, key_code *key)
 		if (matched == -1)
 			continue;
 		if (matched == 0)
-			*key = entry->key | modifiers;
+			*key = (entry->key|modifiers|KEYC_XTERM);
 		return (matched);
 	}
 	return (-1);
@@ -227,8 +227,16 @@ xterm_keys_lookup(key_code key)
 	if (modifiers == 1)
 		return (NULL);
 
+	/*
+	 * If this has the escape modifier, but was not originally an xterm
+	 * key, it may be a genuine escape + key. So don't pass it through as
+	 * an xterm key or programs like vi may be confused.
+	 */
+	if ((key & (KEYC_ESCAPE|KEYC_XTERM)) == KEYC_ESCAPE)
+		return (NULL);
+
 	/* Otherwise, find the key in the table. */
-	key &= ~(KEYC_SHIFT|KEYC_ESCAPE|KEYC_CTRL);
+	key &= KEYC_MASK_KEY;
 	for (i = 0; i < nitems(xterm_keys_table); i++) {
 		entry = &xterm_keys_table[i];
 		if (key == entry->key)