patch 9.0.0985: when using kitty keyboard protocol function keys may not workv9.0.0985

Problem:    When using kitty keyboard protocol function keys may not work.
            (Kovid Goyal)
Solution:   Recognize CSI ending in [ABCDEFHPQRS] also when the termcap
            entries are not specified. (closes #11648)

4 files changed, 249 insertions, 68 deletions

diff --git a/src/term.c b/src/term.c
index f00aac9674..19d4fa1892 100644
--- a/src/term.c
+++ b/src/term.c
@@ -2048,9 +2048,9 @@ set_termname(char_u *term)
 		// things for this terminal
 		if (!gui.in_use)
 		    return FAIL;
-#ifdef HAVE_TGETENT
+# ifdef HAVE_TGETENT
 		break;		// don't try using external termcap
-#endif
+# endif
 	    }
 #endif // FEAT_GUI
 	}
@@ -5088,6 +5088,42 @@ add_key_to_buf(int key, char_u *buf)
 }
 
 /*
+ * Shared between handle_key_with_modifier() and handle_csi_function_key().
+ */
+    static int
+put_key_modifiers_in_typebuf(
+	int	key_arg,
+	int	modifiers_arg,
+	int	csi_len,
+	int	offset,
+	char_u	*buf,
+	int	bufsize,
+	int	*buflen)
+{
+    int key = key_arg;
+    int modifiers = modifiers_arg;
+
+    // Some keys need adjustment when the Ctrl modifier is used.
+    key = may_adjust_key_for_ctrl(modifiers, key);
+
+    // May remove the shift modifier if it's already included in the key.
+    modifiers = may_remove_shift_modifier(modifiers, key);
+
+    // Produce modifiers with K_SPECIAL KS_MODIFIER {mod}
+    char_u string[MAX_KEY_CODE_LEN + 1];
+    int new_slen = modifiers2keycode(modifiers, &key, string);
+
+    // Add the bytes for the key.
+    new_slen += add_key_to_buf(key, string + new_slen);
+
+    string[new_slen] = NUL;
+    if (put_string_in_typebuf(offset, csi_len, string, new_slen,
+						 buf, bufsize, buflen) == FAIL)
+	return -1;
+    return new_slen - csi_len + offset;
+}
+
+/*
  * Handle a sequence with key and modifier, one of:
  *	{lead}27;{modifier};{key}~
  *	{lead}{key};{modifier}u
@@ -5103,10 +5139,6 @@ handle_key_with_modifier(
 	int	bufsize,
 	int	*buflen)
 {
-    int	    key;
-    int	    modifiers;
-    char_u  string[MAX_KEY_CODE_LEN + 1];
-
     // Only set seenModifyOtherKeys for the "{lead}27;" code to avoid setting
     // it for terminals using the kitty keyboard protocol.  Xterm sends
     // the form ending in "u" when the formatOtherKeys resource is set.  We do
@@ -5123,31 +5155,15 @@ handle_key_with_modifier(
 		|| kitty_protocol_state == KKPS_OFF
 		|| kitty_protocol_state == KKPS_AFTER_T_KE)
 	    && term_props[TPR_KITTY].tpr_status != TPR_YES)
+    {
+	ch_log(NULL, "setting seenModifyOtherKeys to TRUE");
 	seenModifyOtherKeys = TRUE;
+    }
 
-    if (trail == 'u')
-	key = arg[0];
-    else
-	key = arg[2];
-
-    modifiers = decode_modifiers(arg[1]);
-
-    // Some keys need adjustment when the Ctrl modifier is used.
-    key = may_adjust_key_for_ctrl(modifiers, key);
-
-    // May remove the shift modifier if it's already included in the key.
-    modifiers = may_remove_shift_modifier(modifiers, key);
-
-    // insert modifiers with KS_MODIFIER
-    int new_slen = modifiers2keycode(modifiers, &key, string);
-
-    // add the bytes for the key
-    new_slen += add_key_to_buf(key, string + new_slen);
-
-    if (put_string_in_typebuf(offset, csi_len, string, new_slen,
-						 buf, bufsize, buflen) == FAIL)
-	return -1;
-    return new_slen - csi_len + offset;
+    int key = trail == 'u' ? arg[0] : arg[2];
+    int modifiers = decode_modifiers(arg[1]);
+    return put_key_modifiers_in_typebuf(key, modifiers,
+					csi_len, offset, buf, bufsize, buflen);
 }
 
 /*
@@ -5186,6 +5202,51 @@ handle_key_without_modifier(
 }
 
 /*
+ * CSI function key without or with modifiers:
+ *	{lead}[ABCDEFHPQRS]
+ *	{lead}1;{modifier}[ABCDEFHPQRS]
+ * Returns zero when nog recognized, a positive number when recognized.
+ */
+    static int
+handle_csi_function_key(
+	int	argc,
+	int	*arg,
+	int	trail,
+	int	csi_len,
+	char_u	*key_name,
+	int	offset,
+	char_u  *buf,
+	int	bufsize,
+	int	*buflen)
+{
+    key_name[0] = 'k';
+    switch (trail)
+    {
+	case 'A': key_name[1] = 'u'; break;  // K_UP
+	case 'B': key_name[1] = 'd'; break;  // K_DOWN
+	case 'C': key_name[1] = 'r'; break;  // K_RIGHT
+	case 'D': key_name[1] = 'l'; break;  // K_LEFT
+
+	// case 'E': keypad BEGIN - not supported
+	case 'F': key_name[0] = '@'; key_name[1] = '7'; break;  // K_END
+	case 'H': key_name[1] = 'h'; break;  // K_HOME
+
+	case 'P': key_name[1] = '1'; break;  // K_F1
+	case 'Q': key_name[1] = '2'; break;  // K_F2
+	case 'R': key_name[1] = '3'; break;  // K_F3
+	case 'S': key_name[1] = '4'; break;  // K_F4
+
+	default: return 0;  // not recognized
+    }
+
+    int key = TERMCAP2KEY(key_name[0], key_name[1]);
+    int modifiers = argc == 2 ? decode_modifiers(arg[1]) : 0;
+    put_key_modifiers_in_typebuf(key, modifiers,
+					csi_len, offset, buf, bufsize, buflen);
+    return csi_len;
+}
+
+/*
  * Handle a CSI escape sequence.
  * - Xterm version string.
  *
@@ -5197,10 +5258,15 @@ handle_key_without_modifier(
  *
  * - window position reply: {lead}3;{x};{y}t
  *
- * - key with modifiers when modifyOtherKeys is enabled:
+ * - key with modifiers when modifyOtherKeys is enabled or the Kitty keyboard
+ *   protocol is used:
  *	    {lead}27;{modifier};{key}~
  *	    {lead}{key};{modifier}u
  *
+ * - function key with or without modifiers:
+ *	{lead}[ABCDEFHPQRS]
+ *	{lead}1;{modifier}[ABCDEFHPQRS]
+ *
  * Return 0 for no match, -1 for partial match, > 0 for full match.
  */
     static int
@@ -5218,7 +5284,7 @@ handle_csi(
     int		first = -1;  // optional char right after {lead}
     int		trail;	     // char that ends CSI sequence
     int		arg[3] = {-1, -1, -1};	// argument numbers
-    int		argc;			// number of arguments
+    int		argc = 0;		// number of arguments
     char_u	*ap = argp;
     int		csi_len;
 
@@ -5226,42 +5292,54 @@ handle_csi(
     if (!VIM_ISDIGIT(*ap))
 	first = *ap++;
 
-    // Find up to three argument numbers.
-    for (argc = 0; argc < 3; )
+    if (ASCII_ISUPPER(first))
     {
-	if (ap >= tp + len)
-	    return -1;
-	if (*ap == ';')
-	    arg[argc++] = -1;  // omitted number
-	else if (VIM_ISDIGIT(*ap))
+	// If "first" is in [ABCDEFHPQRS] then it is actually the "trail" and
+	// no argument follows.
+	trail = first;
+	first = -1;
+	--ap;
+    }
+    else
+    {
+	// Find up to three argument numbers.
+	for (argc = 0; argc < 3; )
 	{
-	    arg[argc] = 0;
-	    for (;;)
+	    if (ap >= tp + len)
+		return -1;
+	    if (*ap == ';')
+		arg[argc++] = -1;  // omitted number
+	    else if (VIM_ISDIGIT(*ap))
 	    {
-		if (ap >= tp + len)
-		    return -1;
-		if (!VIM_ISDIGIT(*ap))
-		    break;
-		arg[argc] = arg[argc] * 10 + (*ap - '0');
-		++ap;
+		arg[argc] = 0;
+		for (;;)
+		{
+		    if (ap >= tp + len)
+			return -1;
+		    if (!VIM_ISDIGIT(*ap))
+			break;
+		    arg[argc] = arg[argc] * 10 + (*ap - '0');
+		    ++ap;
+		}
+		++argc;
 	    }
-	    ++argc;
+	    if (*ap == ';')
+		++ap;
+	    else
+		break;
 	}
-	if (*ap == ';')
+
+	// mrxvt has been reported to have "+" in the version. Assume
+	// the escape sequence ends with a letter or one of "{|}~".
+	while (ap < tp + len
+		&& !(*ap >= '{' && *ap <= '~')
+		&& !ASCII_ISALPHA(*ap))
 	    ++ap;
-	else
-	    break;
+	if (ap >= tp + len)
+	    return -1;
+	trail = *ap;
     }
 
-    // mrxvt has been reported to have "+" in the version. Assume
-    // the escape sequence ends with a letter or one of "{|}~".
-    while (ap < tp + len
-	    && !(*ap >= '{' && *ap <= '~')
-	    && !ASCII_ISALPHA(*ap))
-	++ap;
-    if (ap >= tp + len)
-	return -1;
-    trail = *ap;
     csi_len = (int)(ap - tp) + 1;
 
     // Response to XTQMODKEYS: "CSI > 4 ; Pv m" where Pv indicates the
@@ -5276,11 +5354,22 @@ handle_csi(
 	*slen = csi_len;
     }
 
-    // Cursor position report: Eat it when there are 2 arguments
-    // and it ends in 'R'. Also when u7_status is not "sent", it
-    // may be from a previous Vim that just exited.  But not for
-    // <S-F3>, it sends something similar, check for row and column
-    // to make sense.
+    // Function key starting with CSI:
+    //	{lead}[ABCDEFHPQRS]
+    //	{lead}1;{modifier}[ABCDEFHPQRS]
+    else if (first == -1 && ASCII_ISUPPER(trail)
+	    && (argc == 0 || (argc == 2 && arg[0] == 1)))
+    {
+	int res = handle_csi_function_key(argc, arg, trail,
+			      csi_len, key_name, offset, buf, bufsize, buflen);
+	return res <= 0 ? res : len + res;
+    }
+
+    // Cursor position report: {lead}{row};{col}R
+    // Eat it when there are 2 arguments and it ends in 'R'.
+    // Also when u7_status is not "sent", it may be from a previous Vim that
+    // just exited.  But not for <S-F3>, it sends something similar, check for
+    // row and column to make sense.
     else if (first == -1 && argc == 2 && trail == 'R')
     {
 	handle_u7_response(arg, tp, csi_len);
@@ -5346,6 +5435,7 @@ handle_csi(
 
 	    // Reset seenModifyOtherKeys just in case some key combination has
 	    // been seen that set it before we get the status response.
+	    ch_log(NULL, "setting seenModifyOtherKeys to FALSE");
 	    seenModifyOtherKeys = FALSE;
 	}
 
@@ -5916,7 +6006,9 @@ check_termcode(
 
 	    /*
 	     * Check for responses from the terminal starting with {lead}:
-	     * "<Esc>[" or CSI followed by [0-9>?]
+	     * "<Esc>[" or CSI followed by [0-9>?].
+	     * Also for function keys without a modifier:
+	     * "<Esc>[" or CSI followed by [ABCDEFHPQRS].
 	     *
 	     * - Xterm version string: {lead}>{x};{vers};{y}c
 	     *   Also eat other possible responses to t_RV, rxvt returns
@@ -5935,8 +6027,9 @@ check_termcode(
 	     *	    {lead}{key};{modifier}u
 	     */
 	    if (((tp[0] == ESC && len >= 3 && tp[1] == '[')
-			    || (tp[0] == CSI && len >= 2))
-		    && (VIM_ISDIGIT(*argp) || *argp == '>' || *argp == '?'))
+			|| (tp[0] == CSI && len >= 2))
+		    && vim_strchr((char_u *)"0123456789>?ABCDEFHPQRS",
+								*argp) != NULL)
 	    {
 		int resp = handle_csi(tp, len, argp, offset, buf,
 					     bufsize, buflen, key_name, &slen);
@@ -6424,7 +6517,7 @@ replace_termcodes(
 	    slen = trans_special(&src, result + dlen, FSK_KEYCODE
 			  | ((flags & REPTERM_NO_SIMPLIFY) ? 0 : FSK_SIMPLIFY),
 							   TRUE, did_simplify);
-	    if (slen)
+	    if (slen > 0)
 	    {
 		dlen += slen;
 		continue;
diff --git a/src/testdir/test_termcodes.vim b/src/testdir/test_termcodes.vim
index eb30230140..dd5280fba0 100644
--- a/src/testdir/test_termcodes.vim
+++ b/src/testdir/test_termcodes.vim
@@ -2483,6 +2483,80 @@ func Test_mapping_works_with_unknown_modifiers()
   set timeoutlen&
 endfunc
 
+func RunTest_mapping_funckey(map, func, key, code)
+  call setline(1, '')
+  exe 'inoremap ' .. a:map .. ' xyz'
+  call feedkeys('a' .. a:func(a:key, a:code) .. "\<Esc>", 'Lx!')
+  call assert_equal("xyz", getline(1), 'mapping ' .. a:map)
+  exe 'iunmap ' .. a:map
+endfunc
+
+func Test_mapping_kitty_function_keys()
+  new
+  set timeoutlen=10
+
+  " Function keys made with CSI and ending in [ABCDEFHPQRS].
+  " 'E' is keypad BEGIN, not supported
+  let maps = [
+        \    ['<Up>', 'A', 0],
+        \    ['<S-Up>', 'A', 2],
+        \    ['<C-Up>', 'A', 5],
+        \    ['<C-S-Up>', 'A', 6],
+        \
+        \    ['<Down>', 'B', 0],
+        \    ['<S-Down>', 'B', 2],
+        \    ['<C-Down>', 'B', 5],
+        \    ['<C-S-Down>', 'B', 6],
+        \
+        \    ['<Right>', 'C', 0],
+        \    ['<S-Right>', 'C', 2],
+        \    ['<C-Right>', 'C', 5],
+        \    ['<C-S-Right>', 'C', 6],
+        \
+        \    ['<Left>', 'D', 0],
+        \    ['<S-Left>', 'D', 2],
+        \    ['<C-Left>', 'D', 5],
+        \    ['<C-S-Left>', 'D', 6],
+        \
+        \    ['<End>', 'F', 0],
+        \    ['<S-End>', 'F', 2],
+        \    ['<C-End>', 'F', 5],
+        \    ['<C-S-End>', 'F', 6],
+        \
+        \    ['<Home>', 'H', 0],
+        \    ['<S-Home>', 'H', 2],
+        \    ['<C-Home>', 'H', 5],
+        \    ['<C-S-Home>', 'H', 6],
+        \
+        \    ['<F1>', 'P', 0],
+        \    ['<S-F1>', 'P', 2],
+        \    ['<C-F1>', 'P', 5],
+        \    ['<C-S-F1>', 'P', 6],
+        \
+        \    ['<F2>', 'Q', 0],
+        \    ['<S-F2>', 'Q', 2],
+        \    ['<C-F2>', 'Q', 5],
+        \    ['<C-S-F2>', 'Q', 6],
+        \
+        \    ['<F3>', 'R', 0],
+        \    ['<S-F3>', 'R', 2],
+        \    ['<C-F3>', 'R', 5],
+        \    ['<C-S-F3>', 'R', 6],
+        \
+        \    ['<F4>', 'S', 0],
+        \    ['<S-F4>', 'S', 2],
+        \    ['<C-F4>', 'S', 5],
+        \    ['<C-S-F4>', 'S', 6],
+        \ ]
+
+  for map in maps
+    call RunTest_mapping_funckey(map[0], function('GetEscCodeFunckey'), map[1], map[2])
+  endfor
+
+  bwipe!
+  set timeoutlen&
+endfunc
+
 func Test_insert_literal()
   set timeoutlen=10
 
diff --git a/src/testdir/view_util.vim b/src/testdir/view_util.vim
index a0c1781dd4..d9e740da3f 100644
--- a/src/testdir/view_util.vim
+++ b/src/testdir/view_util.vim
@@ -95,6 +95,18 @@ func GetEscCodeCSIu(key, modifier)
   return "\<Esc>[" .. key .. ';' .. mod .. 'u'
 endfunc
 
+" Return the kitty keyboard protocol encoding for a function key:
+" CSI {key}
+" CSS 1;{modifier} {key}
+func GetEscCodeFunckey(key, modifier)
+  if a:modifier == 0
+    return "\<Esc>[" .. a:key
+  endif
+
+  let mod = printf("%d", a:modifier)
+  return "\<Esc>[1;".. mod .. a:key
+endfunc
+
 " Return the kitty keyboard protocol encoding for "key" without a modifier.
 " Used for the Escape key.
 func GetEscCodeCSIuWithoutModifier(key)
diff --git a/src/version.c b/src/version.c
index 6f303d1dd0..c3fc096483 100644
--- a/src/version.c
+++ b/src/version.c
@@ -696,6 +696,8 @@ static char *(features[]) =
 static int included_patches[] =
 {   /* Add new patch number below this line */
 /**/
+    985,
+/**/
     984,
 /**/
     983,