patch 8.2.4770: cannot easily mix expression and heredocv8.2.4770

Problem:    Cannot easily mix expression and heredoc.
Solution:   Support  in heredoc. (Yegappan Lakshmanan, closes #10138)

6 files changed, 314 insertions, 24 deletions

diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt
index 6e7090acdf..95d14ef5b2 100644
--- a/runtime/doc/eval.txt
+++ b/runtime/doc/eval.txt
@@ -3224,14 +3224,32 @@ declarations and assignments do not use a command.  |vim9-declaration|
 
 						*:let=<<* *:let-heredoc*
 					*E990* *E991* *E172* *E221* *E1145*
-:let {var-name} =<< [trim] {endmarker}
+:let {var-name} =<< [trim] [eval] {endmarker}
 text...
 text...
 {endmarker}
 			Set internal variable {var-name} to a |List|
 			containing the lines of text bounded by the string
-			{endmarker}. The lines of text is used as a
-			|literal-string|.
+			{endmarker}.
+
+			If "eval" is not specified, then each line of text is
+			used as a |literal-string|.  If "eval" is specified,
+			then any Vim expression in the form ``={expr}`` is
+			evaluated and the result replaces the expression.
+			Example where $HOME is expanded: >
+				let lines =<< trim eval END
+				  some text
+				  See the file `=$HOME`/.vimrc
+				  more text
+				END
+<			There can be multiple Vim expressions in a single line
+			but an expression cannot span multiple lines.  If any
+			expression evaluation fails, then the assignment fails.
+			once the "`=" has been found {expr} and a backtick
+			must follow.  {expr} cannot be empty.
+			Currenty, in a compiled function {expr} is evaluated
+			when compiling the function, THIS WILL CHANGE.
+
 			{endmarker} must not contain white space.
 			{endmarker} cannot start with a lower case character.
 			The last line should end only with the {endmarker}
@@ -3281,6 +3299,13 @@ text...
 					1 2 3 4
 					5 6 7 8
 				DATA
+
+				let code =<< trim eval CODE
+				   let v = `=10 + 20`
+				   let h = "`=$HOME`"
+				   let s = "`=Str1()` abc `=Str2()`"
+				   let n = `=MyFunc(3, 4)`
+				CODE
 <
 								*E121*
 :let {var-name}	..	List the value of variable {var-name}.  Multiple
diff --git a/src/evalvars.c b/src/evalvars.c
index 8f8df0cf9d..5f04406265 100644
--- a/src/evalvars.c
+++ b/src/evalvars.c
@@ -603,6 +603,66 @@ list_script_vars(int *first)
 }
 
 /*
+ * Evaluate all the Vim expressions (`=expr`) in string "str" and return the
+ * resulting string.  The caller must free the returned string.
+ */
+    static char_u *
+eval_all_expr_in_str(char_u *str)
+{
+    garray_T	ga;
+    char_u	*s;
+    char_u	*p;
+    char_u	save_c;
+    char_u	*exprval;
+    int		status;
+
+    ga_init2(&ga, 1, 80);
+    p = str;
+
+    // Look for `=expr`, evaluate the expression and replace `=expr` with the
+    // result.
+    while (*p != NUL)
+    {
+	s = p;
+	while (*p != NUL && (*p != '`' || p[1] != '='))
+	    p++;
+	ga_concat_len(&ga, s, p - s);
+	if (*p == NUL)
+	    break;		// no backtick expression found
+
+	s = p;
+	p += 2;		// skip `=
+
+	status = *p == NUL ? OK : skip_expr(&p, NULL);
+	if (status == FAIL || *p != '`')
+	{
+	    // invalid expression or missing ending backtick
+	    if (status != FAIL)
+		emsg(_(e_missing_backtick));
+	    vim_free(ga.ga_data);
+	    return NULL;
+	}
+	s += 2;		// skip `=
+	save_c = *p;
+	*p = NUL;
+	exprval = eval_to_string(s, TRUE);
+	*p = save_c;
+	p++;
+	if (exprval == NULL)
+	{
+	    // expression evaluation failed
+	    vim_free(ga.ga_data);
+	    return NULL;
+	}
+	ga_concat(&ga, exprval);
+	vim_free(exprval);
+    }
+    ga_append(&ga, NUL);
+
+    return ga.ga_data;
+}
+
+/*
  * Get a list of lines from a HERE document. The here document is a list of
  * lines surrounded by a marker.
  *	cmd << {marker}
@@ -619,7 +679,7 @@ list_script_vars(int *first)
  * tcl, mzscheme), script_get is set to TRUE. In this case, if the marker is
  * missing, then '.' is accepted as a marker.
  *
- * Returns a List with {lines} or NULL.
+ * Returns a List with {lines} or NULL on failure.
  */
     list_T *
 heredoc_get(exarg_T *eap, char_u *cmd, int script_get)
@@ -628,11 +688,14 @@ heredoc_get(exarg_T *eap, char_u *cmd, int script_get)
     char_u	*marker;
     list_T	*l;
     char_u	*p;
+    char_u	*str;
     int		marker_indent_len = 0;
     int		text_indent_len = 0;
     char_u	*text_indent = NULL;
     char_u	dot[] = ".";
     int		comment_char = in_vim9script() ? '#' : '"';
+    int		evalstr = FALSE;
+    int		eval_failed = FALSE;
 
     if (eap->getline == NULL)
     {
@@ -642,21 +705,36 @@ heredoc_get(exarg_T *eap, char_u *cmd, int script_get)
 
     // Check for the optional 'trim' word before the marker
     cmd = skipwhite(cmd);
-    if (STRNCMP(cmd, "trim", 4) == 0 && (cmd[4] == NUL || VIM_ISWHITE(cmd[4])))
+
+    while (TRUE)
     {
-	cmd = skipwhite(cmd + 4);
+	if (STRNCMP(cmd, "trim", 4) == 0
+		&& (cmd[4] == NUL || VIM_ISWHITE(cmd[4])))
+	{
+	    cmd = skipwhite(cmd + 4);
+
+	    // Trim the indentation from all the lines in the here document.
+	    // The amount of indentation trimmed is the same as the indentation
+	    // of the first line after the :let command line.  To find the end
+	    // marker the indent of the :let command line is trimmed.
+	    p = *eap->cmdlinep;
+	    while (VIM_ISWHITE(*p))
+	    {
+		p++;
+		marker_indent_len++;
+	    }
+	    text_indent_len = -1;
 
-	// Trim the indentation from all the lines in the here document.
-	// The amount of indentation trimmed is the same as the indentation of
-	// the first line after the :let command line.  To find the end marker
-	// the indent of the :let command line is trimmed.
-	p = *eap->cmdlinep;
-	while (VIM_ISWHITE(*p))
+	    continue;
+	}
+	if (STRNCMP(cmd, "eval", 4) == 0
+		&& (cmd[4] == NUL || VIM_ISWHITE(cmd[4])))
 	{
-	    p++;
-	    marker_indent_len++;
+	    cmd = skipwhite(cmd + 4);
+	    evalstr = TRUE;
+	    continue;
 	}
-	text_indent_len = -1;
+	break;
     }
 
     // The marker is the next word.
@@ -716,6 +794,14 @@ heredoc_get(exarg_T *eap, char_u *cmd, int script_get)
 	    break;
 	}
 
+	// If expression evaluation failed in the heredoc, then skip till the
+	// end marker.
+	if (eval_failed)
+	{
+	    vim_free(theline);
+	    continue;
+	}
+
 	if (text_indent_len == -1 && *theline != NUL)
 	{
 	    // set the text indent from the first line.
@@ -734,12 +820,33 @@ heredoc_get(exarg_T *eap, char_u *cmd, int script_get)
 		if (theline[ti] != text_indent[ti])
 		    break;
 
-	if (list_append_string(l, theline + ti, -1) == FAIL)
+	str = theline + ti;
+	if (evalstr)
+	{
+	    str = eval_all_expr_in_str(str);
+	    if (str == NULL)
+	    {
+		// expression evaluation failed
+		vim_free(theline);
+		eval_failed = TRUE;
+		continue;
+	    }
+	    vim_free(theline);
+	    theline = str;
+	}
+
+	if (list_append_string(l, str, -1) == FAIL)
 	    break;
 	vim_free(theline);
     }
     vim_free(text_indent);
 
+    if (eval_failed)
+    {
+	// expression evaluation in the heredoc failed
+	list_free(l);
+	return NULL;
+    }
     return l;
 }
 
diff --git a/src/testdir/test_let.vim b/src/testdir/test_let.vim
index 5f8879b7e6..2606fa276b 100644
--- a/src/testdir/test_let.vim
+++ b/src/testdir/test_let.vim
@@ -1,5 +1,7 @@
 " Tests for the :let command.
 
+import './vim9.vim' as v9
+
 func Test_let()
   " Test to not autoload when assigning.  It causes internal error.
   set runtimepath+=./sautest
@@ -379,7 +381,8 @@ END
   call assert_equal(['Text', 'with', 'indent'], text)
 endfunc
 
-" Test for the setting a variable using the heredoc syntax
+" Test for the setting a variable using the heredoc syntax.
+" Keep near the end, this messes up highlighting.
 func Test_let_heredoc()
   let var1 =<< END
 Some sample text
@@ -495,4 +498,96 @@ END
   call assert_equal(['     x', '     \y', '     z'], [a, b, c])
 endfunc
 
+" Test for evaluating Vim expressions in a heredoc using `=expr`
+" Keep near the end, this messes up highlighting.
+func Test_let_heredoc_eval()
+  let str = ''
+  let code =<< trim eval END
+    let a = `=5 + 10`
+    let b = `=min([10, 6])` + `=max([4, 6])`
+    `=str`
+    let c = "abc`=str`d"
+  END
+  call assert_equal(['let a = 15', 'let b = 6 + 6', '', 'let c = "abcd"'], code)
+  let $TESTVAR = "Hello"
+  let code =<< eval trim END
+    let s = "`=$TESTVAR`"
+  END
+  call assert_equal(['let s = "Hello"'], code)
+  let code =<< eval END
+    let s = "`=$TESTVAR`"
+END
+  call assert_equal(['    let s = "Hello"'], code)
+  let a = 10
+  let data =<< eval END
+`=a`
+END
+  call assert_equal(['10'], data)
+  let x = 'X'
+  let code =<< eval trim END
+    let a = `abc`
+    let b = `=x`
+    let c = `
+  END
+  call assert_equal(['let a = `abc`', 'let b = X', 'let c = `'], code)
+  let code = 'xxx'
+  let code =<< eval trim END
+    let n = `=5 +
+    6`
+  END
+  call assert_equal('xxx', code)
+  let code =<< eval trim END
+     let n = `=min([1, 2]` + `=max([3, 4])`
+  END
+  call assert_equal('xxx', code)
+
+  let lines =<< trim LINES
+      let text =<< eval trim END
+        let b = `=
+      END
+  LINES
+  call v9.CheckScriptFailure(lines, 'E1083:')
+
+  let lines =<< trim LINES
+      let text =<< eval trim END
+        let b = `=abc
+      END
+  LINES
+  call v9.CheckScriptFailure(lines, 'E1083:')
+
+  let lines =<< trim LINES
+      let text =<< eval trim END
+        let b = `=`
+      END
+  LINES
+  call v9.CheckScriptFailure(lines, 'E15:')
+
+  " Test for sourcing a script containing a heredoc with invalid expression.
+  " Variable assignment should fail, if expression evaluation fails
+  new
+  let g:Xvar = 'test'
+  let g:b = 10
+  let lines =<< trim END
+    let Xvar =<< eval CODE
+    let a = 1
+    let b = `=5+`
+    let c = 2
+    CODE
+    let g:Count += 1
+  END
+  call setline(1, lines)
+  let g:Count = 0
+  call assert_fails('source', 'E15:')
+  call assert_equal(1, g:Count)
+  call setline(3, 'let b = `=abc`')
+  call assert_fails('source', 'E121:')
+  call assert_equal(2, g:Count)
+  call setline(3, 'let b = `=abc` + `=min([9, 4])` + 2')
+  call assert_fails('source', 'E121:')
+  call assert_equal(3, g:Count)
+  call assert_equal('test', g:Xvar)
+  call assert_equal(10, g:b)
+  bw!
+endfunc
+
 " vim: shiftwidth=2 sts=2 expandtab
diff --git a/src/testdir/test_vim9_assign.vim b/src/testdir/test_vim9_assign.vim
index 7e382c22b2..02274b3c55 100644
--- a/src/testdir/test_vim9_assign.vim
+++ b/src/testdir/test_vim9_assign.vim
@@ -740,7 +740,7 @@ def Test_init_in_for_loop()
 enddef
 
 def Test_extend_list()
-  # using uninitilaized list assigns empty list
+  # using uninitialized list assigns empty list
   var lines =<< trim END
       var l1: list<number>
       var l2 = l1
@@ -758,7 +758,7 @@ def Test_extend_list()
   END
   v9.CheckDefAndScriptSuccess(lines)
 
-  # appending to uninitialzed list from a function works
+  # appending to uninitialized list from a function works
   lines =<< trim END
       vim9script
       var list: list<string>
@@ -2637,6 +2637,56 @@ def Test_using_s_var_in_function()
   v9.CheckScriptSuccess(lines)
 enddef
 
+let g:someVar = 'X'
+
+" Test for heredoc with Vim expressions.
+" This messes up highlighting, keep it near the end.
+def Test_heredoc_expr()
+  var code =<< trim eval END
+    var a = `=5 + 10`
+    var b = `=min([10, 6])` + `=max([4, 6])`
+  END
+  assert_equal(['var a = 15', 'var b = 6 + 6'], code)
+
+  code =<< eval trim END
+    var s = "`=$SOME_ENV_VAR`"
+  END
+  assert_equal(['var s = "somemore"'], code)
+
+  code =<< eval END
+    var s = "`=$SOME_ENV_VAR`"
+END
+  assert_equal(['    var s = "somemore"'], code)
+
+  code =<< eval trim END
+    let a = `abc`
+    let b = `=g:someVar`
+    let c = `
+  END
+  assert_equal(['let a = `abc`', 'let b = X', 'let c = `'], code)
+
+  var lines =<< trim LINES
+      var text =<< eval trim END
+        let b = `=
+      END
+  LINES
+  v9.CheckDefAndScriptFailure(lines, 'E1083:')
+
+  lines =<< trim LINES
+      var text =<< eval trim END
+        let b = `=abc
+      END
+  LINES
+  v9.CheckDefAndScriptFailure(lines, 'E1083:')
+
+  lines =<< trim LINES
+      var text =<< eval trim END
+        let b = `=`
+      END
+  LINES
+  v9.CheckDefAndScriptFailure(lines, 'E15:')
+enddef
+
 
 
 " vim: ts=8 sw=2 sts=2 expandtab tw=80 fdm=marker
diff --git a/src/userfunc.c b/src/userfunc.c
index 7ad338856f..155c4ff1c5 100644
--- a/src/userfunc.c
+++ b/src/userfunc.c
@@ -1077,12 +1077,23 @@ get_function_body(
 			    || checkforcmd(&p, "const", 5))))
 		{
 		    p = skipwhite(arg + 3);
-		    if (STRNCMP(p, "trim", 4) == 0)
+		    while (TRUE)
 		    {
-			// Ignore leading white space.
-			p = skipwhite(p + 4);
-			heredoc_trimmed = vim_strnsave(theline,
-						 skipwhite(theline) - theline);
+			if (STRNCMP(p, "trim", 4) == 0)
+			{
+			    // Ignore leading white space.
+			    p = skipwhite(p + 4);
+			    heredoc_trimmed = vim_strnsave(theline,
+				    skipwhite(theline) - theline);
+			    continue;
+			}
+			if (STRNCMP(p, "eval", 4) == 0)
+			{
+			    // Ignore leading white space.
+			    p = skipwhite(p + 4);
+			    continue;
+			}
+			break;
 		    }
 		    skip_until = vim_strnsave(p, skiptowhite(p) - p);
 		    getline_options = GETLINE_NONE;
diff --git a/src/version.c b/src/version.c
index 81e3824357..90b0b12386 100644
--- a/src/version.c
+++ b/src/version.c
@@ -747,6 +747,8 @@ static char *(features[]) =
 static int included_patches[] =
 {   /* Add new patch number below this line */
 /**/
+    4770,
+/**/
     4769,
 /**/
     4768,