patch 8.2.3818: cannot filter or map characters in a stringv8.2.3818

Problem:    Cannot filter or map characters in a string.
Solution:   Make filter() and map() work on a string. (Naruhiko Nishino,
            closes #9327)

5 files changed, 169 insertions, 27 deletions

diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt
index 004f9d4bbe..21cf3f6c17 100644
--- a/runtime/doc/eval.txt
+++ b/runtime/doc/eval.txt
@@ -2628,7 +2628,8 @@ extendnew({expr1}, {expr2} [, {expr3}])
 feedkeys({string} [, {mode}])	Number	add key sequence to typeahead buffer
 filereadable({file})		Number	|TRUE| if {file} is a readable file
 filewritable({file})		Number	|TRUE| if {file} is a writable file
-filter({expr1}, {expr2})	List/Dict  remove items from {expr1} where
+filter({expr1}, {expr2})	List/Dict/Blob/String
+					remove items from {expr1} where
 					{expr2} is 0
 finddir({name} [, {path} [, {count}]])
 				String	find directory {name} in {path}
@@ -2786,14 +2787,16 @@ localtime()			Number	current time
 log({expr})			Float	natural logarithm (base e) of {expr}
 log10({expr})			Float	logarithm of Float {expr} to base 10
 luaeval({expr} [, {expr}])	any	evaluate |Lua| expression
-map({expr1}, {expr2})		List/Dict  change each item in {expr1} to {expr}
+map({expr1}, {expr2})		List/Dict/Blob/String
+					change each item in {expr1} to {expr2}
 maparg({name} [, {mode} [, {abbr} [, {dict}]]])
 				String or Dict
 					rhs of mapping {name} in mode {mode}
 mapcheck({name} [, {mode} [, {abbr}]])
 				String	check for mappings matching {name}
-mapnew({expr1}, {expr2})	List/Dict  like |map()| but creates a new List
-					   or Dictionary
+mapnew({expr1}, {expr2})	List/Dict/Blob/String
+					like |map()| but creates a new List or
+					Dictionary
 mapset({mode}, {abbr}, {dict})	none	restore mapping from |maparg()| result
 match({expr}, {pat} [, {start} [, {count}]])
 				Number	position where {pat} matches in {expr}
@@ -4878,10 +4881,11 @@ filewritable({file})					*filewritable()*
 
 
 filter({expr1}, {expr2})				*filter()*
-		{expr1} must be a |List|, |Blob| or |Dictionary|.
+		{expr1} must be a |List|, |String|, |Blob| or |Dictionary|.
 		For each item in {expr1} evaluate {expr2} and when the result
-		is zero remove the item from the |List| or |Dictionary|. For a
-		|Blob| each byte is removed.
+		is zero or false remove the item from the |List| or
+		|Dictionary|.  Similarly for each byte in a |Blob| and each
+		charactor in a |String|.
 
 		{expr2} must be a |string| or |Funcref|.
 
@@ -4916,14 +4920,16 @@ filter({expr1}, {expr2})				*filter()*
 <		If you do not use "val" you can leave it out: >
 			call filter(myList, {idx -> idx % 2 == 1})
 <
-		The operation is done in-place.  If you want a |List| or
-		|Dictionary| to remain unmodified make a copy first: >
+		For a |List| and a |Dictionary| the operation is done
+		in-place.  If you want it to remain unmodified make a copy
+		first: >
 			:let l = filter(copy(mylist), 'v:val =~ "KEEP"')
 
-<		Returns {expr1}, the |List| , |Blob| or |Dictionary| that was
-		filtered.  When an error is encountered while evaluating
-		{expr2} no further items in {expr1} are processed.  When
-		{expr2} is a Funcref errors inside a function are ignored,
+<		Returns {expr1}, the |List| or |Dictionary| that was filtered,
+		or a new |Blob| or |String|. 
+		When an error is encountered while evaluating {expr2} no
+		further items in {expr1} are processed.
+		When {expr2} is a Funcref errors inside a function are ignored,
 		unless it was defined with the "abort" flag.
 
 		Can also be used as a |method|: >
@@ -7566,16 +7572,19 @@ luaeval({expr} [, {expr}])					*luaeval()*
 <		{only available when compiled with the |+lua| feature}
 
 map({expr1}, {expr2})					*map()*
-		{expr1} must be a |List|, |Blob| or |Dictionary|.
-		Replace each item in {expr1} with the result of evaluating
-		{expr2}.  For a |Blob| each byte is replaced.
+		{expr1} must be a |List|, |String|, |Blob| or |Dictionary|.
+		When {expr1} is a |List|| or |Dictionary|, replace each
+		item in {expr1} with the result of evaluating {expr2}.
+		For a |Blob| each byte is replaced.
+		For a |String|, each character, including composing
+		characters, is replaced.
 		If the item type changes you may want to use |mapnew()| to
 		create a new List or Dictionary.  This is required when using
 		Vim9 script.
 
-		{expr2} must be a |string| or |Funcref|.
+		{expr2} must be a |String| or |Funcref|.
 
-		If {expr2} is a |string|, inside {expr2} |v:val| has the value
+		If {expr2} is a |String|, inside {expr2} |v:val| has the value
 		of the current item.  For a |Dictionary| |v:key| has the key
 		of the current item and for a |List| |v:key| has the index of
 		the current item.  For a |Blob| |v:key| has the index of the
@@ -7605,14 +7614,15 @@ map({expr1}, {expr2})					*map()*
 <		If you do not use "key" you can use a short name: >
 			call map(myDict, {_, val -> 'item: ' . val})
 <
-		The operation is done in-place.  If you want a |List| or
-		|Dictionary| to remain unmodified make a copy first: >
+		The operation is done in-place for a |List| and |Dictionary|.
+		If you want it to remain unmodified make a copy first: >
 			:let tlist = map(copy(mylist), ' v:val . "\t"')
 
-<		Returns {expr1}, the |List|, |Blob| or |Dictionary| that was
-		filtered.  When an error is encountered while evaluating
-		{expr2} no further items in {expr1} are processed.  When
-		{expr2} is a Funcref errors inside a function are ignored,
+<		Returns {expr1}, the |List| or |Dictionary| that was filtered,
+		or a new |Blob| or |String|.
+		When an error is encountered while evaluating {expr2} no
+		further items in {expr1} are processed.
+		When {expr2} is a Funcref errors inside a function are ignored,
 		unless it was defined with the "abort" flag.
 
 		Can also be used as a |method|: >
diff --git a/src/errors.h b/src/errors.h
index 452ff451b3..156e1949fb 100644
--- a/src/errors.h
+++ b/src/errors.h
@@ -758,3 +758,5 @@ EXTERN char e_closure_called_from_invalid_context[]
 	INIT(= N_("E1248: Closure called from invalid context"));
 EXTERN char e_highlight_group_name_too_long[]
 	INIT(= N_("E1249: Highlight group name too long"));
+EXTERN char e_argument_of_str_must_be_list_string_dictionary_or_blob[]
+	INIT(= N_("E1250: Argument of %s must be a List, String, Dictionary or Blob"));
diff --git a/src/list.c b/src/list.c
index ca80e41232..ff22de6fd9 100644
--- a/src/list.c
+++ b/src/list.c
@@ -2329,9 +2329,15 @@ filter_map(typval_T *argvars, typval_T *rettv, filtermap_T filtermap)
 			    && value_check_lock(d->dv_lock, arg_errmsg, TRUE)))
 	    goto theend;
     }
+    else if (argvars[0].v_type == VAR_STRING)
+    {
+	rettv->v_type = VAR_STRING;
+	rettv->vval.v_string = NULL;
+    }
     else
     {
-	semsg(_(e_listdictblobarg), func_name);
+	semsg(_(e_argument_of_str_must_be_list_string_dictionary_or_blob),
+								    func_name);
 	goto theend;
     }
 
@@ -2480,6 +2486,66 @@ filter_map(typval_T *argvars, typval_T *rettv, filtermap_T filtermap)
 		++idx;
 	    }
 	}
+	else if (argvars[0].v_type == VAR_STRING)
+	{
+	    char_u	*p;
+	    typval_T	tv;
+	    garray_T	ga;
+	    char_u	buf[MB_MAXBYTES + 1];
+	    int		len;
+
+	    // set_vim_var_nr() doesn't set the type
+	    set_vim_var_type(VV_KEY, VAR_NUMBER);
+
+	    ga_init2(&ga, (int)sizeof(char), 80);
+	    for (p = tv_get_string(&argvars[0]); *p != NUL; p += len)
+	    {
+	        typval_T newtv;
+
+		if (has_mbyte)
+		    len = mb_ptr2len(p);
+		else
+		    len = 1;
+
+		STRNCPY(buf, p, len);
+		buf[len] = NUL;
+
+		tv.v_type = VAR_STRING;
+		tv.vval.v_string = vim_strsave(buf);
+
+		set_vim_var_nr(VV_KEY, idx);
+		if (filter_map_one(&tv, expr, filtermap, &newtv, &rem) == FAIL
+								   || did_emsg)
+		    break;
+		if (did_emsg)
+		{
+		    clear_tv(&newtv);
+		    clear_tv(&tv);
+		    break;
+		}
+		else if (filtermap != FILTERMAP_FILTER)
+		{
+		    if (newtv.v_type != VAR_STRING)
+		    {
+			clear_tv(&newtv);
+			clear_tv(&tv);
+			emsg(_(e_stringreq));
+			break;
+		    }
+		    else
+			ga_concat(&ga, newtv.vval.v_string);
+		}
+		else if (!rem)
+		    ga_concat(&ga, tv.vval.v_string);
+
+		clear_tv(&newtv);
+		clear_tv(&tv);
+
+	        ++idx;
+	    }
+	    ga_append(&ga, NUL);
+	    rettv->vval.v_string = ga.ga_data;
+	}
 	else // argvars[0].v_type == VAR_LIST
 	{
 	    int	    prev_lock = l->lv_lock;
diff --git a/src/testdir/test_filter_map.vim b/src/testdir/test_filter_map.vim
index a9ff35e1b5..947e6a225a 100644
--- a/src/testdir/test_filter_map.vim
+++ b/src/testdir/test_filter_map.vim
@@ -93,8 +93,6 @@ endfunc
 func Test_map_filter_fails()
   call assert_fails('call map([1], "42 +")', 'E15:')
   call assert_fails('call filter([1], "42 +")', 'E15:')
-  call assert_fails("let l = map('abc', '\"> \" . v:val')", 'E896:')
-  call assert_fails("let l = filter('abc', '\"> \" . v:val')", 'E896:')
   call assert_fails("let l = filter([1, 2, 3], '{}')", 'E728:')
   call assert_fails("let l = filter({'k' : 10}, '{}')", 'E728:')
   call assert_fails("let l = filter([1, 2], {})", 'E731:')
@@ -145,4 +143,68 @@ func Test_mapnew_blob()
   call assert_equal(0z129956, bout)
 endfunc
 
+func Test_filter_map_string()
+  let s = "abc"
+
+  " filter()
+  call filter(s, '"b" != v:val')
+  call assert_equal(s, s)
+  call assert_equal('ac', filter('abc', '"b" != v:val'))
+  call assert_equal('あいうえお', filter('あxいxうxえxお', '"x" != v:val'))
+  call assert_equal('あa😊💕💕b💕', filter('あxax😊x💕💕b💕x', '"x" != v:val'))
+  call assert_equal('xxxx', filter('あxax😊x💕💕b💕x', '"x" == v:val'))
+  let t = "%),:;>?]}’”†‡…‰,‱‼⁇⁈⁉℃℉,、。〉》」,』】〕〗〙〛,！），．：,；？,］｝"
+  let u = "%):;>?]}’”†‡…‰‱‼⁇⁈⁉℃℉、。〉》」』】〕〗〙〛！），．：；？］｝"
+  call assert_equal(u, filter(t, '"," != v:val'))
+  call assert_equal('', filter('abc', '0'))
+  call assert_equal('ac', filter('abc', { i, x -> "b" != x }))
+  call assert_equal('あいうえお', filter('あxいxうxえxお', { i, x -> "x" != x }))
+  call assert_equal('', filter('abc', { i, x -> v:false }))
+
+  " map()
+  call map(s, 'nr2char(char2nr(v:val) + 2)')
+  call assert_equal(s, s)
+  call assert_equal('cde', map('abc', 'nr2char(char2nr(v:val) + 2)'))
+  call assert_equal('[あ][i][う][え][お]', map('あiうえお', '"[" .. v:val .. "]"'))
+  call assert_equal('[あ][a][😊][,][‱][‼][⁇][⁈][⁉][💕][b][💕][c][💕]', map('あa😊,‱‼⁇⁈⁉💕b💕c💕', '"[" .. v:val .. "]"'))
+  call assert_equal('', map('abc', '""'))
+  call assert_equal('cde', map('abc', { i, x -> nr2char(char2nr(x) + 2) }))
+  call assert_equal('[あ][i][う][え][お]', map('あiうえお', { i, x -> '[' .. x .. ']' }))
+  call assert_equal('', map('abc', { i, x -> '' }))
+
+  " mapnew()
+  call mapnew(s, 'nr2char(char2nr(v:val) + 2)')
+  call assert_equal(s, s)
+  call assert_equal('cde', mapnew('abc', 'nr2char(char2nr(v:val) + 2)'))
+  call assert_equal('[あ][i][う][え][お]', mapnew('あiうえお', '"[" .. v:val .. "]"'))
+  call assert_equal('[あ][a][😊][,][‱][‼][⁇][⁈][⁉][💕][b][💕][c][💕]', mapnew('あa😊,‱‼⁇⁈⁉💕b💕c💕', '"[" .. v:val .. "]"'))
+  call assert_equal('', mapnew('abc', '""'))
+  call assert_equal('cde', mapnew('abc', { i, x -> nr2char(char2nr(x) + 2) }))
+  call assert_equal('[あ][i][う][え][お]', mapnew('あiうえお', { i, x -> '[' .. x .. ']' }))
+  call assert_equal('', mapnew('abc', { i, x -> '' }))
+
+  " map() and filter()
+  call assert_equal('[あ][⁈][a][😊][⁉][💕][💕][b][💕]', map(filter('あx⁈ax😊x⁉💕💕b💕x', '"x" != v:val'), '"[" .. v:val .. "]"'))
+
+  " patterns-composing(\Z)
+  call assert_equal('ॠॠ', filter('ऊॠॡ,ऊॠॡ', {i,x -> x =~ '\Z' .. nr2char(0x0960) }))
+  call assert_equal('àà', filter('càt,càt', {i,x -> x =~ '\Za' }))
+  call assert_equal('ÅÅ', filter('Åström,Åström', {i,x -> x =~ '\Z' .. nr2char(0xc5) }))
+  call assert_equal('öö', filter('Åström,Åström', {i,x -> x =~ '\Z' .. nr2char(0xf6) }))
+  call assert_equal('ऊ@ॡ', map('ऊॠॡ', {i,x -> x =~ '\Z' .. nr2char(0x0960) ? '@' : x }))
+  call assert_equal('c@t', map('càt', {i,x -> x =~ '\Za' ? '@' : x }))
+  call assert_equal('@ström', map('Åström', {i,x -> x =~ '\Z' .. nr2char(0xc5) ? '@' : x }))
+  call assert_equal('Åstr@m', map('Åström', {i,x -> x =~ '\Z' .. nr2char(0xf6) ? '@' : x }))
+
+  " patterns-composing(\%C)
+  call assert_equal('ॠॠ', filter('ऊॠॡ,ऊॠॡ', {i,x -> x =~ nr2char(0x0960) .. '\%C' }))
+  call assert_equal('àà', filter('càt,càt', {i,x -> x =~ 'a' .. '\%C' }))
+  call assert_equal('ÅÅ', filter('Åström,Åström', {i,x -> x =~ nr2char(0xc5) .. '\%C' }))
+  call assert_equal('öö', filter('Åström,Åström', {i,x -> x =~ nr2char(0xf6) .. '\%C' }))
+  call assert_equal('ऊ@ॡ', map('ऊॠॡ', {i,x -> x =~ nr2char(0x0960) .. '\%C' ? '@' : x }))
+  call assert_equal('c@t', map('càt', {i,x -> x =~ 'a' .. '\%C' ? '@' : x }))
+  call assert_equal('@ström', map('Åström', {i,x -> x =~ nr2char(0xc5) .. '\%C' ? '@' : x }))
+  call assert_equal('Åstr@m', map('Åström', {i,x -> x =~ nr2char(0xf6) .. '\%C' ? '@' : x }))
+endfunc
+
 " vim: shiftwidth=2 sts=2 expandtab
diff --git a/src/version.c b/src/version.c
index e9836ef412..d48206f18e 100644
--- a/src/version.c
+++ b/src/version.c
@@ -750,6 +750,8 @@ static char *(features[]) =
 static int included_patches[] =
 {   /* Add new patch number below this line */
 /**/
+    3818,
+/**/
     3817,
 /**/
     3816,