patch 9.1.0009: Cannot easily get the list of matchesv9.1.0009

Problem:  Cannot easily get the list of matches
Solution: Add the matchstrlist() and matchbufline() Vim script
          functions (Yegappan Lakshmanan)

closes: #13766

Signed-off-by: Yegappan Lakshmanan <yegappan@yahoo.com>
Signed-off-by: Christian Brabandt <cb@256bit.org>

4 files changed, 455 insertions, 1 deletions

diff --git a/src/errors.h b/src/errors.h
index 828e8de249..dfb6ce644d 100644
--- a/src/errors.h
+++ b/src/errors.h
@@ -1749,9 +1749,11 @@ EXTERN char e_recursive_loop_loading_syncolor_vim[]
 #endif
 EXTERN char e_buffer_nr_invalid_buffer_number[]
 	INIT(= N_("E680: <buffer=%d>: invalid buffer number"));
-#ifdef FEAT_QUICKFIX
+#if defined(FEAT_QUICKFIX) || defined(FEAT_EVAL)
 EXTERN char e_buffer_is_not_loaded[]
 	INIT(= N_("E681: Buffer is not loaded"));
+#endif
+#ifdef FEAT_QUICKFIX
 EXTERN char e_invalid_search_pattern_or_delimiter[]
 	INIT(= N_("E682: Invalid search pattern or delimiter"));
 EXTERN char e_file_name_missing_or_invalid_pattern[]
diff --git a/src/evalfunc.c b/src/evalfunc.c
index 513ddccdbe..e37b3a412a 100644
--- a/src/evalfunc.c
+++ b/src/evalfunc.c
@@ -100,9 +100,11 @@ static void f_line2byte(typval_T *argvars, typval_T *rettv);
 static void f_luaeval(typval_T *argvars, typval_T *rettv);
 #endif
 static void f_match(typval_T *argvars, typval_T *rettv);
+static void f_matchbufline(typval_T *argvars, typval_T *rettv);
 static void f_matchend(typval_T *argvars, typval_T *rettv);
 static void f_matchlist(typval_T *argvars, typval_T *rettv);
 static void f_matchstr(typval_T *argvars, typval_T *rettv);
+static void f_matchstrlist(typval_T *argvars, typval_T *rettv);
 static void f_matchstrpos(typval_T *argvars, typval_T *rettv);
 static void f_max(typval_T *argvars, typval_T *rettv);
 static void f_min(typval_T *argvars, typval_T *rettv);
@@ -1176,6 +1178,8 @@ static argcheck_T arg2_map[] = {arg_list_or_dict_or_blob_or_string_mod, arg_map_
 static argcheck_T arg2_mapnew[] = {arg_list_or_dict_or_blob_or_string, arg_any};
 static argcheck_T arg25_matchadd[] = {arg_string, arg_string, arg_number, arg_number, arg_dict_any};
 static argcheck_T arg25_matchaddpos[] = {arg_string, arg_list_any, arg_number, arg_number, arg_dict_any};
+static argcheck_T arg23_matchstrlist[] = {arg_list_string, arg_string, arg_dict_any};
+static argcheck_T arg45_matchbufline[] = {arg_buffer, arg_string, arg_lnum, arg_lnum, arg_dict_any};
 static argcheck_T arg119_printf[] = {arg_string_or_nr, arg_any, arg_any, arg_any, arg_any, arg_any, arg_any, arg_any, arg_any, arg_any, arg_any, arg_any, arg_any, arg_any, arg_any, arg_any, arg_any, arg_any, arg_any};
 static argcheck_T arg23_reduce[] = {arg_string_list_or_blob, arg_any, arg_any};
 static argcheck_T arg24_remote_expr[] = {arg_string, arg_string, arg_string, arg_number};
@@ -2285,6 +2289,8 @@ static funcentry_T global_functions[] =
 			ret_number,	    f_matchaddpos},
     {"matcharg",	1, 1, FEARG_1,	    arg1_number,
 			ret_list_string,    f_matcharg},
+    {"matchbufline",	4, 5, FEARG_1,	    arg45_matchbufline,
+			ret_list_any,	    f_matchbufline},
     {"matchdelete",	1, 2, FEARG_1,	    arg2_number,
 			ret_number_bool,    f_matchdelete},
     {"matchend",	2, 4, FEARG_1,	    arg24_match_func,
@@ -2297,6 +2303,8 @@ static funcentry_T global_functions[] =
 			ret_list_string,    f_matchlist},
     {"matchstr",	2, 4, FEARG_1,	    arg24_match_func,
 			ret_string,	    f_matchstr},
+    {"matchstrlist",	2, 3, FEARG_1,	    arg23_matchstrlist,
+			ret_list_any,	    f_matchstrlist},
     {"matchstrpos",	2, 4, FEARG_1,	    arg24_match_func,
 			ret_list_any,	    f_matchstrpos},
     {"max",		1, 1, FEARG_1,	    arg1_list_or_dict,
@@ -8024,6 +8032,183 @@ theend:
 }
 
 /*
+ * Return all the matches in string "str" for pattern "rmp".
+ * The matches are returned in the List "mlist".
+ * If "submatches" is TRUE, then submatch information is also returned.
+ * "matchbuf" is TRUE when called for matchbufline().
+ */
+    static int
+get_matches_in_str(
+    char_u	*str,
+    regmatch_T	*rmp,
+    list_T	*mlist,
+    int		idx,
+    int		submatches,
+    int		matchbuf)
+{
+    long	len = (long)STRLEN(str);
+    int		match = 0;
+    colnr_T	startidx = 0;
+
+    for (;;)
+    {
+	match = vim_regexec_nl(rmp, str, startidx);
+	if (!match)
+	    break;
+
+	dict_T *d = dict_alloc();
+	if (d == NULL)
+	    return FAIL;
+	if (list_append_dict(mlist, d) == FAIL)
+	    return FAIL;;
+
+	if (dict_add_number(d, matchbuf ? "lnum" : "idx", idx) == FAIL)
+	    return FAIL;
+
+	if (dict_add_number(d, "byteidx",
+		    (colnr_T)(rmp->startp[0] - str)) == FAIL)
+	    return FAIL;
+
+	if (dict_add_string_len(d, "text", rmp->startp[0],
+		    (int)(rmp->endp[0] - rmp->startp[0])) == FAIL)
+	    return FAIL;
+
+	if (submatches)
+	{
+	    list_T *sml = list_alloc();
+	    if (sml == NULL)
+		return FAIL;
+
+	    if (dict_add_list(d, "submatches", sml) == FAIL)
+		return FAIL;
+
+	    // return a list with the submatches
+	    for (int i = 1; i < NSUBEXP; ++i)
+	    {
+		if (rmp->endp[i] == NULL)
+		{
+		    if (list_append_string(sml, (char_u *)"", 0) == FAIL)
+			return FAIL;
+		}
+		else if (list_append_string(sml, rmp->startp[i],
+			    (int)(rmp->endp[i] - rmp->startp[i])) == FAIL)
+		    return FAIL;
+	    }
+	}
+	startidx = (colnr_T)(rmp->endp[0] - str);
+	if (startidx >= (colnr_T)len || str + startidx <= rmp->startp[0])
+	    break;
+    }
+
+    return OK;
+}
+
+/*
+ * "matchbufline()" function
+ */
+    static void
+f_matchbufline(typval_T *argvars, typval_T *rettv)
+{
+    list_T	*retlist = NULL;
+    char_u	*save_cpo;
+    char_u	patbuf[NUMBUFLEN];
+    regmatch_T	regmatch;
+
+    rettv->vval.v_number = -1;
+    if (rettv_list_alloc(rettv) != OK)
+	return;
+    retlist = rettv->vval.v_list;
+
+    if (check_for_buffer_arg(argvars, 0) == FAIL
+	    || check_for_string_arg(argvars, 1) == FAIL
+	    || check_for_lnum_arg(argvars, 2) == FAIL
+	    || check_for_lnum_arg(argvars, 3) == FAIL
+	    || check_for_opt_dict_arg(argvars, 4) == FAIL)
+	return;
+
+    int prev_did_emsg = did_emsg;
+    buf_T *buf = tv_get_buf(&argvars[0], FALSE);
+    if (buf == NULL)
+    {
+	if (did_emsg == prev_did_emsg)
+	    semsg(_(e_invalid_buffer_name_str), tv_get_string(&argvars[0]));
+	return;
+    }
+    if (buf->b_ml.ml_mfp == NULL)
+    {
+	emsg(_(e_buffer_is_not_loaded));
+	return;
+    }
+
+    char_u *pat = tv_get_string_buf(&argvars[1], patbuf);
+
+    int		did_emsg_before = did_emsg;
+    linenr_T slnum = tv_get_lnum_buf(&argvars[2], buf);
+    if (did_emsg > did_emsg_before)
+	return;
+    if (slnum < 1)
+    {
+	semsg(_(e_invalid_value_for_argument_str), "lnum");
+	return;
+    }
+
+    linenr_T elnum = tv_get_lnum_buf(&argvars[3], buf);
+    if (did_emsg > did_emsg_before)
+	return;
+    if (elnum < 1 || elnum < slnum)
+    {
+	semsg(_(e_invalid_value_for_argument_str), "end_lnum");
+	return;
+    }
+
+    if (elnum > buf->b_ml.ml_line_count)
+	elnum = buf->b_ml.ml_line_count;
+
+    int		submatches = FALSE;
+    if (argvars[4].v_type != VAR_UNKNOWN)
+    {
+	dict_T *d = argvars[4].vval.v_dict;
+	if (d != NULL)
+	{
+	    dictitem_T *di = dict_find(d, (char_u *)"submatches", -1);
+	    if (di != NULL)
+	    {
+		if (di->di_tv.v_type != VAR_BOOL)
+		{
+		    semsg(_(e_invalid_value_for_argument_str), "submatches");
+		    return;
+		}
+		submatches = tv_get_bool(&di->di_tv);
+	    }
+	}
+    }
+
+    // Make 'cpoptions' empty, the 'l' flag should not be used here.
+    save_cpo = p_cpo;
+    p_cpo = empty_option;
+
+    regmatch.regprog = vim_regcomp(pat, RE_MAGIC + RE_STRING);
+    if (regmatch.regprog == NULL)
+	goto theend;
+    regmatch.rm_ic = p_ic;
+
+    while (slnum <= elnum)
+    {
+	char_u *str = ml_get_buf(buf, slnum, FALSE);
+	if (get_matches_in_str(str, &regmatch, retlist, slnum, submatches,
+								TRUE) == FAIL)
+	    goto cleanup;
+	slnum++;
+    }
+
+cleanup:
+    vim_regfree(regmatch.regprog);
+
+theend:
+    p_cpo = save_cpo;
+}
+
+/*
  * "match()" function
  */
     static void
@@ -8060,6 +8245,85 @@ f_matchstr(typval_T *argvars, typval_T *rettv)
 }
 
 /*
+ * "matchstrlist()" function
+ */
+    static void
+f_matchstrlist(typval_T *argvars, typval_T *rettv)
+{
+    list_T	*retlist = NULL;
+    char_u	*save_cpo;
+    list_T	*l = NULL;
+    listitem_T	*li = NULL;
+    char_u	patbuf[NUMBUFLEN];
+    regmatch_T	regmatch;
+
+    rettv->vval.v_number = -1;
+    if (rettv_list_alloc(rettv) != OK)
+	return;
+    retlist = rettv->vval.v_list;
+
+    if (check_for_list_arg(argvars, 0) == FAIL
+	    || check_for_string_arg(argvars, 1) == FAIL
+	    || check_for_opt_dict_arg(argvars, 2) == FAIL)
+	return;
+
+    if ((l = argvars[0].vval.v_list) == NULL)
+	return;
+
+    char_u *pat = tv_get_string_buf_chk(&argvars[1], patbuf);
+    if (pat == NULL)
+	return;
+
+    // Make 'cpoptions' empty, the 'l' flag should not be used here.
+    save_cpo = p_cpo;
+    p_cpo = empty_option;
+
+    regmatch.regprog = vim_regcomp(pat, RE_MAGIC + RE_STRING);
+    if (regmatch.regprog == NULL)
+	goto theend;
+    regmatch.rm_ic = p_ic;
+
+    int		submatches = FALSE;
+    if (argvars[2].v_type != VAR_UNKNOWN)
+    {
+	dict_T *d = argvars[2].vval.v_dict;
+	if (d != NULL)
+	{
+	    dictitem_T *di = dict_find(d, (char_u *)"submatches", -1);
+	    if (di != NULL)
+	    {
+		if (di->di_tv.v_type != VAR_BOOL)
+		{
+		    semsg(_(e_invalid_value_for_argument_str), "submatches");
+		    goto cleanup;
+		}
+		submatches = tv_get_bool(&di->di_tv);
+	    }
+	}
+    }
+
+    int idx = 0;
+    CHECK_LIST_MATERIALIZE(l);
+    FOR_ALL_LIST_ITEMS(l, li)
+    {
+	if (li->li_tv.v_type == VAR_STRING && li->li_tv.vval.v_string != NULL)
+	{
+	    char_u *str = li->li_tv.vval.v_string;
+	    if (get_matches_in_str(str, &regmatch, retlist, idx, submatches,
+								FALSE) == FAIL)
+		goto cleanup;
+	}
+	idx++;
+    }
+
+cleanup:
+    vim_regfree(regmatch.regprog);
+
+theend:
+    p_cpo = save_cpo;
+}
+
+/*
  * "matchstrpos()" function
  */
     static void
diff --git a/src/testdir/test_functions.vim b/src/testdir/test_functions.vim
index 7bfe7fa750..85ccc78b8f 100644
--- a/src/testdir/test_functions.vim
+++ b/src/testdir/test_functions.vim
@@ -1172,6 +1172,192 @@ func Test_matchstrpos()
   call assert_equal(['', -1, -1], matchstrpos(test_null_list(), '\a'))
 endfunc
 
+" Test for matchstrlist()
+func Test_matchstrlist()
+  let lines =<< trim END
+    #" Basic match
+    call assert_equal([{'idx': 0, 'byteidx': 1, 'text': 'bout'},
+          \ {'idx': 1, 'byteidx': 1, 'text': 'bove'}],
+          \ matchstrlist(['about', 'above'], 'bo.*'))
+    #" no match
+    call assert_equal([], matchstrlist(['about', 'above'], 'xy.*'))
+    #" empty string
+    call assert_equal([], matchstrlist([''], '.'))
+    #" empty pattern
+    call assert_equal([{'idx': 0, 'byteidx': 0, 'text': ''}], matchstrlist(['abc'], ''))
+    #" method call
+    call assert_equal([{'idx': 0, 'byteidx': 2, 'text': 'it'}], ['editor']->matchstrlist('ed\zsit\zeor'))
+    #" single character matches
+    call assert_equal([{'idx': 0, 'byteidx': 5, 'text': 'r'}],
+          \ ['editor']->matchstrlist('r'))
+    call assert_equal([{'idx': 0, 'byteidx': 0, 'text': 'a'}], ['a']->matchstrlist('a'))
+    call assert_equal([{'idx': 0, 'byteidx': 0, 'text': ''}],
+          \ matchstrlist(['foobar'], '\zs'))
+    #" string with tabs
+    call assert_equal([{'idx': 0, 'byteidx': 1, 'text': 'foo'}],
+          \ matchstrlist(["\tfoobar"], 'foo'))
+    #" string with multibyte characters
+    call assert_equal([{'idx': 0, 'byteidx': 2, 'text': '😊😊'}],
+          \ matchstrlist(["\t\t😊😊"], '\k\+'))
+
+    #" null string
+    call assert_equal([], matchstrlist(test_null_list(), 'abc'))
+    call assert_equal([], matchstrlist([test_null_string()], 'abc'))
+    call assert_equal([{'idx': 0, 'byteidx': 0, 'text': ''}],
+          \ matchstrlist(['abc'], test_null_string()))
+
+    #" sub matches
+    call assert_equal([{'idx': 0, 'byteidx': 0, 'text': 'acd', 'submatches': ['a', '', 'c', 'd', '', '', '', '', '']}], matchstrlist(['acd'], '\(a\)\?\(b\)\?\(c\)\?\(.*\)', {'submatches': v:true}))
+
+    #" null dict argument
+    call assert_equal([{'idx': 0, 'byteidx': 0, 'text': 'vim'}],
+          \ matchstrlist(['vim'], '\w\+', test_null_dict()))
+
+    #" Error cases
+    call assert_fails("echo matchstrlist('abc', 'a')", 'E1211: List required for argument 1')
+    call assert_fails("echo matchstrlist(['abc'], {})", 'E1174: String required for argument 2')
+    call assert_fails("echo matchstrlist(['abc'], '.', [])", 'E1206: Dictionary required for argument 3')
+    call assert_fails("echo matchstrlist(['abc'], 'a', {'submatches': []})", 'E475: Invalid value for argument submatches')
+    call assert_fails("echo matchstrlist(['abc'], '\\@=')", 'E866: (NFA regexp) Misplaced @')
+  END
+  call v9.CheckLegacyAndVim9Success(lines)
+
+  let lines =<< trim END
+    vim9script
+    # non string items
+    matchstrlist([0z10, {'a': 'x'}], 'x')
+  END
+  call v9.CheckSourceSuccess(lines)
+
+  let lines =<< trim END
+    vim9script
+    def Foo()
+      # non string items
+      assert_equal([], matchstrlist([0z10, {'a': 'x'}], 'x'))
+    enddef
+    Foo()
+  END
+  call v9.CheckSourceFailure(lines, 'E1013: Argument 1: type mismatch, expected list<string> but got list<any>', 2)
+endfunc
+
+" Test for matchbufline()
+func Test_matchbufline()
+  let lines =<< trim END
+    #" Basic match
+    new
+    call setline(1, ['about', 'above', 'below'])
+    VAR bnr = bufnr()
+    wincmd w
+    call assert_equal([{'lnum': 1, 'byteidx': 1, 'text': 'bout'},
+          \ {'lnum': 2, 'byteidx': 1, 'text': 'bove'}],
+          \ matchbufline(bnr, 'bo.*', 1, '$'))
+    #" multiple matches in a line
+    call setbufline(bnr, 1, ['about about', 'above above', 'below'])
+    call assert_equal([{'lnum': 1, 'byteidx': 1, 'text': 'bout'},
+          \ {'lnum': 1, 'byteidx': 7, 'text': 'bout'},
+          \ {'lnum': 2, 'byteidx': 1, 'text': 'bove'},
+          \ {'lnum': 2, 'byteidx': 7, 'text': 'bove'}],
+          \ matchbufline(bnr, 'bo\k\+', 1, '$'))
+    #" no match
+    call assert_equal([], matchbufline(bnr, 'xy.*', 1, '$'))
+    #" match on a particular line
+    call assert_equal([{'lnum': 2, 'byteidx': 7, 'text': 'bove'}],
+          \ matchbufline(bnr, 'bo\k\+$', 2, 2))
+    #" match on a particular line
+    call assert_equal([], matchbufline(bnr, 'bo.*', 3, 3))
+    #" empty string
+    call deletebufline(bnr, 1, '$')
+    call assert_equal([], matchbufline(bnr, '.', 1, '$'))
+    #" empty pattern
+    call setbufline(bnr, 1, 'abc')
+    call assert_equal([{'lnum': 1, 'byteidx': 0, 'text': ''}],
+          \ matchbufline(bnr, '', 1, '$'))
+    #" method call
+    call setbufline(bnr, 1, 'editor')
+    call assert_equal([{'lnum': 1, 'byteidx': 2, 'text': 'it'}],
+          \ bnr->matchbufline('ed\zsit\zeor', 1, 1))
+    #" single character matches
+    call assert_equal([{'lnum': 1, 'byteidx': 5, 'text': 'r'}],
+          \ matchbufline(bnr, 'r', 1, '$'))
+    call setbufline(bnr, 1, 'a')
+    call assert_equal([{'lnum': 1, 'byteidx': 0, 'text': 'a'}],
+          \ matchbufline(bnr, 'a', 1, '$'))
+    #" zero-width match
+    call assert_equal([{'lnum': 1, 'byteidx': 0, 'text': ''}],
+          \ matchbufline(bnr, '\zs', 1, '$'))
+    #" string with tabs
+    call setbufline(bnr, 1, "\tfoobar")
+    call assert_equal([{'lnum': 1, 'byteidx': 1, 'text': 'foo'}],
+          \ matchbufline(bnr, 'foo', 1, '$'))
+    #" string with multibyte characters
+    call setbufline(bnr, 1, "\t\t😊😊")
+    call assert_equal([{'lnum': 1, 'byteidx': 2, 'text': '😊😊'}],
+          \ matchbufline(bnr, '\k\+', 1, '$'))
+    #" empty buffer
+    call deletebufline(bnr, 1, '$')
+    call assert_equal([], matchbufline(bnr, 'abc', 1, '$'))
+
+    #" Non existing buffer
+    call setbufline(bnr, 1, 'abc')
+    call assert_fails("echo matchbufline(5000, 'abc', 1, 1)", 'E158: Invalid buffer name: 5000')
+    #" null string
+    call assert_equal([{'lnum': 1, 'byteidx': 0, 'text': ''}],
+          \ matchbufline(bnr, test_null_string(), 1, 1))
+    #" invalid starting line number
+    call assert_equal([], matchbufline(bnr, 'abc', 100, 100))
+    #" ending line number greater than the last line
+    call assert_equal([{'lnum': 1, 'byteidx': 0, 'text': 'abc'}],
+          \ matchbufline(bnr, 'abc', 1, 100))
+    #" ending line number greater than the starting line number
+    call setbufline(bnr, 1, ['one', 'two'])
+    call assert_fails($"echo matchbufline({bnr}, 'abc', 2, 1)", 'E475: Invalid value for argument end_lnum')
+
+    #" sub matches
+    call deletebufline(bnr, 1, '$')
+    call setbufline(bnr, 1, 'acd')
+    call assert_equal([{'lnum': 1, 'byteidx': 0, 'text': 'acd', 'submatches': ['a', '', 'c', 'd', '', '', '', '', '']}],
+          \ matchbufline(bnr, '\(a\)\?\(b\)\?\(c\)\?\(.*\)', 1, '$', {'submatches': v:true}))
+
+    #" null dict argument
+    call assert_equal([{'lnum': 1, 'byteidx': 0, 'text': 'acd'}],
+          \ matchbufline(bnr, '\w\+', '$', '$', test_null_dict()))
+
+    #" Error cases
+    call assert_fails("echo matchbufline([1], 'abc', 1, 1)", 'E1220: String or Number required for argument 1')
+    call assert_fails("echo matchbufline(1, {}, 1, 1)", 'E1174: String required for argument 2')
+    call assert_fails("echo matchbufline(1, 'abc', {}, 1)", 'E1220: String or Number required for argument 3')
+    call assert_fails("echo matchbufline(1, 'abc', 1, {})", 'E1220: String or Number required for argument 4')
+    call assert_fails($"echo matchbufline({bnr}, 'abc', -1, '$')", 'E475: Invalid value for argument lnum')
+    call assert_fails($"echo matchbufline({bnr}, 'abc', 1, -1)", 'E475: Invalid value for argument end_lnum')
+    call assert_fails($"echo matchbufline({bnr}, '\\@=', 1, 1)", 'E866: (NFA regexp) Misplaced @')
+    call assert_fails($"echo matchbufline({bnr}, 'abc', 1, 1, {{'submatches': []}})", 'E475: Invalid value for argument submatches')
+    :%bdelete!
+    call assert_fails($"echo matchbufline({bnr}, 'abc', 1, '$'))", 'E681: Buffer is not loaded')
+  END
+  call v9.CheckLegacyAndVim9Success(lines)
+
+  call assert_fails($"echo matchbufline('', 'abc', 'abc', 1)", 'E475: Invalid value for argument lnum')
+  call assert_fails($"echo matchbufline('', 'abc', 1, 'abc')", 'E475: Invalid value for argument end_lnum')
+
+  let lines =<< trim END
+    vim9script
+    def Foo()
+      echo matchbufline('', 'abc', 'abc', 1)
+    enddef
+    Foo()
+  END
+  call v9.CheckSourceFailure(lines, 'E1030: Using a String as a Number: "abc"', 1)
+
+  let lines =<< trim END
+    vim9script
+    def Foo()
+      echo matchbufline('', 'abc', 1, 'abc')
+    enddef
+    Foo()
+  END
+  call v9.CheckSourceFailure(lines, 'E1030: Using a String as a Number: "abc"', 1)
+endfunc
+
 func Test_nextnonblank_prevnonblank()
   new
 insert
diff --git a/src/version.c b/src/version.c
index 3839d9c359..2e7f2871ea 100644
--- a/src/version.c
+++ b/src/version.c
@@ -705,6 +705,8 @@ static char *(features[]) =
 static int included_patches[] =
 {   /* Add new patch number below this line */
 /**/
+    9,
+/**/
     8,
 /**/
     7,