Fix --with-nth performance; use simpler regular expression

Related #317

5 files changed, 33 insertions, 15 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f7ec3838..41a50a9d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@ CHANGELOG
       version will treat the given delimiter pattern as a plain string instead
       of a regular expression unless it contains special characters and is
       a valid regular expression.
+    - Simpler regular expression for delimiter for better performance
 
 0.10.2
 ------
diff --git a/src/options.go b/src/options.go
index 35fa8ecf..70900660 100644
--- a/src/options.go
+++ b/src/options.go
@@ -1,7 +1,6 @@
 package fzf
 
 import (
-	"fmt"
 	"io/ioutil"
 	"os"
 	"regexp"
@@ -284,10 +283,6 @@ func delimiterRegexp(str string) Delimiter {
 	}
 
 	// 3. Pattern as regular expression. Slow.
-	rx, e = regexp.Compile(fmt.Sprintf("(?:.*?%s)|(?:.+?$)", str))
-	if e != nil {
-		errorExit("invalid regular expression: " + e.Error())
-	}
 	return Delimiter{regex: rx}
 }
 
diff --git a/src/options_test.go b/src/options_test.go
index e9884f2c..1f96f785 100644
--- a/src/options_test.go
+++ b/src/options_test.go
@@ -2,7 +2,6 @@ package fzf
 
 import (
 	"fmt"
-	"strings"
 	"testing"
 
 	"github.com/junegunn/fzf/src/curses"
@@ -21,7 +20,7 @@ func TestDelimiterRegex(t *testing.T) {
 	}
 	// Valid regex
 	delim = delimiterRegexp("[0-9]")
-	if strings.Index(delim.regex.String(), "[0-9]") < 0 || delim.str != nil {
+	if delim.regex.String() != "[0-9]" || delim.str != nil {
 		t.Error(delim)
 	}
 	// Tab character
@@ -43,20 +42,25 @@ func TestDelimiterRegex(t *testing.T) {
 
 func TestDelimiterRegexString(t *testing.T) {
 	delim := delimiterRegexp("*")
-	tokens := strings.Split("-*--*---**---", *delim.str)
-	if delim.regex != nil || tokens[0] != "-" || tokens[1] != "--" ||
-		tokens[2] != "---" || tokens[3] != "" || tokens[4] != "---" {
+	tokens := Tokenize([]rune("-*--*---**---"), delim)
+	if delim.regex != nil ||
+		string(tokens[0].text) != "-*" ||
+		string(tokens[1].text) != "--*" ||
+		string(tokens[2].text) != "---*" ||
+		string(tokens[3].text) != "*" ||
+		string(tokens[4].text) != "---" {
 		t.Errorf("%s %s %d", delim, tokens, len(tokens))
 	}
 }
 
 func TestDelimiterRegexRegex(t *testing.T) {
 	delim := delimiterRegexp("--\\*")
-	rx := delim.regex
-	tokens := rx.FindAllString("-*--*---**---", -1)
+	tokens := Tokenize([]rune("-*--*---**---"), delim)
 	if delim.str != nil ||
-		tokens[0] != "-*--*" || tokens[1] != "---*" || tokens[2] != "*---" {
-		t.Errorf("%s %s %d", rx, tokens, len(tokens))
+		string(tokens[0].text) != "-*--*" ||
+		string(tokens[1].text) != "---*" ||
+		string(tokens[2].text) != "*---" {
+		t.Errorf("%s %d", tokens, len(tokens))
 	}
 }
 
diff --git a/src/tokenizer.go b/src/tokenizer.go
index 72deb2b1..a8d04003 100644
--- a/src/tokenizer.go
+++ b/src/tokenizer.go
@@ -145,7 +145,16 @@ func Tokenize(runes []rune, delimiter Delimiter) []Token {
 			tokens[i] = tokens[i] + *delimiter.str
 		}
 	} else if delimiter.regex != nil {
-		tokens = delimiter.regex.FindAllString(string(runes), -1)
+		str := string(runes)
+		for len(str) > 0 {
+			loc := delimiter.regex.FindStringIndex(str)
+			if loc == nil {
+				loc = []int{0, len(str)}
+			}
+			last := util.Max(loc[1], 1)
+			tokens = append(tokens, str[:last])
+			str = str[last:]
+		}
 	}
 	asRunes := make([][]rune, len(tokens))
 	for i, token := range tokens {
diff --git a/src/tokenizer_test.go b/src/tokenizer_test.go
index 61017b8c..0f95aa13 100644
--- a/src/tokenizer_test.go
+++ b/src/tokenizer_test.go
@@ -53,6 +53,15 @@ func TestTokenize(t *testing.T) {
 	if string(tokens[0].text) != "  abc:" || tokens[0].prefixLength != 0 {
 		t.Errorf("%s", tokens)
 	}
+
+	// With delimiter regex
+	tokens = Tokenize([]rune(input), delimiterRegexp("\\s+"))
+	if string(tokens[0].text) != "  " || tokens[0].prefixLength != 0 ||
+		string(tokens[1].text) != "abc:  " || tokens[1].prefixLength != 2 ||
+		string(tokens[2].text) != "def:  " || tokens[2].prefixLength != 8 ||
+		string(tokens[3].text) != "ghi  " || tokens[3].prefixLength != 14 {
+		t.Errorf("%s", tokens)
+	}
 }
 
 func TestTransform(t *testing.T) {