hugofs: Add includeFiles and excludeFiles to mount configuration

Fixes #9042

4 files changed, 263 insertions, 120 deletions

diff --git a/hugofs/glob/filename_filter.go b/hugofs/glob/filename_filter.go
new file mode 100644
index 000000000..c4b582bd5
--- /dev/null
+++ b/hugofs/glob/filename_filter.go
@@ -0,0 +1,159 @@
+// Copyright 2021 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package glob
+
+import (
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/gobwas/glob"
+)
+
+type FilenameFilter struct {
+	shouldInclude func(filename string) bool
+	inclusions    []glob.Glob
+	dirInclusions []glob.Glob
+	exclusions    []glob.Glob
+	isWindows     bool
+}
+
+func normalizeFilenameGlobPattern(s string) string {
+	// Use Unix separators even on Windows.
+	s = filepath.ToSlash(s)
+	if !strings.HasPrefix(s, "/") {
+		s = "/" + s
+	}
+	return s
+}
+
+// NewFilenameFilter creates a new Glob where the Match method will
+// return true if the file should be included.
+// Note that the inclusions will be checked first.
+func NewFilenameFilter(inclusions, exclusions []string) (*FilenameFilter, error) {
+	if inclusions == nil && exclusions == nil {
+		return nil, nil
+	}
+	filter := &FilenameFilter{isWindows: isWindows}
+
+	for _, include := range inclusions {
+		include = normalizeFilenameGlobPattern(include)
+		g, err := filenamesGlobCache.GetGlob(include)
+		if err != nil {
+			return nil, err
+		}
+		filter.inclusions = append(filter.inclusions, g)
+
+		// For mounts that do directory walking (e.g. content) we
+		// must make sure that all directories up to this inclusion also
+		// gets included.
+		dir := path.Dir(include)
+		parts := strings.Split(dir, "/")
+		for i, _ := range parts {
+			pattern := "/" + filepath.Join(parts[:i+1]...)
+			g, err := filenamesGlobCache.GetGlob(pattern)
+			if err != nil {
+				return nil, err
+			}
+			filter.dirInclusions = append(filter.dirInclusions, g)
+		}
+	}
+
+	for _, exclude := range exclusions {
+		exclude = normalizeFilenameGlobPattern(exclude)
+		g, err := filenamesGlobCache.GetGlob(exclude)
+		if err != nil {
+			return nil, err
+		}
+		filter.exclusions = append(filter.exclusions, g)
+	}
+
+	return filter, nil
+}
+
+// MustNewFilenameFilter invokes NewFilenameFilter and panics on error.
+func MustNewFilenameFilter(inclusions, exclusions []string) *FilenameFilter {
+	filter, err := NewFilenameFilter(inclusions, exclusions)
+	if err != nil {
+		panic(err)
+	}
+	return filter
+}
+
+// NewFilenameFilterForInclusionFunc create a new filter using the provided inclusion func.
+func NewFilenameFilterForInclusionFunc(shouldInclude func(filename string) bool) *FilenameFilter {
+	return &FilenameFilter{shouldInclude: shouldInclude, isWindows: isWindows}
+}
+
+// Match returns whether filename should be included.
+func (f *FilenameFilter) Match(filename string, isDir bool) bool {
+	if f == nil {
+		return true
+	}
+	return f.doMatch(filename, isDir)
+	/*if f.shouldInclude == nil {
+		fmt.Printf("Match: %q (%t) => %t\n", filename, isDir, isMatch)
+	}
+	return isMatch*/
+}
+
+func (f *FilenameFilter) doMatch(filename string, isDir bool) bool {
+	if f == nil {
+		return true
+	}
+
+	if !strings.HasPrefix(filename, filepathSeparator) {
+		filename = filepathSeparator + filename
+	}
+
+	if f.shouldInclude != nil {
+		if f.shouldInclude(filename) {
+			return true
+		}
+		if f.isWindows {
+			// The Glob matchers below handles this by themselves,
+			// for the shouldInclude we need to take some extra steps
+			// to make this robust.
+			winFilename := filepath.FromSlash(filename)
+			if filename != winFilename {
+				if f.shouldInclude(winFilename) {
+					return true
+				}
+			}
+		}
+
+	}
+
+	for _, inclusion := range f.inclusions {
+		if inclusion.Match(filename) {
+			return true
+		}
+	}
+
+	if isDir && f.inclusions != nil {
+		for _, inclusion := range f.dirInclusions {
+			if inclusion.Match(filename) {
+				return true
+			}
+		}
+	}
+
+	for _, exclusion := range f.exclusions {
+		if exclusion.Match(filename) {
+			return false
+		}
+	}
+
+	return f.inclusions == nil && f.shouldInclude == nil
+}
diff --git a/hugofs/glob/filename_filter_test.go b/hugofs/glob/filename_filter_test.go
new file mode 100644
index 000000000..1fce5b135
--- /dev/null
+++ b/hugofs/glob/filename_filter_test.go
@@ -0,0 +1,70 @@
+// Copyright 2021 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package glob
+
+import (
+	"path/filepath"
+	"strings"
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestFilenameFilter(t *testing.T) {
+	c := qt.New(t)
+
+	excludeAlmostAllJSON, err := NewFilenameFilter([]string{"/a/b/c/foo.json"}, []string{"**.json"})
+	c.Assert(err, qt.IsNil)
+	c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/data/my.json"), false), qt.Equals, false)
+	c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a/b/c/foo.json"), false), qt.Equals, true)
+	c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a/b/c/foo.bar"), false), qt.Equals, false)
+	c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a/b/c"), true), qt.Equals, true)
+	c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a/b"), true), qt.Equals, true)
+	c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a"), true), qt.Equals, true)
+	c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/"), true), qt.Equals, true)
+	c.Assert(excludeAlmostAllJSON.Match("", true), qt.Equals, true)
+
+	excludeAllButFooJSON, err := NewFilenameFilter([]string{"/a/**/foo.json"}, []string{"**.json"})
+	c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/data/my.json"), false), qt.Equals, false)
+	c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/a/b/c/d/e/foo.json"), false), qt.Equals, true)
+	c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/a/b/c"), true), qt.Equals, true)
+	c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/a/b/"), true), qt.Equals, true)
+	c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/"), true), qt.Equals, true)
+	c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/b"), true), qt.Equals, false)
+	c.Assert(err, qt.IsNil)
+
+	nopFilter, err := NewFilenameFilter(nil, nil)
+	c.Assert(err, qt.IsNil)
+	c.Assert(nopFilter.Match("ab.txt", false), qt.Equals, true)
+
+	includeOnlyFilter, err := NewFilenameFilter([]string{"**.json", "**.jpg"}, nil)
+	c.Assert(err, qt.IsNil)
+	c.Assert(includeOnlyFilter.Match("ab.json", false), qt.Equals, true)
+	c.Assert(includeOnlyFilter.Match("ab.jpg", false), qt.Equals, true)
+	c.Assert(includeOnlyFilter.Match("ab.gif", false), qt.Equals, false)
+
+	exlcudeOnlyFilter, err := NewFilenameFilter(nil, []string{"**.json", "**.jpg"})
+	c.Assert(err, qt.IsNil)
+	c.Assert(exlcudeOnlyFilter.Match("ab.json", false), qt.Equals, false)
+	c.Assert(exlcudeOnlyFilter.Match("ab.jpg", false), qt.Equals, false)
+	c.Assert(exlcudeOnlyFilter.Match("ab.gif", false), qt.Equals, true)
+
+	var nilFilter *FilenameFilter
+	c.Assert(nilFilter.Match("ab.gif", false), qt.Equals, true)
+
+	funcFilter := NewFilenameFilterForInclusionFunc(func(s string) bool { return strings.HasSuffix(s, ".json") })
+	c.Assert(funcFilter.Match("ab.json", false), qt.Equals, true)
+	c.Assert(funcFilter.Match("ab.bson", false), qt.Equals, false)
+
+}
diff --git a/hugofs/glob/glob.go b/hugofs/glob/glob.go
index 6dd0df5ed..9e928ec32 100644
--- a/hugofs/glob/glob.go
+++ b/hugofs/glob/glob.go
@@ -1,4 +1,4 @@
-// Copyright 2019 The Hugo Authors. All rights reserved.
+// Copyright 2021 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -14,6 +14,7 @@
 package glob
 
 import (
+	"os"
 	"path"
 	"path/filepath"
 	"runtime"
@@ -24,6 +25,8 @@ import (
 	"github.com/gobwas/glob/syntax"
 )
 
+const filepathSeparator = string(os.PathSeparator)
+
 var (
 	isWindows        = runtime.GOOS == "windows"
 	defaultGlobCache = &globCache{
@@ -33,7 +36,7 @@ var (
 	}
 
 	filenamesGlobCache = &globCache{
-		isCaseSensitive: true, // TODO(bep) bench
+		isCaseSensitive: false, // As long as the search strings are all lower case, this does not allocate.
 		isWindows:       isWindows,
 		cache:           make(map[string]globErr),
 	}
@@ -161,78 +164,3 @@ func HasGlobChar(s string) bool {
 	}
 	return false
 }
-
-type FilenameFilter struct {
-	shouldInclude func(filename string) bool
-	inclusions    []glob.Glob
-	exclusions    []glob.Glob
-	isWindows     bool
-}
-
-// NewFilenameFilter creates a new Glob where the Match method will
-// return true if the file should be exluded.
-// Note that the inclusions will be checked first.
-func NewFilenameFilter(inclusions, exclusions []string) (*FilenameFilter, error) {
-	filter := &FilenameFilter{isWindows: isWindows}
-
-	for _, include := range inclusions {
-		g, err := filenamesGlobCache.GetGlob(filepath.FromSlash(include))
-		if err != nil {
-			return nil, err
-		}
-		filter.inclusions = append(filter.inclusions, g)
-	}
-	for _, exclude := range exclusions {
-		g, err := filenamesGlobCache.GetGlob(filepath.FromSlash(exclude))
-		if err != nil {
-			return nil, err
-		}
-		filter.exclusions = append(filter.exclusions, g)
-	}
-
-	return filter, nil
-}
-
-// NewFilenameFilterForInclusionFunc create a new filter using the provided inclusion func.
-func NewFilenameFilterForInclusionFunc(shouldInclude func(filename string) bool) *FilenameFilter {
-	return &FilenameFilter{shouldInclude: shouldInclude, isWindows: isWindows}
-}
-
-// Match returns whether filename should be included.
-func (f *FilenameFilter) Match(filename string) bool {
-	if f == nil {
-		return true
-	}
-
-	if f.shouldInclude != nil {
-		if f.shouldInclude(filename) {
-			return true
-		}
-		if f.isWindows {
-			// The Glob matchers below handles this by themselves,
-			// for the shouldInclude we need to take some extra steps
-			// to make this robust.
-			winFilename := filepath.FromSlash(filename)
-			if filename != winFilename {
-				if f.shouldInclude(winFilename) {
-					return true
-				}
-			}
-		}
-
-	}
-
-	for _, inclusion := range f.inclusions {
-		if inclusion.Match(filename) {
-			return true
-		}
-	}
-
-	for _, exclusion := range f.exclusions {
-		if exclusion.Match(filename) {
-			return false
-		}
-	}
-
-	return f.inclusions == nil && f.shouldInclude == nil
-}
diff --git a/hugofs/glob/glob_test.go b/hugofs/glob/glob_test.go
index 7ef3fbbed..66efe9e53 100644
--- a/hugofs/glob/glob_test.go
+++ b/hugofs/glob/glob_test.go
@@ -1,4 +1,4 @@
-// Copyright 2019 The Hugo Authors. All rights reserved.
+// Copyright 2021 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -15,7 +15,6 @@ package glob
 
 import (
 	"path/filepath"
-	"strings"
 	"testing"
 
 	qt "github.com/frankban/quicktest"
@@ -67,51 +66,38 @@ func TestNormalizePath(t *testing.T) {
 }
 
 func TestGetGlob(t *testing.T) {
-	c := qt.New(t)
-	g, err := GetGlob("**.JSON")
-	c.Assert(err, qt.IsNil)
-	c.Assert(g.Match("data/my.json"), qt.Equals, true)
+	for _, cache := range []*globCache{defaultGlobCache, filenamesGlobCache} {
+		c := qt.New(t)
+		g, err := cache.GetGlob("**.JSON")
+		c.Assert(err, qt.IsNil)
+		c.Assert(g.Match("data/my.jSon"), qt.Equals, true)
+	}
 }
 
-func TestFilenameFilter(t *testing.T) {
-	c := qt.New(t)
-
-	excludeAlmostAllJSON, err := NewFilenameFilter([]string{"a/b/c/foo.json"}, []string{"**.json"})
-	c.Assert(err, qt.IsNil)
-	c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("data/my.json")), qt.Equals, false)
-	c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("a/b/c/foo.json")), qt.Equals, true)
-	c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("a/b/c/foo.bar")), qt.Equals, false)
-
-	nopFilter, err := NewFilenameFilter(nil, nil)
-	c.Assert(err, qt.IsNil)
-	c.Assert(nopFilter.Match("ab.txt"), qt.Equals, true)
-
-	includeOnlyFilter, err := NewFilenameFilter([]string{"**.json", "**.jpg"}, nil)
-	c.Assert(err, qt.IsNil)
-	c.Assert(includeOnlyFilter.Match("ab.json"), qt.Equals, true)
-	c.Assert(includeOnlyFilter.Match("ab.jpg"), qt.Equals, true)
-	c.Assert(includeOnlyFilter.Match("ab.gif"), qt.Equals, false)
-
-	exlcudeOnlyFilter, err := NewFilenameFilter(nil, []string{"**.json", "**.jpg"})
-	c.Assert(err, qt.IsNil)
-	c.Assert(exlcudeOnlyFilter.Match("ab.json"), qt.Equals, false)
-	c.Assert(exlcudeOnlyFilter.Match("ab.jpg"), qt.Equals, false)
-	c.Assert(exlcudeOnlyFilter.Match("ab.gif"), qt.Equals, true)
-
-	var nilFilter *FilenameFilter
-	c.Assert(nilFilter.Match("ab.gif"), qt.Equals, true)
+func BenchmarkGetGlob(b *testing.B) {
 
-	funcFilter := NewFilenameFilterForInclusionFunc(func(s string) bool { return strings.HasSuffix(s, ".json") })
-	c.Assert(funcFilter.Match("ab.json"), qt.Equals, true)
-	c.Assert(funcFilter.Match("ab.bson"), qt.Equals, false)
+	runBench := func(name string, cache *globCache, search string) {
+		b.Run(name, func(b *testing.B) {
+			g, err := GetGlob("**/foo")
+			if err != nil {
+				b.Fatal(err)
+			}
+			for i := 0; i < b.N; i++ {
+				_ = g.Match(search)
+			}
+		})
+	}
 
-}
+	runBench("Default cache", defaultGlobCache, "abcde")
+	runBench("Filenames cache, lowercase searchs", filenamesGlobCache, "abcde")
+	runBench("Filenames cache, mixed case searchs", filenamesGlobCache, "abCDe")
 
-func BenchmarkGetGlob(b *testing.B) {
-	for i := 0; i < b.N; i++ {
-		_, err := GetGlob("**/foo")
-		if err != nil {
-			b.Fatal(err)
+	b.Run("GetGlob", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			_, err := GetGlob("**/foo")
+			if err != nil {
+				b.Fatal(err)
+			}
 		}
-	}
+	})
 }