From 471ed91c60cd36645794925cb4892cc820eae626 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Sat, 16 Oct 2021 16:24:49 +0200 Subject: hugofs: Add includeFiles and excludeFiles to mount configuration Fixes #9042 --- hugofs/glob/filename_filter.go | 159 ++++++++++++++++++++++++++++++++++++ hugofs/glob/filename_filter_test.go | 70 ++++++++++++++++ hugofs/glob/glob.go | 82 ++----------------- hugofs/glob/glob_test.go | 72 +++++++--------- 4 files changed, 263 insertions(+), 120 deletions(-) create mode 100644 hugofs/glob/filename_filter.go create mode 100644 hugofs/glob/filename_filter_test.go (limited to 'hugofs/glob') diff --git a/hugofs/glob/filename_filter.go b/hugofs/glob/filename_filter.go new file mode 100644 index 000000000..c4b582bd5 --- /dev/null +++ b/hugofs/glob/filename_filter.go @@ -0,0 +1,159 @@ +// Copyright 2021 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package glob + +import ( + "path" + "path/filepath" + "strings" + + "github.com/gobwas/glob" +) + +type FilenameFilter struct { + shouldInclude func(filename string) bool + inclusions []glob.Glob + dirInclusions []glob.Glob + exclusions []glob.Glob + isWindows bool +} + +func normalizeFilenameGlobPattern(s string) string { + // Use Unix separators even on Windows. + s = filepath.ToSlash(s) + if !strings.HasPrefix(s, "/") { + s = "/" + s + } + return s +} + +// NewFilenameFilter creates a new Glob where the Match method will +// return true if the file should be included. +// Note that the inclusions will be checked first. +func NewFilenameFilter(inclusions, exclusions []string) (*FilenameFilter, error) { + if inclusions == nil && exclusions == nil { + return nil, nil + } + filter := &FilenameFilter{isWindows: isWindows} + + for _, include := range inclusions { + include = normalizeFilenameGlobPattern(include) + g, err := filenamesGlobCache.GetGlob(include) + if err != nil { + return nil, err + } + filter.inclusions = append(filter.inclusions, g) + + // For mounts that do directory walking (e.g. content) we + // must make sure that all directories up to this inclusion also + // gets included. + dir := path.Dir(include) + parts := strings.Split(dir, "/") + for i, _ := range parts { + pattern := "/" + filepath.Join(parts[:i+1]...) + g, err := filenamesGlobCache.GetGlob(pattern) + if err != nil { + return nil, err + } + filter.dirInclusions = append(filter.dirInclusions, g) + } + } + + for _, exclude := range exclusions { + exclude = normalizeFilenameGlobPattern(exclude) + g, err := filenamesGlobCache.GetGlob(exclude) + if err != nil { + return nil, err + } + filter.exclusions = append(filter.exclusions, g) + } + + return filter, nil +} + +// MustNewFilenameFilter invokes NewFilenameFilter and panics on error. +func MustNewFilenameFilter(inclusions, exclusions []string) *FilenameFilter { + filter, err := NewFilenameFilter(inclusions, exclusions) + if err != nil { + panic(err) + } + return filter +} + +// NewFilenameFilterForInclusionFunc create a new filter using the provided inclusion func. +func NewFilenameFilterForInclusionFunc(shouldInclude func(filename string) bool) *FilenameFilter { + return &FilenameFilter{shouldInclude: shouldInclude, isWindows: isWindows} +} + +// Match returns whether filename should be included. +func (f *FilenameFilter) Match(filename string, isDir bool) bool { + if f == nil { + return true + } + return f.doMatch(filename, isDir) + /*if f.shouldInclude == nil { + fmt.Printf("Match: %q (%t) => %t\n", filename, isDir, isMatch) + } + return isMatch*/ +} + +func (f *FilenameFilter) doMatch(filename string, isDir bool) bool { + if f == nil { + return true + } + + if !strings.HasPrefix(filename, filepathSeparator) { + filename = filepathSeparator + filename + } + + if f.shouldInclude != nil { + if f.shouldInclude(filename) { + return true + } + if f.isWindows { + // The Glob matchers below handles this by themselves, + // for the shouldInclude we need to take some extra steps + // to make this robust. + winFilename := filepath.FromSlash(filename) + if filename != winFilename { + if f.shouldInclude(winFilename) { + return true + } + } + } + + } + + for _, inclusion := range f.inclusions { + if inclusion.Match(filename) { + return true + } + } + + if isDir && f.inclusions != nil { + for _, inclusion := range f.dirInclusions { + if inclusion.Match(filename) { + return true + } + } + } + + for _, exclusion := range f.exclusions { + if exclusion.Match(filename) { + return false + } + } + + return f.inclusions == nil && f.shouldInclude == nil +} diff --git a/hugofs/glob/filename_filter_test.go b/hugofs/glob/filename_filter_test.go new file mode 100644 index 000000000..1fce5b135 --- /dev/null +++ b/hugofs/glob/filename_filter_test.go @@ -0,0 +1,70 @@ +// Copyright 2021 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package glob + +import ( + "path/filepath" + "strings" + "testing" + + qt "github.com/frankban/quicktest" +) + +func TestFilenameFilter(t *testing.T) { + c := qt.New(t) + + excludeAlmostAllJSON, err := NewFilenameFilter([]string{"/a/b/c/foo.json"}, []string{"**.json"}) + c.Assert(err, qt.IsNil) + c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/data/my.json"), false), qt.Equals, false) + c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a/b/c/foo.json"), false), qt.Equals, true) + c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a/b/c/foo.bar"), false), qt.Equals, false) + c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a/b/c"), true), qt.Equals, true) + c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a/b"), true), qt.Equals, true) + c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a"), true), qt.Equals, true) + c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/"), true), qt.Equals, true) + c.Assert(excludeAlmostAllJSON.Match("", true), qt.Equals, true) + + excludeAllButFooJSON, err := NewFilenameFilter([]string{"/a/**/foo.json"}, []string{"**.json"}) + c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/data/my.json"), false), qt.Equals, false) + c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/a/b/c/d/e/foo.json"), false), qt.Equals, true) + c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/a/b/c"), true), qt.Equals, true) + c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/a/b/"), true), qt.Equals, true) + c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/"), true), qt.Equals, true) + c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/b"), true), qt.Equals, false) + c.Assert(err, qt.IsNil) + + nopFilter, err := NewFilenameFilter(nil, nil) + c.Assert(err, qt.IsNil) + c.Assert(nopFilter.Match("ab.txt", false), qt.Equals, true) + + includeOnlyFilter, err := NewFilenameFilter([]string{"**.json", "**.jpg"}, nil) + c.Assert(err, qt.IsNil) + c.Assert(includeOnlyFilter.Match("ab.json", false), qt.Equals, true) + c.Assert(includeOnlyFilter.Match("ab.jpg", false), qt.Equals, true) + c.Assert(includeOnlyFilter.Match("ab.gif", false), qt.Equals, false) + + exlcudeOnlyFilter, err := NewFilenameFilter(nil, []string{"**.json", "**.jpg"}) + c.Assert(err, qt.IsNil) + c.Assert(exlcudeOnlyFilter.Match("ab.json", false), qt.Equals, false) + c.Assert(exlcudeOnlyFilter.Match("ab.jpg", false), qt.Equals, false) + c.Assert(exlcudeOnlyFilter.Match("ab.gif", false), qt.Equals, true) + + var nilFilter *FilenameFilter + c.Assert(nilFilter.Match("ab.gif", false), qt.Equals, true) + + funcFilter := NewFilenameFilterForInclusionFunc(func(s string) bool { return strings.HasSuffix(s, ".json") }) + c.Assert(funcFilter.Match("ab.json", false), qt.Equals, true) + c.Assert(funcFilter.Match("ab.bson", false), qt.Equals, false) + +} diff --git a/hugofs/glob/glob.go b/hugofs/glob/glob.go index 6dd0df5ed..9e928ec32 100644 --- a/hugofs/glob/glob.go +++ b/hugofs/glob/glob.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. +// Copyright 2021 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,6 +14,7 @@ package glob import ( + "os" "path" "path/filepath" "runtime" @@ -24,6 +25,8 @@ import ( "github.com/gobwas/glob/syntax" ) +const filepathSeparator = string(os.PathSeparator) + var ( isWindows = runtime.GOOS == "windows" defaultGlobCache = &globCache{ @@ -33,7 +36,7 @@ var ( } filenamesGlobCache = &globCache{ - isCaseSensitive: true, // TODO(bep) bench + isCaseSensitive: false, // As long as the search strings are all lower case, this does not allocate. isWindows: isWindows, cache: make(map[string]globErr), } @@ -161,78 +164,3 @@ func HasGlobChar(s string) bool { } return false } - -type FilenameFilter struct { - shouldInclude func(filename string) bool - inclusions []glob.Glob - exclusions []glob.Glob - isWindows bool -} - -// NewFilenameFilter creates a new Glob where the Match method will -// return true if the file should be exluded. -// Note that the inclusions will be checked first. -func NewFilenameFilter(inclusions, exclusions []string) (*FilenameFilter, error) { - filter := &FilenameFilter{isWindows: isWindows} - - for _, include := range inclusions { - g, err := filenamesGlobCache.GetGlob(filepath.FromSlash(include)) - if err != nil { - return nil, err - } - filter.inclusions = append(filter.inclusions, g) - } - for _, exclude := range exclusions { - g, err := filenamesGlobCache.GetGlob(filepath.FromSlash(exclude)) - if err != nil { - return nil, err - } - filter.exclusions = append(filter.exclusions, g) - } - - return filter, nil -} - -// NewFilenameFilterForInclusionFunc create a new filter using the provided inclusion func. -func NewFilenameFilterForInclusionFunc(shouldInclude func(filename string) bool) *FilenameFilter { - return &FilenameFilter{shouldInclude: shouldInclude, isWindows: isWindows} -} - -// Match returns whether filename should be included. -func (f *FilenameFilter) Match(filename string) bool { - if f == nil { - return true - } - - if f.shouldInclude != nil { - if f.shouldInclude(filename) { - return true - } - if f.isWindows { - // The Glob matchers below handles this by themselves, - // for the shouldInclude we need to take some extra steps - // to make this robust. - winFilename := filepath.FromSlash(filename) - if filename != winFilename { - if f.shouldInclude(winFilename) { - return true - } - } - } - - } - - for _, inclusion := range f.inclusions { - if inclusion.Match(filename) { - return true - } - } - - for _, exclusion := range f.exclusions { - if exclusion.Match(filename) { - return false - } - } - - return f.inclusions == nil && f.shouldInclude == nil -} diff --git a/hugofs/glob/glob_test.go b/hugofs/glob/glob_test.go index 7ef3fbbed..66efe9e53 100644 --- a/hugofs/glob/glob_test.go +++ b/hugofs/glob/glob_test.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. +// Copyright 2021 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,7 +15,6 @@ package glob import ( "path/filepath" - "strings" "testing" qt "github.com/frankban/quicktest" @@ -67,51 +66,38 @@ func TestNormalizePath(t *testing.T) { } func TestGetGlob(t *testing.T) { - c := qt.New(t) - g, err := GetGlob("**.JSON") - c.Assert(err, qt.IsNil) - c.Assert(g.Match("data/my.json"), qt.Equals, true) + for _, cache := range []*globCache{defaultGlobCache, filenamesGlobCache} { + c := qt.New(t) + g, err := cache.GetGlob("**.JSON") + c.Assert(err, qt.IsNil) + c.Assert(g.Match("data/my.jSon"), qt.Equals, true) + } } -func TestFilenameFilter(t *testing.T) { - c := qt.New(t) - - excludeAlmostAllJSON, err := NewFilenameFilter([]string{"a/b/c/foo.json"}, []string{"**.json"}) - c.Assert(err, qt.IsNil) - c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("data/my.json")), qt.Equals, false) - c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("a/b/c/foo.json")), qt.Equals, true) - c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("a/b/c/foo.bar")), qt.Equals, false) - - nopFilter, err := NewFilenameFilter(nil, nil) - c.Assert(err, qt.IsNil) - c.Assert(nopFilter.Match("ab.txt"), qt.Equals, true) - - includeOnlyFilter, err := NewFilenameFilter([]string{"**.json", "**.jpg"}, nil) - c.Assert(err, qt.IsNil) - c.Assert(includeOnlyFilter.Match("ab.json"), qt.Equals, true) - c.Assert(includeOnlyFilter.Match("ab.jpg"), qt.Equals, true) - c.Assert(includeOnlyFilter.Match("ab.gif"), qt.Equals, false) - - exlcudeOnlyFilter, err := NewFilenameFilter(nil, []string{"**.json", "**.jpg"}) - c.Assert(err, qt.IsNil) - c.Assert(exlcudeOnlyFilter.Match("ab.json"), qt.Equals, false) - c.Assert(exlcudeOnlyFilter.Match("ab.jpg"), qt.Equals, false) - c.Assert(exlcudeOnlyFilter.Match("ab.gif"), qt.Equals, true) - - var nilFilter *FilenameFilter - c.Assert(nilFilter.Match("ab.gif"), qt.Equals, true) +func BenchmarkGetGlob(b *testing.B) { - funcFilter := NewFilenameFilterForInclusionFunc(func(s string) bool { return strings.HasSuffix(s, ".json") }) - c.Assert(funcFilter.Match("ab.json"), qt.Equals, true) - c.Assert(funcFilter.Match("ab.bson"), qt.Equals, false) + runBench := func(name string, cache *globCache, search string) { + b.Run(name, func(b *testing.B) { + g, err := GetGlob("**/foo") + if err != nil { + b.Fatal(err) + } + for i := 0; i < b.N; i++ { + _ = g.Match(search) + } + }) + } -} + runBench("Default cache", defaultGlobCache, "abcde") + runBench("Filenames cache, lowercase searchs", filenamesGlobCache, "abcde") + runBench("Filenames cache, mixed case searchs", filenamesGlobCache, "abCDe") -func BenchmarkGetGlob(b *testing.B) { - for i := 0; i < b.N; i++ { - _, err := GetGlob("**/foo") - if err != nil { - b.Fatal(err) + b.Run("GetGlob", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _, err := GetGlob("**/foo") + if err != nil { + b.Fatal(err) + } } - } + }) } -- cgit v1.2.3