diff options
author | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2017-07-24 09:00:23 +0200 |
---|---|---|
committer | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2017-12-27 18:44:47 +0100 |
commit | 3cdf19e9b7e46c57a9bb43ff02199177feb55768 (patch) | |
tree | d05e3dc15824c8eeef3e5455193d2d6328621f47 /source | |
parent | 02f2735f68e1bb2e2c412698755d52c4d396f237 (diff) |
:sparkles: Implement Page bundling and image handling
This commit is not the smallest in Hugo's history.
Some hightlights include:
* Page bundles (for complete articles, keeping images and content together etc.).
* Bundled images can be processed in as many versions/sizes as you need with the three methods `Resize`, `Fill` and `Fit`.
* Processed images are cached inside `resources/_gen/images` (default) in your project.
* Symbolic links (both files and dirs) are now allowed anywhere inside /content
* A new table based build summary
* The "Total in nn ms" now reports the total including the handling of the files inside /static. So if it now reports more than you're used to, it is just **more real** and probably faster than before (see below).
A site building benchmark run compared to `v0.31.1` shows that this should be slightly faster and use less memory:
```bash
▶ ./benchSite.sh "TOML,num_langs=.*,num_root_sections=5,num_pages=(500|1000),tags_per_page=5,shortcodes,render"
benchmark old ns/op new ns/op delta
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 101785785 78067944 -23.30%
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 185481057 149159919 -19.58%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 103149918 85679409 -16.94%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 203515478 169208775 -16.86%
benchmark old allocs new allocs delta
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 532464 391539 -26.47%
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 1056549 772702 -26.87%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 555974 406630 -26.86%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 1086545 789922 -27.30%
benchmark old bytes new bytes delta
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 53243246 43598155 -18.12%
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 105811617 86087116 -18.64%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 54558852 44545097 -18.35%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 106903858 86978413 -18.64%
```
Fixes #3651
Closes #3158
Fixes #1014
Closes #2021
Fixes #1240
Updates #3757
Diffstat (limited to 'source')
-rw-r--r-- | source/content_directory_test.go | 11 | ||||
-rw-r--r-- | source/dirs.go | 11 | ||||
-rw-r--r-- | source/file.go | 172 | ||||
-rw-r--r-- | source/fileInfo.go | 213 | ||||
-rw-r--r-- | source/fileInfo_test.go (renamed from source/inmemory.go) | 15 | ||||
-rw-r--r-- | source/file_test.go | 62 | ||||
-rw-r--r-- | source/filesystem.go | 108 | ||||
-rw-r--r-- | source/filesystem_test.go | 66 | ||||
-rw-r--r-- | source/lazy_file_reader.go | 170 | ||||
-rw-r--r-- | source/lazy_file_reader_test.go | 236 | ||||
-rw-r--r-- | source/sourceSpec.go | 117 |
11 files changed, 390 insertions, 791 deletions
diff --git a/source/content_directory_test.go b/source/content_directory_test.go index 4ff12af8d..9874acec2 100644 --- a/source/content_directory_test.go +++ b/source/content_directory_test.go @@ -14,6 +14,7 @@ package source import ( + "path/filepath" "testing" "github.com/gohugoio/hugo/hugofs" @@ -41,21 +42,21 @@ func TestIgnoreDotFilesAndDirectories(t *testing.T) { {"foobar/bar~foo.md", false, nil}, {"foobar/foo.md", true, []string{"\\.md$", "\\.boo$"}}, {"foobar/foo.html", false, []string{"\\.md$", "\\.boo$"}}, - {"foobar/foo.md", true, []string{"^foo"}}, - {"foobar/foo.md", false, []string{"*", "\\.md$", "\\.boo$"}}, + {"foobar/foo.md", true, []string{"foo.md$"}}, + {"foobar/foo.md", true, []string{"*", "\\.md$", "\\.boo$"}}, {"foobar/.#content.md", true, []string{"/\\.#"}}, {".#foobar.md", true, []string{"^\\.#"}}, } - for _, test := range tests { + for i, test := range tests { v := viper.New() v.Set("ignoreFiles", test.ignoreFilesRegexpes) s := NewSourceSpec(v, hugofs.NewMem(v)) - if ignored := s.isNonProcessablePath(test.path); test.ignore != ignored { - t.Errorf("File not ignored. Expected: %t, got: %t", test.ignore, ignored) + if ignored := s.IgnoreFile(filepath.FromSlash(test.path)); test.ignore != ignored { + t.Errorf("[%d] File not ignored", i) } } } diff --git a/source/dirs.go b/source/dirs.go index 1e6850da7..49a849453 100644 --- a/source/dirs.go +++ b/source/dirs.go @@ -38,7 +38,7 @@ type Dirs struct { staticDirs []string AbsStaticDirs []string - publishDir string + Language *helpers.Language } // NewDirs creates a new dirs with the given configuration and filesystem. @@ -48,7 +48,12 @@ func NewDirs(fs *hugofs.Fs, cfg config.Provider, logger *jww.Notepad) (*Dirs, er return nil, err } - d := &Dirs{pathSpec: ps, logger: logger} + var l *helpers.Language + if language, ok := cfg.(*helpers.Language); ok { + l = language + } + + d := &Dirs{Language: l, pathSpec: ps, logger: logger} return d, d.init(cfg) @@ -96,8 +101,6 @@ func (d *Dirs) init(cfg config.Provider) error { d.AbsStaticDirs[i] = d.pathSpec.AbsPathify(di) + helpers.FilePathSeparator } - d.publishDir = d.pathSpec.AbsPathify(cfg.GetString("publishDir")) + helpers.FilePathSeparator - return nil } diff --git a/source/file.go b/source/file.go deleted file mode 100644 index a630431c6..000000000 --- a/source/file.go +++ /dev/null @@ -1,172 +0,0 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package source - -import ( - "io" - "path/filepath" - "strings" - - "github.com/gohugoio/hugo/hugofs" - - "github.com/gohugoio/hugo/config" - "github.com/gohugoio/hugo/helpers" -) - -// SourceSpec abstracts language-specific file creation. -type SourceSpec struct { - Cfg config.Provider - Fs *hugofs.Fs - - languages map[string]interface{} - defaultContentLanguage string -} - -// NewSourceSpec initializes SourceSpec using languages from a given configuration. -func NewSourceSpec(cfg config.Provider, fs *hugofs.Fs) SourceSpec { - defaultLang := cfg.GetString("defaultContentLanguage") - languages := cfg.GetStringMap("languages") - return SourceSpec{Cfg: cfg, Fs: fs, languages: languages, defaultContentLanguage: defaultLang} -} - -// File represents a source content file. -// All paths are relative from the source directory base -type File struct { - relpath string // Original relative path, e.g. section/foo.txt - logicalName string // foo.txt - baseName string // `post` for `post.md`, also `post.en` for `post.en.md` - Contents io.Reader - section string // The first directory - dir string // The relative directory Path (minus file name) - ext string // Just the ext (eg txt) - uniqueID string // MD5 of the file's path - - translationBaseName string // `post` for `post.es.md` (if `Multilingual` is enabled.) - lang string // The language code if `Multilingual` is enabled -} - -// UniqueID is the MD5 hash of the file's path and is for most practical applications, -// Hugo content files being one of them, considered to be unique. -func (f *File) UniqueID() string { - return f.uniqueID -} - -// String returns the file's content as a string. -func (f *File) String() string { - return helpers.ReaderToString(f.Contents) -} - -// Bytes returns the file's content as a byte slice. -func (f *File) Bytes() []byte { - return helpers.ReaderToBytes(f.Contents) -} - -// BaseFileName is a filename without extension. -func (f *File) BaseFileName() string { - return f.baseName -} - -// TranslationBaseName is a filename with no extension, -// not even the optional language extension part. -func (f *File) TranslationBaseName() string { - return f.translationBaseName -} - -// Lang for this page, if `Multilingual` is enabled on your site. -func (f *File) Lang() string { - return f.lang -} - -// Section is first directory below the content root. -func (f *File) Section() string { - return f.section -} - -// LogicalName is filename and extension of the file. -func (f *File) LogicalName() string { - return f.logicalName -} - -// SetDir sets the relative directory where this file lives. -// TODO(bep) Get rid of this. -func (f *File) SetDir(dir string) { - f.dir = dir -} - -// Dir gets the name of the directory that contains this file. -// The directory is relative to the content root. -func (f *File) Dir() string { - return f.dir -} - -// Extension gets the file extension, i.e "myblogpost.md" will return "md". -func (f *File) Extension() string { - return f.ext -} - -// Ext is an alias for Extension. -func (f *File) Ext() string { - return f.Extension() -} - -// Path gets the relative path including file name and extension. -// The directory is relative to the content root. -func (f *File) Path() string { - return f.relpath -} - -// NewFileWithContents creates a new File pointer with the given relative path and -// content. The language defaults to "en". -func (sp SourceSpec) NewFileWithContents(relpath string, content io.Reader) *File { - file := sp.NewFile(relpath) - file.Contents = content - file.lang = "en" - return file -} - -// NewFile creates a new File pointer with the given relative path. -func (sp SourceSpec) NewFile(relpath string) *File { - f := &File{ - relpath: relpath, - } - - f.dir, f.logicalName = filepath.Split(f.relpath) - f.ext = strings.TrimPrefix(filepath.Ext(f.LogicalName()), ".") - f.baseName = helpers.Filename(f.LogicalName()) - - lang := strings.TrimPrefix(filepath.Ext(f.baseName), ".") - if _, ok := sp.languages[lang]; lang == "" || !ok { - f.lang = sp.defaultContentLanguage - f.translationBaseName = f.baseName - } else { - f.lang = lang - f.translationBaseName = helpers.Filename(f.baseName) - } - - f.section = helpers.GuessSection(f.Dir()) - f.uniqueID = helpers.Md5String(filepath.ToSlash(f.relpath)) - - return f -} - -// NewFileFromAbs creates a new File pointer with the given full file path path and -// content. -func (sp SourceSpec) NewFileFromAbs(base, fullpath string, content io.Reader) (f *File, err error) { - var name string - if name, err = helpers.GetRelativePath(fullpath, base); err != nil { - return nil, err - } - - return sp.NewFileWithContents(name, content), nil -} diff --git a/source/fileInfo.go b/source/fileInfo.go new file mode 100644 index 000000000..e4b4a80fb --- /dev/null +++ b/source/fileInfo.go @@ -0,0 +1,213 @@ +// Copyright 2017-present The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package source + +import ( + "io" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/gohugoio/hugo/helpers" +) + +// fileInfo implements the File interface. +var ( + _ File = (*FileInfo)(nil) + _ ReadableFile = (*FileInfo)(nil) +) + +type File interface { + + // Filename gets the full path and filename to the file. + Filename() string + + // Path gets the relative path including file name and extension. + // The directory is relative to the content root. + Path() string + + // Dir gets the name of the directory that contains this file. + // The directory is relative to the content root. + Dir() string + + // Extension gets the file extension, i.e "myblogpost.md" will return "md". + Extension() string + // Ext is an alias for Extension. + Ext() string // Hmm... Deprecate Extension + + // Lang for this page, if `Multilingual` is enabled on your site. + Lang() string + + // LogicalName is filename and extension of the file. + LogicalName() string + + // Section is first directory below the content root. + Section() string + + // BaseFileName is a filename without extension. + BaseFileName() string + + // TranslationBaseName is a filename with no extension, + // not even the optional language extension part. + TranslationBaseName() string + + // UniqueID is the MD5 hash of the file's path and is for most practical applications, + // Hugo content files being one of them, considered to be unique. + UniqueID() string + + FileInfo() os.FileInfo + + String() string + + // Deprecated + Bytes() []byte +} + +// A ReadableFile is a File that is readable. +type ReadableFile interface { + File + Open() (io.ReadCloser, error) +} + +type FileInfo struct { + + // Absolute filename to the file on disk. + filename string + fi os.FileInfo + + // Derived from filename + ext string // Extension without any "." + lang string + + name string + + dir string + relDir string + relPath string + baseName string + translationBaseName string + section string + + uniqueID string + + sp *SourceSpec + + lazyInit sync.Once +} + +func (fi *FileInfo) Filename() string { return fi.filename } +func (fi *FileInfo) Path() string { return fi.relPath } +func (fi *FileInfo) Dir() string { return fi.relDir } +func (fi *FileInfo) Extension() string { return fi.Ext() } +func (fi *FileInfo) Ext() string { return fi.ext } +func (fi *FileInfo) Lang() string { return fi.lang } +func (fi *FileInfo) LogicalName() string { return fi.name } +func (fi *FileInfo) BaseFileName() string { return fi.baseName } +func (fi *FileInfo) TranslationBaseName() string { return fi.translationBaseName } + +func (fi *FileInfo) Section() string { + fi.init() + return fi.section +} + +func (fi *FileInfo) UniqueID() string { + fi.init() + return fi.uniqueID +} +func (fi *FileInfo) FileInfo() os.FileInfo { + return fi.fi +} + +func (fi *FileInfo) Bytes() []byte { + // Remove in Hugo 0.34 + helpers.Deprecated("File", "Bytes", "", false) + return []byte("") +} + +func (fi *FileInfo) String() string { return fi.BaseFileName() } + +// We create a lot of these FileInfo objects, but there are parts of it used only +// in some cases that is slightly expensive to construct. +func (fi *FileInfo) init() { + fi.lazyInit.Do(func() { + parts := strings.Split(fi.relDir, helpers.FilePathSeparator) + var section string + if len(parts) == 1 { + section = parts[0] + } else if len(parts) > 1 { + if parts[0] == "" { + section = parts[1] + } else { + section = parts[0] + } + } + + fi.section = section + + fi.uniqueID = helpers.MD5String(filepath.ToSlash(fi.relPath)) + + }) +} + +func (sp *SourceSpec) NewFileInfo(baseDir, filename string, fi os.FileInfo) *FileInfo { + dir, name := filepath.Split(filename) + + dir = strings.TrimSuffix(dir, helpers.FilePathSeparator) + baseDir = strings.TrimSuffix(baseDir, helpers.FilePathSeparator) + + relDir := "" + if dir != baseDir { + relDir = strings.TrimPrefix(dir, baseDir) + } + + relDir = strings.TrimPrefix(relDir, helpers.FilePathSeparator) + + relPath := filepath.Join(relDir, name) + + ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(name), ".")) + baseName := helpers.Filename(name) + + lang := strings.TrimPrefix(filepath.Ext(baseName), ".") + var translationBaseName string + + if _, ok := sp.Languages[lang]; lang == "" || !ok { + lang = sp.DefaultContentLanguage + translationBaseName = baseName + } else { + translationBaseName = helpers.Filename(baseName) + } + + f := &FileInfo{ + sp: sp, + filename: filename, + fi: fi, + lang: lang, + ext: ext, + dir: dir, + relDir: relDir, + relPath: relPath, + name: name, + baseName: baseName, + translationBaseName: translationBaseName, + } + + return f + +} + +// Open implements ReadableFile. +func (fi *FileInfo) Open() (io.ReadCloser, error) { + return fi.sp.Fs.Source.Open(fi.Filename()) +} diff --git a/source/inmemory.go b/source/fileInfo_test.go index 387bde3b8..3f99497ad 100644 --- a/source/inmemory.go +++ b/source/fileInfo_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. +// Copyright 2017-present The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -13,13 +13,10 @@ package source -// ByteSource represents a source's name and content. -// It's currently only used for testing purposes. -type ByteSource struct { - Name string - Content []byte -} +import ( + "testing" +) + +func TestFileInfo(t *testing.T) { -func (b *ByteSource) String() string { - return b.Name + " " + string(b.Content) } diff --git a/source/file_test.go b/source/file_test.go deleted file mode 100644 index 64ad6fb46..000000000 --- a/source/file_test.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package source - -import ( - "path/filepath" - "strings" - "testing" - - "github.com/gohugoio/hugo/hugofs" - "github.com/spf13/viper" - - "github.com/stretchr/testify/assert" -) - -func TestFileUniqueID(t *testing.T) { - ss := newTestSourceSpec() - - f1 := File{uniqueID: "123"} - f2 := ss.NewFile("a") - - assert.Equal(t, "123", f1.UniqueID()) - assert.Equal(t, "0cc175b9c0f1b6a831c399e269772661", f2.UniqueID()) - - f3 := ss.NewFile(filepath.FromSlash("test1/index.md")) - f4 := ss.NewFile(filepath.FromSlash("test2/index.md")) - - assert.NotEqual(t, f3.UniqueID(), f4.UniqueID()) - - f5l := ss.NewFile("test3/index.md") - f5w := ss.NewFile(filepath.FromSlash("test3/index.md")) - - assert.Equal(t, f5l.UniqueID(), f5w.UniqueID()) -} - -func TestFileString(t *testing.T) { - ss := newTestSourceSpec() - assert.Equal(t, "abc", ss.NewFileWithContents("a", strings.NewReader("abc")).String()) - assert.Equal(t, "", ss.NewFile("a").String()) -} - -func TestFileBytes(t *testing.T) { - ss := newTestSourceSpec() - assert.Equal(t, []byte("abc"), ss.NewFileWithContents("a", strings.NewReader("abc")).Bytes()) - assert.Equal(t, []byte(""), ss.NewFile("a").Bytes()) -} - -func newTestSourceSpec() SourceSpec { - v := viper.New() - return SourceSpec{Fs: hugofs.NewMem(v), Cfg: v} -} diff --git a/source/filesystem.go b/source/filesystem.go index e6e354e99..a5f2988e9 100644 --- a/source/filesystem.go +++ b/source/filesystem.go @@ -14,73 +14,52 @@ package source import ( - "io" "os" "path/filepath" - "regexp" "runtime" - "strings" + "sync" "github.com/gohugoio/hugo/helpers" - "github.com/spf13/cast" jww "github.com/spf13/jwalterweatherman" "golang.org/x/text/unicode/norm" ) -type Input interface { - Files() []*File -} - type Filesystem struct { - files []*File - Base string - AvoidPaths []string + files []ReadableFile + filesInit sync.Once + + Base string SourceSpec } -func (sp SourceSpec) NewFilesystem(base string, avoidPaths ...string) *Filesystem { - return &Filesystem{SourceSpec: sp, Base: base, AvoidPaths: avoidPaths} +type Input interface { + Files() []ReadableFile } -func (f *Filesystem) FilesByExts(exts ...string) []*File { - var newFiles []*File - - if len(exts) == 0 { - return f.Files() - } - - for _, x := range f.Files() { - for _, e := range exts { - if x.Ext() == strings.TrimPrefix(e, ".") { - newFiles = append(newFiles, x) - } - } - } - return newFiles +func (sp SourceSpec) NewFilesystem(base string) *Filesystem { + return &Filesystem{SourceSpec: sp, Base: base} } -func (f *Filesystem) Files() []*File { - if len(f.files) < 1 { +func (f *Filesystem) Files() []ReadableFile { + f.filesInit.Do(func() { f.captureFiles() - } + }) return f.files } // add populates a file in the Filesystem.files -func (f *Filesystem) add(name string, reader io.Reader) (err error) { - var file *File +func (f *Filesystem) add(name string, fi os.FileInfo) (err error) { + var file ReadableFile if runtime.GOOS == "darwin" { // When a file system is HFS+, its filepath is in NFD form. name = norm.NFC.String(name) } - file, err = f.SourceSpec.NewFileFromAbs(f.Base, name, reader) + file = f.SourceSpec.NewFileInfo(f.Base, name, fi) + f.files = append(f.files, file) - if err == nil { - f.files = append(f.files, file) - } return err } @@ -90,16 +69,12 @@ func (f *Filesystem) captureFiles() { return nil } - b, err := f.ShouldRead(filePath, fi) + b, err := f.shouldRead(filePath, fi) if err != nil { return err } if b { - rd, err := NewLazyFileReader(f.Fs.Source, filePath) - if err != nil { - return err - } - f.add(filePath, rd) + f.add(filePath, fi) } return err } @@ -118,11 +93,11 @@ func (f *Filesystem) captureFiles() { } -func (f *Filesystem) ShouldRead(filePath string, fi os.FileInfo) (bool, error) { +func (f *Filesystem) shouldRead(filename string, fi os.FileInfo) (bool, error) { if fi.Mode()&os.ModeSymlink == os.ModeSymlink { - link, err := filepath.EvalSymlinks(filePath) + link, err := filepath.EvalSymlinks(filename) if err != nil { - jww.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", filePath, err) + jww.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", filename, err) return false, nil } linkfi, err := f.Fs.Source.Stat(link) @@ -130,52 +105,25 @@ func (f *Filesystem) ShouldRead(filePath string, fi os.FileInfo) (bool, error) { jww.ERROR.Printf("Cannot stat '%s', error was: %s", link, err) return false, nil } + if !linkfi.Mode().IsRegular() { - jww.ERROR.Printf("Symbolic links for directories not supported, skipping '%s'", filePath) + jww.ERROR.Printf("Symbolic links for directories not supported, skipping '%s'", filename) } return false, nil } + ignore := f.SourceSpec.IgnoreFile(filename) + if fi.IsDir() { - if f.avoid(filePath) || f.isNonProcessablePath(filePath) { + if ignore { return false, filepath.SkipDir } return false, nil } - if f.isNonProcessablePath(filePath) { + if ignore { return false, nil } - return true, nil -} - -func (f *Filesystem) avoid(filePath string) bool { - for _, avoid := range f.AvoidPaths { - if avoid == filePath { - return true - } - } - return false -} -func (sp SourceSpec) isNonProcessablePath(filePath string) bool { - base := filepath.Base(filePath) - if strings.HasPrefix(base, ".") || - strings.HasPrefix(base, "#") || - strings.HasSuffix(base, "~") { - return true - } - ignoreFiles := cast.ToStringSlice(sp.Cfg.Get("ignoreFiles")) - if len(ignoreFiles) > 0 { - for _, ignorePattern := range ignoreFiles { - match, err := regexp.MatchString(ignorePattern, filePath) - if err != nil { - helpers.DistinctErrorLog.Printf("Invalid regexp '%s' in ignoreFiles: %s", ignorePattern, err) - return false - } else if match { - return true - } - } - } - return false + return true, nil } diff --git a/source/filesystem_test.go b/source/filesystem_test.go index 90512ce3f..25ce0268f 100644 --- a/source/filesystem_test.go +++ b/source/filesystem_test.go @@ -14,11 +14,13 @@ package source import ( - "bytes" - "path/filepath" + "os" "runtime" - "strings" "testing" + + "github.com/gohugoio/hugo/hugofs" + + "github.com/spf13/viper" ) func TestEmptySourceFilesystem(t *testing.T) { @@ -37,54 +39,6 @@ type TestPath struct { dir string } -func TestAddFile(t *testing.T) { - ss := newTestSourceSpec() - tests := platformPaths - for _, test := range tests { - base := platformBase - srcDefault := ss.NewFilesystem("") - srcWithBase := ss.NewFilesystem(base) - - for _, src := range []*Filesystem{srcDefault, srcWithBase} { - - p := test.filename - if !filepath.IsAbs(test.filename) { - p = filepath.Join(src.Base, test.filename) - } - - if err := src.add(p, bytes.NewReader([]byte(test.content))); err != nil { - if err.Error() == "source: missing base directory" { - continue - } - t.Fatalf("%s add returned an error: %s", p, err) - } - - if len(src.Files()) != 1 { - t.Fatalf("%s Files() should return 1 file", p) - } - - f := src.Files()[0] - if f.LogicalName() != test.logical { - t.Errorf("Filename (Base: %q) expected: %q, got: %q", src.Base, test.logical, f.LogicalName()) - } - - b := new(bytes.Buffe |