diff options
author | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2017-07-24 09:00:23 +0200 |
---|---|---|
committer | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2017-12-27 18:44:47 +0100 |
commit | 3cdf19e9b7e46c57a9bb43ff02199177feb55768 (patch) | |
tree | d05e3dc15824c8eeef3e5455193d2d6328621f47 /hugolib | |
parent | 02f2735f68e1bb2e2c412698755d52c4d396f237 (diff) |
:sparkles: Implement Page bundling and image handling
This commit is not the smallest in Hugo's history.
Some hightlights include:
* Page bundles (for complete articles, keeping images and content together etc.).
* Bundled images can be processed in as many versions/sizes as you need with the three methods `Resize`, `Fill` and `Fit`.
* Processed images are cached inside `resources/_gen/images` (default) in your project.
* Symbolic links (both files and dirs) are now allowed anywhere inside /content
* A new table based build summary
* The "Total in nn ms" now reports the total including the handling of the files inside /static. So if it now reports more than you're used to, it is just **more real** and probably faster than before (see below).
A site building benchmark run compared to `v0.31.1` shows that this should be slightly faster and use less memory:
```bash
▶ ./benchSite.sh "TOML,num_langs=.*,num_root_sections=5,num_pages=(500|1000),tags_per_page=5,shortcodes,render"
benchmark old ns/op new ns/op delta
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 101785785 78067944 -23.30%
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 185481057 149159919 -19.58%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 103149918 85679409 -16.94%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 203515478 169208775 -16.86%
benchmark old allocs new allocs delta
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 532464 391539 -26.47%
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 1056549 772702 -26.87%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 555974 406630 -26.86%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 1086545 789922 -27.30%
benchmark old bytes new bytes delta
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 53243246 43598155 -18.12%
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 105811617 86087116 -18.64%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 54558852 44545097 -18.35%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 106903858 86978413 -18.64%
```
Fixes #3651
Closes #3158
Fixes #1014
Closes #2021
Fixes #1240
Updates #3757
Diffstat (limited to 'hugolib')
43 files changed, 3206 insertions, 2006 deletions
diff --git a/hugolib/alias.go b/hugolib/alias.go index a3fe5c24a..dbb864384 100644 --- a/hugolib/alias.go +++ b/hugolib/alias.go @@ -109,7 +109,7 @@ func (s *Site) publishDestAlias(allowRoot bool, path, permalink string, p *Page) return err } - return s.publish(targetPath, aliasContent) + return s.publish(&s.PathSpec.ProcessingStats.Aliases, targetPath, aliasContent) } diff --git a/hugolib/alias_test.go b/hugolib/alias_test.go index 1d6824dba..abbda5f35 100644 --- a/hugolib/alias_test.go +++ b/hugolib/alias_test.go @@ -51,7 +51,9 @@ func TestAlias(t *testing.T) { writeSource(t, fs, filepath.Join("content", "page.md"), pageWithAlias) writeSource(t, fs, filepath.Join("layouts", "_default", "single.html"), basicTemplate) - buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{}) + s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{}) + + require.Len(t, s.rawAllPages, 1) // the real page th.assertFileContent(filepath.Join("public", "page", "index.html"), "For some moments the old man") diff --git a/hugolib/config.go b/hugolib/config.go index 62098aeea..c83f38cce 100644 --- a/hugolib/config.go +++ b/hugolib/config.go @@ -64,22 +64,21 @@ func LoadConfig(fs afero.Fs, relativeSourcePath, configFilename string) (*viper. v.RegisterAlias("indexes", "taxonomies") - // Remove these in Hugo 0.23. + // Remove these in Hugo 0.33. if v.IsSet("disable404") { - helpers.Deprecated("site config", "disable404", "Use disableKinds=[\"404\"]", false) + helpers.Deprecated("site config", "disable404", "Use disableKinds=[\"404\"]", true) } if v.IsSet("disableRSS") { - helpers.Deprecated("site config", "disableRSS", "Use disableKinds=[\"RSS\"]", false) + helpers.Deprecated("site config", "disableRSS", "Use disableKinds=[\"RSS\"]", true) } if v.IsSet("disableSitemap") { - // NOTE: Do not remove this until Hugo 0.24, ERROR in 0.23. - helpers.Deprecated("site config", "disableSitemap", "Use disableKinds= [\"sitemap\"]", false) + helpers.Deprecated("site config", "disableSitemap", "Use disableKinds= [\"sitemap\"]", true) } if v.IsSet("disableRobotsTXT") { - helpers.Deprecated("site config", "disableRobotsTXT", "Use disableKinds= [\"robotsTXT\"]", false) + helpers.Deprecated("site config", "disableRobotsTXT", "Use disableKinds= [\"robotsTXT\"]", true) } if err := loadDefaultSettingsFor(v); err != nil { @@ -176,6 +175,7 @@ func loadDefaultSettingsFor(v *viper.Viper) error { v.SetDefault("contentDir", "content") v.SetDefault("layoutDir", "layouts") v.SetDefault("staticDir", "static") + v.SetDefault("resourceDir", "resources") v.SetDefault("archetypeDir", "archetypes") v.SetDefault("publishDir", "public") v.SetDefault("dataDir", "data") diff --git a/hugolib/datafiles_test.go b/hugolib/datafiles_test.go index b62fb197d..cdc5e0684 100644 --- a/hugolib/datafiles_test.go +++ b/hugolib/datafiles_test.go @@ -27,16 +27,15 @@ import ( jww "github.com/spf13/jwalterweatherman" "github.com/gohugoio/hugo/parser" - "github.com/gohugoio/hugo/source" "github.com/stretchr/testify/require" ) func TestDataDirJSON(t *testing.T) { t.Parallel() - sources := []source.ByteSource{ - {Name: filepath.FromSlash("data/test/foo.json"), Content: []byte(`{ "bar": "foofoo" }`)}, - {Name: filepath.FromSlash("data/test.json"), Content: []byte(`{ "hello": [ { "world": "foo" } ] }`)}, + sources := [][2]string{ + {filepath.FromSlash("data/test/foo.json"), `{ "bar": "foofoo" }`}, + {filepath.FromSlash("data/test.json"), `{ "hello": [ { "world": "foo" } ] }`}, } expected, err := parser.HandleJSONMetaData([]byte(`{ "test": { "hello": [{ "world": "foo" }] , "foo": { "bar":"foofoo" } } }`)) @@ -51,8 +50,8 @@ func TestDataDirJSON(t *testing.T) { func TestDataDirToml(t *testing.T) { t.Parallel() - sources := []source.ByteSource{ - {Name: filepath.FromSlash("data/test/kung.toml"), Content: []byte("[foo]\nbar = 1")}, + sources := [][2]string{ + {"data/test/kung.toml", "[foo]\nbar = 1"}, } expected, err := parser.HandleTOMLMetaData([]byte("[test]\n[test.kung]\n[test.kung.foo]\nbar = 1")) @@ -67,12 +66,12 @@ func TestDataDirToml(t *testing.T) { func TestDataDirYAMLWithOverridenValue(t *testing.T) { t.Parallel() - sources := []source.ByteSource{ + sources := [][2]string{ // filepath.Walk walks the files in lexical order, '/' comes before '.'. Simulate this: - {Name: filepath.FromSlash("data/a.yaml"), Content: []byte("a: 1")}, - {Name: filepath.FromSlash("data/test/v1.yaml"), Content: []byte("v1-2: 2")}, - {Name: filepath.FromSlash("data/test/v2.yaml"), Content: []byte("v2:\n- 2\n- 3")}, - {Name: filepath.FromSlash("data/test.yaml"), Content: []byte("v1: 1")}, + {filepath.FromSlash("data/a.yaml"), "a: 1"}, + {filepath.FromSlash("data/test/v1.yaml"), "v1-2: 2"}, + {filepath.FromSlash("data/test/v2.yaml"), "v2:\n- 2\n- 3"}, + {filepath.FromSlash("data/test.yaml"), "v1: 1"}, } expected := map[string]interface{}{"a": map[string]interface{}{"a": 1}, @@ -85,10 +84,10 @@ func TestDataDirYAMLWithOverridenValue(t *testing.T) { func TestDataDirMultipleSources(t *testing.T) { t.Parallel() - sources := []source.ByteSource{ - {Name: filepath.FromSlash("data/test/first.toml"), Content: []byte("bar = 1")}, - {Name: filepath.FromSlash("themes/mytheme/data/test/first.toml"), Content: []byte("bar = 2")}, - {Name: filepath.FromSlash("data/test/second.toml"), Content: []byte("tender = 2")}, + sources := [][2]string{ + {filepath.FromSlash("data/test/first.toml"), "bar = 1"}, + {filepath.FromSlash("themes/mytheme/data/test/first.toml"), "bar = 2"}, + {filepath.FromSlash("data/test/second.toml"), "tender = 2"}, } expected, _ := parser.HandleTOMLMetaData([]byte("[test.first]\nbar = 1\n[test.second]\ntender=2")) @@ -98,7 +97,7 @@ func TestDataDirMultipleSources(t *testing.T) { } -func doTestDataDir(t *testing.T, expected interface{}, sources []source.ByteSource, configKeyValues ...interface{}) { +func doTestDataDir(t *testing.T, expected interface{}, sources [][2]string, configKeyValues ...interface{}) { var ( cfg, fs = newTestCfg() ) diff --git a/hugolib/fileInfo.go b/hugolib/fileInfo.go new file mode 100644 index 000000000..14dd8dbf9 --- /dev/null +++ b/hugolib/fileInfo.go @@ -0,0 +1,109 @@ +// Copyright 2017-present The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "os" + "strings" + + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/source" +) + +// fileInfo implements the File and ReadableFile interface. +var ( + _ source.File = (*fileInfo)(nil) + _ source.ReadableFile = (*fileInfo)(nil) +) + +type fileInfo struct { + bundleTp bundleDirType + source.ReadableFile + overriddenLang string +} + +func (fi *fileInfo) Lang() string { + if fi.overriddenLang != "" { + return fi.overriddenLang + } + return fi.ReadableFile.Lang() +} + +func (fi *fileInfo) isOwner() bool { + return fi.bundleTp > bundleNot +} + +func isContentFile(filename string) bool { + return contentFileExtensionsSet[strings.TrimPrefix(helpers.Ext(filename), ".")] +} + +func (fi *fileInfo) isContentFile() bool { + return contentFileExtensionsSet[fi.Ext()] +} + +func newFileInfo(sp *source.SourceSpec, baseDir, filename string, fi os.FileInfo, tp bundleDirType) *fileInfo { + + baseFi := sp.NewFileInfo(baseDir, filename, fi) + f := &fileInfo{ + bundleTp: tp, + ReadableFile: baseFi, + } + + return f + +} + +type bundleDirType int + +const ( + bundleNot bundleDirType = iota + + // All from here are bundles in one form or another. + bundleLeaf + bundleBranch +) + +// Returns the given file's name's bundle type and whether it is a content +// file or not. +func classifyBundledFile(name string) (bundleDirType, bool) { + if !isContentFile(name) { + return bundleNot, false + } + if strings.HasPrefix(name, "_index.") { + return bundleBranch, true + } + + if strings.HasPrefix(name, "index.") { + return bundleLeaf, true + } + + return bundleNot, true +} + +func (b bundleDirType) String() string { + switch b { + case bundleNot: + return "Not a bundle" + case bundleLeaf: + return "Regular bundle" + case bundleBranch: + return "Branch bundle" + } + + return "" +} + +func (b bundleDirType) isBundle() bool { + return b > bundleNot +} diff --git a/hugolib/fileInfo_test.go b/hugolib/fileInfo_test.go new file mode 100644 index 000000000..18579c078 --- /dev/null +++ b/hugolib/fileInfo_test.go @@ -0,0 +1,61 @@ +// Copyright 2017-present The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "testing" + + "path/filepath" + + "github.com/gohugoio/hugo/source" + "github.com/stretchr/testify/require" +) + +func TestBundleFileInfo(t *testing.T) { + t.Parallel() + + assert := require.New(t) + cfg, fs := newTestBundleSourcesMultilingual(t) + sourceSpec := source.NewSourceSpec(cfg, fs) + + for _, this := range []struct { + filename string + check func(f *fileInfo) + }{ + {"/path/to/file.md", func(fi *fileInfo) { + assert.Equal("md", fi.Ext()) + assert.Equal("en", fi.Lang()) + assert.False(fi.isOwner()) + assert.True(fi.isContentFile()) + }}, + {"/path/to/file.JPG", func(fi *fileInfo) { + assert.Equal("jpg", fi.Ext()) + assert.False(fi.isContentFile()) + }}, + {"/path/to/file.nn.png", func(fi *fileInfo) { + assert.Equal("png", fi.Ext()) + assert.Equal("nn", fi.Lang()) + assert.Equal("file", fi.TranslationBaseName()) + assert.False(fi.isContentFile()) + }}, + } { + fi := newFileInfo( + sourceSpec, + filepath.FromSlash("/work/base"), + filepath.FromSlash(this.filename), + nil, bundleNot) + this.check(fi) + } + +} diff --git a/hugolib/handler_base.go b/hugolib/handler_base.go deleted file mode 100644 index d7e4a63a3..000000000 --- a/hugolib/handler_base.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "github.com/gohugoio/hugo/source" -) - -// Handler is used for processing files of a specific type. -type Handler interface { - FileConvert(*source.File, *Site) HandledResult - PageConvert(*Page) HandledResult - Read(*source.File, *Site) HandledResult - Extensions() []string -} - -// Handle identifies functionality associated with certain file extensions. -type Handle struct { - extensions []string -} - -// Extensions returns a list of extensions. -func (h Handle) Extensions() []string { - return h.extensions -} - -// HandledResult describes the results of a file handling operation. -type HandledResult struct { - page *Page - file *source.File - err error -} - -// HandledResult is an error -func (h HandledResult) Error() string { - if h.err != nil { - if h.page != nil { - return "Error: " + h.err.Error() + " for " + h.page.File.LogicalName() - } - if h.file != nil { - return "Error: " + h.err.Error() + " for " + h.file.LogicalName() - } - } - return h.err.Error() -} - -func (h HandledResult) String() string { - return h.Error() -} - -// Page returns the affected page. -func (h HandledResult) Page() *Page { - return h.page -} diff --git a/hugolib/handler_file.go b/hugolib/handler_file.go deleted file mode 100644 index 82ea85fb2..000000000 --- a/hugolib/handler_file.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "bytes" - - "github.com/dchest/cssmin" - "github.com/gohugoio/hugo/source" -) - -func init() { - RegisterHandler(new(cssHandler)) - RegisterHandler(new(defaultHandler)) -} - -type basicFileHandler Handle - -func (h basicFileHandler) Read(f *source.File, s *Site) HandledResult { - return HandledResult{file: f} -} - -func (h basicFileHandler) PageConvert(*Page) HandledResult { - return HandledResult{} -} - -type defaultHandler struct{ basicFileHandler } - -func (h defaultHandler) Extensions() []string { return []string{"*"} } -func (h defaultHandler) FileConvert(f *source.File, s *Site) HandledResult { - err := s.publish(f.Path(), f.Contents) - if err != nil { - return HandledResult{err: err} - } - return HandledResult{file: f} -} - -type cssHandler struct{ basicFileHandler } - -func (h cssHandler) Extensions() []string { return []string{"css"} } -func (h cssHandler) FileConvert(f *source.File, s *Site) HandledResult { - x := cssmin.Minify(f.Bytes()) - err := s.publish(f.Path(), bytes.NewReader(x)) - if err != nil { - return HandledResult{err: err} - } - return HandledResult{file: f} -} diff --git a/hugolib/handler_meta.go b/hugolib/handler_meta.go deleted file mode 100644 index c1aaf5f8c..000000000 --- a/hugolib/handler_meta.go +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "errors" - - "fmt" - - "github.com/gohugoio/hugo/source" -) - - |