diff options
author | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2018-08-05 11:13:49 +0200 |
---|---|---|
committer | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2018-08-06 19:58:41 +0200 |
commit | 789ef8c639e4621abd36da530bcb5942ac9297da (patch) | |
tree | f225fc3663affc49805f1d309b77b096d40fc8f6 | |
parent | 71931b30b1813b146aaa60f5cdab16c0f9ebebdb (diff) |
Add support for minification of final output
Hugo Pipes added minification support for resources fetched via ´resources.Get` and similar.
This also adds support for minification of the final output for supported output formats: HTML, XML, SVG, CSS, JavaScript, JSON.
To enable, run Hugo with the `--minify` flag:
```bash
hugo --minify
```
This commit is also a major spring cleaning of the `transform` package to allow the new minification step fit into that processing chain.
Fixes #1251
32 files changed, 964 insertions, 522 deletions
diff --git a/commands/commands.go b/commands/commands.go index 88939e600..17c9e15cc 100644 --- a/commands/commands.go +++ b/commands/commands.go @@ -177,6 +177,7 @@ Complete documentation is available at http://gohugo.io/.`, cc.cmd.Flags().BoolVarP(&cc.buildWatch, "watch", "w", false, "watch filesystem for changes and recreate as needed") cc.cmd.Flags().Bool("renderToMemory", false, "render to memory (only useful for benchmark testing)") + cc.cmd.Flags().Bool("minify", false, "minify any supported output format (HTML, XML etc.)") // Set bash-completion _ = cc.cmd.PersistentFlags().SetAnnotation("logFile", cobra.BashCompFilenameExt, []string{}) diff --git a/commands/hugo.go b/commands/hugo.go index 312767499..1381bf210 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -209,16 +209,25 @@ func initializeFlags(cmd *cobra.Command, cfg config.Provider) { "verboseLog", } + // Will set a value even if it is the default. + flagKeysForced := []string{ + "minify", + } + for _, key := range persFlagKeys { - setValueFromFlag(cmd.PersistentFlags(), key, cfg, "") + setValueFromFlag(cmd.PersistentFlags(), key, cfg, "", false) } for _, key := range flagKeys { - setValueFromFlag(cmd.Flags(), key, cfg, "") + setValueFromFlag(cmd.Flags(), key, cfg, "", false) + } + + for _, key := range flagKeysForced { + setValueFromFlag(cmd.Flags(), key, cfg, "", true) } // Set some "config aliases" - setValueFromFlag(cmd.Flags(), "destination", cfg, "publishDir") - setValueFromFlag(cmd.Flags(), "i18n-warnings", cfg, "logI18nWarnings") + setValueFromFlag(cmd.Flags(), "destination", cfg, "publishDir", false) + setValueFromFlag(cmd.Flags(), "i18n-warnings", cfg, "logI18nWarnings", false) } @@ -229,9 +238,9 @@ var deprecatedFlags = map[string]bool{ strings.ToLower("canonifyURLs"): true, } -func setValueFromFlag(flags *flag.FlagSet, key string, cfg config.Provider, targetKey string) { +func setValueFromFlag(flags *flag.FlagSet, key string, cfg config.Provider, targetKey string, force bool) { key = strings.TrimSpace(key) - if flags.Changed(key) { + if (force && flags.Lookup(key) != nil) || flags.Changed(key) { if _, deprecated := deprecatedFlags[strings.ToLower(key)]; deprecated { msg := fmt.Sprintf(`Set "%s = true" in your config.toml. If you need to set this configuration value from the command line, set it via an OS environment variable: "HUGO_%s=true hugo"`, key, strings.ToUpper(key)) diff --git a/deps/deps.go b/deps/deps.go index a0904483b..3eecffca2 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -162,7 +162,7 @@ func New(cfg DepsCfg) (*Deps, error) { return nil, err } - resourceSpec, err := resource.NewSpec(ps, logger, cfg.MediaTypes) + resourceSpec, err := resource.NewSpec(ps, logger, cfg.OutputFormats, cfg.MediaTypes) if err != nil { return nil, err } @@ -224,7 +224,7 @@ func (d Deps) ForLanguage(cfg DepsCfg) (*Deps, error) { // The resource cache is global so reuse. // TODO(bep) clean up these inits. resourceCache := d.ResourceSpec.ResourceCache - d.ResourceSpec, err = resource.NewSpec(d.PathSpec, d.Log, cfg.MediaTypes) + d.ResourceSpec, err = resource.NewSpec(d.PathSpec, d.Log, cfg.OutputFormats, cfg.MediaTypes) if err != nil { return nil, err } @@ -267,6 +267,9 @@ type DepsCfg struct { // The media types configured. MediaTypes media.Types + // The output formats configured. + OutputFormats output.Formats + // Template handling. TemplateProvider ResourceProvider WithTemplate func(templ tpl.TemplateHandler) error diff --git a/helpers/path.go b/helpers/path.go index 92ce4079f..134a20527 100644 --- a/helpers/path.go +++ b/helpers/path.go @@ -515,6 +515,25 @@ func WriteToDisk(inpath string, r io.Reader, fs afero.Fs) (err error) { return afero.WriteReader(fs, inpath, r) } +// OpenFileForWriting opens or creates the given file. If the target directory +// does not exist, it gets created. +func OpenFileForWriting(fs afero.Fs, filename string) (afero.File, error) { + filename = filepath.Clean(filename) + // Create will truncate if file already exists. + f, err := fs.Create(filename) + if err != nil { + if !os.IsNotExist(err) { + return nil, err + } + if err = fs.MkdirAll(filepath.Dir(filename), 0755); err != nil { + return nil, err + } + f, err = fs.Create(filename) + } + + return f, err +} + // GetTempDir returns a temporary directory with the given sub path. func GetTempDir(subPath string, fs afero.Fs) string { return afero.GetTempDir(fs, subPath) diff --git a/hugolib/alias.go b/hugolib/alias.go index 3b053130e..b2b296143 100644 --- a/hugolib/alias.go +++ b/hugolib/alias.go @@ -22,6 +22,8 @@ import ( "runtime" "strings" + "github.com/gohugoio/hugo/output" + "github.com/gohugoio/hugo/publisher" "github.com/gohugoio/hugo/tpl" jww "github.com/spf13/jwalterweatherman" @@ -89,11 +91,11 @@ func (a aliasHandler) renderAlias(isXHTML bool, permalink string, page *Page) (i return buffer, nil } -func (s *Site) writeDestAlias(path, permalink string, p *Page) (err error) { - return s.publishDestAlias(false, path, permalink, p) +func (s *Site) writeDestAlias(path, permalink string, outputFormat output.Format, p *Page) (err error) { + return s.publishDestAlias(false, path, permalink, outputFormat, p) } -func (s *Site) publishDestAlias(allowRoot bool, path, permalink string, p *Page) (err error) { +func (s *Site) publishDestAlias(allowRoot bool, path, permalink string, outputFormat output.Format, p *Page) (err error) { handler := newAliasHandler(s.Tmpl, s.Log, allowRoot) isXHTML := strings.HasSuffix(path, ".xhtml") @@ -110,7 +112,14 @@ func (s *Site) publishDestAlias(allowRoot bool, path, permalink string, p *Page) return err } - return s.publish(&s.PathSpec.ProcessingStats.Aliases, targetPath, aliasContent) + pd := publisher.Descriptor{ + Src: aliasContent, + TargetPath: targetPath, + StatCounter: &s.PathSpec.ProcessingStats.Aliases, + OutputFormat: outputFormat, + } + + return s.publisher.Publish(pd) } diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index 859e0da7c..6ce6657fa 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -24,6 +24,7 @@ import ( "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/langs" + "github.com/gohugoio/hugo/publisher" "github.com/gohugoio/hugo/i18n" "github.com/gohugoio/hugo/tpl" @@ -182,6 +183,7 @@ func applyDeps(cfg deps.DepsCfg, sites ...*Site) error { cfg.Language = s.Language cfg.MediaTypes = s.mediaTypesConfig + cfg.OutputFormats = s.outputFormatsConfig if d == nil { cfg.WithTemplate = s.withSiteTemplates(cfg.WithTemplate) @@ -208,6 +210,9 @@ func applyDeps(cfg deps.DepsCfg, sites ...*Site) error { s.Deps = d } + // Set up the main publishing chain. + s.publisher = publisher.NewDestinationPublisher(d.PathSpec.BaseFs.PublishFs, s.outputFormatsConfig, s.mediaTypesConfig, cfg.Cfg.GetBool("minify")) + if err := s.initializeSiteInfo(); err != nil { return err } diff --git a/hugolib/minify_publisher_test.go b/hugolib/minify_publisher_test.go new file mode 100644 index 000000000..ce183343b --- /dev/null +++ b/hugolib/minify_publisher_test.go @@ -0,0 +1,71 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "testing" + + "github.com/spf13/viper" + + "github.com/stretchr/testify/require" +) + +func TestMinifyPublisher(t *testing.T) { + t.Parallel() + assert := require.New(t) + + v := viper.New() + v.Set("minify", true) + v.Set("baseURL", "https://example.org/") + + htmlTemplate := ` +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="utf-8"> + <title>HTML5 boilerplate – all you really need…</title> + <link rel="stylesheet" href="css/style.css"> + <!--[if IE]> + <script src="http://html5shiv.googlecode.com/svn/trunk/html5.js"></script> + <![endif]--> +</head> + +<body id="home"> + + <h1>{{ .Page.Title }}</h1> + +</body> +</html> +` + + b := newTestSitesBuilder(t) + b.WithViper(v).WithContent("page.md", pageWithAlias) + b.WithTemplates("_default/list.html", htmlTemplate, "_default/single.html", htmlTemplate, "alias.html", htmlTemplate) + b.CreateSites().Build(BuildCfg{}) + + assert.Equal(1, len(b.H.Sites)) + require.Len(t, b.H.Sites[0].RegularPages, 1) + + // Check minification + // HTML + b.AssertFileContent("public/page/index.html", "<!doctype html><html lang=en><head><meta charset=utf-8><title>HTML5 boilerplate – all you really need…</title><link rel=stylesheet href=css/style.css></head><body id=home><h1>Has Alias</h1></body></html>") + // HTML alias. Note the custom template which does no redirect. + b.AssertFileContent("public/foo/bar/index.html", "<!doctype html><html lang=en><head><meta charset=utf-8><title>HTML5 boilerplate ") + + // RSS + b.AssertFileContent("public/index.xml", "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\"?><rss version=\"2.0\" xmlns:atom=\"http://www.w3.org/2005/Atom\"><channel><title/><link>https://example.org/</link>") + + // Sitemap + b.AssertFileContent("public/sitemap.xml", "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\"?><urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\" xmlns:xhtml=\"http://www.w3.org/1999/xhtml\"><url><loc>https://example.org/</loc><priority>0</priority></url><url>") +} diff --git a/hugolib/resource_chain_test.go b/hugolib/resource_chain_test.go index 3e199d318..0a4c3bd1f 100644 --- a/hugolib/resource_chain_test.go +++ b/hugolib/resource_chain_test.go @@ -223,7 +223,7 @@ Min HTML: {{ ( resources.Get "mydata/html1.html" | resources.Minify ).Content | b.AssertFileContent("public/index.html", `Min XML: <hello><world>Hugo Rocks!</<world></hello>`) b.AssertFileContent("public/index.html", `Min SVG: <svg height="100" width="100"><path d="M5 10 20 40z"/></svg>`) b.AssertFileContent("public/index.html", `Min SVG again: <svg height="100" width="100"><path d="M5 10 20 40z"/></svg>`) - b.AssertFileContent("public/index.html", `Min HTML: <a href=#>Cool</a>`) + b.AssertFileContent("public/index.html", `Min HTML: <html><a href=#>Cool</a></html>`) }}, {"concat", func() bool { return true }, func(b *sitesBuilder) { diff --git a/hugolib/site.go b/hugolib/site.go index 4cca648f3..b55b6040b 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -29,6 +29,7 @@ import ( "time" "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/publisher" "github.com/gohugoio/hugo/resource" "github.com/gohugoio/hugo/langs" @@ -54,15 +55,12 @@ import ( "github.com/gohugoio/hugo/related" "github.com/gohugoio/hugo/source" "github.com/gohugoio/hugo/tpl" - "github.com/gohugoio/hugo/transform" "github.com/spf13/afero" "github.com/spf13/cast" "github.com/spf13/nitro" "github.com/spf13/viper" ) -var _ = transform.AbsURL - // used to indicate if run as a test. var testMode bool @@ -151,6 +149,8 @@ type Site struct { relatedDocsHandler *relatedDocsHandler siteRefLinker + + publisher publisher.Publisher } type siteRenderingContext struct { @@ -195,6 +195,7 @@ func (s *Site) reset() *Site { mediaTypesConfig: s.mediaTypesConfig, Language: s.Language, owner: s.owner, + publisher: s.publisher, siteConfig: s.siteConfig, PageCollections: newPageCollections()} @@ -759,8 +760,9 @@ func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) { site := sites[i] var err error depsCfg := deps.DepsCfg{ - Language: site.Language, - MediaTypes: site.mediaTypesConfig, + Language: site.Language, + MediaTypes: site.mediaTypesConfig, + OutputFormats: site.outputFormatsConfig, } site.Deps, err = first.Deps.ForLanguage(depsCfg) if err != nil { @@ -1637,8 +1639,8 @@ func (s *Site) permalink(link string) string { } -func (s *Site) renderAndWriteXML(statCounter *uint64, name string, dest string, d interface{}, layouts ...string) error { - s.Log.DEBUG.Printf("Render XML for %q to %q", name, dest) +func (s *Site) renderAndWriteXML(statCounter *uint64, name string, targetPath string, d interface{}, layouts ...string) error { + s.Log.DEBUG.Printf("Render XML for %q to %q", name, targetPath) renderBuffer := bp.GetBuffer() defer bp.PutBuffer(renderBuffer) renderBuffer.WriteString("<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\" ?>\n") @@ -1648,30 +1650,32 @@ func (s *Site) renderAndWriteXML(statCounter *uint64, name string, dest string, return nil } - outBuffer := bp.GetBuffer() - defer bp.PutBuffer(outBuffer) - - var path []byte + var path string if s.Info.relativeURLs { - path = []byte(helpers.GetDottedRelativePath(dest)) + path = helpers.GetDottedRelativePath(targetPath) } else { s := s.PathSpec.BaseURL.String() if !strings.HasSuffix(s, "/") { s += "/" } - path = []byte(s) + path = s } - transformer := transform.NewChain(transform.AbsURLInXML) - if err := transformer.Apply(outBuffer, renderBuffer, path); err != nil { - s.DistinctErrorLog.Println(err) - return nil + + pd := publisher.Descriptor{ + Src: renderBuffer, + TargetPath: targetPath, + StatCounter: statCounter, + // For the minification part of XML, + // we currently only use the MIME type. + OutputFormat: output.RSSFormat, + AbsURLPath: path, } - return s.publish(statCounter, dest, outBuffer) + return s.publisher.Publish(pd) } -func (s *Site) renderAndWritePage(statCounter *uint64, name string, dest string, p *PageOutput, layouts ...string) error { +func (s *Site) renderAndWritePage(statCounter *uint64, name string, targetPath string, p *PageOutput, layouts ...string) error { renderBuffer := bp.GetBuffer() defer bp.PutBuffer(renderBuffer) @@ -1684,49 +1688,44 @@ func (s *Site) renderAndWritePage(statCounter *uint64, name string, dest string, return nil } - outBuffer := bp.GetBuffer() - defer bp.PutBuffer(outBuffer) + isHTML := p.outputFormat.IsHTML - transformLinks := transform.NewEmptyTransforms() + var path string - isHTML := p.outputFormat.IsHTML + if s.Info.relativeURLs { + path = helpers.GetDottedRelativePath(targetPath) + } else if s.Info.canonifyURLs { + url := s.PathSpec.BaseURL.String() + if !strings.HasSuffix(url, "/") { + url += "/" + } + path = url + } + + pd := publisher.Descriptor{ + Src: renderBuffer, + TargetPath: targetPath, + StatCounter: statCounter, + OutputFormat: p.outputFormat, + } if isHTML { if s.Info.relativeURLs || s.Info.canonifyURLs { - transformLinks = append(transformLinks, transform.AbsURL) + pd.AbsURLPath = path } if s.running() && s.Cfg.GetBool("watch") && !s.Cfg.GetBool("disableLiveReload") { - transformLinks = append(transformLinks, transform.LiveReloadInject(s.Cfg.GetInt("liveReloadPort"))) + pd.LiveReloadPort = s.Cfg.GetInt("liveReloadPort") } // For performance reasons we only inject the Hugo generator tag on the home page. if p.IsHome() { - if !s.Cfg.GetBool("disableHugoGeneratorInject") { - transformLinks = append(transformLinks, transform.HugoGeneratorInject) - } - } - } - - var path []byte - - if s.Info.relativeURLs { - path = []byte(helpers.GetDottedRelativePath(dest)) - } else if s.Info.canonifyURLs { - url := s.PathSpec.BaseURL.String() - if !strings.HasSuffix(url, "/") { - url += "/" + pd.AddHugoGeneratorTag = !s.Cfg.GetBool("disableHugoGeneratorInject") } - path = []byte(url) - } - transformer := transform.NewChain(transformLinks...) - if err := transformer.Apply(outBuffer, renderBuffer, path); err != nil { - s.DistinctErrorLog.Println(err) - return nil } - return s.publish(statCounter, dest, outBuffer) + return s.publisher.Publish(pd) } func (s *Site) renderForLayouts(name string, d interface{}, w io.Writer, layouts ...string) (err error) { diff --git a/hugolib/site_render.go b/hugolib/site_render.go index 2da4064b4..a0d6506e2 100644 --- a/hugolib/site_render.go +++ b/hugolib/site_render.go @@ -195,7 +195,7 @@ func (s *Site) renderPaginator(p *PageOutput) error { // TODO(bep) do better link := newOutputFormat(p.Page, p.outputFormat).Permalink() - if err := s.writeDestAlias(target, link, nil); err != nil { + if err := s.writeDestAlias(target, link, p.outputFormat, nil); err != nil { return err } @@ -417,7 +417,7 @@ func (s *Site) renderAliases() error { a = path.Join(lang, a) } - if err := s.writeDestAlias(a, plink, p); err != nil { + if err := s.writeDestAlias(a, plink, f, p); err != nil { return err } } @@ -425,18 +425,21 @@ func (s *Site) renderAliases() error { } if s.owner.multilingual.enabled() && !s.owner.IsMultihost() { - mainLang := s.owner.multilingual.DefaultLang - if s.Info.defaultContentLanguageInSubdir { - mainLangURL := s.PathSpec.AbsURL(mainLang.Lang, false) - s.Log.DEBUG.Printf("Write redirect to main language %s: %s", mainLang, mainLangURL) - if err := s.publishDestAlias(true, "/", mainLangURL, nil); err != nil { - return err - } - } else { - mainLangURL := s.PathSpec.AbsURL("", false) - s.Log.DEBUG.Printf("Write redirect to main language %s: %s", mainLang, mainLangURL) - if err := s.publishDestAlias(true, mainLang.Lang, mainLangURL, nil); err != nil { - return err + html, found := s.outputFormatsConfig.GetByName("HTML") + if found { + mainLang := s.owner.multilingual.DefaultLang + if s.Info.defaultContentLanguageInSubdir { + mainLangURL := s.PathSpec.AbsURL(mainLang.Lang, false) + s.Log.DEBUG.Printf("Write redirect to main language %s: %s", mainLang, mainLangURL) + if err := s.publishDestAlias(true, "/", mainLangURL, html, nil); err != nil { + return err + } + } else { + mainLangURL := s.PathSpec.AbsURL("", false) + s.Log.DEBUG.Printf("Write redirect to main language %s: %s", mainLang, mainLangURL) + if err := s.publishDestAlias(true, mainLang.Lang, mainLangURL, html, nil); err != nil { + return err + } } } } diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go index cee1f8c4c..4ba951449 100644 --- a/hugolib/testhelpers_test.go +++ b/hugolib/testhelpers_test.go @@ -130,6 +130,7 @@ func (s *sitesBuilder) WithConfigTemplate(data interface{}, format, configTempla func (s *sitesBuilder) WithViper(v *viper.Viper) *sitesBuilder { loadDefaultSettingsFor(v) s.Cfg = v + return s } diff --git a/minifiers/minifiers.go b/minifiers/minifiers.go new file mode 100644 index 000000000..70d42843f --- /dev/null +++ b/minifiers/minifiers.go @@ -0,0 +1,126 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package minifiers contains minifiers mapped to MIME types. This package is used +// in both the resource transformation, i.e. resources.Minify, and in the publishing +// chain. +package minifiers + +import ( |