diff options
46 files changed, 1258 insertions, 1372 deletions
diff --git a/commands/commandeer.go b/commands/commandeer.go index c9059dd0c..761d79912 100644 --- a/commands/commandeer.go +++ b/commands/commandeer.go @@ -133,7 +133,7 @@ func (c *commandeer) getErrorWithContext() interface{} { if c.h.verbose { var b bytes.Buffer - herrors.FprintStackTrace(&b, c.buildErr) + herrors.FprintStackTraceFromErr(&b, c.buildErr) m["StackTrace"] = b.String() } diff --git a/commands/hugo.go b/commands/hugo.go index d319dda8f..b2b0981a9 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -716,7 +716,7 @@ func (c *commandeer) handleBuildErr(err error, msg string) { c.logger.ERROR.Print(msg + ":\n\n") c.logger.ERROR.Println(helpers.FirstUpper(err.Error())) if !c.h.quiet && c.h.verbose { - herrors.PrintStackTrace(err) + herrors.PrintStackTraceFromErr(err) } } diff --git a/commands/server.go b/commands/server.go index 64409ee18..728847492 100644 --- a/commands/server.go +++ b/commands/server.go @@ -416,7 +416,7 @@ func (c *commandeer) serve(s *serverCmd) error { roots = []string{""} } - templ, err := c.hugo().TextTmpl.Parse("__default_server_error", buildErrorTemplate) + templ, err := c.hugo().TextTmpl().Parse("__default_server_error", buildErrorTemplate) if err != nil { return err } @@ -428,7 +428,7 @@ func (c *commandeer) serve(s *serverCmd) error { s: s, errorTemplate: func(ctx interface{}) (io.Reader, error) { b := &bytes.Buffer{} - err := c.hugo().Tmpl.Execute(templ, b, ctx) + err := c.hugo().Tmpl().Execute(templ, b, ctx) return b, err }, } diff --git a/common/herrors/errors.go b/common/herrors/errors.go index ff8eab116..5fae6fcae 100644 --- a/common/herrors/errors.go +++ b/common/herrors/errors.go @@ -15,11 +15,14 @@ package herrors import ( + "bytes" "errors" "fmt" "io" "os" + "runtime" "runtime/debug" + "strconv" _errors "github.com/pkg/errors" ) @@ -33,13 +36,13 @@ type stackTracer interface { StackTrace() _errors.StackTrace } -// PrintStackTrace prints the error's stack trace to stdoud. -func PrintStackTrace(err error) { - FprintStackTrace(os.Stdout, err) +// PrintStackTraceFromErr prints the error's stack trace to stdoud. +func PrintStackTraceFromErr(err error) { + FprintStackTraceFromErr(os.Stdout, err) } -// FprintStackTrace prints the error's stack trace to w. -func FprintStackTrace(w io.Writer, err error) { +// FprintStackTraceFromErr prints the error's stack trace to w. +func FprintStackTraceFromErr(w io.Writer, err error) { if err, ok := err.(stackTracer); ok { for _, f := range err.StackTrace() { fmt.Fprintf(w, "%+s:%d\n", f, f) @@ -47,6 +50,13 @@ func FprintStackTrace(w io.Writer, err error) { } } +// PrintStackTrace prints the current stacktrace to w. +func PrintStackTrace(w io.Writer) { + buf := make([]byte, 1<<16) + runtime.Stack(buf, true) + fmt.Fprintf(w, "%s", buf) +} + // Recover is a helper function that can be used to capture panics. // Put this at the top of a method/function that crashes in a template: // defer herrors.Recover() @@ -56,7 +66,16 @@ func Recover(args ...interface{}) { args = append(args, "stacktrace from panic: \n"+string(debug.Stack()), "\n") fmt.Println(args...) } +} +// Get the current goroutine id. Used only for debugging. +func GetGID() uint64 { + b := make([]byte, 64) + b = b[:runtime.Stack(b, false)] + b = bytes.TrimPrefix(b, []byte("goroutine ")) + b = b[:bytes.IndexByte(b, ' ')] + n, _ := strconv.ParseUint(string(b), 10, 64) + return n } // ErrFeatureNotAvailable denotes that a feature is unavailable. diff --git a/common/types/types.go b/common/types/types.go index f03031439..04a27766e 100644 --- a/common/types/types.go +++ b/common/types/types.go @@ -21,6 +21,12 @@ import ( "github.com/spf13/cast" ) +// RLocker represents the read locks in sync.RWMutex. +type RLocker interface { + RLock() + RUnlock() +} + // KeyValueStr is a string tuple. type KeyValueStr struct { Key string diff --git a/create/content_template_handler.go b/create/content_template_handler.go index b70cf02eb..e4cddedf5 100644 --- a/create/content_template_handler.go +++ b/create/content_template_handler.go @@ -129,9 +129,9 @@ func executeArcheTypeAsTemplate(s *hugolib.Site, name, kind, targetPath, archety archetypeTemplate = []byte(archetypeShortcodeReplacementsPre.Replace(string(archetypeTemplate))) // Reuse the Hugo template setup to get the template funcs properly set up. - templateHandler := s.Deps.Tmpl.(tpl.TemplateManager) - templateName := "_text/" + helpers.Filename(archetypeFilename) - if err := templateHandler.AddTemplate(templateName, string(archetypeTemplate)); err != nil { + templateHandler := s.Deps.Tmpl().(tpl.TemplateManager) + templateName := helpers.Filename(archetypeFilename) + if err := templateHandler.AddTemplate("_text/"+templateName, string(archetypeTemplate)); err != nil { return nil, errors.Wrapf(err, "Failed to parse archetype file %q:", archetypeFilename) } diff --git a/deps/deps.go b/deps/deps.go index ecbba2e56..092a0b887 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -5,6 +5,7 @@ import ( "time" "github.com/pkg/errors" + "go.uber.org/atomic" "github.com/gohugoio/hugo/cache/filecache" "github.com/gohugoio/hugo/common/loggers" @@ -38,10 +39,10 @@ type Deps struct { DistinctWarningLog *helpers.DistinctLogger // The templates to use. This will usually implement the full tpl.TemplateManager. - Tmpl tpl.TemplateHandler `json:"-"` + tmpl tpl.TemplateHandler // We use this to parse and execute ad-hoc text templates. - TextTmpl tpl.TemplateParseFinder `json:"-"` + textTmpl tpl.TemplateParseFinder // The file systems to use. Fs *hugofs.Fs `json:"-"` @@ -92,6 +93,9 @@ type Deps struct { // BuildStartListeners will be notified before a build starts. BuildStartListeners *Listeners + // Atomic flags set during a build. + BuildFlags BuildFlags + *globalErrHandler } @@ -153,9 +157,20 @@ type ResourceProvider interface { Clone(deps *Deps) error } -// TemplateHandler returns the used tpl.TemplateFinder as tpl.TemplateHandler. -func (d *Deps) TemplateHandler() tpl.TemplateManager { - return d.Tmpl.(tpl.TemplateManager) +func (d *Deps) Tmpl() tpl.TemplateHandler { + return d.tmpl +} + +func (d *Deps) TextTmpl() tpl.TemplateParseFinder { + return d.textTmpl +} + +func (d *Deps) SetTmpl(tmpl tpl.TemplateHandler) { + d.tmpl = tmpl +} + +func (d *Deps) SetTextTmpl(tmpl tpl.TemplateParseFinder) { + d.textTmpl = tmpl } // LoadResources loads translations and templates. @@ -315,6 +330,7 @@ func (d Deps) ForLanguage(cfg DepsCfg, onCreated func(d *Deps) error) (*Deps, er } d.BuildStartListeners = &Listeners{} + d.BuildFlags = BuildFlags{} return &d, nil @@ -358,3 +374,14 @@ type DepsCfg struct { // Whether we are in running (server) mode Running bool } + +// BuildFlags are flags that may be turned on during a build. +type BuildFlags struct { + HasLateTemplate atomic.Bool +} + +func NewBuildFlags() BuildFlags { + return BuildFlags{ + //HasLateTemplate: atomic.NewBool(false), + } +} diff --git a/deps/deps_test.go b/deps/deps_test.go new file mode 100644 index 000000000..e2dca0ecc --- /dev/null +++ b/deps/deps_test.go @@ -0,0 +1,28 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package deps + +import ( + "testing" + + qt "github.com/frankban/quicktest" +) + +func TestBuildFlags(t *testing.T) { + c := qt.New(t) + var bf BuildFlags + c.Assert(bf.HasLateTemplate.Load(), qt.Equals, false) + bf.HasLateTemplate.Store(true) + c.Assert(bf.HasLateTemplate.Load(), qt.Equals, true) +} @@ -55,6 +55,7 @@ require ( github.com/yuin/goldmark v1.1.21 github.com/yuin/goldmark-highlighting v0.0.0-20191202084645-78f32c8dd6d5 go.opencensus.io v0.22.0 // indirect + go.uber.org/atomic v1.4.0 gocloud.dev v0.15.0 golang.org/x/image v0.0.0-20191214001246-9130b4cfad52 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect @@ -421,6 +421,7 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= diff --git a/hugolib/alias.go b/hugolib/alias.go index c80e7d0d2..9eba8b335 100644 --- a/hugolib/alias.go +++ b/hugolib/alias.go @@ -81,7 +81,7 @@ func (s *Site) writeDestAlias(path, permalink string, outputFormat output.Format } func (s *Site) publishDestAlias(allowRoot bool, path, permalink string, outputFormat output.Format, p page.Page) (err error) { - handler := newAliasHandler(s.Tmpl, s.Log, allowRoot) + handler := newAliasHandler(s.Tmpl(), s.Log, allowRoot) s.Log.DEBUG.Println("creating alias:", path, "redirecting to", permalink) diff --git a/hugolib/cascade_test.go b/hugolib/cascade_test.go index 6b176ad64..be243e39b 100644 --- a/hugolib/cascade_test.go +++ b/hugolib/cascade_test.go @@ -68,12 +68,12 @@ func TestCascade(t *testing.T) { 42|taxonomy|tags/blue|blue|home.png|tags|HTML-| 42|section|sect3|Cascade Home|home.png|sect3|HTML-| 42|taxonomyTerm|tags|Cascade Home|home.png|tags|HTML-| - 42|page|p2.md|Cascade Home|home.png|page|HTML-| + 42|page|p2.md|Cascade Home|home.png||HTML-| 42|page|sect2/p2.md|Cascade Home|home.png|sect2|HTML-| 42|page|sect3/p1.md|Cascade Home|home.png|sect3|HTML-| 42|taxonomy|tags/green|green|home.png|tags|HTML-| - 42|home|_index.md|Home|home.png|page|HTML-| - 42|page|p1.md|p1|home.png|page|HTML-| + 42|home|_index.md|Home|home.png||HTML-| + 42|page|p1.md|p1|home.png||HTML-| 42|section|sect1/_index.md|Sect1|sect1.png|stype|HTML-| 42|section|sect1/s1_2/_index.md|Sect1_2|sect1.png|stype|HTML-| 42|page|sect1/s1_2/p1.md|Sect1_2_p1|sect1.png|stype|HTML-| diff --git a/hugolib/content_render_hooks_test.go b/hugolib/content_render_hooks_test.go index ee7a02074..8aba1dd8c 100644 --- a/hugolib/content_render_hooks_test.go +++ b/hugolib/content_render_hooks_test.go @@ -13,7 +13,9 @@ package hugolib -import "testing" +import ( + "testing" +) func TestRenderHooks(t *testing.T) { config := ` diff --git a/hugolib/hugo_modules_test.go b/hugolib/hugo_modules_test.go index 14085e2c0..5c2b46b30 100644 --- a/hugolib/hugo_modules_test.go +++ b/hugolib/hugo_modules_test.go @@ -340,6 +340,8 @@ b = "B param" } func TestModulesIncompatible(t *testing.T) { + t.Parallel() + b := newTestSitesBuilder(t).WithWorkingDir("/site").WithConfigFile("toml", ` baseURL="https://example.org" @@ -518,6 +520,7 @@ weight = 2 } func TestMountsProject(t *testing.T) { + t.Parallel() config := ` @@ -547,6 +550,7 @@ title: "My Page" // https://github.com/gohugoio/hugo/issues/6684 func TestMountsContentFile(t *testing.T) { + t.Parallel() c := qt.New(t) workingDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-modules-content-file") c.Assert(err, qt.IsNil) diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index 4e1623b2e..a0c62f01e 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -121,6 +121,9 @@ type hugoSitesInit struct { // Loads the data from all of the /data folders. data *lazy.Init + // Performs late initialization (before render) of the templates. + layouts *lazy.Init + // Loads the Git info for all the pages if enabled. gitInfo *lazy.Init @@ -130,6 +133,7 @@ type hugoSitesInit struct { func (h *hugoSitesInit) Reset() { h.data.Reset() + h.layouts.Reset() h.gitInfo.Reset() h.translations.Reset() } @@ -271,6 +275,7 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) { Sites: sites, init: &hugoSitesInit{ data: lazy.New(), + layouts: lazy.New(), gitInfo: lazy.New(), translations: lazy.New(), }, @@ -289,6 +294,15 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) { return nil, nil }) + h.init.layouts.Add(func() (interface{}, error) { + for _, s := range h.Sites { + if err := s.Tmpl().(tpl.TemplateManager).MarkReady(); err != nil { + return nil, err + } + } + return nil, nil + }) + h.init.translations.Add(func() (interface{}, error) { if len(h.Sites) > 1 { allTranslations := pagesToTranslationsMap(h.Sites) @@ -429,10 +443,6 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) { func (s *Site) withSiteTemplates(withTemplates ...func(templ tpl.TemplateManager) error) func(templ tpl.TemplateManager) error { return func(templ tpl.TemplateManager) error { - if err := templ.LoadTemplates(""); err != nil { - return err - } - for _, wt := range withTemplates { if wt == nil { continue @@ -619,10 +629,10 @@ func (h *HugoSites) renderCrossSitesArtifacts() error { s := h.Sites[0] - smLayouts := []string{"sitemapindex.xml", "_default/sitemapindex.xml", "_internal/_default/sitemapindex.xml"} + templ := s.lookupLayouts("sitemapindex.xml", "_default/sitemapindex.xml", "_internal/_default/sitemapindex.xml") return s.renderAndWriteXML(&s.PathSpec.ProcessingStats.Sitemaps, "sitemapindex", - s.siteCfg.sitemap.Filename, h.toSiteInfos(), smLayouts...) + s.siteCfg.sitemap.Filename, h.toSiteInfos(), templ) } func (h *HugoSites) removePageByFilename(filename string) { @@ -832,7 +842,7 @@ func (h *HugoSites) resetPageStateFromEvents(idset identity.Identities) { if po.cp == nil { continue } - for id, _ := range idset { + for id := range idset { if po.cp.dependencyTracker.Search(id) != nil { po.cp.Reset() continue OUTPUTS @@ -841,7 +851,7 @@ func (h *HugoSites) resetPageStateFromEvents(idset identity.Identities) { } for _, s := range p.shortcodeState.shortcodes { - for id, _ := range idset { + for id := range idset { if idm, ok := s.info.(identity.Manager); ok && idm.Search(id) != nil { for _, po := range p.pageOutputs { if po.cp != nil { diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index d749ff581..901941bda 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -291,6 +291,10 @@ func (h *HugoSites) assemble(bcfg *BuildCfg) error { } func (h *HugoSites) render(config *BuildCfg) error { + if _, err := h.init.layouts.Do(); err != nil { + return err + } + siteRenderContext := &siteRenderContext{cfg: config, multihost: h.multihost} if !config.PartialReRender { @@ -312,11 +316,6 @@ func (h *HugoSites) render(config *BuildCfg) error { case <-h.Done(): return nil default: - // For the non-renderable pages, we use the content iself as - // template and we may have to re-parse and execute it for - // each output format. - h.TemplateHandler().RebuildClone() - for _, s2 := range h.Sites { // We render site by site, but since the content is lazily rendered // and a site can "borrow" content from other sites, every site diff --git a/hugolib/hugo_sites_build_errors_test.go b/hugolib/hugo_sites_build_errors_test.go index 21b745ccd..d90a8b364 100644 --- a/hugolib/hugo_sites_build_errors_test.go +++ b/hugolib/hugo_sites_build_errors_test.go @@ -27,7 +27,7 @@ func (t testSiteBuildErrorAsserter) getFileError(err error) *herrors.ErrorWithFi func (t testSiteBuildErrorAsserter) assertLineNumber(lineNumber int, err error) { fe := t.getFileError(err) - t.c.Assert(fe.Position().LineNumber, qt.Equals, lineNumber) + t.c.Assert(fe.Position().LineNumber, qt.Equals, lineNumber, qt.Commentf(err.Error())) } func (t testSiteBuildErrorAsserter) assertErrorMessage(e1, e2 string) { @@ -65,7 +65,8 @@ func TestSiteBuildErrors(t *testing.T) { fileFixer: func(content string) string { return strings.Replace(content, ".Title }}", ".Title }", 1) }, - assertCreateError: func(a testSiteBuildErrorAsserter, err error) { + // Base templates gets parsed at build time. + assertBuildError: func(a testSiteBuildErrorAsserter, err error) { a.assertLineNumber(4, err) }, }, @@ -90,7 +91,7 @@ func TestSiteBuildErrors(t *testing.T) { a.c.Assert(fe.Position().LineNumber, qt.Equals, 5) a.c.Assert(fe.Position().ColumnNumber, qt.Equals, 1) a.c.Assert(fe.ChromaLexer, qt.Equals, "go-html-template") - a.assertErrorMessage("\"layouts/_default/single.html:5:1\": parse failed: template: _default/single.html:5: unexpected \"}\" in operand", fe.Error()) + a.assertErrorMessage("\"layouts/foo/single.html:5:1\": parse failed: template: foo/single.html:5: unexpected \"}\" in operand", fe.Error()) }, }, @@ -258,6 +259,13 @@ SINGLE L5: {{ .Title }} {{ .Content }} {{ end }} `)) + b.WithTemplatesAdded("layouts/foo/single.html", f(single, ` +SINGLE L2: +SINGLE L3: +SINGLE L4: +SINGLE L5: {{ .Title }} {{ .Content }} +`)) + b.WithContent("myyaml.md", f(yamlcontent, `--- title: "The YAML" --- diff --git a/hugolib/hugo_sites_rebuild_test.go b/hugolib/hugo_sites_rebuild_test.go< |