diff options
author | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2016-07-28 09:30:58 +0200 |
---|---|---|
committer | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2016-09-06 18:32:16 +0300 |
commit | 708bc78770a0b0361908f6404f57264c53252a95 (patch) | |
tree | 9b7e3a05b1e83a768bfa0dd96b61b07dd7917cfd /hugolib/hugo_sites.go | |
parent | f023dfd7636f73b11c94e86a05c6273941d52c58 (diff) |
Optimize the multilanguage build process
Work In Progress!
This commit makes a rework of the build and rebuild process to better suit a multi-site setup.
This also includes a complete overhaul of the site tests. Previous these were a messy mix that
were testing just small parts of the build chain, some of it testing code-paths not even used in
"real life". Now all tests that depends on a built site follows the same and real production code path.
See #2309
Closes #2211
Closes #477
Closes #1744
Diffstat (limited to 'hugolib/hugo_sites.go')
-rw-r--r-- | hugolib/hugo_sites.go | 251 |
1 files changed, 231 insertions, 20 deletions
diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index dd8d3e5d2..2dd1bb9be 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -14,42 +14,119 @@ package hugolib import ( + "errors" + "strings" "time" - "github.com/fsnotify/fsnotify" + "github.com/spf13/viper" + "github.com/fsnotify/fsnotify" + "github.com/spf13/hugo/source" + "github.com/spf13/hugo/tpl" jww "github.com/spf13/jwalterweatherman" ) // HugoSites represents the sites to build. Each site represents a language. -type HugoSites []*Site +type HugoSites struct { + Sites []*Site + + Multilingual *Multilingual +} + +func NewHugoSites(sites ...*Site) (*HugoSites, error) { + languages := make(Languages, len(sites)) + for i, s := range sites { + if s.Language == nil { + return nil, errors.New("Missing language for site") + } + languages[i] = s.Language + } + defaultLang := viper.GetString("DefaultContentLanguage") + if defaultLang == "" { + defaultLang = "en" + } + langConfig := &Multilingual{Languages: languages, DefaultLang: NewLanguage(defaultLang)} + + return &HugoSites{Multilingual: langConfig, Sites: sites}, nil +} // Reset resets the sites, making it ready for a full rebuild. // TODO(bep) multilingo func (h HugoSites) Reset() { - for i, s := range h { - h[i] = s.Reset() + for i, s := range h.Sites { + h.Sites[i] = s.Reset() } } +type BuildCfg struct { + // Whether we are in watch (server) mode + Watching bool + // Print build stats at the end of a build + PrintStats bool + // Skip rendering. Useful for testing. + skipRender bool + // Use this to add templates to use for rendering. + // Useful for testing. + withTemplate func(templ tpl.Template) error +} + // Build builds all sites. -func (h HugoSites) Build(watching, printStats bool) error { +func (h HugoSites) Build(config BuildCfg) error { + + if h.Sites == nil || len(h.Sites) == 0 { + return errors.New("No site(s) to build") + } + t0 := time.Now() - for _, site := range h { - t1 := time.Now() + // We should probably refactor the Site and pull up most of the logic from there to here, + // but that seems like a daunting task. + // So for now, if there are more than one site (language), + // we pre-process the first one, then configure all the sites based on that. + firstSite := h.Sites[0] + + for _, s := range h.Sites { + // TODO(bep) ml + s.Multilingual = h.Multilingual + s.RunMode.Watching = config.Watching + } + + if err := firstSite.PreProcess(config); err != nil { + return err + } - site.RunMode.Watching = watching + h.setupTranslations(firstSite) - if err := site.Build(); err != nil { + if len(h.Sites) > 1 { + // Initialize the rest + for _, site := range h.Sites[1:] { + site.Tmpl = firstSite.Tmpl + site.initializeSiteInfo() + } + } + + for _, s := range h.Sites { + + if err := s.PostProcess(); err != nil { return err } - if printStats { - site.Stats(t1) + + if !config.skipRender { + if err := s.Render(); err != nil { + return err + } + + } + + if config.PrintStats { + s.Stats() } + + // TODO(bep) ml lang in site.Info? + // TODO(bep) ml Page sorting? } - if printStats { + if config.PrintStats { jww.FEEDBACK.Printf("total in %v ms\n", int(1000*time.Since(t0).Seconds())) } @@ -58,25 +135,159 @@ func (h HugoSites) Build(watching, printStats bool) error { } // Rebuild rebuilds all sites. -func (h HugoSites) Rebuild(events []fsnotify.Event, printStats bool) error { +func (h HugoSites) Rebuild(config BuildCfg, events ...fsnotify.Event) error { t0 := time.Now() - for _, site := range h { - t1 := time.Now() + firstSite := h.Sites[0] - if err := site.ReBuild(events); err != nil { - return err + for _, s := range h.Sites { + s.resetBuildState() + } + + sourceChanged, err := firstSite.ReBuild(events) + + if err != nil { + return err + } + + // Assign pages to sites per translation. + h.setupTranslations(firstSite) + + for _, s := range h.Sites { + + if sourceChanged { + if err := s.PostProcess(); err != nil { + return err + } } - if printStats { - site.Stats(t1) + if !config.skipRender { + if err := s.Render(); err != nil { + return err + } + } + + if config.PrintStats { + s.Stats() } } - if printStats { + if config.PrintStats { jww.FEEDBACK.Printf("total in %v ms\n", int(1000*time.Since(t0).Seconds())) } return nil } + +func (s *HugoSites) setupTranslations(master *Site) { + + for _, p := range master.rawAllPages { + if p.Lang() == "" { + panic("Page language missing: " + p.Title) + } + + shouldBuild := p.shouldBuild() + + for i, site := range s.Sites { + if strings.HasPrefix(site.Language.Lang, p.Lang()) { + site.updateBuildStats(p) + if shouldBuild { + site.Pages = append(site.Pages, p) + p.Site = &site.Info + } + } + + if !shouldBuild { + continue + } + + if i == 0 { + site.AllPages = append(site.AllPages, p) + } + } + + for i := 1; i < len(s.Sites); i++ { + s.Sites[i].AllPages = s.Sites[0].AllPages + } + } + + if len(s.Sites) > 1 { + pages := s.Sites[0].AllPages + allTranslations := pagesToTranslationsMap(s.Multilingual, pages) + assignTranslationsToPages(allTranslations, pages) + } +} + +func (s *Site) updateBuildStats(page *Page) { + if page.IsDraft() { + s.draftCount++ + } + + if page.IsFuture() { + s.futureCount++ + } + + if page.IsExpired() { + s.expiredCount++ + } +} + +// Convenience func used in tests to build a single site/language excluding render phase. +func buildSiteSkipRender(s *Site, additionalTemplates ...string) error { + return doBuildSite(s, false, additionalTemplates...) +} + +// Convenience func used in tests to build a single site/language including render phase. +func buildAndRenderSite(s *Site, additionalTemplates ...string) error { + return doBuildSite(s, true, additionalTemplates...) +} + +// Convenience func used in tests to build a single site/language. +func doBuildSite(s *Site, render bool, additionalTemplates ...string) error { + sites, err := NewHugoSites(s) + if err != nil { + return err + } + + addTemplates := func(templ tpl.Template) error { + for i := 0; i < len(additionalTemplates); i += 2 { + err := templ.AddTemplate(additionalTemplates[i], additionalTemplates[i+1]) + if err != nil { + return err + } + } + return nil + } + + config := BuildCfg{skipRender: !render, withTemplate: addTemplates} + return sites.Build(config) +} + +// Convenience func used in tests. +func newHugoSitesFromSourceAndLanguages(input []source.ByteSource, languages Languages) (*HugoSites, error) { + if len(languages) == 0 { + panic("Must provide at least one language") + } + first := &Site{ + Source: &source.InMemorySource{ByteSource: input}, + Language: languages[0], + } + if len(languages) == 1 { + return NewHugoSites(first) + } + + sites := make([]*Site, len(languages)) + sites[0] = first + for i := 1; i < len(languages); i++ { + sites[i] = &Site{Language: languages[i]} + } + + return NewHugoSites(sites...) + +} + +// Convenience func used in tests. +func newHugoSitesFromLanguages(languages Languages) (*HugoSites, error) { + return newHugoSitesFromSourceAndLanguages(nil, languages) +} |