diff options
author | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2019-09-10 11:26:34 +0200 |
---|---|---|
committer | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2020-02-18 09:49:42 +0100 |
commit | eada236f87d9669885da1ff647672bb3dc6b4954 (patch) | |
tree | a0303f067b2cbe06c55637013dbd7702a551c64f /hugolib/site.go | |
parent | e5329f13c02b87f0c30f8837759c810cd90ff8da (diff) |
Introduce a tree map for all content
This commit introduces a new data structure to store pages and their resources.
This data structure is backed by radix trees.
This simplies tree operations, makes all pages a bundle, and paves the way for #6310.
It also solves a set of annoying issues (see list below).
Not a motivation behind this, but this commit also makes Hugo in general a little bit faster and more memory effective (see benchmarks). Especially for partial rebuilds on content edits, but also when taxonomies is in use.
```
name old time/op new time/op delta
SiteNew/Bundle_with_image/Edit-16 1.32ms ± 8% 1.00ms ± 9% -24.42% (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file/Edit-16 1.28ms ± 0% 0.94ms ± 0% -26.26% (p=0.029 n=4+4)
SiteNew/Tags_and_categories/Edit-16 33.9ms ± 2% 21.8ms ± 1% -35.67% (p=0.029 n=4+4)
SiteNew/Canonify_URLs/Edit-16 40.6ms ± 1% 37.7ms ± 3% -7.20% (p=0.029 n=4+4)
SiteNew/Deep_content_tree/Edit-16 56.7ms ± 0% 51.7ms ± 1% -8.82% (p=0.029 n=4+4)
SiteNew/Many_HTML_templates/Edit-16 19.9ms ± 2% 18.3ms ± 3% -7.64% (p=0.029 n=4+4)
SiteNew/Page_collections/Edit-16 37.9ms ± 4% 34.0ms ± 2% -10.28% (p=0.029 n=4+4)
SiteNew/Bundle_with_image-16 10.7ms ± 0% 10.6ms ± 0% -1.15% (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file-16 10.8ms ± 0% 10.7ms ± 0% -1.05% (p=0.029 n=4+4)
SiteNew/Tags_and_categories-16 43.2ms ± 1% 39.6ms ± 1% -8.35% (p=0.029 n=4+4)
SiteNew/Canonify_URLs-16 47.6ms ± 1% 47.3ms ± 0% ~ (p=0.057 n=4+4)
SiteNew/Deep_content_tree-16 73.0ms ± 1% 74.2ms ± 1% ~ (p=0.114 n=4+4)
SiteNew/Many_HTML_templates-16 37.9ms ± 0% 38.1ms ± 1% ~ (p=0.114 n=4+4)
SiteNew/Page_collections-16 53.6ms ± 1% 54.7ms ± 1% +2.09% (p=0.029 n=4+4)
name old alloc/op new alloc/op delta
SiteNew/Bundle_with_image/Edit-16 486kB ± 0% 430kB ± 0% -11.47% (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file/Edit-16 265kB ± 0% 209kB ± 0% -21.06% (p=0.029 n=4+4)
SiteNew/Tags_and_categories/Edit-16 13.6MB ± 0% 8.8MB ± 0% -34.93% (p=0.029 n=4+4)
SiteNew/Canonify_URLs/Edit-16 66.5MB ± 0% 63.9MB ± 0% -3.95% (p=0.029 n=4+4)
SiteNew/Deep_content_tree/Edit-16 28.8MB ± 0% 25.8MB ± 0% -10.55% (p=0.029 n=4+4)
SiteNew/Many_HTML_templates/Edit-16 6.16MB ± 0% 5.56MB ± 0% -9.86% (p=0.029 n=4+4)
SiteNew/Page_collections/Edit-16 16.9MB ± 0% 16.0MB ± 0% -5.19% (p=0.029 n=4+4)
SiteNew/Bundle_with_image-16 2.28MB ± 0% 2.29MB ± 0% +0.35% (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file-16 2.07MB ± 0% 2.07MB ± 0% ~ (p=0.114 n=4+4)
SiteNew/Tags_and_categories-16 14.3MB ± 0% 13.2MB ± 0% -7.30% (p=0.029 n=4+4)
SiteNew/Canonify_URLs-16 69.1MB ± 0% 69.0MB ± 0% ~ (p=0.343 n=4+4)
SiteNew/Deep_content_tree-16 31.3MB ± 0% 31.8MB ± 0% +1.49% (p=0.029 n=4+4)
SiteNew/Many_HTML_templates-16 10.8MB ± 0% 10.9MB ± 0% +1.11% (p=0.029 n=4+4)
SiteNew/Page_collections-16 21.4MB ± 0% 21.6MB ± 0% +1.15% (p=0.029 n=4+4)
name old allocs/op new allocs/op delta
SiteNew/Bundle_with_image/Edit-16 4.74k ± 0% 3.86k ± 0% -18.57% (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file/Edit-16 4.73k ± 0% 3.85k ± 0% -18.58% (p=0.029 n=4+4)
SiteNew/Tags_and_categories/Edit-16 301k ± 0% 198k ± 0% -34.14% (p=0.029 n=4+4)
SiteNew/Canonify_URLs/Edit-16 389k ± 0% 373k ± 0% -4.07% (p=0.029 n=4+4)
SiteNew/Deep_content_tree/Edit-16 338k ± 0% 262k ± 0% -22.63% (p=0.029 n=4+4)
SiteNew/Many_HTML_templates/Edit-16 102k ± 0% 88k ± 0% -13.81% (p=0.029 n=4+4)
SiteNew/Page_collections/Edit-16 176k ± 0% 152k ± 0% -13.32% (p=0.029 n=4+4)
SiteNew/Bundle_with_image-16 26.8k ± 0% 26.8k ± 0% +0.05% (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file-16 26.8k ± 0% 26.8k ± 0% +0.05% (p=0.029 n=4+4)
SiteNew/Tags_and_categories-16 273k ± 0% 245k ± 0% -10.36% (p=0.029 n=4+4)
SiteNew/Canonify_URLs-16 396k ± 0% 398k ± 0% +0.39% (p=0.029 n=4+4)
SiteNew/Deep_content_tree-16 317k ± 0% 325k ± 0% +2.53% (p=0.029 n=4+4)
SiteNew/Many_HTML_templates-16 146k ± 0% 147k ± 0% +0.98% (p=0.029 n=4+4)
SiteNew/Page_collections-16 210k ± 0% 215k ± 0% +2.44% (p=0.029 n=4+4)
```
Fixes #6312
Fixes #6087
Fixes #6738
Fixes #6412
Fixes #6743
Fixes #6875
Fixes #6034
Fixes #6902
Fixes #6173
Fixes #6590
Diffstat (limited to 'hugolib/site.go')
-rw-r--r-- | hugolib/site.go | 275 |
1 files changed, 176 insertions, 99 deletions
diff --git a/hugolib/site.go b/hugolib/site.go index bbcbcd27a..34e5ad156 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -100,10 +100,10 @@ type Site struct { *PageCollections - Taxonomies TaxonomyList + taxonomies TaxonomyList Sections Taxonomy - Info SiteInfo + Info *SiteInfo language *langs.Language @@ -163,9 +163,28 @@ type Site struct { init *siteInit } +func (s *Site) Taxonomies() TaxonomyList { + s.init.taxonomies.Do() + return s.taxonomies +} + +type taxonomiesConfig map[string]string + +func (t taxonomiesConfig) Values() []viewName { + var vals []viewName + for k, v := range t { + vals = append(vals, viewName{singular: k, plural: v}) + } + sort.Slice(vals, func(i, j int) bool { + return vals[i].plural < vals[j].plural + }) + + return vals +} + type siteConfigHolder struct { sitemap config.Sitemap - taxonomiesConfig map[string]string + taxonomiesConfig taxonomiesConfig timeout time.Duration hasCJKLanguage bool enableEmoji bool @@ -176,12 +195,14 @@ type siteInit struct { prevNext *lazy.Init prevNextInSection *lazy.Init menus *lazy.Init + taxonomies *lazy.Init } func (init *siteInit) Reset() { init.prevNext.Reset() init.prevNextInSection.Reset() init.menus.Reset() + init.taxonomies.Reset() } func (s *Site) initInit(init *lazy.Init, pctx pageContext) bool { @@ -198,65 +219,87 @@ func (s *Site) prepareInits() { var init lazy.Init s.init.prevNext = init.Branch(func() (interface{}, error) { - regularPages := s.findWorkPagesByKind(page.KindPage) + regularPages := s.RegularPages() for i, p := range regularPages { - if p.posNextPrev == nil { + np, ok := p.(nextPrevProvider) + if !ok { + continue + } + + pos := np.getNextPrev() + if pos == nil { continue } - p.posNextPrev.nextPage = nil - p.posNextPrev.prevPage = nil + + pos.nextPage = nil + pos.prevPage = nil if i > 0 { - p.posNextPrev.nextPage = regularPages[i-1] + pos.nextPage = regularPages[i-1] } if i < len(regularPages)-1 { - p.posNextPrev.prevPage = regularPages[i+1] + pos.prevPage = regularPages[i+1] } } return nil, nil }) s.init.prevNextInSection = init.Branch(func() (interface{}, error) { - var rootSection []int - // TODO(bep) cm attach this to the bucket. - for i, p1 := range s.workAllPages { - if p1.IsPage() && p1.Section() == "" { - rootSection = append(rootSection, i) - } - if p1.IsSection() { - sectionPages := p1.RegularPages() - for i, p2 := range sectionPages { - p2s := p2.(*pageState) - if p2s.posNextPrevSection == nil { - continue - } - p2s.posNextPrevSection.nextPage = nil - p2s.posNextPrevSection.prevPage = nil + var sections page.Pages + s.home.treeRef.m.collectSectionsRecursiveIncludingSelf(s.home.treeRef.key, func(n *contentNode) { + sections = append(sections, n.p) + }) - if i > 0 { - p2s.posNextPrevSection.nextPage = sectionPages[i-1] - } + setNextPrev := func(pas page.Pages) { + for i, p := range pas { + np, ok := p.(nextPrevInSectionProvider) + if !ok { + continue + } - if i < len(sectionPages)-1 { - p2s.posNextPrevSection.prevPage = sectionPages[i+1] - } + pos := np.getNextPrevInSection() + if pos == nil { + continue + } + + pos.nextPage = nil + pos.prevPage = nil + + if i > 0 { + pos.nextPage = pas[i-1] + } + + if i < len(pas)-1 { + pos.prevPage = pas[i+1] } } } - for i, j := range rootSection { - p := s.workAllPages[j] - if i > 0 { - p.posNextPrevSection.nextPage = s.workAllPages[rootSection[i-1]] - } + for _, sect := range sections { + treeRef := sect.(treeRefProvider).getTreeRef() - if i < len(rootSection)-1 { - p.posNextPrevSection.prevPage = s.workAllPages[rootSection[i+1]] - } + var pas page.Pages + treeRef.m.collectPages(treeRef.key+cmBranchSeparator, func(c *contentNode) { + pas = append(pas, c.p) + }) + page.SortByDefault(pas) + + setNextPrev(pas) } + // The root section only goes one level down. + treeRef := s.home.getTreeRef() + + var pas page.Pages + treeRef.m.collectPages(treeRef.key+cmBranchSeparator, func(c *contentNode) { + pas = append(pas, c.p) + }) + page.SortByDefault(pas) + + setNextPrev(pas) + return nil, nil }) @@ -265,6 +308,11 @@ func (s *Site) prepareInits() { return nil, nil }) + s.init.taxonomies = init.Branch(func() (interface{}, error) { + err := s.pageMap.assembleTaxonomies() + return nil, err + }) + } type siteRenderingContext struct { @@ -279,14 +327,15 @@ func (s *Site) Menus() navigation.Menus { func (s *Site) initRenderFormats() { formatSet := make(map[string]bool) formats := output.Formats{} - for _, p := range s.workAllPages { - for _, f := range p.m.configuredOutputFormats { + s.pageMap.pageTrees.WalkRenderable(func(s string, n *contentNode) bool { + for _, f := range n.p.m.configuredOutputFormats { if !formatSet[f.Name] { formats = append(formats, f) formatSet[f.Name] = true } } - } + return false + }) // Add the per kind configured output formats for _, kind := range allKindsInPages { @@ -345,8 +394,6 @@ func (s *Site) reset() *Site { // newSite creates a new site with the given configuration. func newSite(cfg deps.DepsCfg) (*Site, error) { - c := newPageCollections() - if cfg.Language == nil { cfg.Language = langs.NewDefaultLanguage(cfg.Cfg) } @@ -385,6 +432,17 @@ func newSite(cfg deps.DepsCfg) (*Site, error) { return nil, err } + if disabledKinds[kindRSS] { + // Legacy + tmp := siteOutputFormatsConfig[:0] + for _, x := range siteOutputFormatsConfig { + if !strings.EqualFold(x.Name, "rss") { + tmp = append(tmp, x) + } + } + siteOutputFormatsConfig = tmp + } + outputFormats, err := createSiteOutputFormats(siteOutputFormatsConfig, cfg.Language) if err != nil { return nil, err @@ -435,18 +493,23 @@ func newSite(cfg deps.DepsCfg) (*Site, error) { } s := &Site{ - PageCollections: c, - language: cfg.Language, - disabledKinds: disabledKinds, - titleFunc: titleFunc, - relatedDocsHandler: page.NewRelatedDocsHandler(relatedContentConfig), - outputFormats: outputFormats, - rc: &siteRenderingContext{output.HTMLFormat}, - outputFormatsConfig: siteOutputFormatsConfig, - mediaTypesConfig: siteMediaTypesConfig, - frontmatterHandler: frontMatterHandler, + + language: cfg.Language, + disabledKinds: disabledKinds, + + outputFormats: outputFormats, + outputFormatsConfig: siteOutputFormatsConfig, + mediaTypesConfig: siteMediaTypesConfig, + enableInlineShortcodes: cfg.Language.GetBool("enableInlineShortcodes"), siteCfg: siteConfig, + + titleFunc: titleFunc, + + rc: &siteRenderingContext{output.HTMLFormat}, + + frontmatterHandler: frontMatterHandler, + relatedDocsHandler: page.NewRelatedDocsHandler(relatedContentConfig), } s.prepareInits() @@ -595,7 +658,7 @@ func (s *SiteInfo) Menus() navigation.Menus { // TODO(bep) type func (s *SiteInfo) Taxonomies() interface{} { - return s.s.Taxonomies + return s.s.Taxonomies() } func (s *SiteInfo) Params() maps.Params { @@ -734,7 +797,7 @@ func (s *siteRefLinker) refLink(ref string, source interface{}, relative bool, o if refURL.Path != "" { var err error - target, err = s.s.getPageNew(p, refURL.Path) + target, err = s.s.getPageRef(p, refURL.Path) var pos text.Position if err != nil || target == nil { if p, ok := source.(text.Positioner); ok { @@ -988,7 +1051,7 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro OutputFormats: site.outputFormatsConfig, } site.Deps, err = first.Deps.ForLanguage(depsCfg, func(d *deps.Deps) error { - d.Site = &site.Info + d.Site = site.Info return nil }) if err != nil { @@ -1189,7 +1252,7 @@ func (s *Site) initializeSiteInfo() error { } } - s.Info = SiteInfo{ + s.Info = &SiteInfo{ title: lang.GetString("title"), Author: lang.GetStringMap("author"), Social: lang.GetStringMapString("social"), @@ -1231,11 +1294,17 @@ func (s *Site) eventToIdentity(e fsnotify.Event) (identity.PathIdentity, bool) { func (s *Site) readAndProcessContent(filenames ...string) error { sourceSpec := source.NewSourceSpec(s.PathSpec, s.BaseFs.Content.Fs) - proc := newPagesProcessor(s.h, sourceSpec, len(filenames) > 0) + proc := newPagesProcessor(s.h, sourceSpec) - c := newPagesCollector(sourceSpec, s.Log, s.h.ContentChanges, proc, filenames...) + c := newPagesCollector(sourceSpec, s.h.content, s.Log, s.h.ContentChanges, proc, filenames...) - return c.Collect() + if err := c.Collect(); err != nil { + return err + } + + s.h.content = newPageMaps(s.h) + + return nil } func (s *Site) getMenusFromConfig() navigation.Menus { @@ -1309,35 +1378,45 @@ func (s *Site) assembleMenus() { sectionPagesMenu := s.Info.sectionPagesMenu if sectionPagesMenu != "" { - for _, p := range s.workAllPages { - if p.Kind() == page.KindSection { - // From Hugo 0.22 we have nested sections, but until we get a - // feel of how that would work in this setting, let us keep - // this menu for the top level only. - id := p.Section() - if _, ok := flat[twoD{sectionPagesMenu, id}]; ok { - continue - } - - me := navigation.MenuEntry{Identifier: id, - Name: p.LinkTitle(), - Weight: p.Weight(), - Page: p} - flat[twoD{sectionPagesMenu, me.KeyName()}] = &me + s.pageMap.sections.Walk(func(s string, v interface{}) bool { + p := v.(*contentNode).p + if p.IsHome() { + return false } - } + // From Hugo 0.22 we have nested sections, but until we get a + // feel of how that would work in this setting, let us keep + // this menu for the top level only. + id := p.Section() + if _, ok := flat[twoD{sectionPagesMenu, id}]; ok { + return false + } + + me := navigation.MenuEntry{Identifier: id, + Name: p.LinkTitle(), + Weight: p.Weight(), + Page: p} + flat[twoD{sectionPagesMenu, me.KeyName()}] = &me + + return false + }) + } // Add menu entries provided by pages - for _, p := range s.workAllPages { + s.pageMap.pageTrees.WalkRenderable(func(ss string, n *contentNode) bool { + p := n.p + for name, me := range p.pageMenus.menus() { if _, ok := flat[twoD{name, me.KeyName()}]; ok { - s.SendError(p.wrapError(errors.Errorf("duplicate menu entry with identifier %q in menu %q", me.KeyName(), name))) + err := p.wrapError(errors.Errorf("duplicate menu entry with identifier %q in menu %q", me.KeyName(), name)) + s.Log.WARN.Println(err) continue } flat[twoD{name, me.KeyName()}] = me } - } + + return false + }) // Create Children Menus First for _, e := range flat { @@ -1410,15 +1489,17 @@ func (s *Site) resetBuildState(sourceChanged bool) { s.init.Reset() if sourceChanged { - s.PageCollections = newPageCollectionsFromPages(s.rawAllPages) - for _, p := range s.rawAllPages { + s.PageCollections = newPageCollections(s.pageMap) + s.pageMap.withEveryBundlePage(func(p *pageState) bool { p.pagePages = &pagePages{} p.parent = nil p.Scratcher = maps.NewScratcher() - } + return false + }) } else { - s.pagesMap.withEveryPage(func(p *pageState) { + s.pageMap.withEveryBundlePage(func(p *pageState) bool { p.Scratcher = maps.NewScratcher() + return false }) } } @@ -1613,6 +1694,7 @@ func (s *Site) kindFromFileInfoOrSections(fi *fileInfo, sections []string) strin return s.kindFromSections(sections) } + return page.KindPage } @@ -1640,26 +1722,21 @@ func (s *Site) kindFromSectionPath(sectionPath string) string { return page.KindSection } -func (s *Site) newTaxonomyPage(title string, sections ...string) *pageState { - p, err := newPageFromMeta( - map[string]interface{}{"title": title}, - &pageMeta{ - s: s, - kind: page.KindTaxonomy, - sections: sections, - }) +func (s *Site) newPage( + n *contentNode, + parentbBucket *pagesMapBucket, + kind, title string, + sections ...string) *pageState { - if err != nil { - panic(err) + m := map[string]interface{}{} + if title != "" { + m["title"] = title } - return p - -} - -func (s *Site) newPage(kind string, sections ...string) *pageState { p, err := newPageFromMeta( - map[string]interface{}{}, + n, + parentbBucket, + m, &pageMeta{ s: s, kind: kind, |