diff options
author | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2019-09-10 11:26:34 +0200 |
---|---|---|
committer | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2020-02-18 09:49:42 +0100 |
commit | eada236f87d9669885da1ff647672bb3dc6b4954 (patch) | |
tree | a0303f067b2cbe06c55637013dbd7702a551c64f /hugolib/disableKinds_test.go | |
parent | e5329f13c02b87f0c30f8837759c810cd90ff8da (diff) |
Introduce a tree map for all content
This commit introduces a new data structure to store pages and their resources.
This data structure is backed by radix trees.
This simplies tree operations, makes all pages a bundle, and paves the way for #6310.
It also solves a set of annoying issues (see list below).
Not a motivation behind this, but this commit also makes Hugo in general a little bit faster and more memory effective (see benchmarks). Especially for partial rebuilds on content edits, but also when taxonomies is in use.
```
name old time/op new time/op delta
SiteNew/Bundle_with_image/Edit-16 1.32ms ± 8% 1.00ms ± 9% -24.42% (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file/Edit-16 1.28ms ± 0% 0.94ms ± 0% -26.26% (p=0.029 n=4+4)
SiteNew/Tags_and_categories/Edit-16 33.9ms ± 2% 21.8ms ± 1% -35.67% (p=0.029 n=4+4)
SiteNew/Canonify_URLs/Edit-16 40.6ms ± 1% 37.7ms ± 3% -7.20% (p=0.029 n=4+4)
SiteNew/Deep_content_tree/Edit-16 56.7ms ± 0% 51.7ms ± 1% -8.82% (p=0.029 n=4+4)
SiteNew/Many_HTML_templates/Edit-16 19.9ms ± 2% 18.3ms ± 3% -7.64% (p=0.029 n=4+4)
SiteNew/Page_collections/Edit-16 37.9ms ± 4% 34.0ms ± 2% -10.28% (p=0.029 n=4+4)
SiteNew/Bundle_with_image-16 10.7ms ± 0% 10.6ms ± 0% -1.15% (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file-16 10.8ms ± 0% 10.7ms ± 0% -1.05% (p=0.029 n=4+4)
SiteNew/Tags_and_categories-16 43.2ms ± 1% 39.6ms ± 1% -8.35% (p=0.029 n=4+4)
SiteNew/Canonify_URLs-16 47.6ms ± 1% 47.3ms ± 0% ~ (p=0.057 n=4+4)
SiteNew/Deep_content_tree-16 73.0ms ± 1% 74.2ms ± 1% ~ (p=0.114 n=4+4)
SiteNew/Many_HTML_templates-16 37.9ms ± 0% 38.1ms ± 1% ~ (p=0.114 n=4+4)
SiteNew/Page_collections-16 53.6ms ± 1% 54.7ms ± 1% +2.09% (p=0.029 n=4+4)
name old alloc/op new alloc/op delta
SiteNew/Bundle_with_image/Edit-16 486kB ± 0% 430kB ± 0% -11.47% (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file/Edit-16 265kB ± 0% 209kB ± 0% -21.06% (p=0.029 n=4+4)
SiteNew/Tags_and_categories/Edit-16 13.6MB ± 0% 8.8MB ± 0% -34.93% (p=0.029 n=4+4)
SiteNew/Canonify_URLs/Edit-16 66.5MB ± 0% 63.9MB ± 0% -3.95% (p=0.029 n=4+4)
SiteNew/Deep_content_tree/Edit-16 28.8MB ± 0% 25.8MB ± 0% -10.55% (p=0.029 n=4+4)
SiteNew/Many_HTML_templates/Edit-16 6.16MB ± 0% 5.56MB ± 0% -9.86% (p=0.029 n=4+4)
SiteNew/Page_collections/Edit-16 16.9MB ± 0% 16.0MB ± 0% -5.19% (p=0.029 n=4+4)
SiteNew/Bundle_with_image-16 2.28MB ± 0% 2.29MB ± 0% +0.35% (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file-16 2.07MB ± 0% 2.07MB ± 0% ~ (p=0.114 n=4+4)
SiteNew/Tags_and_categories-16 14.3MB ± 0% 13.2MB ± 0% -7.30% (p=0.029 n=4+4)
SiteNew/Canonify_URLs-16 69.1MB ± 0% 69.0MB ± 0% ~ (p=0.343 n=4+4)
SiteNew/Deep_content_tree-16 31.3MB ± 0% 31.8MB ± 0% +1.49% (p=0.029 n=4+4)
SiteNew/Many_HTML_templates-16 10.8MB ± 0% 10.9MB ± 0% +1.11% (p=0.029 n=4+4)
SiteNew/Page_collections-16 21.4MB ± 0% 21.6MB ± 0% +1.15% (p=0.029 n=4+4)
name old allocs/op new allocs/op delta
SiteNew/Bundle_with_image/Edit-16 4.74k ± 0% 3.86k ± 0% -18.57% (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file/Edit-16 4.73k ± 0% 3.85k ± 0% -18.58% (p=0.029 n=4+4)
SiteNew/Tags_and_categories/Edit-16 301k ± 0% 198k ± 0% -34.14% (p=0.029 n=4+4)
SiteNew/Canonify_URLs/Edit-16 389k ± 0% 373k ± 0% -4.07% (p=0.029 n=4+4)
SiteNew/Deep_content_tree/Edit-16 338k ± 0% 262k ± 0% -22.63% (p=0.029 n=4+4)
SiteNew/Many_HTML_templates/Edit-16 102k ± 0% 88k ± 0% -13.81% (p=0.029 n=4+4)
SiteNew/Page_collections/Edit-16 176k ± 0% 152k ± 0% -13.32% (p=0.029 n=4+4)
SiteNew/Bundle_with_image-16 26.8k ± 0% 26.8k ± 0% +0.05% (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file-16 26.8k ± 0% 26.8k ± 0% +0.05% (p=0.029 n=4+4)
SiteNew/Tags_and_categories-16 273k ± 0% 245k ± 0% -10.36% (p=0.029 n=4+4)
SiteNew/Canonify_URLs-16 396k ± 0% 398k ± 0% +0.39% (p=0.029 n=4+4)
SiteNew/Deep_content_tree-16 317k ± 0% 325k ± 0% +2.53% (p=0.029 n=4+4)
SiteNew/Many_HTML_templates-16 146k ± 0% 147k ± 0% +0.98% (p=0.029 n=4+4)
SiteNew/Page_collections-16 210k ± 0% 215k ± 0% +2.44% (p=0.029 n=4+4)
```
Fixes #6312
Fixes #6087
Fixes #6738
Fixes #6412
Fixes #6743
Fixes #6875
Fixes #6034
Fixes #6902
Fixes #6173
Fixes #6590
Diffstat (limited to 'hugolib/disableKinds_test.go')
-rw-r--r-- | hugolib/disableKinds_test.go | 384 |
1 files changed, 226 insertions, 158 deletions
diff --git a/hugolib/disableKinds_test.go b/hugolib/disableKinds_test.go index b26385cc3..f5b2c65a1 100644 --- a/hugolib/disableKinds_test.go +++ b/hugolib/disableKinds_test.go @@ -13,188 +13,256 @@ package hugolib import ( - "strings" "testing" "fmt" qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/resources/page" - - "github.com/gohugoio/hugo/helpers" ) -func TestDisableKindsNoneDisabled(t *testing.T) { - t.Parallel() - doTestDisableKinds(t) -} - -func TestDisableKindsSomeDisabled(t *testing.T) { - t.Parallel() - doTestDisableKinds(t, page.KindSection, kind404) -} +func TestDisable(t *testing.T) { + c := qt.New(t) -func TestDisableKindsOneDisabled(t *testing.T) { - t.Parallel() - for _, kind := range allKinds { - if kind == page.KindPage { - // Turning off regular page generation have some side-effects - // not handled by the assertions below (no sections), so - // skip that for now. - continue - } - doTestDisableKinds(t, kind) - } -} - -func TestDisableKindsAllDisabled(t *testing.T) { - t.Parallel() - doTestDisableKinds(t, allKinds...) -} - -func doTestDisableKinds(t *testing.T, disabled ...string) { - siteConfigTemplate := ` + newSitesBuilder := func(c *qt.C, disableKind string) *sitesBuilder { + config := fmt.Sprintf(` baseURL = "http://example.com/blog" enableRobotsTXT = true -disableKinds = %s - -paginate = 1 -defaultContentLanguage = "en" - -[Taxonomies] -tag = "tags" -category = "categories" -` - - pageTemplate := `--- -title: "%s" -tags: -%s -categories: -- Hugo ---- -# Doc -` +disableKinds = [%q] +`, disableKind) - disabledStr := "[]" - - if len(disabled) > 0 { - disabledStr = strings.Replace(fmt.Sprintf("%#v", disabled), "[]string{", "[", -1) - disabledStr = strings.Replace(disabledStr, "}", "]", -1) - } + b := newTestSitesBuilder(c) + b.WithConfigFile("toml", config).WithContent("sect/page.md", ` +--- +title: Page +categories: ["mycat"] +tags: ["mytag"] +--- - siteConfig := fmt.Sprintf(siteConfigTemplate, disabledStr) +`, "sect/no-list.md", ` +--- +title: No List +_build: + list: false +--- - b := newTestSitesBuilder(t).WithConfigFile("toml", siteConfig) +`, "sect/no-render.md", ` +--- +title: No List +_build: + render: false +--- +`, "sect/no-publishresources/index.md", ` +--- +title: No Publish Resources +_build: + publishResources: false +--- - b.WithTemplates( - "index.html", "Home|{{ .Title }}|{{ .Content }}", - "_default/single.html", "Single|{{ .Title }}|{{ .Content }}", - "_default/list.html", "List|{{ .Title }}|{{ .Content }}", - "_default/terms.html", "Terms List|{{ .Title }}|{{ .Content }}", - "layouts/404.html", "Page Not Found", - ) +`, "sect/headlessbundle/index.md", ` +--- +title: Headless +headless: true +--- - b.WithContent( - "sect/p1.md", fmt.Sprintf(pageTemplate, "P1", "- tag1"), - "categories/_index.md", newTestPage("Category Terms", "2017-01-01", 10), - "tags/tag1/_index.md", newTestPage("Tag1 List", "2017-01-01", 10), - ) +`) + b.WithSourceFile("content/sect/headlessbundle/data.json", "DATA") + b.WithSourceFile("content/sect/no-publishresources/data.json", "DATA") - b.Build(BuildCfg{}) - h := b.H + return b - assertDisabledKinds(b, h.Sites[0], disabled...) + } -} + getPage := func(b *sitesBuilder, ref string) page.Page { + b.Helper() + p, err := b.H.Sites[0].getPageNew(nil, ref) + b.Assert(err, qt.IsNil) + return p + } -func assertDisabledKinds(b *sitesBuilder, s *Site, disabled ...string) { - assertDisabledKind(b, - func(isDisabled bool) bool { - if isDisabled { - return len(s.RegularPages()) == 0 - } - return len(s.RegularPages()) > 0 - }, disabled, page.KindPage, "public/sect/p1/index.html", "Single|P1") - assertDisabledKind(b, - func(isDisabled bool) bool { - p := s.getPage(page.KindHome) - if isDisabled { - return p == nil - } - return p != nil - }, disabled, page.KindHome, "public/index.html", "Home") - assertDisabledKind(b, - func(isDisabled bool) bool { - p := s.getPage(page.KindSection, "sect") - if isDisabled { - return p == nil - } - return p != nil - }, disabled, page.KindSection, "public/sect/index.html", "Sects") - assertDisabledKind(b, - func(isDisabled bool) bool { - p := s.getPage(page.KindTaxonomy, "tags", "tag1") - - if isDisabled { - return p == nil - } - return p != nil - - }, disabled, page.KindTaxonomy, "public/tags/tag1/index.html", "Tag1") - assertDisabledKind(b, - func(isDisabled bool) bool { - p := s.getPage(page.KindTaxonomyTerm, "tags") - if isDisabled { - return p == nil + getPageInSitePages := func(b *sitesBuilder, ref string) page.Page { + b.Helper() + for _, pages := range []page.Pages{b.H.Sites[0].Pages(), b.H.Sites[0].RegularPages()} { + for _, p := range pages { + if ref == p.(*pageState).sourceRef() { + return p + } } - return p != nil - - }, disabled, page.KindTaxonomyTerm, "public/tags/index.html", "Tags") - assertDisabledKind(b, - func(isDisabled bool) bool { - p := s.getPage(page.KindTaxonomyTerm, "categories") + } + return nil + } - if isDisabled { - return p == nil + getPageInPagePages := func(p page.Page, ref string) page.Page { + for _, pages := range []page.Pages{p.Pages(), p.RegularPages(), p.Sections()} { + for _, p := range pages { + if ref == p.(*pageState).sourceRef() { + return p + } } - return p != nil - - }, disabled, page.KindTaxonomyTerm, "public/categories/index.html", "Category Terms") - assertDisabledKind(b, - func(isDisabled bool) bool { - p := s.getPage(page.KindTaxonomy, "categories", "hugo") - if isDisabled { - return p == nil - } - return p != nil - - }, disabled, page.KindTaxonomy, "public/categories/hugo/index.html", "Hugo") - // The below have no page in any collection. - assertDisabledKind(b, func(isDisabled bool) bool { return true }, disabled, kindRSS, "public/index.xml", "<link>") - assertDisabledKind(b, func(isDisabled bool) bool { return true }, disabled, kindSitemap, "public/sitemap.xml", "sitemap") - assertDisabledKind(b, func(isDisabled bool) bool { return true }, disabled, kindRobotsTXT, "public/robots.txt", "User-agent") - assertDisabledKind(b, func(isDisabled bool) bool { return true }, disabled, kind404, "public/404.html", "Page Not Found") -} - -func assertDisabledKind(b *sitesBuilder, kindAssert func(bool) bool, disabled []string, kind, path, matcher string) { - isDisabled := stringSliceContains(kind, disabled...) - b.Assert(kindAssert(isDisabled), qt.Equals, true) - - if kind == kindRSS && !isDisabled { - // If the home page is also disabled, there is not RSS to look for. - if stringSliceContains(page.KindHome, disabled...) { - isDisabled = true } + return nil } - if isDisabled { - // Path should not exist - fileExists, err := helpers.Exists(path, b.Fs.Destination) - b.Assert(err, qt.IsNil) - b.Assert(fileExists, qt.Equals, false) - - } else { - b.AssertFileContent(path, matcher) - } + disableKind := page.KindPage + c.Run("Disable "+disableKind, func(c *qt.C) { + b := newSitesBuilder(c, disableKind) + b.Build(BuildCfg{}) + s := b.H.Sites[0] + b.Assert(getPage(b, "/sect/page.md"), qt.IsNil) + b.Assert(b.CheckExists("public/sect/page/index.html"), qt.Equals, false) + b.Assert(getPageInSitePages(b, "/sect/page.md"), qt.IsNil) + b.Assert(getPageInPagePages(getPage(b, "/"), "/sect/page.md"), qt.IsNil) + + // Also check the side effects + b.Assert(b.CheckExists("public/categories/mycat/index.html"), qt.Equals, false) + b.Assert(len(s.Taxonomies()["categories"]), qt.Equals, 0) + }) + + disableKind = page.KindTaxonomy + c.Run("Disable "+disableKind, func(c *qt.C) { + b := newSitesBuilder(c, disableKind) + b.Build(BuildCfg{}) + s := b.H.Sites[0] + b.Assert(b.CheckExists("public/categories/index.html"), qt.Equals, true) + b.Assert(b.CheckExists("public/categories/mycat/index.html"), qt.Equals, false) + b.Assert(len(s.Taxonomies()["categories"]), qt.Equals, 0) + b.Assert(getPage(b, "/categories"), qt.Not(qt.IsNil)) + b.Assert(getPage(b, "/categories/mycat"), qt.IsNil) + }) + + disableKind = page.KindTaxonomyTerm + c.Run("Disable "+disableKind, func(c *qt.C) { + b := newSitesBuilder(c, disableKind) + b.Build(BuildCfg{}) + s := b.H.Sites[0] + b.Assert(b.CheckExists("public/categories/mycat/index.html"), qt.Equals, true) + b.Assert(b.CheckExists("public/categories/index.html"), qt.Equals, false) + b.Assert(len(s.Taxonomies()["categories"]), qt.Equals, 1) + b.Assert(getPage(b, "/categories/mycat"), qt.Not(qt.IsNil)) + categories := getPage(b, "/categories") + b.Assert(categories, qt.Not(qt.IsNil)) + b.Assert(categories.RelPermalink(), qt.Equals, "") + b.Assert(getPageInSitePages(b, "/categories"), qt.IsNil) + b.Assert(getPageInPagePages(getPage(b, "/"), "/categories"), qt.IsNil) + }) + + disableKind = page.KindHome + c.Run("Disable "+disableKind, func(c *qt.C) { + b := newSitesBuilder(c, disableKind) + b.Build(BuildCfg{}) + b.Assert(b.CheckExists("public/index.html"), qt.Equals, false) + home := getPage(b, "/") + b.Assert(home, qt.Not(qt.IsNil)) + b.Assert(home.RelPermalink(), qt.Equals, "") + b.Assert(getPageInSitePages(b, "/"), qt.IsNil) + b.Assert(getPageInPagePages(home, "/"), qt.IsNil) + b.Assert(getPage(b, "/sect/page.md"), qt.Not(qt.IsNil)) + }) + + disableKind = page.KindSection + c.Run("Disable "+disableKind, func(c *qt.C) { + b := newSitesBuilder(c, disableKind) + b.Build(BuildCfg{}) + b.Assert(b.CheckExists("public/sect/index.html"), qt.Equals, false) + sect := getPage(b, "/sect") + b.Assert(sect, qt.Not(qt.IsNil)) + b.Assert(sect.RelPermalink(), qt.Equals, "") + b.Assert(getPageInSitePages(b, "/sect"), qt.IsNil) + home := getPage(b, "/") + b.Assert(getPageInPagePages(home, "/sect"), qt.IsNil) + b.Assert(home.OutputFormats(), qt.HasLen, 2) + page := getPage(b, "/sect/page.md") + b.Assert(page, qt.Not(qt.IsNil)) + b.Assert(page.CurrentSection(), qt.Equals, sect) + b.Assert(getPageInPagePages(sect, "/sect/page.md"), qt.Not(qt.IsNil)) + b.AssertFileContent("public/sitemap.xml", "sitemap") + b.AssertFileContent("public/index.xml", "rss") + + }) + + disableKind = kindRSS + c.Run("Disable "+disableKind, func(c *qt.C) { + b := newSitesBuilder(c, disableKind) + b.Build(BuildCfg{}) + b.Assert(b.CheckExists("public/index.xml"), qt.Equals, false) + home := getPage(b, "/") + b.Assert(home.OutputFormats(), qt.HasLen, 1) + }) + + disableKind = kindSitemap + c.Run("Disable "+disableKind, func(c *qt.C) { + b := newSitesBuilder(c, disableKind) + b.Build(BuildCfg{}) + b.Assert(b.CheckExists("public/sitemap.xml"), qt.Equals, false) + }) + + disableKind = kind404 + c.Run("Disable "+disableKind, func(c *qt.C) { + b := newSitesBuilder(c, disableKind) + b.Build(BuildCfg{}) + b.Assert(b.CheckExists("public/404.html"), qt.Equals, false) + }) + + disableKind = kindRobotsTXT + c.Run("Disable "+disableKind, func(c *qt.C) { + b := newSitesBuilder(c, disableKind) + b.WithTemplatesAdded("robots.txt", "myrobots") + b.Build(BuildCfg{}) + b.Assert(b.CheckExists("public/robots.txt"), qt.Equals, false) + }) + + c.Run("Headless bundle", func(c *qt.C) { + b := newSitesBuilder(c, disableKind) + b.Build(BuildCfg{}) + b.Assert(b.CheckExists("public/sect/headlessbundle/index.html"), qt.Equals, false) + b.Assert(b.CheckExists("public/sect/headlessbundle/data.json"), qt.Equals, true) + bundle := getPage(b, "/sect/headlessbundle/index.md") + b.Assert(bundle, qt.Not(qt.IsNil)) + b.Assert(bundle.RelPermalink(), qt.Equals, "") + resource := bundle.Resources()[0] + b.Assert(resource.RelPermalink(), qt.Equals, "/blog/sect/headlessbundle/data.json") + b.Assert(bundle.OutputFormats(), qt.HasLen, 0) + b.Assert(bundle.AlternativeOutputFormats(), qt.HasLen, 0) + }) + + c.Run("Build config, no list", func(c *qt.C) { + b := newSitesBuilder(c, disableKind) + b.Build(BuildCfg{}) + ref := "/sect/no-list.md" + b.Assert(b.CheckExists("public/sect/no-list/index.html"), qt.Equals, true) + p := getPage(b, ref) + b.Assert(p, qt.Not(qt.IsNil)) + b.Assert(p.RelPermalink(), qt.Equals, "/blog/sect/no-list/") + b.Assert(getPageInSitePages(b, ref), qt.IsNil) + sect := getPage(b, "/sect") + b.Assert(getPageInPagePages(sect, ref), qt.IsNil) + + }) + + c.Run("Build config, no render", func(c *qt.C) { + b := newSitesBuilder(c, disableKind) + b.Build(BuildCfg{}) + ref := "/sect/no-render.md" + b.Assert(b.CheckExists("public/sect/no-render/index.html"), qt.Equals, false) + p := getPage(b, ref) + b.Assert(p, qt.Not(qt.IsNil)) + b.Assert(p.RelPermalink(), qt.Equals, "") + b.Assert(p.OutputFormats(), qt.HasLen, 0) + b.Assert(getPageInSitePages(b, ref), qt.Not(qt.IsNil)) + sect := getPage(b, "/sect") + b.Assert(getPageInPagePages(sect, ref), qt.Not(qt.IsNil)) + }) + + c.Run("Build config, no publish resources", func(c *qt.C) { + b := newSitesBuilder(c, disableKind) + b.Build(BuildCfg{}) + b.Assert(b.CheckExists("public/sect/no-publishresources/index.html"), qt.Equals, true) + b.Assert(b.CheckExists("public/sect/no-publishresources/data.json"), qt.Equals, false) + bundle := getPage(b, "/sect/no-publishresources/index.md") + b.Assert(bundle, qt.Not(qt.IsNil)) + b.Assert(bundle.RelPermalink(), qt.Equals, "/blog/sect/no-publishresources/") + b.Assert(bundle.Resources(), qt.HasLen, 1) + resource := bundle.Resources()[0] + b.Assert(resource.RelPermalink(), qt.Equals, "/blog/sect/no-publishresources/data.json") + }) } |