summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>2024-06-19 12:52:10 +0200
committerBjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>2024-06-19 15:18:43 +0200
commit478a9107a68aeb5dac9ea0cec0a347fadb708b64 (patch)
tree1125ee7892a418445cedd0ad2ad55a68984a0fa2
parentb46d101d5a6069674d17a6ee5ea3c6c7ae6b2fba (diff)
Speed up GetTerms
```text name old time/op new time/op delta TaxonomiesGetTerms/pages_100-10 5.25ms 5% 5.13ms 4% ~ (p=0.486 n=4+4) TaxonomiesGetTerms/pages_1000-10 30.1ms 1% 26.8ms 1% -11.13% (p=0.029 n=4+4) TaxonomiesGetTerms/pages_10000-10 1.33s 24% 0.29s 2% -78.42% (p=0.029 n=4+4) TaxonomiesGetTerms/pages_20000-10 5.50s 12% 0.83s 28% -84.88% (p=0.029 n=4+4) name old alloc/op new alloc/op delta TaxonomiesGetTerms/pages_100-10 4.08MB 0% 4.06MB 0% -0.59% (p=0.029 n=4+4) TaxonomiesGetTerms/pages_1000-10 25.1MB 0% 24.9MB 0% -0.87% (p=0.029 n=4+4) TaxonomiesGetTerms/pages_10000-10 238MB 2% 233MB 0% -1.94% (p=0.029 n=4+4) TaxonomiesGetTerms/pages_20000-10 469MB 0% 465MB 0% -1.00% (p=0.029 n=4+4) name old allocs/op new allocs/op delta TaxonomiesGetTerms/pages_100-10 49.5k 0% 48.9k 0% -1.17% (p=0.029 n=4+4) TaxonomiesGetTerms/pages_1000-10 304k 0% 298k 0% -1.97% (p=0.029 n=4+4) TaxonomiesGetTerms/pages_10000-10 3.02M 7% 2.81M 0% -7.09% (p=0.029 n=4+4) TaxonomiesGetTerms/pages_20000-10 5.77M 1% 5.59M 0% -3.19% (p=0.029 n=4+4) ``` Note that the numbers above represents a full site build, but GetTerms is a big part of the site in question. Fixes #12610
-rw-r--r--hugolib/content_map_page.go20
-rw-r--r--hugolib/taxonomy_test.go55
2 files changed, 66 insertions, 9 deletions
diff --git a/hugolib/content_map_page.go b/hugolib/content_map_page.go
index 0ce43ea68..f9709df15 100644
--- a/hugolib/content_map_page.go
+++ b/hugolib/content_map_page.go
@@ -98,6 +98,7 @@ type pageMap struct {
cachePages1 *dynacache.Partition[string, page.Pages]
cachePages2 *dynacache.Partition[string, page.Pages]
cacheResources *dynacache.Partition[string, resource.Resources]
+ cacheGetTerms *dynacache.Partition[string, map[string]page.Pages]
cacheContentRendered *dynacache.Partition[string, *resources.StaleValue[contentSummary]]
cacheContentPlain *dynacache.Partition[string, *resources.StaleValue[contentPlainPlainWords]]
contentTableOfContents *dynacache.Partition[string, *resources.StaleValue[contentTableOfContents]]
@@ -448,16 +449,13 @@ func (m *pageMap) getPagesWithTerm(q pageMapQueryPagesBelowPath) page.Pages {
func (m *pageMap) getTermsForPageInTaxonomy(path, taxonomy string) page.Pages {
prefix := paths.AddLeadingSlash(taxonomy)
- v, err := m.cachePages1.GetOrCreate(prefix+path, func(string) (page.Pages, error) {
- var pas page.Pages
-
+ termPages, err := m.cacheGetTerms.GetOrCreate(prefix, func(string) (map[string]page.Pages, error) {
+ mm := make(map[string]page.Pages)
err := m.treeTaxonomyEntries.WalkPrefix(
doctree.LockTypeNone,
paths.AddTrailingSlash(prefix),
func(s string, n *weightedContentNode) (bool, error) {
- if strings.HasSuffix(s, path) {
- pas = append(pas, n.term)
- }
+ mm[n.n.Path()] = append(mm[n.n.Path()], n.term)
return false, nil
},
)
@@ -465,15 +463,18 @@ func (m *pageMap) getTermsForPageInTaxonomy(path, taxonomy string) page.Pages {
return nil, err
}
- page.SortByDefault(pas)
+ // Sort the terms.
+ for _, v := range mm {
+ page.SortByDefault(v)
+ }
- return pas, nil
+ return mm, nil
})
if err != nil {
panic(err)
}
- return v
+ return termPages[path]
}
func (m *pageMap) forEachResourceInPage(
@@ -898,6 +899,7 @@ func newPageMap(i int, s *Site, mcache *dynacache.Cache, pageTrees *pageTrees) *
pageTrees: pageTrees.Shape(0, i),
cachePages1: dynacache.GetOrCreatePartition[string, page.Pages](mcache, fmt.Sprintf("/pag1/%d", i), dynacache.OptionsPartition{Weight: 10, ClearWhen: dynacache.ClearOnRebuild}),
cachePages2: dynacache.GetOrCreatePartition[string, page.Pages](mcache, fmt.Sprintf("/pag2/%d", i), dynacache.OptionsPartition{Weight: 10, ClearWhen: dynacache.ClearOnRebuild}),
+ cacheGetTerms: dynacache.GetOrCreatePartition[string, map[string]page.Pages](mcache, fmt.Sprintf("/gett/%d", i), dynacache.OptionsPartition{Weight: 5, ClearWhen: dynacache.ClearOnRebuild}),
cacheResources: dynacache.GetOrCreatePartition[string, resource.Resources](mcache, fmt.Sprintf("/ress/%d", i), dynacache.OptionsPartition{Weight: 60, ClearWhen: dynacache.ClearOnRebuild}),
cacheContentRendered: dynacache.GetOrCreatePartition[string, *resources.StaleValue[contentSummary]](mcache, fmt.Sprintf("/cont/ren/%d", i), dynacache.OptionsPartition{Weight: 70, ClearWhen: dynacache.ClearOnChange}),
cacheContentPlain: dynacache.GetOrCreatePartition[string, *resources.StaleValue[contentPlainPlainWords]](mcache, fmt.Sprintf("/cont/pla/%d", i), dynacache.OptionsPartition{Weight: 70, ClearWhen: dynacache.ClearOnChange}),
diff --git a/hugolib/taxonomy_test.go b/hugolib/taxonomy_test.go
index bfdfd8dfd..e96b82d39 100644
--- a/hugolib/taxonomy_test.go
+++ b/hugolib/taxonomy_test.go
@@ -970,3 +970,58 @@ title: p1
b.AssertFileExists("public/ja/s1/index.html", false) // failing test
b.AssertFileExists("public/ja/s1/category/index.html", true)
}
+
+func BenchmarkTaxonomiesGetTerms(b *testing.B) {
+ createBuilders := func(b *testing.B, numPages int) []*IntegrationTestBuilder {
+ files := `
+-- hugo.toml --
+baseURL = "https://example.com"
+disableKinds = ["RSS", "sitemap", "section"]
+[taxononomies]
+tag = "tags"
+-- layouts/_default/list.html --
+List.
+-- layouts/_default/single.html --
+GetTerms.tags: {{ range .GetTerms "tags" }}{{ .Title }}|{{ end }}
+-- content/_index.md --
+`
+
+ tagsVariants := []string{
+ "tags: ['a']",
+ "tags: ['a', 'b']",
+ "tags: ['a', 'b', 'c']",
+ "tags: ['a', 'b', 'c', 'd']",
+ "tags: ['a', 'b', 'd', 'e']",
+ "tags: ['a', 'b', 'c', 'd', 'e']",
+ "tags: ['a', 'd']",
+ "tags: ['a', 'f']",
+ }
+
+ for i := 1; i < numPages; i++ {
+ tags := tagsVariants[i%len(tagsVariants)]
+ files += fmt.Sprintf("\n-- content/posts/p%d.md --\n---\n%s\n---", i+1, tags)
+ }
+ cfg := IntegrationTestConfig{
+ T: b,
+ TxtarString: files,
+ }
+ builders := make([]*IntegrationTestBuilder, b.N)
+
+ for i := range builders {
+ builders[i] = NewIntegrationTestBuilder(cfg)
+ }
+
+ b.ResetTimer()
+
+ return builders
+ }
+
+ for _, numPages := range []int{100, 1000, 10000, 20000} {
+ b.Run(fmt.Sprintf("pages_%d", numPages), func(b *testing.B) {
+ builders := createBuilders(b, numPages)
+ for i := 0; i < b.N; i++ {
+ builders[i].Build()
+ }
+ })
+ }
+}