Introduce a tree map for all content

This commit introduces a new data structure to store pages and their resources.

This data structure is backed by radix trees.

This simplies tree operations, makes all pages a bundle,  and paves the way for #6310.

It also solves a set of annoying issues (see list below).

Not a motivation behind this, but this commit also makes Hugo in general a little bit faster and more memory effective (see benchmarks). Especially for partial rebuilds on content edits, but also when taxonomies is in use.

```
name                                   old time/op    new time/op    delta
SiteNew/Bundle_with_image/Edit-16        1.32ms ± 8%    1.00ms ± 9%  -24.42%  (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file/Edit-16    1.28ms ± 0%    0.94ms ± 0%  -26.26%  (p=0.029 n=4+4)
SiteNew/Tags_and_categories/Edit-16      33.9ms ± 2%    21.8ms ± 1%  -35.67%  (p=0.029 n=4+4)
SiteNew/Canonify_URLs/Edit-16            40.6ms ± 1%    37.7ms ± 3%   -7.20%  (p=0.029 n=4+4)
SiteNew/Deep_content_tree/Edit-16        56.7ms ± 0%    51.7ms ± 1%   -8.82%  (p=0.029 n=4+4)
SiteNew/Many_HTML_templates/Edit-16      19.9ms ± 2%    18.3ms ± 3%   -7.64%  (p=0.029 n=4+4)
SiteNew/Page_collections/Edit-16         37.9ms ± 4%    34.0ms ± 2%  -10.28%  (p=0.029 n=4+4)
SiteNew/Bundle_with_image-16             10.7ms ± 0%    10.6ms ± 0%   -1.15%  (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file-16         10.8ms ± 0%    10.7ms ± 0%   -1.05%  (p=0.029 n=4+4)
SiteNew/Tags_and_categories-16           43.2ms ± 1%    39.6ms ± 1%   -8.35%  (p=0.029 n=4+4)
SiteNew/Canonify_URLs-16                 47.6ms ± 1%    47.3ms ± 0%     ~     (p=0.057 n=4+4)
SiteNew/Deep_content_tree-16             73.0ms ± 1%    74.2ms ± 1%     ~     (p=0.114 n=4+4)
SiteNew/Many_HTML_templates-16           37.9ms ± 0%    38.1ms ± 1%     ~     (p=0.114 n=4+4)
SiteNew/Page_collections-16              53.6ms ± 1%    54.7ms ± 1%   +2.09%  (p=0.029 n=4+4)

name                                   old alloc/op   new alloc/op   delta
SiteNew/Bundle_with_image/Edit-16         486kB ± 0%     430kB ± 0%  -11.47%  (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file/Edit-16     265kB ± 0%     209kB ± 0%  -21.06%  (p=0.029 n=4+4)
SiteNew/Tags_and_categories/Edit-16      13.6MB ± 0%     8.8MB ± 0%  -34.93%  (p=0.029 n=4+4)
SiteNew/Canonify_URLs/Edit-16            66.5MB ± 0%    63.9MB ± 0%   -3.95%  (p=0.029 n=4+4)
SiteNew/Deep_content_tree/Edit-16        28.8MB ± 0%    25.8MB ± 0%  -10.55%  (p=0.029 n=4+4)
SiteNew/Many_HTML_templates/Edit-16      6.16MB ± 0%    5.56MB ± 0%   -9.86%  (p=0.029 n=4+4)
SiteNew/Page_collections/Edit-16         16.9MB ± 0%    16.0MB ± 0%   -5.19%  (p=0.029 n=4+4)
SiteNew/Bundle_with_image-16             2.28MB ± 0%    2.29MB ± 0%   +0.35%  (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file-16         2.07MB ± 0%    2.07MB ± 0%     ~     (p=0.114 n=4+4)
SiteNew/Tags_and_categories-16           14.3MB ± 0%    13.2MB ± 0%   -7.30%  (p=0.029 n=4+4)
SiteNew/Canonify_URLs-16                 69.1MB ± 0%    69.0MB ± 0%     ~     (p=0.343 n=4+4)
SiteNew/Deep_content_tree-16             31.3MB ± 0%    31.8MB ± 0%   +1.49%  (p=0.029 n=4+4)
SiteNew/Many_HTML_templates-16           10.8MB ± 0%    10.9MB ± 0%   +1.11%  (p=0.029 n=4+4)
SiteNew/Page_collections-16              21.4MB ± 0%    21.6MB ± 0%   +1.15%  (p=0.029 n=4+4)

name                                   old allocs/op  new allocs/op  delta
SiteNew/Bundle_with_image/Edit-16         4.74k ± 0%     3.86k ± 0%  -18.57%  (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file/Edit-16     4.73k ± 0%     3.85k ± 0%  -18.58%  (p=0.029 n=4+4)
SiteNew/Tags_and_categories/Edit-16        301k ± 0%      198k ± 0%  -34.14%  (p=0.029 n=4+4)
SiteNew/Canonify_URLs/Edit-16              389k ± 0%      373k ± 0%   -4.07%  (p=0.029 n=4+4)
SiteNew/Deep_content_tree/Edit-16          338k ± 0%      262k ± 0%  -22.63%  (p=0.029 n=4+4)
SiteNew/Many_HTML_templates/Edit-16        102k ± 0%       88k ± 0%  -13.81%  (p=0.029 n=4+4)
SiteNew/Page_collections/Edit-16           176k ± 0%      152k ± 0%  -13.32%  (p=0.029 n=4+4)
SiteNew/Bundle_with_image-16              26.8k ± 0%     26.8k ± 0%   +0.05%  (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file-16          26.8k ± 0%     26.8k ± 0%   +0.05%  (p=0.029 n=4+4)
SiteNew/Tags_and_categories-16             273k ± 0%      245k ± 0%  -10.36%  (p=0.029 n=4+4)
SiteNew/Canonify_URLs-16                   396k ± 0%      398k ± 0%   +0.39%  (p=0.029 n=4+4)
SiteNew/Deep_content_tree-16               317k ± 0%      325k ± 0%   +2.53%  (p=0.029 n=4+4)
SiteNew/Many_HTML_templates-16             146k ± 0%      147k ± 0%   +0.98%  (p=0.029 n=4+4)
SiteNew/Page_collections-16                210k ± 0%      215k ± 0%   +2.44%  (p=0.029 n=4+4)
```

Fixes #6312
Fixes #6087
Fixes #6738
Fixes #6412
Fixes #6743
Fixes #6875
Fixes #6034
Fixes #6902
Fixes #6173
Fixes #6590

1 files changed, 82 insertions, 40 deletions

diff --git a/hugolib/page__meta.go b/hugolib/page__meta.go
index 60952c831..8ecffbda3 100644
--- a/hugolib/page__meta.go
+++ b/hugolib/page__meta.go
@@ -61,7 +61,10 @@ type pageMeta struct {
 	// a fixed pageOutput.
 	standalone bool
 
-	bundleType string
+	draft       bool // Only published when running with -D flag
+	buildConfig pagemeta.BuildConfig
+
+	bundleType files.ContentClass
 
 	// Params contains configuration defined in the params section of page frontmatter.
 	params map[string]interface{}
@@ -85,8 +88,6 @@ type pageMeta struct {
 
 	aliases []string
 
-	draft bool
-
 	description string
 	keywords    []string
 
@@ -94,13 +95,6 @@ type pageMeta struct {
 
 	resource.Dates
 
-	// This is enabled if it is a leaf bundle (the "index.md" type) and it is marked as headless in front matter.
-	// Being headless means that
-	// 1. The page itself is not rendered to disk
-	// 2. It is not available in .Site.Pages etc.
-	// 3. But you can get it via .Site.GetPage
-	headless bool
-
 	// Set if this page is bundled inside another.
 	bundled bool
 
@@ -160,7 +154,7 @@ func (p *pageMeta) Authors() page.AuthorList {
 	return al
 }
 
-func (p *pageMeta) BundleType() string {
+func (p *pageMeta) BundleType() files.ContentClass {
 	return p.bundleType
 }
 
@@ -309,40 +303,53 @@ func (p *pageMeta) Weight() int {
 	return p.weight
 }
 
-func (pm *pageMeta) setMetadata(bucket *pagesMapBucket, p *pageState, frontmatter map[string]interface{}) error {
-	if frontmatter == nil && bucket.cascade == nil {
-		return errors.New("missing frontmatter data")
+func (pm *pageMeta) mergeBucketCascades(b1, b2 *pagesMapBucket) {
+	if b1.cascade == nil {
+		b1.cascade = make(map[string]interface{})
+	}
+	if b2 != nil && b2.cascade != nil {
+		for k, v := range b2.cascade {
+			if _, found := b1.cascade[k]; !found {
+				b1.cascade[k] = v
+			}
+		}
 	}
+}
 
+func (pm *pageMeta) setMetadata(parentBucket *pagesMapBucket, p *pageState, frontmatter map[string]interface{}) error {
 	pm.params = make(maps.Params)
 
+	if frontmatter == nil && (parentBucket == nil || parentBucket.cascade == nil) {
+		return nil
+	}
+
 	if frontmatter != nil {
 		// Needed for case insensitive fetching of params values
 		maps.ToLower(frontmatter)
-		if p.IsNode() {
+		if p.bucket != nil {
 			// Check for any cascade define on itself.
 			if cv, found := frontmatter["cascade"]; found {
-				cvm := maps.ToStringMap(cv)
-				if bucket.cascade == nil {
-					bucket.cascade = cvm
-				} else {
-					for k, v := range cvm {
-						bucket.cascade[k] = v
-					}
-				}
-			}
-		}
-
-		if bucket != nil && bucket.cascade != nil {
-			for k, v := range bucket.cascade {
-				if _, found := frontmatter[k]; !found {
-					frontmatter[k] = v
-				}
+				p.bucket.cascade = maps.ToStringMap(cv)
 			}
 		}
 	} else {
 		frontmatter = make(map[string]interface{})
-		for k, v := range bucket.cascade {
+	}
+
+	var cascade map[string]interface{}
+
+	if p.bucket != nil {
+		if parentBucket != nil {
+			// Merge missing keys from parent into this.
+			pm.mergeBucketCascades(p.bucket, parentBucket)
+		}
+		cascade = p.bucket.cascade
+	} else if parentBucket != nil {
+		cascade = parentBucket.cascade
+	}
+
+	for k, v := range cascade {
+		if _, found := frontmatter[k]; !found {
 			frontmatter[k] = v
 		}
 	}
@@ -379,6 +386,11 @@ func (pm *pageMeta) setMetadata(bucket *pagesMapBucket, p *pageState, frontmatte
 		p.s.Log.ERROR.Printf("Failed to handle dates for page %q: %s", p.pathOrTitle(), err)
 	}
 
+	pm.buildConfig, err = pagemeta.DecodeBuildConfig(frontmatter["_build"])
+	if err != nil {
+		return err
+	}
+
 	var sitemapSet bool
 
 	var draft, published, isCJKLanguage *bool
@@ -439,12 +451,15 @@ func (pm *pageMeta) setMetadata(bucket *pagesMapBucket, p *pageState, frontmatte
 			pm.keywords = cast.ToStringSlice(v)
 			pm.params[loki] = pm.keywords
 		case "headless":
-			// For now, only the leaf bundles ("index.md") can be headless (i.e. produce no output).
-			// We may expand on this in the future, but that gets more complex pretty fast.
-			if p.File().TranslationBaseName() == "index" {
-				pm.headless = cast.ToBool(v)
+			// Legacy setting for leaf bundles.
+			// This is since Hugo 0.63 handled in a more general way for all
+			// pages.
+			isHeadless := cast.ToBool(v)
+			pm.params[loki] = isHeadless
+			if p.File().TranslationBaseName() == "index" && isHeadless {
+				pm.buildConfig.List = false
+				pm.buildConfig.Render = false
 			}
-			pm.params[loki] = pm.headless
 		case "outputs":
 			o := cast.ToStringSlice(v)
 			if len(o) > 0 {
@@ -594,7 +609,23 @@ func (pm *pageMeta) setMetadata(bucket *pagesMapBucket, p *pageState, frontmatte
 	return nil
 }
 
-func (p *pageMeta) applyDefaultValues(ps *pageState) error {
+func (p *pageMeta) noList() bool {
+	return !p.buildConfig.List
+}
+
+func (p *pageMeta) noRender() bool {
+	return !p.buildConfig.Render
+}
+
+func (p *pageMeta) applyDefaultValues(n *contentNode) error {
+	if p.buildConfig.IsZero() {
+		p.buildConfig, _ = pagemeta.DecodeBuildConfig(nil)
+	}
+
+	if !p.s.isEnabled(p.Kind()) {
+		(&p.buildConfig).Disable()
+	}
+
 	if p.markup == "" {
 		if !p.File().IsZero() {
 			// Fall back to file extension
@@ -610,13 +641,21 @@ func (p *pageMeta) applyDefaultValues(ps *pageState) error {
 		case page.KindHome:
 			p.title = p.s.Info.title
 		case page.KindSection:
-			sectionName := helpers.FirstUpper(p.sections[0])
+			var sectionName string
+			if n != nil {
+				sectionName = n.rootSection()
+			} else {
+				sectionName = p.sections[0]
+			}
+
+			sectionName = helpers.FirstUpper(sectionName)
 			if p.s.Cfg.GetBool("pluralizeListTitles") {
 				p.title = inflect.Pluralize(sectionName)
 			} else {
 				p.title = sectionName
 			}
 		case page.KindTaxonomy:
+			// TODO(bep) improve
 			key := p.sections[len(p.sections)-1]
 			p.title = strings.Replace(p.s.titleFunc(key), "-", " ", -1)
 		case page.KindTaxonomyTerm:
@@ -653,7 +692,7 @@ func (p *pageMeta) applyDefaultValues(ps *pageState) error {
 			markup = "markdown"
 		}
 
-		cp, err := p.newContentConverter(ps, markup, renderingConfigOverrides)
+		cp, err := p.newContentConverter(n.p, markup, renderingConfigOverrides)
 		if err != nil {
 			return err
 		}
@@ -665,6 +704,9 @@ func (p *pageMeta) applyDefaultValues(ps *pageState) error {
 }
 
 func (p *pageMeta) newContentConverter(ps *pageState, markup string, renderingConfigOverrides map[string]interface{}) (converter.Converter, error) {
+	if ps == nil {
+		panic("no Page provided")
+	}
 	cp := p.s.ContentSpec.Converters.Get(markup)
 	if cp == nil {
 		return nil, errors.Errorf("no content renderer found for markup %q", p.markup)