Allow headless bundles to list pages via $page.Pages and $page.RegularPages

Fixes #7075

7 files changed, 177 insertions, 67 deletions

diff --git a/hugolib/content_map.go b/hugolib/content_map.go
index 2ea36ff31..7b28965ac 100644
--- a/hugolib/content_map.go
+++ b/hugolib/content_map.go
@@ -789,6 +789,12 @@ func (t contentTrees) DeletePrefix(prefix string) int {
 
 type contentTreeNodeCallback func(s string, n *contentNode) bool
 
+func newContentTreeFilter(fn func(n *contentNode) bool) contentTreeNodeCallback {
+	return func(s string, n *contentNode) bool {
+		return fn(n)
+	}
+}
+
 var (
 	contentTreeNoListFilter = func(s string, n *contentNode) bool {
 		if n.p == nil {
@@ -805,43 +811,36 @@ var (
 	}
 )
 
-func (c *contentTree) WalkPrefixListable(prefix string, fn contentTreeNodeCallback) {
-	c.WalkPrefixFilter(prefix, contentTreeNoListFilter, fn)
-}
-
-func (c *contentTree) WalkPrefixFilter(prefix string, filter, walkFn contentTreeNodeCallback) {
-	c.WalkPrefix(prefix, func(s string, v interface{}) bool {
-		n := v.(*contentNode)
-		if filter(s, n) {
-			return false
-		}
-		return walkFn(s, n)
-	})
-}
+func (c *contentTree) WalkQuery(query pageMapQuery, walkFn contentTreeNodeCallback) {
+	filter := query.Filter
+	if filter == nil {
+		filter = contentTreeNoListFilter
+	}
+	if query.Prefix != "" {
+		c.WalkPrefix(query.Prefix, func(s string, v interface{}) bool {
+			n := v.(*contentNode)
+			if filter != nil && filter(s, n) {
+				return false
+			}
+			return walkFn(s, n)
+		})
 
-func (c *contentTree) WalkListable(fn contentTreeNodeCallback) {
-	c.WalkFilter(contentTreeNoListFilter, fn)
-}
+		return
+	}
 
-func (c *contentTree) WalkFilter(filter, walkFn contentTreeNodeCallback) {
 	c.Walk(func(s string, v interface{}) bool {
 		n := v.(*contentNode)
-		if filter(s, n) {
+		if filter != nil && filter(s, n) {
 			return false
 		}
 		return walkFn(s, n)
 	})
 }
 
-func (c contentTrees) WalkListable(fn contentTreeNodeCallback) {
-	for _, tree := range c {
-		tree.WalkListable(fn)
-	}
-}
-
 func (c contentTrees) WalkRenderable(fn contentTreeNodeCallback) {
+	query := pageMapQuery{Filter: contentTreeNoRenderFilter}
 	for _, tree := range c {
-		tree.WalkFilter(contentTreeNoRenderFilter, fn)
+		tree.WalkQuery(query, fn)
 	}
 }
 
@@ -931,44 +930,73 @@ func (c *contentTreeRef) getSection() (string, *contentNode) {
 	return c.m.getSection(c.key)
 }
 
-func (c *contentTreeRef) collectPages() page.Pages {
+func (c *contentTreeRef) getPages() page.Pages {
 	var pas page.Pages
-	c.m.collectPages(c.key+cmBranchSeparator, func(c *contentNode) {
-		pas = append(pas, c.p)
-	})
+	c.m.collectPages(
+		pageMapQuery{
+			Prefix: c.key + cmBranchSeparator,
+			Filter: c.n.p.m.getListFilter(true),
+		},
+		func(c *contentNode) {
+			pas = append(pas, c.p)
+		},
+	)
 	page.SortByDefault(pas)
 
 	return pas
 }
 
-func (c *contentTreeRef) collectPagesRecursive() page.Pages {
+func (c *contentTreeRef) getPagesRecursive() page.Pages {
 	var pas page.Pages
-	c.m.collectPages(c.key+cmBranchSeparator, func(c *contentNode) {
+
+	query := pageMapQuery{
+		Filter: c.n.p.m.getListFilter(true),
+	}
+
+	query.Prefix = c.key + cmBranchSeparator
+	c.m.collectPages(query, func(c *contentNode) {
 		pas = append(pas, c.p)
 	})
-	c.m.collectPages(c.key+"/", func(c *contentNode) {
+
+	query.Prefix = c.key + "/"
+	c.m.collectPages(query, func(c *contentNode) {
 		pas = append(pas, c.p)
 	})
+
 	page.SortByDefault(pas)
 
 	return pas
 }
 
-func (c *contentTreeRef) collectPagesAndSections() page.Pages {
+func (c *contentTreeRef) getPagesAndSections() page.Pages {
 	var pas page.Pages
-	c.m.collectPagesAndSections(c.key, func(c *contentNode) {
+
+	query := pageMapQuery{
+		Filter: c.n.p.m.getListFilter(true),
+		Prefix: c.key,
+	}
+
+	c.m.collectPagesAndSections(query, func(c *contentNode) {
 		pas = append(pas, c.p)
 	})
+
 	page.SortByDefault(pas)
 
 	return pas
 }
 
-func (c *contentTreeRef) collectSections() page.Pages {
+func (c *contentTreeRef) getSections() page.Pages {
 	var pas page.Pages
-	c.m.collectSections(c.key, func(c *contentNode) {
+
+	query := pageMapQuery{
+		Filter: c.n.p.m.getListFilter(true),
+		Prefix: c.key,
+	}
+
+	c.m.collectSections(query, func(c *contentNode) {
 		pas = append(pas, c.p)
 	})
+
 	page.SortByDefault(pas)
 
 	return pas
diff --git a/hugolib/content_map_page.go b/hugolib/content_map_page.go
index 4ba97f511..bcab9ffa9 100644
--- a/hugolib/content_map_page.go
+++ b/hugolib/content_map_page.go
@@ -606,36 +606,47 @@ func (m *pageMap) attachPageToViews(s string, b *contentNode) {
 	}
 }
 
-func (m *pageMap) collectPages(prefix string, fn func(c *contentNode)) error {
-	m.pages.WalkPrefixListable(prefix, func(s string, n *contentNode) bool {
+type pageMapQuery struct {
+	Prefix string
+	Filter contentTreeNodeCallback
+}
+
+func (m *pageMap) collectPages(query pageMapQuery, fn func(c *contentNode)) error {
+	if query.Filter == nil {
+		query.Filter = contentTreeNoListFilter
+	}
+
+	m.pages.WalkQuery(query, func(s string, n *contentNode) bool {
 		fn(n)
 		return false
 	})
+
 	return nil
 }
 
-func (m *pageMap) collectPagesAndSections(prefix string, fn func(c *contentNode)) error {
-	if err := m.collectSections(prefix, fn); err != nil {
+func (m *pageMap) collectPagesAndSections(query pageMapQuery, fn func(c *contentNode)) error {
+	if err := m.collectSections(query, fn); err != nil {
 		return err
 	}
 
-	if err := m.collectPages(prefix+cmBranchSeparator, fn); err != nil {
+	query.Prefix = query.Prefix + cmBranchSeparator
+	if err := m.collectPages(query, fn); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-func (m *pageMap) collectSections(prefix string, fn func(c *contentNode)) error {
+func (m *pageMap) collectSections(query pageMapQuery, fn func(c *contentNode)) error {
 	var level int
-	isHome := prefix == "/"
+	isHome := query.Prefix == "/"
 
 	if !isHome {
-		level = strings.Count(prefix, "/")
+		level = strings.Count(query.Prefix, "/")
 	}
 
-	return m.collectSectionsFn(prefix, func(s string, c *contentNode) bool {
-		if s == prefix {
+	return m.collectSectionsFn(query, func(s string, c *contentNode) bool {
+		if s == query.Prefix {
 			return false
 		}
 
@@ -649,27 +660,28 @@ func (m *pageMap) collectSections(prefix string, fn func(c *contentNode)) error
 	})
 }
 
-func (m *pageMap) collectSectionsFn(prefix string, fn func(s string, c *contentNode) bool) error {
-	if !strings.HasSuffix(prefix, "/") {
-		prefix += "/"
+func (m *pageMap) collectSectionsFn(query pageMapQuery, fn func(s string, c *contentNode) bool) error {
+
+	if !strings.HasSuffix(query.Prefix, "/") {
+		query.Prefix += "/"
 	}
 
-	m.sections.WalkPrefixListable(prefix, func(s string, n *contentNode) bool {
+	m.sections.WalkQuery(query, func(s string, n *contentNode) bool {
 		return fn(s, n)
 	})
 
 	return nil
 }
 
-func (m *pageMap) collectSectionsRecursiveIncludingSelf(prefix string, fn func(c *contentNode)) error {
-	return m.collectSectionsFn(prefix, func(s string, c *contentNode) bool {
+func (m *pageMap) collectSectionsRecursiveIncludingSelf(query pageMapQuery, fn func(c *contentNode)) error {
+	return m.collectSectionsFn(query, func(s string, c *contentNode) bool {
 		fn(c)
 		return false
 	})
 }
 
 func (m *pageMap) collectTaxonomies(prefix string, fn func(c *contentNode)) error {
-	m.taxonomies.WalkPrefixListable(prefix, func(s string, n *contentNode) bool {
+	m.taxonomies.WalkQuery(pageMapQuery{Prefix: prefix}, func(s string, n *contentNode) bool {
 		fn(n)
 		return false
 	})
@@ -797,21 +809,21 @@ type pagesMapBucket struct {
 
 func (b *pagesMapBucket) getPages() page.Pages {
 	b.pagesInit.Do(func() {
-		b.pages = b.owner.treeRef.collectPages()
+		b.pages = b.owner.treeRef.getPages()
 		page.SortByDefault(b.pages)
 	})
 	return b.pages
 }
 
 func (b *pagesMapBucket) getPagesRecursive() page.Pages {
-	pages := b.owner.treeRef.collectPagesRecursive()
+	pages := b.owner.treeRef.getPagesRecursive()
 	page.SortByDefault(pages)
 	return pages
 }
 
 func (b *pagesMapBucket) getPagesAndSections() page.Pages {
 	b.pagesAndSectionsInit.Do(func() {
-		b.pagesAndSections = b.owner.treeRef.collectPagesAndSections()
+		b.pagesAndSections = b.owner.treeRef.getPagesAndSections()
 	})
 	return b.pagesAndSections
 }
@@ -821,7 +833,7 @@ func (b *pagesMapBucket) getSections() page.Pages {
 		if b.owner.treeRef == nil {
 			return
 		}
-		b.sections = b.owner.treeRef.collectSections()
+		b.sections = b.owner.treeRef.getSections()
 	})
 
 	return b.sections
diff --git a/hugolib/disableKinds_test.go b/hugolib/disableKinds_test.go
index b1c00b48e..9ac30442e 100644
--- a/hugolib/disableKinds_test.go
+++ b/hugolib/disableKinds_test.go
@@ -66,7 +66,26 @@ title: Headless
 headless: true
 ---
 
-`)
+
+`, "headless-local/_index.md", `
+---
+title: Headless Local Lists
+cascade:
+    _build:
+        render: false
+        list: local
+        publishResources: false
+---
+
+`, "headless-local/headless-local-page.md", "---\ntitle: Headless Local Page\n---",
+			"headless-local/sub/_index.md", `
+---
+title: Headless Local Lists Sub
+---
+
+`, "headless-local/sub/headless-local-sub-page.md", "---\ntitle: Headless Local Sub Page\n---",
+		)
+
 		b.WithSourceFile("content/sect/headlessbundle/data.json", "DATA")
 		b.WithSourceFile("content/sect/no-publishresources/data.json", "DATA")
 
@@ -93,8 +112,11 @@ headless: true
 		return nil
 	}
 
-	getPageInPagePages := func(p page.Page, ref string) page.Page {
-		for _, pages := range []page.Pages{p.Pages(), p.RegularPages(), p.Sections()} {
+	getPageInPagePages := func(p page.Page, ref string, pageCollections ...page.Pages) page.Page {
+		if len(pageCollections) == 0 {
+			pageCollections = []page.Pages{p.Pages(), p.RegularPages(), p.RegularPagesRecursive(), p.Sections()}
+		}
+		for _, pages := range pageCollections {
 			for _, p := range pages {
 				if ref == p.(*pageState).sourceRef() {
 					return p
@@ -240,6 +262,33 @@ headless: true
 
 	})
 
+	c.Run("Build config, local list", func(c *qt.C) {
+		b := newSitesBuilder(c, disableKind)
+		b.Build(BuildCfg{})
+		ref := "/headless-local"
+		sect := getPage(b, ref)
+		b.Assert(sect, qt.Not(qt.IsNil))
+		b.Assert(getPageInSitePages(b, ref), qt.IsNil)
+		b.Assert(getPageInSitePages(b, ref+"/headless-local-page"), qt.IsNil)
+		for i, p := range sect.RegularPages() {
+			fmt.Println("REG", i, p.(*pageState).sourceRef())
+		}
+
+		localPageRef := ref + "/headless-local-page.md"
+
+		b.Assert(getPageInPagePages(sect, localPageRef, sect.RegularPages()), qt.Not(qt.IsNil))
+		b.Assert(getPageInPagePages(sect, localPageRef, sect.RegularPagesRecursive()), qt.Not(qt.IsNil))
+		b.Assert(getPageInPagePages(sect, localPageRef, sect.Pages()), qt.Not(qt.IsNil))
+
+		ref = "/headless-local/sub"
+
+		sect = getPage(b, ref)
+		b.Assert(sect, qt.Not(qt.IsNil))
+
+		localPageRef = ref + "/headless-local-sub-page.md"
+		b.Assert(getPageInPagePages(sect, localPageRef), qt.Not(qt.IsNil))
+	})
+
 	c.Run("Build config, no render", func(c *qt.C) {
 		b := newSitesBuilder(c, disableKind)
 		b.Build(BuildCfg{})
diff --git a/hugolib/page.go b/hugolib/page.go
index f71c4d9b8..fddc25fa0 100644
--- a/hugolib/page.go
+++ b/hugolib/page.go
@@ -147,7 +147,7 @@ func (p *pageState) GetTerms(taxonomy string) page.Pages {
 
 	var pas page.Pages
 
-	m.taxonomies.WalkPrefixListable(prefix, func(s string, n *contentNode) bool {
+	m.taxonomies.WalkQuery(pageMapQuery{Prefix: prefix}, func(s string, n *contentNode) bool {
 		if _, found := m.taxonomyEntries.Get(s + self); found {
 			pas = append(pas, n.p)
 		}
diff --git a/hugolib/page__meta.go b/hugolib/page__meta.go
index 7eb7cbfe1..87e955103 100644
--- a/hugolib/page__meta.go
+++ b/hugolib/page__meta.go
@@ -460,7 +460,7 @@ func (pm *pageMeta) setMetadata(parentBucket *pagesMapBucket, p *pageState, fron
 			isHeadless := cast.ToBool(v)
 			pm.params[loki] = isHeadless
 			if p.File().TranslationBaseName() == "index" && isHeadless {
-				pm.buildConfig.List = false
+				pm.buildConfig.List = pagemeta.Never
 				pm.buildConfig.Render = false
 			}
 		case "outputs":
@@ -613,7 +613,28 @@ func (pm *pageMeta) setMetadata(parentBucket *pagesMapBucket, p *pageState, fron
 }
 
 func (p *pageMeta) noList() bool {
-	return !p.buildConfig.List
+	return !p.buildConfig.ShouldList()
+}
+
+func (p *pageMeta) getListFilter(local bool) contentTreeNodeCallback {
+
+	return newContentTreeFilter(func(n *contentNode) bool {
+		if n == nil {
+			return true
+		}
+
+		var shouldList bool
+		switch n.p.m.buildConfig.List {
+		case pagemeta.Always:
+			shouldList = true
+		case pagemeta.Never:
+			shouldList = false
+		case pagemeta.ListLocally:
+			shouldList = local
+		}
+
+		return !shouldList
+	})
 }
 
 func (p *pageMeta) noRender() bool {
diff --git a/hugolib/site.go b/hugolib/site.go
index f85dc47e2..56fa654db 100644
--- a/hugolib/site.go
+++ b/hugolib/site.go
@@ -248,7 +248,7 @@ func (s *Site) prepareInits() {
 	s.init.prevNextInSection = init.Branch(func() (interface{}, error) {
 
 		var sections page.Pages
-		s.home.treeRef.m.collectSectionsRecursiveIncludingSelf(s.home.treeRef.key, func(n *contentNode) {
+		s.home.treeRef.m.collectSectionsRecursiveIncludingSelf(pageMapQuery{Prefix: s.home.treeRef.key}, func(n *contentNode) {
 			sections = append(sections, n.p)
 		})
 
@@ -281,7 +281,7 @@ func (s *Site) prepareInits() {
 			treeRef := sect.(treeRefProvider).getTreeRef()
 
 			var pas page.Pages
-			treeRef.m.collectPages(treeRef.key+cmBranchSeparator, func(c *contentNode) {
+			treeRef.m.collectPages(pageMapQuery{Prefix: treeRef.key + cmBranchSeparator}, func(c *contentNode) {
 				pas = append(pas, c.p)
 			})
 			page.SortByDefault(pas)
@@ -293,7 +293,7 @@ func (s *Site) prepareInits() {
 		treeRef := s.home.getTreeRef()
 
 		var pas page.Pages
-		treeRef.m.collectPages(treeRef.key+cmBranchSeparator, func(c *contentNode) {
+		treeRef.m.collectPages(pageMapQuery{Prefix: treeRef.key + cmBranchSeparator}, func(c *contentNode) {
 			pas = append(pas, c.p)
 		})
 		page.SortByDefault(pas)
diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go
index 8ecff189a..2815c6d99 100644
--- a/hugolib/testhelpers_test.go
+++ b/hugolib/testhelpers_test.go
@@ -1021,7 +1021,7 @@ func printStringIndexes(s string) {
 }
 
 func isCI() bool {
-	return os.Getenv("CI") != "" && os.Getenv("CIRCLE_BRANCH") == ""
+	return (os.Getenv("CI") != "" || os.Getenv("CI_LOCAL") != "") && os.Getenv("CIRCLE_BRANCH") == ""
 }
 
 // See https://github.com/golang/go/issues/19280