Add headless bundle support

This commit adds  support for `headless bundles` for the `index` bundle type.

So:

```toml
headless = true
```

In front matter means that

* It will have no `Permalink` and no rendered HTML in /public
* It will not be part of `.Site.RegularPages` etc.

But you can get it by:

* `.Site.GetPage ...`

The use cases are many:

* Shared media galleries
* Reusable page content "snippets"
* ...

Fixes #4311

6 files changed, 164 insertions, 21 deletions

diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go
index fbaf27aa4..e917c3209 100644
--- a/hugolib/hugo_sites.go
+++ b/hugolib/hugo_sites.go
@@ -508,7 +508,11 @@ func (h *HugoSites) setupTranslations() {
 			shouldBuild := p.shouldBuild()
 			s.updateBuildStats(p)
 			if shouldBuild {
-				s.Pages = append(s.Pages, p)
+				if p.headless {
+					s.headlessPages = append(s.headlessPages, p)
+				} else {
+					s.Pages = append(s.Pages, p)
+				}
 			}
 		}
 	}
@@ -560,6 +564,10 @@ func (s *Site) preparePagesForRender(cfg *BuildCfg) {
 		pageChan <- p
 	}
 
+	for _, p := range s.headlessPages {
+		pageChan <- p
+	}
+
 	close(pageChan)
 
 	wg.Wait()
diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go
index b2b394eb5..8f03f589f 100644
--- a/hugolib/hugo_sites_build.go
+++ b/hugolib/hugo_sites_build.go
@@ -179,19 +179,26 @@ func (h *HugoSites) assemble(config *BuildCfg) error {
 	}
 
 	for _, s := range h.Sites {
-		for _, p := range s.Pages {
-			// May have been set in front matter
-			if len(p.outputFormats) == 0 {
-				p.outputFormats = s.outputFormats[p.Kind]
-			}
-			for _, r := range p.Resources.ByType(pageResourceType) {
-				r.(*Page).outputFormats = p.outputFormats
-			}
+		for _, pages := range []Pages{s.Pages, s.headlessPages} {
+			for _, p := range pages {
+				// May have been set in front matter
+				if len(p.outputFormats) == 0 {
+					p.outputFormats = s.outputFormats[p.Kind]
+				}
 
-			if err := p.initPaths(); err != nil {
-				return err
-			}
+				if p.headless {
+					// headless = 1 output format only
+					p.outputFormats = p.outputFormats[:1]
+				}
+				for _, r := range p.Resources.ByType(pageResourceType) {
+					r.(*Page).outputFormats = p.outputFormats
+				}
 
+				if err := p.initPaths(); err != nil {
+					return err
+				}
+
+			}
 		}
 		s.assembleMenus()
 		s.refreshPageCaches()
diff --git a/hugolib/page.go b/hugolib/page.go
index 2502faa08..4df681661 100644
--- a/hugolib/page.go
+++ b/hugolib/page.go
@@ -237,6 +237,13 @@ type Page struct {
 	// Is set to a forward slashed path if this is a Page resources living in a folder below its owner.
 	resourcePath string
 
+	// This is enabled if it is a leaf bundle (the "index.md" type) and it is marked as headless in front matter.
+	// Being headless means that
+	// 1. The page itself is not rendered to disk
+	// 2. It is not available in .Site.Pages etc.
+	// 3. But you can get it via .Site.GetPage
+	headless bool
+
 	layoutDescriptor output.LayoutDescriptor
 
 	scratch *Scratch
@@ -986,11 +993,17 @@ func (p *Page) URL() string {
 
 // Permalink returns the absolute URL to this Page.
 func (p *Page) Permalink() string {
+	if p.headless {
+		return ""
+	}
 	return p.permalink
 }
 
 // RelPermalink gets a URL to the resource relative to the host.
 func (p *Page) RelPermalink() string {
+	if p.headless {
+		return ""
+	}
 	return p.relPermalink
 }
 
@@ -1150,6 +1163,13 @@ func (p *Page) update(f interface{}) error {
 				p.s.Log.ERROR.Printf("Failed to parse date '%v' in page %s", v, p.File.Path())
 			}
 			p.params[loki] = p.Date
+		case "headless":
+			// For now, only the leaf bundles ("index.md") can be headless (i.e. produce no output).
+			// We may expand on this in the future, but that gets more complex pretty fast.
+			if p.TranslationBaseName() == "index" {
+				p.headless = cast.ToBool(v)
+			}
+			p.params[loki] = p.headless
 		case "lastmod":
 			p.Lastmod, err = cast.ToTimeE(v)
 			if err != nil {
diff --git a/hugolib/page_bundler_test.go b/hugolib/page_bundler_test.go
index 5b4bb3530..ab268dee3 100644
--- a/hugolib/page_bundler_test.go
+++ b/hugolib/page_bundler_test.go
@@ -224,6 +224,87 @@ func TestPageBundlerSiteWitSymbolicLinksInContent(t *testing.T) {
 
 }
 
+func TestPageBundlerHeadless(t *testing.T) {
+	t.Parallel()
+
+	cfg, fs := newTestCfg()
+	assert := require.New(t)
+
+	workDir := "/work"
+	cfg.Set("workingDir", workDir)
+	cfg.Set("contentDir", "base")
+	cfg.Set("baseURL", "https://example.com")
+
+	pageContent := `---
+title: "Bundle Galore"
+slug: s1
+date: 2017-01-23
+---
+
+TheContent.
+
+{{< myShort >}}
+`
+
+	writeSource(t, fs, filepath.Join(workDir, "layouts", "_default", "single.html"), "single {{ .Content }}")
+	writeSource(t, fs, filepath.Join(workDir, "layouts", "_default", "list.html"), "list")
+	writeSource(t, fs, filepath.Join(workDir, "layouts", "shortcodes", "myShort.html"), "SHORTCODE")
+
+	writeSource(t, fs, filepath.Join(workDir, "base", "a", "index.md"), pageContent)
+	writeSource(t, fs, filepath.Join(workDir, "base", "a", "l1.png"), "PNG image")
+	writeSource(t, fs, filepath.Join(workDir, "base", "a", "l2.png"), "PNG image")
+
+	writeSource(t, fs, filepath.Join(workDir, "base", "b", "index.md"), `---
+title: "Headless Bundle in Topless Bar"
+slug: s2
+headless: true
+date: 2017-01-23
+---
+
+TheContent.
+HEADLESS {{< myShort >}}
+`)
+	writeSource(t, fs, filepath.Join(workDir, "base", "b", "l1.png"), "PNG image")
+	writeSource(t, fs, filepath.Join(workDir, "base", "b", "l2.png"), "PNG image")
+	writeSource(t, fs, filepath.Join(workDir, "base", "b", "p1.md"), pageContent)
+
+	s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{})
+
+	assert.Equal(1, len(s.RegularPages))
+	assert.Equal(1, len(s.headlessPages))
+
+	regular := s.getPage(KindPage, "a/index")
+	assert.Equal("/a/s1/", regular.RelPermalink())
+
+	headless := s.getPage(KindPage, "b/index")
+	assert.NotNil(headless)
+	assert.True(headless.headless)
+	assert.Equal("Headless Bundle in Topless Bar", headless.Title())
+	assert.Equal("", headless.RelPermalink())
+	assert.Equal("", headless.Permalink())
+	assert.Contains(headless.Content, "HEADLESS SHORTCODE")
+
+	headlessResources := headless.Resources
+	assert.Equal(3, len(headlessResources))
+	assert.Equal(2, len(headlessResources.Match("l*")))
+	pageResource := headlessResources.GetMatch("p*")
+	assert.NotNil(pageResource)
+	assert.IsType(&Page{}, pageResource)
+	p := pageResource.(*Page)
+	assert.Contains(p.Content, "SHORTCODE")
+	assert.Equal("p1.md", p.Name())
+
+	th := testHelper{s.Cfg, s.Fs, t}
+
+	th.assertFileContent(filepath.FromSlash(workDir+"/public/a/s1/index.html"), "TheContent")
+	th.assertFileContent(filepath.FromSlash(workDir+"/public/a/s1/l1.png"), "PNG")
+
+	th.assertFileNotExist(workDir + "/public/b/s2/index.html")
+	// But the bundled resources needs to be published
+	th.assertFileContent(filepath.FromSlash(workDir+"/public/b/s2/l1.png"), "PNG")
+
+}
+
 func newTestBundleSources(t *testing.T) (*viper.Viper, *hugofs.Fs) {
 	cfg, fs := newTestCfg()
 	assert := require.New(t)
diff --git a/hugolib/page_collections.go b/hugolib/page_collections.go
index f6e8eae07..a9335ad41 100644
--- a/hugolib/page_collections.go
+++ b/hugolib/page_collections.go
@@ -43,6 +43,9 @@ type PageCollections struct {
 	// Includes absolute all pages (of all types), including drafts etc.
 	rawAllPages Pages
 
+	// Includes headless bundles, i.e. bundles that produce no output for its content page.
+	headlessPages Pages
+
 	pageCache *cache.PartitionedLazyCache
 }
 
@@ -66,15 +69,17 @@ func (c *PageCollections) refreshPageCaches() {
 				// in this cache, as we intend to use this in the ref and relref
 				// shortcodes. If the user says "sect/doc1.en.md", he/she knows
 				// what he/she is looking for.
-				for _, p := range c.AllRegularPages {
-					cache[filepath.ToSlash(p.Source.Path())] = p
-					// Ref/Relref supports this potentially ambiguous lookup.
-					cache[p.Source.LogicalName()] = p
-
-					if s != nil && p.s == s {
-						// We need a way to get to the current language version.
-						pathWithNoExtensions := path.Join(filepath.ToSlash(p.Source.Dir()), p.Source.TranslationBaseName())
-						cache[pathWithNoExtensions] = p
+				for _, pageCollection := range []Pages{c.AllRegularPages, c.headlessPages} {
+					for _, p := range pageCollection {
+						cache[filepath.ToSlash(p.Source.Path())] = p
+						// Ref/Relref supports this potentially ambiguous lookup.
+						cache[p.Source.LogicalName()] = p
+
+						if s != nil && p.s == s {
+							// We need a way to get to the current language version.
+							pathWithNoExtensions := path.Join(filepath.ToSlash(p.Source.Dir()), p.Source.TranslationBaseName())
+							cache[pathWithNoExtensions] = p
+						}
 					}
 
 				}
diff --git a/hugolib/site_render.go b/hugolib/site_render.go
index 43019619b..bde4ef1f3 100644
--- a/hugolib/site_render.go
+++ b/hugolib/site_render.go
@@ -45,6 +45,12 @@ func (s *Site) renderPages(filter map[string]bool) error {
 		go pageRenderer(s, pages, results, wg)
 	}
 
+	if len(s.headlessPages) > 0 {
+		wg.Add(1)
+		go headlessPagesPublisher(s, wg)
+
+	}
+
 	hasFilter := filter != nil && len(filter) > 0
 
 	for _, page := range s.Pages {
@@ -67,6 +73,22 @@ func (s *Site) renderPages(filter map[string]bool) error {
 	return nil
 }
 
+func headlessPagesPublisher(s *Site, wg *sync.WaitGroup) {
+	defer wg.Done()
+	for _, page := range s.headlessPages {
+		outFormat := page.outputFormats[0] // There is only one
+		pageOutput, err := newPageOutput(page, false, outFormat)
+		if err == nil {
+			page.mainPageOutput = pageOutput
+			err = pageOutput.renderResources()
+		}
+
+		if err != nil {
+			s.Log.ERROR.Printf("Failed to render resources for headless page %q: %s", page, err)
+		}
+	}
+}
+
 func pageRenderer(s *Site, pages <-chan *Page, results chan<- error, wg *sync.WaitGroup) {
 	defer wg.Done()