Create pages from _content.gotmpl

Closes #12427
Closes #12485
Closes #6310
Closes #5074

5 files changed, 212 insertions, 27 deletions

diff --git a/media/builtin.go b/media/builtin.go
index aafe245c9..ee35b7d08 100644
--- a/media/builtin.go
+++ b/media/builtin.go
@@ -34,8 +34,12 @@ type BuiltinTypes struct {
 	OpenTypeFontType Type
 
 	// Common document types
-	PDFType      Type
-	MarkdownType Type
+	PDFType              Type
+	MarkdownType         Type
+	EmacsOrgModeType     Type
+	AsciiDocType         Type
+	PandocType           Type
+	ReStructuredTextType Type
 
 	// Common video types
 	AVIType  Type
@@ -85,8 +89,12 @@ var Builtin = BuiltinTypes{
 	OpenTypeFontType: Type{Type: "font/otf"},
 
 	// Common document types
-	PDFType:      Type{Type: "application/pdf"},
-	MarkdownType: Type{Type: "text/markdown"},
+	PDFType:              Type{Type: "application/pdf"},
+	MarkdownType:         Type{Type: "text/markdown"},
+	AsciiDocType:         Type{Type: "text/asciidoc"}, // https://github.com/asciidoctor/asciidoctor/issues/2502
+	PandocType:           Type{Type: "text/pandoc"},
+	ReStructuredTextType: Type{Type: "text/rst"}, // https://docutils.sourceforge.io/FAQ.html#what-s-the-official-mime-type-for-restructuredtext-data
+	EmacsOrgModeType:     Type{Type: "text/org"},
 
 	// Common video types
 	AVIType:  Type{Type: "video/x-msvideo"},
@@ -108,7 +116,7 @@ var defaultMediaTypesConfig = map[string]any{
 	"text/x-scss":     map[string]any{"suffixes": []string{"scss"}},
 	"text/x-sass":     map[string]any{"suffixes": []string{"sass"}},
 	"text/csv":        map[string]any{"suffixes": []string{"csv"}},
-	"text/html":       map[string]any{"suffixes": []string{"html"}},
+	"text/html":       map[string]any{"suffixes": []string{"html", "htm"}},
 	"text/javascript": map[string]any{"suffixes": []string{"js", "jsm", "mjs"}},
 	"text/typescript": map[string]any{"suffixes": []string{"ts"}},
 	"text/tsx":        map[string]any{"suffixes": []string{"tsx"}},
@@ -137,7 +145,11 @@ var defaultMediaTypesConfig = map[string]any{
 
 	// Common document types
 	"application/pdf": map[string]any{"suffixes": []string{"pdf"}},
-	"text/markdown":   map[string]any{"suffixes": []string{"md", "markdown"}},
+	"text/markdown":   map[string]any{"suffixes": []string{"md", "mdown", "markdown"}},
+	"text/asciidoc":   map[string]any{"suffixes": []string{"adoc", "asciidoc", "ad"}},
+	"text/pandoc":     map[string]any{"suffixes": []string{"pandoc", "pdc"}},
+	"text/rst":        map[string]any{"suffixes": []string{"rst"}},
+	"text/org":        map[string]any{"suffixes": []string{"org"}},
 
 	// Common video types
 	"video/x-msvideo": map[string]any{"suffixes": []string{"avi"}},
@@ -152,10 +164,3 @@ var defaultMediaTypesConfig = map[string]any{
 
 	"application/octet-stream": map[string]any{},
 }
-
-func init() {
-	// Apply delimiter to all.
-	for _, m := range defaultMediaTypesConfig {
-		m.(map[string]any)["delimiter"] = "."
-	}
-}
diff --git a/media/config.go b/media/config.go
index cdec2e438..18e983369 100644
--- a/media/config.go
+++ b/media/config.go
@@ -14,13 +14,14 @@
 package media
 
 import (
-	"errors"
 	"fmt"
+	"path/filepath"
 	"reflect"
 	"sort"
 	"strings"
 
 	"github.com/gohugoio/hugo/common/maps"
+	"github.com/gohugoio/hugo/common/paths"
 	"github.com/gohugoio/hugo/config"
 
 	"github.com/mitchellh/mapstructure"
@@ -31,6 +32,11 @@ import (
 var DefaultTypes Types
 
 func init() {
+	// Apply delimiter to all.
+	for _, m := range defaultMediaTypesConfig {
+		m.(map[string]any)["delimiter"] = "."
+	}
+
 	ns, err := DecodeTypes(nil)
 	if err != nil {
 		panic(err)
@@ -39,17 +45,122 @@ func init() {
 
 	// Initialize the Builtin types with values from DefaultTypes.
 	v := reflect.ValueOf(&Builtin).Elem()
+
 	for i := 0; i < v.NumField(); i++ {
 		f := v.Field(i)
+		fieldName := v.Type().Field(i).Name
 		builtinType := f.Interface().(Type)
+		if builtinType.Type == "" {
+			panic(fmt.Errorf("builtin type %q is empty", fieldName))
+		}
 		defaultType, found := DefaultTypes.GetByType(builtinType.Type)
 		if !found {
-			panic(errors.New("missing default type for builtin type: " + builtinType.Type))
+			panic(fmt.Errorf("missing default type for field builtin type: %q", fieldName))
 		}
 		f.Set(reflect.ValueOf(defaultType))
 	}
 }
 
+func init() {
+	DefaultContentTypes = ContentTypes{
+		HTML:             Builtin.HTMLType,
+		Markdown:         Builtin.MarkdownType,
+		AsciiDoc:         Builtin.AsciiDocType,
+		Pandoc:           Builtin.PandocType,
+		ReStructuredText: Builtin.ReStructuredTextType,
+		EmacsOrgMode:     Builtin.EmacsOrgModeType,
+	}
+
+	DefaultContentTypes.init()
+}
+
+var DefaultContentTypes ContentTypes
+
+// ContentTypes holds the media types that are considered content in Hugo.
+type ContentTypes struct {
+	HTML             Type
+	Markdown         Type
+	AsciiDoc         Type
+	Pandoc           Type
+	ReStructuredText Type
+	EmacsOrgMode     Type
+
+	// Created in init().
+	types        Types
+	extensionSet map[string]bool
+}
+
+func (t *ContentTypes) init() {
+	t.types = Types{t.HTML, t.Markdown, t.AsciiDoc, t.Pandoc, t.ReStructuredText, t.EmacsOrgMode}
+	t.extensionSet = make(map[string]bool)
+	for _, mt := range t.types {
+		for _, suffix := range mt.Suffixes() {
+			t.extensionSet[suffix] = true
+		}
+	}
+}
+
+func (t ContentTypes) IsContentSuffix(suffix string) bool {
+	return t.extensionSet[suffix]
+}
+
+// IsContentFile returns whether the given filename is a content file.
+func (t ContentTypes) IsContentFile(filename string) bool {
+	return t.IsContentSuffix(strings.TrimPrefix(filepath.Ext(filename), "."))
+}
+
+// IsIndexContentFile returns whether the given filename is an index content file.
+func (t ContentTypes) IsIndexContentFile(filename string) bool {
+	if !t.IsContentFile(filename) {
+		return false
+	}
+
+	base := filepath.Base(filename)
+
+	return strings.HasPrefix(base, "index.") || strings.HasPrefix(base, "_index.")
+}
+
+// IsHTMLSuffix returns whether the given suffix is a HTML media type.
+func (t ContentTypes) IsHTMLSuffix(suffix string) bool {
+	for _, s := range t.HTML.Suffixes() {
+		if s == suffix {
+			return true
+		}
+	}
+	return false
+}
+
+// Types is a slice of media types.
+func (t ContentTypes) Types() Types {
+	return t.types
+}
+
+// FromTypes creates a new ContentTypes updated with the values from the given Types.
+func (t ContentTypes) FromTypes(types Types) ContentTypes {
+	if tt, ok := types.GetByType(t.HTML.Type); ok {
+		t.HTML = tt
+	}
+	if tt, ok := types.GetByType(t.Markdown.Type); ok {
+		t.Markdown = tt
+	}
+	if tt, ok := types.GetByType(t.AsciiDoc.Type); ok {
+		t.AsciiDoc = tt
+	}
+	if tt, ok := types.GetByType(t.Pandoc.Type); ok {
+		t.Pandoc = tt
+	}
+	if tt, ok := types.GetByType(t.ReStructuredText.Type); ok {
+		t.ReStructuredText = tt
+	}
+	if tt, ok := types.GetByType(t.EmacsOrgMode.Type); ok {
+		t.EmacsOrgMode = tt
+	}
+
+	t.init()
+
+	return t
+}
+
 // Hold the configuration for a given media type.
 type MediaTypeConfig struct {
 	// The file suffixes used for this media type.
@@ -105,3 +216,10 @@ func DecodeTypes(in map[string]any) (*config.ConfigNamespace[map[string]MediaTyp
 	}
 	return ns, nil
 }
+
+// TODO(bep) get rid of this.
+var DefaultPathParser = &paths.PathParser{
+	IsContentExt: func(ext string) bool {
+		return DefaultContentTypes.IsContentSuffix(ext)
+	},
+}
diff --git a/media/config_test.go b/media/config_test.go
index 4803eb42a..634686060 100644
--- a/media/config_test.go
+++ b/media/config_test.go
@@ -114,7 +114,7 @@ func TestDefaultTypes(t *testing.T) {
 		tp               Type
 		expectedMainType string
 		expectedSubType  string
-		expectedSuffix   string
+		expectedSuffixes string
 		expectedType     string
 		expectedString   string
 	}{
@@ -122,29 +122,34 @@ func TestDefaultTypes(t *testing.T) {
 		{Builtin.CSSType, "text", "css", "css", "text/css", "text/css"},
 		{Builtin.SCSSType, "text", "x-scss", "scss", "text/x-scss", "text/x-scss"},
 		{Builtin.CSVType, "text", "csv", "csv", "text/csv", "text/csv"},
-		{Builtin.HTMLType, "text", "html", "html", "text/html", "text/html"},
-		{Builtin.JavascriptType, "text", "javascript", "js", "text/javascript", "text/javascript"},
+		{Builtin.HTMLType, "text", "html", "html,htm", "text/html", "text/html"},
+		{Builtin.MarkdownType, "text", "markdown", "md,mdown,markdown", "text/markdown", "text/markdown"},
+		{Builtin.EmacsOrgModeType, "text", "org", "org", "text/org", "text/org"},
+		{Builtin.PandocType, "text", "pandoc", "pandoc,pdc", "text/pandoc", "text/pandoc"},
+		{Builtin.ReStructuredTextType, "text", "rst", "rst", "text/rst", "text/rst"},
+		{Builtin.AsciiDocType, "text", "asciidoc", "adoc,asciidoc,ad", "text/asciidoc", "text/asciidoc"},
+		{Builtin.JavascriptType, "text", "javascript", "js,jsm,mjs", "text/javascript", "text/javascript"},
 		{Builtin.TypeScriptType, "text", "typescript", "ts", "text/typescript", "text/typescript"},
 		{Builtin.TSXType, "text", "tsx", "tsx", "text/tsx", "text/tsx"},
 		{Builtin.JSXType, "text", "jsx", "jsx", "text/jsx", "text/jsx"},
 		{Builtin.JSONType, "application", "json", "json", "application/json", "application/json"},
-		{Builtin.RSSType, "application", "rss", "xml", "application/rss+xml", "application/rss+xml"},
+		{Builtin.RSSType, "application", "rss", "xml,rss", "application/rss+xml", "application/rss+xml"},
 		{Builtin.SVGType, "image", "svg", "svg", "image/svg+xml", "image/svg+xml"},
 		{Builtin.TextType, "text", "plain", "txt", "text/plain", "text/plain"},
 		{Builtin.XMLType, "application", "xml", "xml", "application/xml", "application/xml"},
 		{Builtin.TOMLType, "application", "toml", "toml", "application/toml", "application/toml"},
-		{Builtin.YAMLType, "application", "yaml", "yaml", "application/yaml", "application/yaml"},
+		{Builtin.YAMLType, "application", "yaml", "yaml,yml", "application/yaml", "application/yaml"},
 		{Builtin.PDFType, "application", "pdf", "pdf", "application/pdf", "application/pdf"},
 		{Builtin.TrueTypeFontType, "font", "ttf", "ttf", "font/ttf", "font/ttf"},
 		{Builtin.OpenTypeFontType, "font", "otf", "otf", "font/otf", "font/otf"},
 	} {
 		c.Assert(test.tp.MainType, qt.Equals, test.expectedMainType)
 		c.Assert(test.tp.SubType, qt.Equals, test.expectedSubType)
-
+		c.Assert(test.tp.SuffixesCSV, qt.Equals, test.expectedSuffixes)
 		c.Assert(test.tp.Type, qt.Equals, test.expectedType)
 		c.Assert(test.tp.String(), qt.Equals, test.expectedString)
 
 	}
 
-	c.Assert(len(DefaultTypes), qt.Equals, 36)
+	c.Assert(len(DefaultTypes), qt.Equals, 40)
 }
diff --git a/media/mediaType.go b/media/mediaType.go
index 367c8ecc9..a7ba1309a 100644
--- a/media/mediaType.go
+++ b/media/mediaType.go
@@ -117,13 +117,16 @@ func FromContent(types Types, extensionHints []string, content []byte) Type {
 	return m
 }
 
-// FromStringAndExt creates a Type from a MIME string and a given extension.
-func FromStringAndExt(t, ext string) (Type, error) {
+// FromStringAndExt creates a Type from a MIME string and a given extensions
+func FromStringAndExt(t string, ext ...string) (Type, error) {
 	tp, err := FromString(t)
 	if err != nil {
 		return tp, err
 	}
-	tp.SuffixesCSV = strings.TrimPrefix(ext, ".")
+	for i, e := range ext {
+		ext[i] = strings.TrimPrefix(e, ".")
+	}
+	tp.SuffixesCSV = strings.Join(ext, ",")
 	tp.Delimiter = DefaultDelimiter
 	tp.init()
 	return tp, nil
@@ -187,6 +190,16 @@ func (m Type) IsText() bool {
 	return false
 }
 
+// For internal use.
+func (m Type) IsHTML() bool {
+	return m.SubType == Builtin.HTMLType.SubType
+}
+
+// For internal use.
+func (m Type) IsMarkdown() bool {
+	return m.SubType == Builtin.MarkdownType.SubType
+}
+
 func InitMediaType(m *Type) {
 	m.init()
 }
@@ -221,6 +234,26 @@ func (t Types) Len() int           { return len(t) }
 func (t Types) Swap(i, j int)      { t[i], t[j] = t[j], t[i] }
 func (t Types) Less(i, j int) bool { return t[i].Type < t[j].Type }
 
+// GetBestMatch returns the best match for the given media type string.
+func (t Types) GetBestMatch(s string) (Type, bool) {
+	// First try an exact match.
+	if mt, found := t.GetByType(s); found {
+		return mt, true
+	}
+
+	// Try main type.
+	if mt, found := t.GetBySubType(s); found {
+		return mt, true
+	}
+
+	// Try extension.
+	if mt, _, found := t.GetFirstBySuffix(s); found {
+		return mt, true
+	}
+
+	return Type{}, false
+}
+
 // GetByType returns a media type for tp.
 func (t Types) GetByType(tp string) (Type, bool) {
 	for _, tt := range t {
@@ -324,6 +357,22 @@ func (t Types) GetByMainSubType(mainType, subType string) (tp Type, found bool)
 	return
 }
 
+// GetBySubType gets a media type given a sub type e.g. "plain".
+func (t Types) GetBySubType(subType string) (tp Type, found bool) {
+	for _, tt := range t {
+		if strings.EqualFold(subType, tt.SubType) {
+			if found {
+				// ambiguous
+				found = false
+				return
+			}
+			tp = tt
+			found = true
+		}
+	}
+	return
+}
+
 // IsZero reports whether this Type represents a zero value.
 // For internal use.
 func (m Type) IsZero() bool {
diff --git a/media/mediaType_test.go b/media/mediaType_test.go
index 2e3a4a914..fb3eb664f 100644
--- a/media/mediaType_test.go
+++ b/media/mediaType_test.go
@@ -115,10 +115,10 @@ func TestFromTypeString(t *testing.T) {
 
 func TestFromStringAndExt(t *testing.T) {
 	c := qt.New(t)
-	f, err := FromStringAndExt("text/html", "html")
+	f, err := FromStringAndExt("text/html", "html", "htm")
 	c.Assert(err, qt.IsNil)
 	c.Assert(f, qt.Equals, Builtin.HTMLType)
-	f, err = FromStringAndExt("text/html", ".html")
+	f, err = FromStringAndExt("text/html", ".html", ".htm")
 	c.Assert(err, qt.IsNil)
 	c.Assert(f, qt.Equals, Builtin.HTMLType)
 }
@@ -214,3 +214,11 @@ func BenchmarkTypeOps(b *testing.B) {
 
 	}
 }
+
+func TestIsContentFile(t *testing.T) {
+	c := qt.New(t)
+
+	c.Assert(DefaultContentTypes.IsContentFile(filepath.FromSlash("my/file.md")), qt.Equals, true)
+	c.Assert(DefaultContentTypes.IsContentFile(filepath.FromSlash("my/file.ad")), qt.Equals, true)
+	c.Assert(DefaultContentTypes.IsContentFile(filepath.FromSlash("textfile.txt")), qt.Equals, false)
+}