Prepare for Goldmark

This commmit prepares for the addition of Goldmark as the new Markdown renderer in Hugo.

This introduces a new `markup` package with some common interfaces and each implementation in its own package.

See #5963

10 files changed, 109 insertions, 63 deletions

diff --git a/hugolib/config.go b/hugolib/config.go
index 35724ec41..8a9c42b3c 100644
--- a/hugolib/config.go
+++ b/hugolib/config.go
@@ -564,11 +564,6 @@ func (configLoader) mergeStringMapKeepLeft(rootKey, key string, v1, v2 config.Pr
 
 func loadDefaultSettingsFor(v *viper.Viper) error {
 
-	c, err := helpers.NewContentSpec(v)
-	if err != nil {
-		return err
-	}
-
 	v.RegisterAlias("indexes", "taxonomies")
 
 	/*
@@ -616,7 +611,6 @@ func loadDefaultSettingsFor(v *viper.Viper) error {
 	v.SetDefault("paginate", 10)
 	v.SetDefault("paginatePath", "page")
 	v.SetDefault("summaryLength", 70)
-	v.SetDefault("blackfriday", c.BlackFriday)
 	v.SetDefault("rssLimit", -1)
 	v.SetDefault("sectionPagesMenu", "")
 	v.SetDefault("disablePathToLower", false)
diff --git a/hugolib/page.go b/hugolib/page.go
index 306ca7b0f..b0e8c4359 100644
--- a/hugolib/page.go
+++ b/hugolib/page.go
@@ -23,6 +23,8 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/gohugoio/hugo/markup/converter"
+
 	"github.com/gohugoio/hugo/common/maps"
 
 	"github.com/gohugoio/hugo/hugofs/files"
@@ -65,7 +67,7 @@ var (
 type pageContext interface {
 	posOffset(offset int) text.Position
 	wrapError(err error) error
-	getRenderingConfig() *helpers.BlackFriday
+	getContentConverter() converter.Converter
 }
 
 // wrapErr adds some context to the given error if possible.
@@ -299,13 +301,6 @@ func (p *pageState) Translations() page.Pages {
 	return p.translations
 }
 
-func (p *pageState) getRenderingConfig() *helpers.BlackFriday {
-	if p.m.renderingConfig == nil {
-		return p.s.ContentSpec.BlackFriday
-	}
-	return p.m.renderingConfig
-}
-
 func (ps *pageState) initCommonProviders(pp pagePaths) error {
 	if ps.IsPage() {
 		ps.posNextPrev = &nextPrev{init: ps.s.init.prevNext}
@@ -516,6 +511,10 @@ func (p *pageState) wrapError(err error) error {
 	return err
 }
 
+func (p *pageState) getContentConverter() converter.Converter {
+	return p.m.contentConverter
+}
+
 func (p *pageState) addResources(r ...resource.Resource) {
 	p.resources = append(p.resources, r...)
 }
diff --git a/hugolib/page__meta.go b/hugolib/page__meta.go
index e8ef13bfd..d137ac340 100644
--- a/hugolib/page__meta.go
+++ b/hugolib/page__meta.go
@@ -21,6 +21,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/gohugoio/hugo/markup/converter"
+
 	"github.com/gohugoio/hugo/hugofs/files"
 
 	"github.com/gohugoio/hugo/common/hugo"
@@ -29,7 +31,6 @@ import (
 
 	"github.com/gohugoio/hugo/source"
 	"github.com/markbates/inflect"
-	"github.com/mitchellh/mapstructure"
 	"github.com/pkg/errors"
 
 	"github.com/gohugoio/hugo/common/maps"
@@ -123,7 +124,7 @@ type pageMeta struct {
 
 	s *Site
 
-	renderingConfig *helpers.BlackFriday
+	contentConverter converter.Converter
 }
 
 func (p *pageMeta) Aliases() []string {
@@ -598,7 +599,7 @@ func (p *pageMeta) applyDefaultValues() error {
 			p.markup = helpers.GuessType(p.File().Ext())
 		}
 		if p.markup == "" {
-			p.markup = "unknown"
+			p.markup = "markdown"
 		}
 	}
 
@@ -637,17 +638,28 @@ func (p *pageMeta) applyDefaultValues() error {
 		}
 	}
 
-	bfParam := getParamToLower(p, "blackfriday")
-	if bfParam != nil {
-		p.renderingConfig = p.s.ContentSpec.BlackFriday
+	if !p.f.IsZero() && p.markup != "html" {
+		var renderingConfigOverrides map[string]interface{}
+		bfParam := getParamToLower(p, "blackfriday")
+		if bfParam != nil {
+			renderingConfigOverrides = cast.ToStringMap(bfParam)
+		}
+
+		cp := p.s.ContentSpec.Converters.Get(p.markup)
+		if cp == nil {
+			return errors.Errorf("no content renderer found for markup %q", p.markup)
+		}
+
+		cpp, err := cp.New(converter.DocumentContext{
+			DocumentID:      p.f.UniqueID(),
+			DocumentName:    p.f.Path(),
+			ConfigOverrides: renderingConfigOverrides,
+		})
 
-		// Create a copy so we can modify it.
-		bf := *p.s.ContentSpec.BlackFriday
-		p.renderingConfig = &bf
-		pageParam := cast.ToStringMap(bfParam)
-		if err := mapstructure.Decode(pageParam, &p.renderingConfig); err != nil {
-			return errors.WithMessage(err, "failed to decode rendering config")
+		if err != nil {
+			return err
 		}
+		p.contentConverter = cpp
 	}
 
 	return nil
diff --git a/hugolib/page__output.go b/hugolib/page__output.go
index 619ac0d77..764c46a93 100644
--- a/hugolib/page__output.go
+++ b/hugolib/page__output.go
@@ -45,8 +45,10 @@ func newPageOutput(
 		paginatorProvider = pag
 	}
 
-	var contentProvider page.ContentProvider = page.NopPage
-	var tableOfContentsProvider page.TableOfContentsProvider = page.NopPage
+	var (
+		contentProvider         page.ContentProvider         = page.NopPage
+		tableOfContentsProvider page.TableOfContentsProvider = page.NopPage
+	)
 
 	if cp != nil {
 		contentProvider = cp
diff --git a/hugolib/page__per_output.go b/hugolib/page__per_output.go
index 6a1262703..ef2419eca 100644
--- a/hugolib/page__per_output.go
+++ b/hugolib/page__per_output.go
@@ -23,6 +23,8 @@ import (
 	"sync"
 	"unicode/utf8"
 
+	"github.com/gohugoio/hugo/markup/converter"
+
 	"github.com/gohugoio/hugo/lazy"
 
 	bp "github.com/gohugoio/hugo/bufferpool"
@@ -97,7 +99,12 @@ func newPageContentOutput(p *pageState) func(f output.Format) (*pageContentOutpu
 
 			if p.renderable {
 				if !isHTML {
-					cp.workContent = cp.renderContent(p, cp.workContent)
+					r, err := cp.renderContent(cp.workContent)
+					if err != nil {
+						return err
+					}
+					cp.workContent = r.Bytes()
+
 					tmpContent, tmpTableOfContents := helpers.ExtractTOC(cp.workContent)
 					cp.tableOfContents = helpers.BytesToHTML(tmpTableOfContents)
 					cp.workContent = tmpContent
@@ -140,13 +147,16 @@ func newPageContentOutput(p *pageState) func(f output.Format) (*pageContentOutpu
 						}
 					}
 				} else if cp.p.m.summary != "" {
-					html := cp.p.s.ContentSpec.RenderBytes(&helpers.RenderingContext{
-						Content: []byte(cp.p.m.summary), RenderTOC: false, PageFmt: cp.p.m.markup,
-						Cfg:        p.Language(),
-						BaseFs:     p.s.BaseFs,
-						DocumentID: p.File().UniqueID(), DocumentName: p.File().Path(),
-						Config: cp.p.getRenderingConfig()})
-					html = cp.p.s.ContentSpec.TrimShortHTML(html)
+					b, err := cp.p.getContentConverter().Convert(
+						converter.RenderContext{
+							Src: []byte(cp.p.m.summary),
+						},
+					)
+
+					if err != nil {
+						return err
+					}
+					html := cp.p.s.ContentSpec.TrimShortHTML(b.Bytes())
 					cp.summary = helpers.BytesToHTML(html)
 				}
 			}
@@ -311,13 +321,12 @@ func (p *pageContentOutput) setAutoSummary() error {
 
 }
 
-func (cp *pageContentOutput) renderContent(p page.Page, content []byte) []byte {
-	return cp.p.s.ContentSpec.RenderBytes(&helpers.RenderingContext{
-		Content: content, RenderTOC: true, PageFmt: cp.p.m.markup,
-		Cfg:        p.Language(),
-		BaseFs:     cp.p.s.BaseFs,
-		DocumentID: p.File().UniqueID(), DocumentName: p.File().Path(),
-		Config: cp.p.getRenderingConfig()})
+func (cp *pageContentOutput) renderContent(content []byte) (converter.Result, error) {
+	return cp.p.getContentConverter().Convert(
+		converter.RenderContext{
+			Src:       content,
+			RenderTOC: true,
+		})
 }
 
 func (p *pageContentOutput) setWordCounts(isCJKLanguage bool) {
diff --git a/hugolib/page_test.go b/hugolib/page_test.go
index 6cf03b895..6b9c4193d 100644
--- a/hugolib/page_test.go
+++ b/hugolib/page_test.go
@@ -18,6 +18,10 @@ import (
 	"html/template"
 	"os"
 
+	"github.com/gohugoio/hugo/markup/rst"
+
+	"github.com/gohugoio/hugo/markup/asciidoc"
+
 	"github.com/gohugoio/hugo/config"
 
 	"github.com/gohugoio/hugo/common/loggers"
@@ -378,8 +382,8 @@ func testAllMarkdownEnginesForPages(t *testing.T,
 	}{
 		{"md", func() bool { return true }},
 		{"mmark", func() bool { return true }},
-		{"ad", func() bool { return helpers.HasAsciidoc() }},
-		{"rst", func() bool { return helpers.HasRst() }},
+		{"ad", func() bool { return asciidoc.Supports() }},
+		{"rst", func() bool { return rst.Supports() }},
 	}
 
 	for _, e := range engines {
diff --git a/hugolib/shortcode.go b/hugolib/shortcode.go
index d0cdf3950..700ac5bd8 100644
--- a/hugolib/shortcode.go
+++ b/hugolib/shortcode.go
@@ -21,6 +21,8 @@ import (
 	"html/template"
 	"path"
 
+	"github.com/gohugoio/hugo/markup/converter"
+
 	"github.com/gohugoio/hugo/common/herrors"
 	"github.com/pkg/errors"
 
@@ -43,7 +45,6 @@ import (
 	"github.com/gohugoio/hugo/output"
 
 	bp "github.com/gohugoio/hugo/bufferpool"
-	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/tpl"
 )
 
@@ -347,13 +348,19 @@ func renderShortcode(
 		// Pre Hugo 0.55 this was the behaviour even for the outer-most
 		// shortcode.
 		if sc.doMarkup && (level > 0 || sc.info.Config.Version == 1) {
-			newInner := s.ContentSpec.RenderBytes(&helpers.RenderingContext{
-				Content:      []byte(inner),
-				PageFmt:      p.m.markup,
-				Cfg:          p.Language(),
-				DocumentID:   p.File().UniqueID(),
-				DocumentName: p.File().Path(),
-				Config:       p.getRenderingConfig()})
+			var err error
+
+			b, err := p.getContentConverter().Convert(
+				converter.RenderContext{
+					Src: []byte(inner),
+				},
+			)
+
+			if err != nil {
+				return "", false, err
+			}
+
+			newInner := b.Bytes()
 
 			// If the type is “” (unknown) or “markdown”, we assume the markdown
 			// generation has been performed. Given the input: `a line`, markdown
diff --git a/hugolib/shortcode_test.go b/hugolib/shortcode_test.go
index 614b8f060..01fa61512 100644
--- a/hugolib/shortcode_test.go
+++ b/hugolib/shortcode_test.go
@@ -18,6 +18,9 @@ import (
 	"path/filepath"
 	"reflect"
 
+	"github.com/gohugoio/hugo/markup/asciidoc"
+	"github.com/gohugoio/hugo/markup/rst"
+
 	"github.com/spf13/viper"
 
 	"github.com/gohugoio/hugo/parser/pageparser"
@@ -27,7 +30,6 @@ import (
 	"testing"
 
 	"github.com/gohugoio/hugo/deps"
-	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/tpl"
 	"github.com/spf13/cast"
 
@@ -538,6 +540,19 @@ title: "Foo"
 			"<h1>Hugo!</h1>"},
 	}
 
+	temp := tests[:0]
+	for _, test := range tests {
+		if strings.HasSuffix(test.contentPath, ".ad") && !asciidoc.Supports() {
+			t.Log("Skip Asciidoc test case as no Asciidoc present.")
+			continue
+		} else if strings.HasSuffix(test.contentPath, ".rst") && !rst.Supports() {
+			t.Log("Skip Rst test case as no rst2html present.")
+			continue
+		}
+		temp = append(temp, test)
+	}
+	tests = temp
+
 	sources := make([][2]string, len(tests))
 
 	for i, test := range tests {
@@ -578,11 +593,6 @@ title: "Foo"
 		test := test
 		t.Run(fmt.Sprintf("test=%d;contentPath=%s", i, test.contentPath), func(t *testing.T) {
 			t.Parallel()
-			if strings.HasSuffix(test.contentPath, ".ad") && !helpers.HasAsciidoc() {
-				t.Skip("Skip Asciidoc test case as no Asciidoc present.")
-			} else if strings.HasSuffix(test.contentPath, ".rst") && !helpers.HasRst() {
-				t.Skip("Skip Rst test case as no rst2html present.")
-			}
 
 			th := newTestHelper(s.Cfg, s.Fs, t)
 
diff --git a/hugolib/site.go b/hugolib/site.go
index b9ec64224..db0cd2ea5 100644
--- a/hugolib/site.go
+++ b/hugolib/site.go
@@ -28,6 +28,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/gohugoio/hugo/markup/converter"
+
 	"github.com/gohugoio/hugo/hugofs/files"
 
 	"github.com/gohugoio/hugo/common/maps"
@@ -758,17 +760,23 @@ func (s *siteRefLinker) refLink(ref string, source interface{}, relative bool, o
 	}
 
 	if refURL.Fragment != "" {
+		_ = target
 		link = link + "#" + refURL.Fragment
 
-		if pctx, ok := target.(pageContext); ok && !target.File().IsZero() && !pctx.getRenderingConfig().PlainIDAnchors {
+		if pctx, ok := target.(pageContext); ok {
 			if refURL.Path != "" {
-				link = link + ":" + target.File().UniqueID()
+				if di, ok := pctx.getContentConverter().(converter.DocumentInfo); ok {
+					link = link + di.AnchorSuffix()
+				}
+			}
+		} else if pctx, ok := p.(pageContext); ok {
+			if di, ok := pctx.getContentConverter().(converter.DocumentInfo); ok {
+				link = link + di.AnchorSuffix()
 			}
-		} else if pctx, ok := p.(pageContext); ok && !p.File().IsZero() && !pctx.getRenderingConfig().PlainIDAnchors {
-			link = link + ":" + p.File().UniqueID()
 		}
 
 	}
+
 	return link, nil
 }
 
diff --git a/hugolib/site_test.go b/hugolib/site_test.go
index 6cbcbf8d5..995664da4 100644
--- a/hugolib/site_test.go
+++ b/hugolib/site_test.go
@@ -1018,6 +1018,7 @@ func TestRefLinking(t *testing.T) {
 }
 
 func checkLinkCase(site *Site, link string, currentPage page.Page, relative bool, outputFormat string, expected string, t *testing.T, i int) {
+	t.Helper()
 	if out, err := site.refLink(link, currentPage, relative, outputFormat); err != nil || out != expected {
 		t.Fatalf("[%d] Expected %q from %q to resolve to %q, got %q - error: %s", i, link, currentPage.Path(), expected, out, err)
 	}