From 5f6b6ec68936ebbbf590894c02a1a3ecad30735f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Fri, 16 Aug 2019 15:55:03 +0200 Subject: Prepare for Goldmark This commmit prepares for the addition of Goldmark as the new Markdown renderer in Hugo. This introduces a new `markup` package with some common interfaces and each implementation in its own package. See #5963 --- helpers/content.go | 524 +++------------------------------------ helpers/content_renderer.go | 108 -------- helpers/content_renderer_test.go | 141 ----------- helpers/content_test.go | 235 +----------------- helpers/pygments_test.go | 10 +- helpers/testhelpers_test.go | 4 +- 6 files changed, 47 insertions(+), 975 deletions(-) delete mode 100644 helpers/content_renderer.go delete mode 100644 helpers/content_renderer_test.go (limited to 'helpers') diff --git a/helpers/content.go b/helpers/content.go index fe96ce7d2..357bd48e7 100644 --- a/helpers/content.go +++ b/helpers/content.go @@ -19,22 +19,18 @@ package helpers import ( "bytes" - "fmt" "html/template" - "os/exec" - "runtime" "unicode" "unicode/utf8" - "github.com/gohugoio/hugo/common/maps" - "github.com/gohugoio/hugo/hugolib/filesystems" - "github.com/niklasfasching/go-org/org" + "github.com/gohugoio/hugo/common/loggers" + + "github.com/gohugoio/hugo/markup/converter" + + "github.com/gohugoio/hugo/markup" bp "github.com/gohugoio/hugo/bufferpool" "github.com/gohugoio/hugo/config" - "github.com/miekg/mmark" - "github.com/mitchellh/mapstructure" - "github.com/russross/blackfriday" "github.com/spf13/afero" jww "github.com/spf13/jwalterweatherman" @@ -52,9 +48,9 @@ var ( // ContentSpec provides functionality to render markdown content. type ContentSpec struct { - BlackFriday *BlackFriday - footnoteAnchorPrefix string - footnoteReturnLinkContents string + Converters markup.ConverterProvider + MardownConverter converter.Converter // Markdown converter with no document context + // SummaryLength is the length of the summary that Hugo extracts from a content. summaryLength int @@ -70,16 +66,13 @@ type ContentSpec struct { // NewContentSpec returns a ContentSpec initialized // with the appropriate fields from the given config.Provider. -func NewContentSpec(cfg config.Provider) (*ContentSpec, error) { - bf := newBlackfriday(cfg.GetStringMap("blackfriday")) +func NewContentSpec(cfg config.Provider, logger *loggers.Logger, contentFs afero.Fs) (*ContentSpec, error) { + spec := &ContentSpec{ - BlackFriday: bf, - footnoteAnchorPrefix: cfg.GetString("footnoteAnchorPrefix"), - footnoteReturnLinkContents: cfg.GetString("footnoteReturnLinkContents"), - summaryLength: cfg.GetInt("summaryLength"), - BuildFuture: cfg.GetBool("buildFuture"), - BuildExpired: cfg.GetBool("buildExpired"), - BuildDrafts: cfg.GetBool("buildDrafts"), + summaryLength: cfg.GetInt("summaryLength"), + BuildFuture: cfg.GetBool("buildFuture"), + BuildExpired: cfg.GetBool("buildExpired"), + BuildDrafts: cfg.GetBool("buildDrafts"), Cfg: cfg, } @@ -109,99 +102,29 @@ func NewContentSpec(cfg config.Provider) (*ContentSpec, error) { spec.Highlight = h.chromaHighlight } - return spec, nil -} - -// BlackFriday holds configuration values for BlackFriday rendering. -type BlackFriday struct { - Smartypants bool - SmartypantsQuotesNBSP bool - AngledQuotes bool - Fractions bool - HrefTargetBlank bool - NofollowLinks bool - NoreferrerLinks bool - SmartDashes bool - LatexDashes bool - TaskLists bool - PlainIDAnchors bool - Extensions []string - ExtensionsMask []string - SkipHTML bool -} - -// NewBlackfriday creates a new Blackfriday filled with site config or some sane defaults. -func newBlackfriday(config map[string]interface{}) *BlackFriday { - defaultParam := map[string]interface{}{ - "smartypants": true, - "angledQuotes": false, - "smartypantsQuotesNBSP": false, - "fractions": true, - "hrefTargetBlank": false, - "nofollowLinks": false, - "noreferrerLinks": false, - "smartDashes": true, - "latexDashes": true, - "plainIDAnchors": true, - "taskLists": true, - "skipHTML": false, - } - - maps.ToLower(defaultParam) - - siteConfig := make(map[string]interface{}) - - for k, v := range defaultParam { - siteConfig[k] = v - } - - for k, v := range config { - siteConfig[k] = v + converterProvider, err := markup.NewConverterProvider(converter.ProviderConfig{ + Cfg: cfg, + ContentFs: contentFs, + Logger: logger, + Highlight: spec.Highlight, + }) + if err != nil { + return nil, err } - combinedConfig := &BlackFriday{} - if err := mapstructure.Decode(siteConfig, combinedConfig); err != nil { - jww.FATAL.Printf("Failed to get site rendering config\n%s", err.Error()) + spec.Converters = converterProvider + p := converterProvider.Get("markdown") + conv, err := p.New(converter.DocumentContext{}) + if err != nil { + return nil, err } + spec.MardownConverter = conv - return combinedConfig -} - -var blackfridayExtensionMap = map[string]int{ - "noIntraEmphasis": blackfriday.EXTENSION_NO_INTRA_EMPHASIS, - "tables": blackfriday.EXTENSION_TABLES, - "fencedCode": blackfriday.EXTENSION_FENCED_CODE, - "autolink": blackfriday.EXTENSION_AUTOLINK, - "strikethrough": blackfriday.EXTENSION_STRIKETHROUGH, - "laxHtmlBlocks": blackfriday.EXTENSION_LAX_HTML_BLOCKS, - "spaceHeaders": blackfriday.EXTENSION_SPACE_HEADERS, - "hardLineBreak": blackfriday.EXTENSION_HARD_LINE_BREAK, - "tabSizeEight": blackfriday.EXTENSION_TAB_SIZE_EIGHT, - "footnotes": blackfriday.EXTENSION_FOOTNOTES, - "noEmptyLineBeforeBlock": blackfriday.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK, - "headerIds": blackfriday.EXTENSION_HEADER_IDS, - "titleblock": blackfriday.EXTENSION_TITLEBLOCK, - "autoHeaderIds": blackfriday.EXTENSION_AUTO_HEADER_IDS, - "backslashLineBreak": blackfriday.EXTENSION_BACKSLASH_LINE_BREAK, - "definitionLists": blackfriday.EXTENSION_DEFINITION_LISTS, - "joinLines": blackfriday.EXTENSION_JOIN_LINES, + return spec, nil } var stripHTMLReplacer = strings.NewReplacer("\n", " ", "

", "\n", "
", "\n", "
", "\n") -var mmarkExtensionMap = map[string]int{ - "tables": mmark.EXTENSION_TABLES, - "fencedCode": mmark.EXTENSION_FENCED_CODE, - "autolink": mmark.EXTENSION_AUTOLINK, - "laxHtmlBlocks": mmark.EXTENSION_LAX_HTML_BLOCKS, - "spaceHeaders": mmark.EXTENSION_SPACE_HEADERS, - "hardLineBreak": mmark.EXTENSION_HARD_LINE_BREAK, - "footnotes": mmark.EXTENSION_FOOTNOTES, - "noEmptyLineBeforeBlock": mmark.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK, - "headerIds": mmark.EXTENSION_HEADER_IDS, - "autoHeaderIds": mmark.EXTENSION_AUTO_HEADER_IDS, -} - // StripHTML accepts a string, strips out all HTML tags and returns it. func StripHTML(s string) string { @@ -250,181 +173,6 @@ func BytesToHTML(b []byte) template.HTML { return template.HTML(string(b)) } -// getHTMLRenderer creates a new Blackfriday HTML Renderer with the given configuration. -func (c *ContentSpec) getHTMLRenderer(defaultFlags int, ctx *RenderingContext) blackfriday.Renderer { - renderParameters := blackfriday.HtmlRendererParameters{ - FootnoteAnchorPrefix: c.footnoteAnchorPrefix, - FootnoteReturnLinkContents: c.footnoteReturnLinkContents, - } - - b := len(ctx.DocumentID) != 0 - - if ctx.Config == nil { - panic(fmt.Sprintf("RenderingContext of %q doesn't have a config", ctx.DocumentID)) - } - - if b && !ctx.Config.PlainIDAnchors { - renderParameters.FootnoteAnchorPrefix = ctx.DocumentID + ":" + renderParameters.FootnoteAnchorPrefix - renderParameters.HeaderIDSuffix = ":" + ctx.DocumentID - } - - htmlFlags := defaultFlags - htmlFlags |= blackfriday.HTML_USE_XHTML - htmlFlags |= blackfriday.HTML_FOOTNOTE_RETURN_LINKS - - if ctx.Config.Smartypants { - htmlFlags |= blackfriday.HTML_USE_SMARTYPANTS - } - - if ctx.Config.SmartypantsQuotesNBSP { - htmlFlags |= blackfriday.HTML_SMARTYPANTS_QUOTES_NBSP - } - - if ctx.Config.AngledQuotes { - htmlFlags |= blackfriday.HTML_SMARTYPANTS_ANGLED_QUOTES - } - - if ctx.Config.Fractions { - htmlFlags |= blackfriday.HTML_SMARTYPANTS_FRACTIONS - } - - if ctx.Config.HrefTargetBlank { - htmlFlags |= blackfriday.HTML_HREF_TARGET_BLANK - } - - if ctx.Config.NofollowLinks { - htmlFlags |= blackfriday.HTML_NOFOLLOW_LINKS - } - - if ctx.Config.NoreferrerLinks { - htmlFlags |= blackfriday.HTML_NOREFERRER_LINKS - } - - if ctx.Config.SmartDashes { - htmlFlags |= blackfriday.HTML_SMARTYPANTS_DASHES - } - - if ctx.Config.LatexDashes { - htmlFlags |= blackfriday.HTML_SMARTYPANTS_LATEX_DASHES - } - - if ctx.Config.SkipHTML { - htmlFlags |= blackfriday.HTML_SKIP_HTML - } - - return &HugoHTMLRenderer{ - cs: c, - RenderingContext: ctx, - Renderer: blackfriday.HtmlRendererWithParameters(htmlFlags, "", "", renderParameters), - } -} - -func getMarkdownExtensions(ctx *RenderingContext) int { - // Default Blackfriday common extensions - commonExtensions := 0 | - blackfriday.EXTENSION_NO_INTRA_EMPHASIS | - blackfriday.EXTENSION_TABLES | - blackfriday.EXTENSION_FENCED_CODE | - blackfriday.EXTENSION_AUTOLINK | - blackfriday.EXTENSION_STRIKETHROUGH | - blackfriday.EXTENSION_SPACE_HEADERS | - blackfriday.EXTENSION_HEADER_IDS | - blackfriday.EXTENSION_BACKSLASH_LINE_BREAK | - blackfriday.EXTENSION_DEFINITION_LISTS - - // Extra Blackfriday extensions that Hugo enables by default - flags := commonExtensions | - blackfriday.EXTENSION_AUTO_HEADER_IDS | - blackfriday.EXTENSION_FOOTNOTES - - if ctx.Config == nil { - panic(fmt.Sprintf("RenderingContext of %q doesn't have a config", ctx.DocumentID)) - } - - for _, extension := range ctx.Config.Extensions { - if flag, ok := blackfridayExtensionMap[extension]; ok { - flags |= flag - } - } - for _, extension := range ctx.Config.ExtensionsMask { - if flag, ok := blackfridayExtensionMap[extension]; ok { - flags &= ^flag - } - } - return flags -} - -func (c *ContentSpec) markdownRender(ctx *RenderingContext) []byte { - if ctx.RenderTOC { - return blackfriday.Markdown(ctx.Content, - c.getHTMLRenderer(blackfriday.HTML_TOC, ctx), - getMarkdownExtensions(ctx)) - } - return blackfriday.Markdown(ctx.Content, c.getHTMLRenderer(0, ctx), - getMarkdownExtensions(ctx)) -} - -// getMmarkHTMLRenderer creates a new mmark HTML Renderer with the given configuration. -func (c *ContentSpec) getMmarkHTMLRenderer(defaultFlags int, ctx *RenderingContext) mmark.Renderer { - renderParameters := mmark.HtmlRendererParameters{ - FootnoteAnchorPrefix: c.footnoteAnchorPrefix, - FootnoteReturnLinkContents: c.footnoteReturnLinkContents, - } - - b := len(ctx.DocumentID) != 0 - - if ctx.Config == nil { - panic(fmt.Sprintf("RenderingContext of %q doesn't have a config", ctx.DocumentID)) - } - - if b && !ctx.Config.PlainIDAnchors { - renderParameters.FootnoteAnchorPrefix = ctx.DocumentID + ":" + renderParameters.FootnoteAnchorPrefix - // renderParameters.HeaderIDSuffix = ":" + ctx.DocumentId - } - - htmlFlags := defaultFlags - htmlFlags |= mmark.HTML_FOOTNOTE_RETURN_LINKS - - return &HugoMmarkHTMLRenderer{ - cs: c, - Renderer: mmark.HtmlRendererWithParameters(htmlFlags, "", "", renderParameters), - Cfg: c.Cfg, - } -} - -func getMmarkExtensions(ctx *RenderingContext) int { - flags := 0 - flags |= mmark.EXTENSION_TABLES - flags |= mmark.EXTENSION_FENCED_CODE - flags |= mmark.EXTENSION_AUTOLINK - flags |= mmark.EXTENSION_SPACE_HEADERS - flags |= mmark.EXTENSION_CITATION - flags |= mmark.EXTENSION_TITLEBLOCK_TOML - flags |= mmark.EXTENSION_HEADER_IDS - flags |= mmark.EXTENSION_AUTO_HEADER_IDS - flags |= mmark.EXTENSION_UNIQUE_HEADER_IDS - flags |= mmark.EXTENSION_FOOTNOTES - flags |= mmark.EXTENSION_SHORT_REF - flags |= mmark.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK - flags |= mmark.EXTENSION_INCLUDE - - if ctx.Config == nil { - panic(fmt.Sprintf("RenderingContext of %q doesn't have a config", ctx.DocumentID)) - } - - for _, extension := range ctx.Config.Extensions { - if flag, ok := mmarkExtensionMap[extension]; ok { - flags |= flag - } - } - return flags -} - -func (c *ContentSpec) mmarkRender(ctx *RenderingContext) []byte { - return mmark.Parse(ctx.Content, c.getMmarkHTMLRenderer(0, ctx), - getMmarkExtensions(ctx)).Bytes() -} - // ExtractTOC extracts Table of Contents from content. func ExtractTOC(content []byte) (newcontent []byte, toc []byte) { if !bytes.Contains(content, []byte("