From 5f6b6ec68936ebbbf590894c02a1a3ecad30735f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Fri, 16 Aug 2019 15:55:03 +0200 Subject: Prepare for Goldmark This commmit prepares for the addition of Goldmark as the new Markdown renderer in Hugo. This introduces a new `markup` package with some common interfaces and each implementation in its own package. See #5963 --- markup/blackfriday/convert.go | 224 +++++++++++++++++++++++++++++++++++++ markup/blackfriday/convert_test.go | 194 ++++++++++++++++++++++++++++++++ markup/blackfriday/renderer.go | 85 ++++++++++++++ 3 files changed, 503 insertions(+) create mode 100644 markup/blackfriday/convert.go create mode 100644 markup/blackfriday/convert_test.go create mode 100644 markup/blackfriday/renderer.go (limited to 'markup/blackfriday') diff --git a/markup/blackfriday/convert.go b/markup/blackfriday/convert.go new file mode 100644 index 000000000..f9d957a4e --- /dev/null +++ b/markup/blackfriday/convert.go @@ -0,0 +1,224 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package blackfriday converts Markdown to HTML using Blackfriday v1. +package blackfriday + +import ( + "github.com/gohugoio/hugo/markup/converter" + "github.com/gohugoio/hugo/markup/internal" + "github.com/russross/blackfriday" +) + +// Provider is the package entry point. +var Provider converter.NewProvider = provider{} + +type provider struct { +} + +func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) { + defaultBlackFriday, err := internal.NewBlackfriday(cfg) + if err != nil { + return nil, err + } + + defaultExtensions := getMarkdownExtensions(defaultBlackFriday) + + pygmentsCodeFences := cfg.Cfg.GetBool("pygmentsCodeFences") + pygmentsCodeFencesGuessSyntax := cfg.Cfg.GetBool("pygmentsCodeFencesGuessSyntax") + pygmentsOptions := cfg.Cfg.GetString("pygmentsOptions") + + var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) { + b := defaultBlackFriday + extensions := defaultExtensions + + if ctx.ConfigOverrides != nil { + var err error + b, err = internal.UpdateBlackFriday(b, ctx.ConfigOverrides) + if err != nil { + return nil, err + } + extensions = getMarkdownExtensions(b) + } + + return &blackfridayConverter{ + ctx: ctx, + bf: b, + extensions: extensions, + cfg: cfg, + + pygmentsCodeFences: pygmentsCodeFences, + pygmentsCodeFencesGuessSyntax: pygmentsCodeFencesGuessSyntax, + pygmentsOptions: pygmentsOptions, + }, nil + } + + return n, nil + +} + +type blackfridayConverter struct { + ctx converter.DocumentContext + bf *internal.BlackFriday + extensions int + + pygmentsCodeFences bool + pygmentsCodeFencesGuessSyntax bool + pygmentsOptions string + + cfg converter.ProviderConfig +} + +func (c *blackfridayConverter) AnchorSuffix() string { + if c.bf.PlainIDAnchors { + return "" + } + return ":" + c.ctx.DocumentID +} + +func (c *blackfridayConverter) Convert(ctx converter.RenderContext) (converter.Result, error) { + r := c.getHTMLRenderer(ctx.RenderTOC) + + return converter.Bytes(blackfriday.Markdown(ctx.Src, r, c.extensions)), nil + +} + +func (c *blackfridayConverter) getHTMLRenderer(renderTOC bool) blackfriday.Renderer { + flags := getFlags(renderTOC, c.bf) + + documentID := c.ctx.DocumentID + + renderParameters := blackfriday.HtmlRendererParameters{ + FootnoteAnchorPrefix: c.bf.FootnoteAnchorPrefix, + FootnoteReturnLinkContents: c.bf.FootnoteReturnLinkContents, + } + + if documentID != "" && !c.bf.PlainIDAnchors { + renderParameters.FootnoteAnchorPrefix = documentID + ":" + renderParameters.FootnoteAnchorPrefix + renderParameters.HeaderIDSuffix = ":" + documentID + } + + return &hugoHTMLRenderer{ + c: c, + Renderer: blackfriday.HtmlRendererWithParameters(flags, "", "", renderParameters), + } +} + +func getFlags(renderTOC bool, cfg *internal.BlackFriday) int { + + var flags int + + if renderTOC { + flags = blackfriday.HTML_TOC + } + + flags |= blackfriday.HTML_USE_XHTML + flags |= blackfriday.HTML_FOOTNOTE_RETURN_LINKS + + if cfg.Smartypants { + flags |= blackfriday.HTML_USE_SMARTYPANTS + } + + if cfg.SmartypantsQuotesNBSP { + flags |= blackfriday.HTML_SMARTYPANTS_QUOTES_NBSP + } + + if cfg.AngledQuotes { + flags |= blackfriday.HTML_SMARTYPANTS_ANGLED_QUOTES + } + + if cfg.Fractions { + flags |= blackfriday.HTML_SMARTYPANTS_FRACTIONS + } + + if cfg.HrefTargetBlank { + flags |= blackfriday.HTML_HREF_TARGET_BLANK + } + + if cfg.NofollowLinks { + flags |= blackfriday.HTML_NOFOLLOW_LINKS + } + + if cfg.NoreferrerLinks { + flags |= blackfriday.HTML_NOREFERRER_LINKS + } + + if cfg.SmartDashes { + flags |= blackfriday.HTML_SMARTYPANTS_DASHES + } + + if cfg.LatexDashes { + flags |= blackfriday.HTML_SMARTYPANTS_LATEX_DASHES + } + + if cfg.SkipHTML { + flags |= blackfriday.HTML_SKIP_HTML + } + + return flags +} + +func getMarkdownExtensions(cfg *internal.BlackFriday) int { + // Default Blackfriday common extensions + commonExtensions := 0 | + blackfriday.EXTENSION_NO_INTRA_EMPHASIS | + blackfriday.EXTENSION_TABLES | + blackfriday.EXTENSION_FENCED_CODE | + blackfriday.EXTENSION_AUTOLINK | + blackfriday.EXTENSION_STRIKETHROUGH | + blackfriday.EXTENSION_SPACE_HEADERS | + blackfriday.EXTENSION_HEADER_IDS | + blackfriday.EXTENSION_BACKSLASH_LINE_BREAK | + blackfriday.EXTENSION_DEFINITION_LISTS + + // Extra Blackfriday extensions that Hugo enables by default + flags := commonExtensions | + blackfriday.EXTENSION_AUTO_HEADER_IDS | + blackfriday.EXTENSION_FOOTNOTES + + for _, extension := range cfg.Extensions { + if flag, ok := blackfridayExtensionMap[extension]; ok { + flags |= flag + } + } + for _, extension := range cfg.ExtensionsMask { + if flag, ok := blackfridayExtensionMap[extension]; ok { + flags &= ^flag + } + } + return flags +} + +var blackfridayExtensionMap = map[string]int{ + "noIntraEmphasis": blackfriday.EXTENSION_NO_INTRA_EMPHASIS, + "tables": blackfriday.EXTENSION_TABLES, + "fencedCode": blackfriday.EXTENSION_FENCED_CODE, + "autolink": blackfriday.EXTENSION_AUTOLINK, + "strikethrough": blackfriday.EXTENSION_STRIKETHROUGH, + "laxHtmlBlocks": blackfriday.EXTENSION_LAX_HTML_BLOCKS, + "spaceHeaders": blackfriday.EXTENSION_SPACE_HEADERS, + "hardLineBreak": blackfriday.EXTENSION_HARD_LINE_BREAK, + "tabSizeEight": blackfriday.EXTENSION_TAB_SIZE_EIGHT, + "footnotes": blackfriday.EXTENSION_FOOTNOTES, + "noEmptyLineBeforeBlock": blackfriday.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK, + "headerIds": blackfriday.EXTENSION_HEADER_IDS, + "titleblock": blackfriday.EXTENSION_TITLEBLOCK, + "autoHeaderIds": blackfriday.EXTENSION_AUTO_HEADER_IDS, + "backslashLineBreak": blackfriday.EXTENSION_BACKSLASH_LINE_BREAK, + "definitionLists": blackfriday.EXTENSION_DEFINITION_LISTS, + "joinLines": blackfriday.EXTENSION_JOIN_LINES, +} + +var ( + _ converter.DocumentInfo = (*blackfridayConverter)(nil) +) diff --git a/markup/blackfriday/convert_test.go b/markup/blackfriday/convert_test.go new file mode 100644 index 000000000..094edf35f --- /dev/null +++ b/markup/blackfriday/convert_test.go @@ -0,0 +1,194 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package blackfriday + +import ( + "testing" + + "github.com/spf13/viper" + + "github.com/gohugoio/hugo/markup/internal" + + "github.com/gohugoio/hugo/markup/converter" + + qt "github.com/frankban/quicktest" + "github.com/russross/blackfriday" +) + +func TestGetMarkdownExtensionsMasksAreRemovedFromExtensions(t *testing.T) { + c := qt.New(t) + b, err := internal.NewBlackfriday(converter.ProviderConfig{Cfg: viper.New()}) + c.Assert(err, qt.IsNil) + + b.Extensions = []string{"headerId"} + b.ExtensionsMask = []string{"noIntraEmphasis"} + + actualFlags := getMarkdownExtensions(b) + if actualFlags&blackfriday.EXTENSION_NO_INTRA_EMPHASIS == blackfriday.EXTENSION_NO_INTRA_EMPHASIS { + t.Errorf("Masked out flag {%v} found amongst returned extensions.", blackfriday.EXTENSION_NO_INTRA_EMPHASIS) + } +} + +func TestGetMarkdownExtensionsByDefaultAllExtensionsAreEnabled(t *testing.T) { + type data struct { + testFlag int + } + + c := qt.New(t) + b, err := internal.NewBlackfriday(converter.ProviderConfig{Cfg: viper.New()}) + c.Assert(err, qt.IsNil) + + b.Extensions = []string{""} + b.ExtensionsMask = []string{""} + allExtensions := []data{ + {blackfriday.EXTENSION_NO_INTRA_EMPHASIS}, + {blackfriday.EXTENSION_TABLES}, + {blackfriday.EXTENSION_FENCED_CODE}, + {blackfriday.EXTENSION_AUTOLINK}, + {blackfriday.EXTENSION_STRIKETHROUGH}, + // {blackfriday.EXTENSION_LAX_HTML_BLOCKS}, + {blackfriday.EXTENSION_SPACE_HEADERS}, + // {blackfriday.EXTENSION_HARD_LINE_BREAK}, + // {blackfriday.EXTENSION_TAB_SIZE_EIGHT}, + {blackfriday.EXTENSION_FOOTNOTES}, + // {blackfriday.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK}, + {blackfriday.EXTENSION_HEADER_IDS}, + // {blackfriday.EXTENSION_TITLEBLOCK}, + {blackfriday.EXTENSION_AUTO_HEADER_IDS}, + {blackfriday.EXTENSION_BACKSLASH_LINE_BREAK}, + {blackfriday.EXTENSION_DEFINITION_LISTS}, + } + + actualFlags := getMarkdownExtensions(b) + for _, e := range allExtensions { + if actualFlags&e.testFlag != e.testFlag { + t.Errorf("Flag %v was not found in the list of extensions.", e) + } + } +} + +func TestGetMarkdownExtensionsAddingFlagsThroughRenderingContext(t *testing.T) { + c := qt.New(t) + b, err := internal.NewBlackfriday(converter.ProviderConfig{Cfg: viper.New()}) + c.Assert(err, qt.IsNil) + + b.Extensions = []string{"definitionLists"} + b.ExtensionsMask = []string{""} + + actualFlags := getMarkdownExtensions(b) + if actualFlags&blackfriday.EXTENSION_DEFINITION_LISTS != blackfriday.EXTENSION_DEFINITION_LISTS { + t.Errorf("Masked out flag {%v} found amongst returned extensions.", blackfriday.EXTENSION_DEFINITION_LISTS) + } +} + +func TestGetFlags(t *testing.T) { + c := qt.New(t) + cfg := converter.ProviderConfig{Cfg: viper.New()} + b, err := internal.NewBlackfriday(cfg) + c.Assert(err, qt.IsNil) + flags := getFlags(false, b) + if flags&blackfriday.HTML_USE_XHTML != blackfriday.HTML_USE_XHTML { + t.Errorf("Test flag: %d was not found amongs set flags:%d; Result: %d", blackfriday.HTML_USE_XHTML, flags, flags&blackfriday.HTML_USE_XHTML) + } +} + +func TestGetAllFlags(t *testing.T) { + c := qt.New(t) + cfg := converter.ProviderConfig{Cfg: viper.New()} + b, err := internal.NewBlackfriday(cfg) + c.Assert(err, qt.IsNil) + + type data struct { + testFlag int + } + + allFlags := []data{ + {blackfriday.HTML_USE_XHTML}, + {blackfriday.HTML_FOOTNOTE_RETURN_LINKS}, + {blackfriday.HTML_USE_SMARTYPANTS}, + {blackfriday.HTML_SMARTYPANTS_QUOTES_NBSP}, + {blackfriday.HTML_SMARTYPANTS_ANGLED_QUOTES}, + {blackfriday.HTML_SMARTYPANTS_FRACTIONS}, + {blackfriday.HTML_HREF_TARGET_BLANK}, + {blackfriday.HTML_NOFOLLOW_LINKS}, + {blackfriday.HTML_NOREFERRER_LINKS}, + {blackfriday.HTML_SMARTYPANTS_DASHES}, + {blackfriday.HTML_SMARTYPANTS_LATEX_DASHES}, + } + + b.AngledQuotes = true + b.Fractions = true + b.HrefTargetBlank = true + b.NofollowLinks = true + b.NoreferrerLinks = true + b.LatexDashes = true + b.PlainIDAnchors = true + b.SmartDashes = true + b.Smartypants = true + b.SmartypantsQuotesNBSP = true + + actualFlags := getFlags(false, b) + + var expectedFlags int + //OR-ing flags together... + for _, d := range allFlags { + expectedFlags |= d.testFlag + } + if expectedFlags != actualFlags { + t.Errorf("Expected flags (%d) did not equal actual (%d) flags.", expectedFlags, actualFlags) + } +} + +func TestConvert(t *testing.T) { + c := qt.New(t) + p, err := Provider.New(converter.ProviderConfig{ + Cfg: viper.New(), + }) + c.Assert(err, qt.IsNil) + conv, err := p.New(converter.DocumentContext{}) + c.Assert(err, qt.IsNil) + b, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")}) + c.Assert(err, qt.IsNil) + c.Assert(string(b.Bytes()), qt.Equals, "

testContent

\n") +} + +func TestGetHTMLRendererAnchors(t *testing.T) { + c := qt.New(t) + p, err := Provider.New(converter.ProviderConfig{ + Cfg: viper.New(), + }) + c.Assert(err, qt.IsNil) + conv, err := p.New(converter.DocumentContext{ + DocumentID: "testid", + ConfigOverrides: map[string]interface{}{ + "plainIDAnchors": false, + "footnotes": true, + }, + }) + c.Assert(err, qt.IsNil) + b, err := conv.Convert(converter.RenderContext{Src: []byte(`# Header + +This is a footnote.[^1] And then some. + + +[^1]: Footnote text. + +`)}) + + c.Assert(err, qt.IsNil) + s := string(b.Bytes()) + c.Assert(s, qt.Contains, "

Header

") + c.Assert(s, qt.Contains, "This is a footnote.1") + c.Assert(s, qt.Contains, "[return]") +} diff --git a/markup/blackfriday/renderer.go b/markup/blackfriday/renderer.go new file mode 100644 index 000000000..9f4d44e02 --- /dev/null +++ b/markup/blackfriday/renderer.go @@ -0,0 +1,85 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package blackfriday + +import ( + "bytes" + "strings" + + "github.com/russross/blackfriday" +) + +// hugoHTMLRenderer wraps a blackfriday.Renderer, typically a blackfriday.Html +// adding some custom behaviour. +type hugoHTMLRenderer struct { + c *blackfridayConverter + blackfriday.Renderer +} + +// BlockCode renders a given text as a block of code. +// Pygments is used if it is setup to handle code fences. +func (r *hugoHTMLRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string) { + if r.c.pygmentsCodeFences && (lang != "" || r.c.pygmentsCodeFencesGuessSyntax) { + opts := r.c.pygmentsOptions + str := strings.Trim(string(text), "\n\r") + highlighted, _ := r.c.cfg.Highlight(str, lang, opts) + out.WriteString(highlighted) + } else { + r.Renderer.BlockCode(out, text, lang) + } +} + +// ListItem adds task list support to the Blackfriday renderer. +func (r *hugoHTMLRenderer) ListItem(out *bytes.Buffer, text []byte, flags int) { + if !r.c.bf.TaskLists { + r.Renderer.ListItem(out, text, flags) + return + } + + switch { + case bytes.HasPrefix(text, []byte("[ ] ")): + text = append([]byte(``)...) + + case bytes.HasPrefix(text, []byte("[x] ")) || bytes.HasPrefix(text, []byte("[X] ")): + text = append([]byte(``)...) + } + + r.Renderer.ListItem(out, text, flags) +} + +// List adds task list support to the Blackfriday renderer. +func (r *hugoHTMLRenderer) List(out *bytes.Buffer, text func() bool, flags int) { + if !r.c.bf.TaskLists { + r.Renderer.List(out, text, flags) + return + } + marker := out.Len() + r.Renderer.List(out, text, flags) + if out.Len() > marker { + list := out.Bytes()[marker:] + if bytes.Contains(list, []byte("task-list-item")) { + // Find the index of the first >, it might be 3 or 4 depending on whether + // there is a new line at the start, but this is safer than just hardcoding it. + closingBracketIndex := bytes.Index(list, []byte(">")) + // Rewrite the buffer from the marker + out.Truncate(marker) + // Safely assuming closingBracketIndex won't be -1 since there is a list + // May be either dl, ul or ol + list := append(list[:closingBracketIndex], append([]byte(` class="task-list"`), list[closingBracketIndex:]...)...) + out.Write(list) + } + } +} -- cgit v1.2.3