diff options
39 files changed, 1735 insertions, 982 deletions
diff --git a/deps/deps.go b/deps/deps.go index aaed900e5..d7b381ce9 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -223,7 +223,7 @@ func New(cfg DepsCfg) (*Deps, error) { return nil, err } - contentSpec, err := helpers.NewContentSpec(cfg.Language) + contentSpec, err := helpers.NewContentSpec(cfg.Language, logger, ps.BaseFs.Content.Fs) if err != nil { return nil, err } @@ -277,7 +277,7 @@ func (d Deps) ForLanguage(cfg DepsCfg, onCreated func(d *Deps) error) (*Deps, er return nil, err } - d.ContentSpec, err = helpers.NewContentSpec(l) + d.ContentSpec, err = helpers.NewContentSpec(l, d.Log, d.BaseFs.Content.Fs) if err != nil { return nil, err } diff --git a/helpers/content.go b/helpers/content.go index fe96ce7d2..357bd48e7 100644 --- a/helpers/content.go +++ b/helpers/content.go @@ -19,22 +19,18 @@ package helpers import ( "bytes" - "fmt" "html/template" - "os/exec" - "runtime" "unicode" "unicode/utf8" - "github.com/gohugoio/hugo/common/maps" - "github.com/gohugoio/hugo/hugolib/filesystems" - "github.com/niklasfasching/go-org/org" + "github.com/gohugoio/hugo/common/loggers" + + "github.com/gohugoio/hugo/markup/converter" + + "github.com/gohugoio/hugo/markup" bp "github.com/gohugoio/hugo/bufferpool" "github.com/gohugoio/hugo/config" - "github.com/miekg/mmark" - "github.com/mitchellh/mapstructure" - "github.com/russross/blackfriday" "github.com/spf13/afero" jww "github.com/spf13/jwalterweatherman" @@ -52,9 +48,9 @@ var ( // ContentSpec provides functionality to render markdown content. type ContentSpec struct { - BlackFriday *BlackFriday - footnoteAnchorPrefix string - footnoteReturnLinkContents string + Converters markup.ConverterProvider + MardownConverter converter.Converter // Markdown converter with no document context + // SummaryLength is the length of the summary that Hugo extracts from a content. summaryLength int @@ -70,16 +66,13 @@ type ContentSpec struct { // NewContentSpec returns a ContentSpec initialized // with the appropriate fields from the given config.Provider. -func NewContentSpec(cfg config.Provider) (*ContentSpec, error) { - bf := newBlackfriday(cfg.GetStringMap("blackfriday")) +func NewContentSpec(cfg config.Provider, logger *loggers.Logger, contentFs afero.Fs) (*ContentSpec, error) { + spec := &ContentSpec{ - BlackFriday: bf, - footnoteAnchorPrefix: cfg.GetString("footnoteAnchorPrefix"), - footnoteReturnLinkContents: cfg.GetString("footnoteReturnLinkContents"), - summaryLength: cfg.GetInt("summaryLength"), - BuildFuture: cfg.GetBool("buildFuture"), - BuildExpired: cfg.GetBool("buildExpired"), - BuildDrafts: cfg.GetBool("buildDrafts"), + summaryLength: cfg.GetInt("summaryLength"), + BuildFuture: cfg.GetBool("buildFuture"), + BuildExpired: cfg.GetBool("buildExpired"), + BuildDrafts: cfg.GetBool("buildDrafts"), Cfg: cfg, } @@ -109,99 +102,29 @@ func NewContentSpec(cfg config.Provider) (*ContentSpec, error) { spec.Highlight = h.chromaHighlight } - return spec, nil -} - -// BlackFriday holds configuration values for BlackFriday rendering. -type BlackFriday struct { - Smartypants bool - SmartypantsQuotesNBSP bool - AngledQuotes bool - Fractions bool - HrefTargetBlank bool - NofollowLinks bool - NoreferrerLinks bool - SmartDashes bool - LatexDashes bool - TaskLists bool - PlainIDAnchors bool - Extensions []string - ExtensionsMask []string - SkipHTML bool -} - -// NewBlackfriday creates a new Blackfriday filled with site config or some sane defaults. -func newBlackfriday(config map[string]interface{}) *BlackFriday { - defaultParam := map[string]interface{}{ - "smartypants": true, - "angledQuotes": false, - "smartypantsQuotesNBSP": false, - "fractions": true, - "hrefTargetBlank": false, - "nofollowLinks": false, - "noreferrerLinks": false, - "smartDashes": true, - "latexDashes": true, - "plainIDAnchors": true, - "taskLists": true, - "skipHTML": false, - } - - maps.ToLower(defaultParam) - - siteConfig := make(map[string]interface{}) - - for k, v := range defaultParam { - siteConfig[k] = v - } - - for k, v := range config { - siteConfig[k] = v + converterProvider, err := markup.NewConverterProvider(converter.ProviderConfig{ + Cfg: cfg, + ContentFs: contentFs, + Logger: logger, + Highlight: spec.Highlight, + }) + if err != nil { + return nil, err } - combinedConfig := &BlackFriday{} - if err := mapstructure.Decode(siteConfig, combinedConfig); err != nil { - jww.FATAL.Printf("Failed to get site rendering config\n%s", err.Error()) + spec.Converters = converterProvider + p := converterProvider.Get("markdown") + conv, err := p.New(converter.DocumentContext{}) + if err != nil { + return nil, err } + spec.MardownConverter = conv - return combinedConfig -} - -var blackfridayExtensionMap = map[string]int{ - "noIntraEmphasis": blackfriday.EXTENSION_NO_INTRA_EMPHASIS, - "tables": blackfriday.EXTENSION_TABLES, - "fencedCode": blackfriday.EXTENSION_FENCED_CODE, - "autolink": blackfriday.EXTENSION_AUTOLINK, - "strikethrough": blackfriday.EXTENSION_STRIKETHROUGH, - "laxHtmlBlocks": blackfriday.EXTENSION_LAX_HTML_BLOCKS, - "spaceHeaders": blackfriday.EXTENSION_SPACE_HEADERS, - "hardLineBreak": blackfriday.EXTENSION_HARD_LINE_BREAK, - "tabSizeEight": blackfriday.EXTENSION_TAB_SIZE_EIGHT, - "footnotes": blackfriday.EXTENSION_FOOTNOTES, - "noEmptyLineBeforeBlock": blackfriday.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK, - "headerIds": blackfriday.EXTENSION_HEADER_IDS, - "titleblock": blackfriday.EXTENSION_TITLEBLOCK, - "autoHeaderIds": blackfriday.EXTENSION_AUTO_HEADER_IDS, - "backslashLineBreak": blackfriday.EXTENSION_BACKSLASH_LINE_BREAK, - "definitionLists": blackfriday.EXTENSION_DEFINITION_LISTS, - "joinLines": blackfriday.EXTENSION_JOIN_LINES, + return spec, nil } var stripHTMLReplacer = strings.NewReplacer("\n", " ", "</p>", "\n", "<br>", "\n", "<br />", "\n") -var mmarkExtensionMap = map[string]int{ - "tables": mmark.EXTENSION_TABLES, - "fencedCode": mmark.EXTENSION_FENCED_CODE, - "autolink": mmark.EXTENSION_AUTOLINK, - "laxHtmlBlocks": mmark.EXTENSION_LAX_HTML_BLOCKS, - "spaceHeaders": mmark.EXTENSION_SPACE_HEADERS, - "hardLineBreak": mmark.EXTENSION_HARD_LINE_BREAK, - "footnotes": mmark.EXTENSION_FOOTNOTES, - "noEmptyLineBeforeBlock": mmark.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK, - "headerIds": mmark.EXTENSION_HEADER_IDS, - "autoHeaderIds": mmark.EXTENSION_AUTO_HEADER_IDS, -} - // StripHTML accepts a string, strips out all HTML tags and returns it. func StripHTML(s string) string { @@ -250,181 +173,6 @@ func BytesToHTML(b []byte) template.HTML { return template.HTML(string(b)) } -// getHTMLRenderer creates a new Blackfriday HTML Renderer with the given configuration. -func (c *ContentSpec) getHTMLRenderer(defaultFlags int, ctx *RenderingContext) blackfriday.Renderer { - renderParameters := blackfriday.HtmlRendererParameters{ - FootnoteAnchorPrefix: c.footnoteAnchorPrefix, - FootnoteReturnLinkContents: c.footnoteReturnLinkContents, - } - - b := len(ctx.DocumentID) != 0 - - if ctx.Config == nil { - panic(fmt.Sprintf("RenderingContext of %q doesn't have a config", ctx.DocumentID)) - } - - if b && !ctx.Config.PlainIDAnchors { - renderParameters.FootnoteAnchorPrefix = ctx.DocumentID + ":" + renderParameters.FootnoteAnchorPrefix - renderParameters.HeaderIDSuffix = ":" + ctx.DocumentID - } - - htmlFlags := defaultFlags - htmlFlags |= blackfriday.HTML_USE_XHTML - htmlFlags |= blackfriday.HTML_FOOTNOTE_RETURN_LINKS - - if ctx.Config.Smartypants { - htmlFlags |= blackfriday.HTML_USE_SMARTYPANTS - } - - if ctx.Config.SmartypantsQuotesNBSP { - htmlFlags |= blackfriday.HTML_SMARTYPANTS_QUOTES_NBSP - } - - if ctx.Config.AngledQuotes { - htmlFlags |= blackfriday.HTML_SMARTYPANTS_ANGLED_QUOTES - } - - if ctx.Config.Fractions { - htmlFlags |= blackfriday.HTML_SMARTYPANTS_FRACTIONS - } - - if ctx.Config.HrefTargetBlank { - htmlFlags |= blackfriday.HTML_HREF_TARGET_BLANK - } - - if ctx.Config.NofollowLinks { - htmlFlags |= blackfriday.HTML_NOFOLLOW_LINKS - } - - if ctx.Config.NoreferrerLinks { - htmlFlags |= blackfriday.HTML_NOREFERRER_LINKS - } - - if ctx.Config.SmartDashes { - htmlFlags |= blackfriday.HTML_SMARTYPANTS_DASHES - } - - if ctx.Config.LatexDashes { - htmlFlags |= blackfriday.HTML_SMARTYPANTS_LATEX_DASHES - } - - if ctx.Config.SkipHTML { - htmlFlags |= blackfriday.HTML_SKIP_HTML - } - - return &HugoHTMLRenderer{ - cs: c, - RenderingContext: ctx, - Renderer: blackfriday.HtmlRendererWithParameters(htmlFlags, "", "", renderParameters), - } -} - -func getMarkdownExtensions(ctx *RenderingContext) int { - // Default Blackfriday common extensions - commonExtensions := 0 | - blackfriday.EXTENSION_NO_INTRA_EMPHASIS | - blackfriday.EXTENSION_TABLES | - blackfriday.EXTENSION_FENCED_CODE | - blackfriday.EXTENSION_AUTOLINK | - blackfriday.EXTENSION_STRIKETHROUGH | - blackfriday.EXTENSION_SPACE_HEADERS | - blackfriday.EXTENSION_HEADER_IDS | - blackfriday.EXTENSION_BACKSLASH_LINE_BREAK | - blackfriday.EXTENSION_DEFINITION_LISTS - - // Extra Blackfriday extensions that Hugo enables by default - flags := commonExtensions | - blackfriday.EXTENSION_AUTO_HEADER_IDS | - blackfriday.EXTENSION_FOOTNOTES - - if ctx.Config == nil { - panic(fmt.Sprintf("RenderingContext of %q doesn't have a config", ctx.DocumentID)) - } - - for _, extension := range ctx.Config.Extensions { - if flag, ok := blackfridayExtensionMap[extension]; ok { - flags |= flag - } - } - for _, extension := range ctx.Config.ExtensionsMask { - if flag, ok := blackfridayExtensionMap[extension]; ok { - flags &= ^flag - } - } - return flags -} - -func (c *ContentSpec) markdownRender(ctx *RenderingContext) []byte { - if ctx.RenderTOC { - return blackfriday.Markdown(ctx.Content, - c.getHTMLRenderer(blackfriday.HTML_TOC, ctx), - getMarkdownExtensions(ctx)) - } - return blackfriday.Markdown(ctx.Content, c.getHTMLRenderer(0, ctx), - getMarkdownExtensions(ctx)) -} - -// getMmarkHTMLRenderer creates a new mmark HTML Renderer with the given configuration. -func (c *ContentSpec) getMmarkHTMLRenderer(defaultFlags int, ctx *RenderingContext) mmark.Renderer { - renderParameters := mmark.HtmlRendererParameters{ - FootnoteAnchorPrefix: c.footnoteAnchorPrefix, - FootnoteReturnLinkContents: c.footnoteReturnLinkContents, - } - - b := len(ctx.DocumentID) != 0 - - if ctx.Config == nil { - panic(fmt.Sprintf("RenderingContext of %q doesn't have a config", ctx.DocumentID)) - } - - if b && !ctx.Config.PlainIDAnchors { - renderParameters.FootnoteAnchorPrefix = ctx.DocumentID + ":" + renderParameters.FootnoteAnchorPrefix - // renderParameters.HeaderIDSuffix = ":" + ctx.DocumentId - } - - htmlFlags := defaultFlags - htmlFlags |= mmark.HTML_FOOTNOTE_RETURN_LINKS - - return &HugoMmarkHTMLRenderer{ - cs: c, - Renderer: mmark.HtmlRendererWithParameters(htmlFlags, "", "", renderParameters), - Cfg: c.Cfg, - } -} - -func getMmarkExtensions(ctx *RenderingContext) int { - flags := 0 - flags |= mmark.EXTENSION_TABLES - flags |= mmark.EXTENSION_FENCED_CODE - flags |= mmark.EXTENSION_AUTOLINK - flags |= mmark.EXTENSION_SPACE_HEADERS - flags |= mmark.EXTENSION_CITATION - flags |= mmark.EXTENSION_TITLEBLOCK_TOML - flags |= mmark.EXTENSION_HEADER_IDS - flags |= mmark.EXTENSION_AUTO_HEADER_IDS - flags |= mmark.EXTENSION_UNIQUE_HEADER_IDS - flags |= mmark.EXTENSION_FOOTNOTES - flags |= mmark.EXTENSION_SHORT_REF - flags |= mmark.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK - flags |= mmark.EXTENSION_INCLUDE - - if ctx.Config == nil { - panic(fmt.Sprintf("RenderingContext of %q doesn't have a config", ctx.DocumentID)) - } - - for _, extension := range ctx.Config.Extensions { - if flag, ok := mmarkExtensionMap[extension]; ok { - flags |= flag - } - } - return flags -} - -func (c *ContentSpec) mmarkRender(ctx *RenderingContext) []byte { - return mmark.Parse(ctx.Content, c.getMmarkHTMLRenderer(0, ctx), - getMmarkExtensions(ctx)).Bytes() -} - // ExtractTOC extracts Table of Contents from content. func ExtractTOC(content []byte) (newcontent []byte, toc []byte) { if !bytes.Contains(content, []byte("<nav>")) { @@ -464,38 +212,12 @@ func ExtractTOC(content []byte) (newcontent []byte, toc []byte) { return } -// RenderingContext holds contextual information, like content and configuration, -// for a given content rendering. -// By creating you must set the Config, otherwise it will panic. -type RenderingContext struct { - BaseFs *filesystems.BaseFs - Content []byte - PageFmt string - DocumentID string - DocumentName string - Config *BlackFriday - RenderTOC bool - Cfg config.Provider -} - -// RenderBytes renders a []byte. -func (c *ContentSpec) RenderBytes(ctx *RenderingContext) []byte { - switch ctx.PageFmt { - default: - return c.markdownRender(ctx) - case "markdown": - return c.markdownRender(ctx) - case "asciidoc": - return getAsciidocContent(ctx) - case "mmark": - return c.mmarkRender(ctx) - case "rst": - return getRstContent(ctx) - case "org": - return orgRender(ctx, c) - case "pandoc": - return getPandocContent(ctx) +func (c *ContentSpec) RenderMarkdown(src []byte) ([]byte, error) { + b, err := c.MardownConverter.Convert(converter.RenderContext{Src: src}) + if err != nil { + return nil, err } + return b.Bytes(), nil } // TotalWords counts instance of one or more consecutive white space @@ -622,181 +344,3 @@ func (c *ContentSpec) truncateWordsToWholeSentenceOld(content string) (string, b return strings.Join(words[:c.summaryLength], " "), true } - -func getAsciidocExecPath() string { - path, err := exec.LookPath("asciidoc") - if err != nil { - return "" - } - return path -} - -func getAsciidoctorExecPath() string { - path, err := exec.LookPath("asciidoctor") - if err != nil { - return "" - } - return path -} - -// HasAsciidoc returns whether Asciidoc or Asciidoctor is installed on this computer. -func HasAsciidoc() bool { - return (getAsciidoctorExecPath() != "" || - getAsciidocExecPath() != "") -} - -// getAsciidocContent calls asciidoctor or asciidoc as an external helper -// to convert AsciiDoc content to HTML. -func getAsciidocContent(ctx *RenderingContext) []byte { - var isAsciidoctor bool - path := getAsciidoctorExecPath() - if path == "" { - path = getAsciidocExecPath() - if path == "" { - jww.ERROR.Println("asciidoctor / asciidoc not found in $PATH: Please install.\n", - " Leaving AsciiDoc content unrendered.") - return ctx.Content - } - } else { - isAsciidoctor = true - } - - jww.INFO.Println("Rendering", ctx.DocumentName, "with", path, "...") - args := []string{"--no-header-footer", "--safe"} - if isAsciidoctor { - // asciidoctor-specific arg to show stack traces on errors - args = append(args, "--trace") - } - args = append(args, "-") - return externallyRenderContent(ctx, path, args) -} - -// HasRst returns whether rst2html is installed on this computer. -func HasRst() bool { - return getRstExecPath() != "" -} - -func getRstExecPath() string { - path, err := exec.LookPath("rst2html") - if err != nil { - path, err = exec.LookPath("rst2html.py") - if err != nil { - return "" - } - } - return path -} - -func getPythonExecPath() string { - path, err := exec.LookPath("python") - if err != nil { - path, err = exec.LookPath("python.exe") - if err != nil { - return "" - } - } - return path -} - -// getRstContent calls the Python script rst2html as an external helper -// to convert reStructuredText content to HTML. -func getRstContent(ctx *RenderingContext) []byte { - path := getRstExecPath() - - if path == "" { - jww.ERROR.Println("rst2html / rst2html.py not found in $PATH: Please install.\n", - " Leaving reStructuredText content unrendered.") - return ctx.Content - - } - jww.INFO.Println("Rendering", ctx.DocumentName, "with", path, "...") - var result []byte - // certain *nix based OSs wrap executables in scripted launchers - // invoking binaries on these OSs via python interpreter causes SyntaxError - // invoke directly so that shebangs work as expected - // handle Windows manually because it doesn't do shebangs - if runtime.GOOS == "windows" { - python := getPythonExecPath() - args := []string{path, "--leave-comments", "--initial-header-level=2"} - result = externallyRenderContent(ctx, python, args) - } else { - args := []string{"--leave-comments", "--initial-header-level=2"} - result = externallyRenderContent(ctx, path, args) - } - // TODO(bep) check if rst2html has a body only option. - bodyStart := bytes.Index(result, []byte("<body>\n")) - if bodyStart < 0 { - bodyStart = -7 //compensate for length - } - - bodyEnd := bytes.Index(result, []byte("\n</body>")) - if bodyEnd < 0 || bodyEnd >= len(result) { - bodyEnd = len(result) - 1 - if bodyEnd < 0 { - bodyEnd = 0 - } - } - - return result[bodyStart+7 : bodyEnd] -} - -// getPandocContent calls pandoc as an external helper to convert pandoc markdown to HTML. -func getPandocContent(ctx *RenderingContext) []byte { - path, err := exec.LookPath("pandoc") - if err != nil { - jww.ERROR.Println("pandoc not found in $PATH: Please install.\n", - " Leaving pandoc content unrendered.") - return ctx.Content - } - args := []string{"--mathjax"} - return externallyRenderContent(ctx, path, args) -} - -func orgRender(ctx *RenderingContext, c *ContentSpec) []byte { - config := org.New() - config.Log = jww.WARN - config.ReadFile = func(filename string) ([]byte, error) { - return afero.ReadFile(ctx.BaseFs.Content.Fs, filename) - } - writer := org.NewHTMLWriter() - writer.HighlightCodeBlock = func(source, lang string) string { - highlightedSource, err := c.Highlight(source, lang, "") - if err != nil { - jww.ERROR.Printf("Could not highlight source as lang %s. Using raw source.", lang) - return source - } - return highlightedSource - } - - html, err := config.Parse(bytes.NewReader(ctx.Content), ctx.DocumentName).Write(writer) - if err != nil { - jww.ERROR.Printf("Could not render org: %s. Using unrendered content.", err) - return ctx.Content - } - return []byte(html) -} - -func externallyRenderContent(ctx *RenderingContext, path string, args []string) []byte { - content := ctx.Content - cleanContent := bytes.Replace(content, SummaryDivider, []byte(""), 1) - - cmd := exec.Command(path, args...) - cmd.Stdin = bytes.NewReader(cleanContent) - var out, cmderr bytes.Buffer - cmd.Stdout = &out - cmd.Stderr = &cmderr - err := cmd.Run() - // Most external helpers exit w/ non-zero exit code only if severe, i.e. - // halting errors occurred. -> log stderr output regardless of state of err - for _, item := range strings.Split(cmderr.String(), "\n") { - item := strings.TrimSpace(item) - if item != "" { - jww.ERROR.Printf("%s: %s", ctx.DocumentName, item) - } - } - if err != nil { - jww.ERROR.Printf("%s rendering %s: %v", path, ctx.DocumentName, err) - } - - return normalizeExternalHelperLineFeeds(out.Bytes()) -} diff --git a/helpers/content_renderer_test.go b/helpers/content_renderer_test.go deleted file mode 100644 index 40acd89e1..000000000 --- a/helpers/content_renderer_test.go +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package helpers - -import ( - "bytes" - "regexp" - "testing" - - qt "github.com/frankban/quicktest" - "github.com/spf13/viper" -) - -// Renders a codeblock using Blackfriday -func (c *ContentSpec) render(input string) string { - ctx := &RenderingContext{Cfg: c.Cfg, Config: c.BlackFriday} - render := c.getHTMLRenderer(0, ctx) - - buf := &bytes.Buffer{} - render.BlockCode(buf, []byte(input), "html") - return buf.String() -} - -// Renders a codeblock using Mmark -func (c *ContentSpec) renderWithMmark(input string) string { - ctx := &RenderingContext{Cfg: c.Cfg, Config: c.BlackFriday} - render := c.getMmarkHTMLRenderer(0, ctx) - - buf := &bytes.Buffer{} - render.BlockCode(buf, []byte(input), "html", []byte(""), false, false) - return buf.String() -} - -func TestCode |