From 5f6b6ec68936ebbbf590894c02a1a3ecad30735f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Fri, 16 Aug 2019 15:55:03 +0200 Subject: Prepare for Goldmark This commmit prepares for the addition of Goldmark as the new Markdown renderer in Hugo. This introduces a new `markup` package with some common interfaces and each implementation in its own package. See #5963 --- markup/asciidoc/convert.go | 97 ++++++++++++++++ markup/asciidoc/convert_test.go | 38 +++++++ markup/blackfriday/convert.go | 224 +++++++++++++++++++++++++++++++++++++ markup/blackfriday/convert_test.go | 194 ++++++++++++++++++++++++++++++++ markup/blackfriday/renderer.go | 85 ++++++++++++++ markup/converter/converter.go | 83 ++++++++++++++ markup/internal/blackfriday.go | 108 ++++++++++++++++++ markup/internal/external.go | 52 +++++++++ markup/markup.go | 83 ++++++++++++++ markup/markup_test.go | 41 +++++++ markup/mmark/convert.go | 143 +++++++++++++++++++++++ markup/mmark/convert_test.go | 77 +++++++++++++ markup/mmark/renderer.go | 44 ++++++++ markup/org/convert.go | 69 ++++++++++++ markup/org/convert_test.go | 35 ++++++ markup/pandoc/convert.go | 76 +++++++++++++ markup/pandoc/convert_test.go | 38 +++++++ markup/rst/convert.go | 109 ++++++++++++++++++ markup/rst/convert_test.go | 38 +++++++ 19 files changed, 1634 insertions(+) create mode 100644 markup/asciidoc/convert.go create mode 100644 markup/asciidoc/convert_test.go create mode 100644 markup/blackfriday/convert.go create mode 100644 markup/blackfriday/convert_test.go create mode 100644 markup/blackfriday/renderer.go create mode 100644 markup/converter/converter.go create mode 100644 markup/internal/blackfriday.go create mode 100644 markup/internal/external.go create mode 100644 markup/markup.go create mode 100644 markup/markup_test.go create mode 100644 markup/mmark/convert.go create mode 100644 markup/mmark/convert_test.go create mode 100644 markup/mmark/renderer.go create mode 100644 markup/org/convert.go create mode 100644 markup/org/convert_test.go create mode 100644 markup/pandoc/convert.go create mode 100644 markup/pandoc/convert_test.go create mode 100644 markup/rst/convert.go create mode 100644 markup/rst/convert_test.go (limited to 'markup') diff --git a/markup/asciidoc/convert.go b/markup/asciidoc/convert.go new file mode 100644 index 000000000..9e63911d8 --- /dev/null +++ b/markup/asciidoc/convert.go @@ -0,0 +1,97 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package asciidoc converts Asciidoc to HTML using Asciidoc or Asciidoctor +// external binaries. +package asciidoc + +import ( + "os/exec" + + "github.com/gohugoio/hugo/markup/internal" + + "github.com/gohugoio/hugo/markup/converter" +) + +// Provider is the package entry point. +var Provider converter.NewProvider = provider{} + +type provider struct { +} + +func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) { + var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) { + return &asciidocConverter{ + ctx: ctx, + cfg: cfg, + }, nil + } + return n, nil +} + +type asciidocConverter struct { + ctx converter.DocumentContext + cfg converter.ProviderConfig +} + +func (a *asciidocConverter) Convert(ctx converter.RenderContext) (converter.Result, error) { + return converter.Bytes(a.getAsciidocContent(ctx.Src, a.ctx)), nil +} + +// getAsciidocContent calls asciidoctor or asciidoc as an external helper +// to convert AsciiDoc content to HTML. +func (a *asciidocConverter) getAsciidocContent(src []byte, ctx converter.DocumentContext) []byte { + var isAsciidoctor bool + path := getAsciidoctorExecPath() + if path == "" { + path = getAsciidocExecPath() + if path == "" { + a.cfg.Logger.ERROR.Println("asciidoctor / asciidoc not found in $PATH: Please install.\n", + " Leaving AsciiDoc content unrendered.") + return src + } + } else { + isAsciidoctor = true + } + + a.cfg.Logger.INFO.Println("Rendering", ctx.DocumentName, "with", path, "...") + args := []string{"--no-header-footer", "--safe"} + if isAsciidoctor { + // asciidoctor-specific arg to show stack traces on errors + args = append(args, "--trace") + } + args = append(args, "-") + return internal.ExternallyRenderContent(a.cfg, ctx, src, path, args) +} + +func getAsciidocExecPath() string { + path, err := exec.LookPath("asciidoc") + if err != nil { + return "" + } + return path +} + +func getAsciidoctorExecPath() string { + path, err := exec.LookPath("asciidoctor") + if err != nil { + return "" + } + return path +} + +// Supports returns whether Asciidoc or Asciidoctor is installed on this computer. +func Supports() bool { + return (getAsciidoctorExecPath() != "" || + getAsciidocExecPath() != "") +} diff --git a/markup/asciidoc/convert_test.go b/markup/asciidoc/convert_test.go new file mode 100644 index 000000000..1c53f4f25 --- /dev/null +++ b/markup/asciidoc/convert_test.go @@ -0,0 +1,38 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package asciidoc + +import ( + "testing" + + "github.com/gohugoio/hugo/common/loggers" + + "github.com/gohugoio/hugo/markup/converter" + + qt "github.com/frankban/quicktest" +) + +func TestConvert(t *testing.T) { + if !Supports() { + t.Skip("asciidoc/asciidoctor not installed") + } + c := qt.New(t) + p, err := Provider.New(converter.ProviderConfig{Logger: loggers.NewErrorLogger()}) + c.Assert(err, qt.IsNil) + conv, err := p.New(converter.DocumentContext{}) + c.Assert(err, qt.IsNil) + b, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")}) + c.Assert(err, qt.IsNil) + c.Assert(string(b.Bytes()), qt.Equals, "
\n

testContent

\n
\n") +} diff --git a/markup/blackfriday/convert.go b/markup/blackfriday/convert.go new file mode 100644 index 000000000..f9d957a4e --- /dev/null +++ b/markup/blackfriday/convert.go @@ -0,0 +1,224 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package blackfriday converts Markdown to HTML using Blackfriday v1. +package blackfriday + +import ( + "github.com/gohugoio/hugo/markup/converter" + "github.com/gohugoio/hugo/markup/internal" + "github.com/russross/blackfriday" +) + +// Provider is the package entry point. +var Provider converter.NewProvider = provider{} + +type provider struct { +} + +func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) { + defaultBlackFriday, err := internal.NewBlackfriday(cfg) + if err != nil { + return nil, err + } + + defaultExtensions := getMarkdownExtensions(defaultBlackFriday) + + pygmentsCodeFences := cfg.Cfg.GetBool("pygmentsCodeFences") + pygmentsCodeFencesGuessSyntax := cfg.Cfg.GetBool("pygmentsCodeFencesGuessSyntax") + pygmentsOptions := cfg.Cfg.GetString("pygmentsOptions") + + var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) { + b := defaultBlackFriday + extensions := defaultExtensions + + if ctx.ConfigOverrides != nil { + var err error + b, err = internal.UpdateBlackFriday(b, ctx.ConfigOverrides) + if err != nil { + return nil, err + } + extensions = getMarkdownExtensions(b) + } + + return &blackfridayConverter{ + ctx: ctx, + bf: b, + extensions: extensions, + cfg: cfg, + + pygmentsCodeFences: pygmentsCodeFences, + pygmentsCodeFencesGuessSyntax: pygmentsCodeFencesGuessSyntax, + pygmentsOptions: pygmentsOptions, + }, nil + } + + return n, nil + +} + +type blackfridayConverter struct { + ctx converter.DocumentContext + bf *internal.BlackFriday + extensions int + + pygmentsCodeFences bool + pygmentsCodeFencesGuessSyntax bool + pygmentsOptions string + + cfg converter.ProviderConfig +} + +func (c *blackfridayConverter) AnchorSuffix() string { + if c.bf.PlainIDAnchors { + return "" + } + return ":" + c.ctx.DocumentID +} + +func (c *blackfridayConverter) Convert(ctx converter.RenderContext) (converter.Result, error) { + r := c.getHTMLRenderer(ctx.RenderTOC) + + return converter.Bytes(blackfriday.Markdown(ctx.Src, r, c.extensions)), nil + +} + +func (c *blackfridayConverter) getHTMLRenderer(renderTOC bool) blackfriday.Renderer { + flags := getFlags(renderTOC, c.bf) + + documentID := c.ctx.DocumentID + + renderParameters := blackfriday.HtmlRendererParameters{ + FootnoteAnchorPrefix: c.bf.FootnoteAnchorPrefix, + FootnoteReturnLinkContents: c.bf.FootnoteReturnLinkContents, + } + + if documentID != "" && !c.bf.PlainIDAnchors { + renderParameters.FootnoteAnchorPrefix = documentID + ":" + renderParameters.FootnoteAnchorPrefix + renderParameters.HeaderIDSuffix = ":" + documentID + } + + return &hugoHTMLRenderer{ + c: c, + Renderer: blackfriday.HtmlRendererWithParameters(flags, "", "", renderParameters), + } +} + +func getFlags(renderTOC bool, cfg *internal.BlackFriday) int { + + var flags int + + if renderTOC { + flags = blackfriday.HTML_TOC + } + + flags |= blackfriday.HTML_USE_XHTML + flags |= blackfriday.HTML_FOOTNOTE_RETURN_LINKS + + if cfg.Smartypants { + flags |= blackfriday.HTML_USE_SMARTYPANTS + } + + if cfg.SmartypantsQuotesNBSP { + flags |= blackfriday.HTML_SMARTYPANTS_QUOTES_NBSP + } + + if cfg.AngledQuotes { + flags |= blackfriday.HTML_SMARTYPANTS_ANGLED_QUOTES + } + + if cfg.Fractions { + flags |= blackfriday.HTML_SMARTYPANTS_FRACTIONS + } + + if cfg.HrefTargetBlank { + flags |= blackfriday.HTML_HREF_TARGET_BLANK + } + + if cfg.NofollowLinks { + flags |= blackfriday.HTML_NOFOLLOW_LINKS + } + + if cfg.NoreferrerLinks { + flags |= blackfriday.HTML_NOREFERRER_LINKS + } + + if cfg.SmartDashes { + flags |= blackfriday.HTML_SMARTYPANTS_DASHES + } + + if cfg.LatexDashes { + flags |= blackfriday.HTML_SMARTYPANTS_LATEX_DASHES + } + + if cfg.SkipHTML { + flags |= blackfriday.HTML_SKIP_HTML + } + + return flags +} + +func getMarkdownExtensions(cfg *internal.BlackFriday) int { + // Default Blackfriday common extensions + commonExtensions := 0 | + blackfriday.EXTENSION_NO_INTRA_EMPHASIS | + blackfriday.EXTENSION_TABLES | + blackfriday.EXTENSION_FENCED_CODE | + blackfriday.EXTENSION_AUTOLINK | + blackfriday.EXTENSION_STRIKETHROUGH | + blackfriday.EXTENSION_SPACE_HEADERS | + blackfriday.EXTENSION_HEADER_IDS | + blackfriday.EXTENSION_BACKSLASH_LINE_BREAK | + blackfriday.EXTENSION_DEFINITION_LISTS + + // Extra Blackfriday extensions that Hugo enables by default + flags := commonExtensions | + blackfriday.EXTENSION_AUTO_HEADER_IDS | + blackfriday.EXTENSION_FOOTNOTES + + for _, extension := range cfg.Extensions { + if flag, ok := blackfridayExtensionMap[extension]; ok { + flags |= flag + } + } + for _, extension := range cfg.ExtensionsMask { + if flag, ok := blackfridayExtensionMap[extension]; ok { + flags &= ^flag + } + } + return flags +} + +var blackfridayExtensionMap = map[string]int{ + "noIntraEmphasis": blackfriday.EXTENSION_NO_INTRA_EMPHASIS, + "tables": blackfriday.EXTENSION_TABLES, + "fencedCode": blackfriday.EXTENSION_FENCED_CODE, + "autolink": blackfriday.EXTENSION_AUTOLINK, + "strikethrough": blackfriday.EXTENSION_STRIKETHROUGH, + "laxHtmlBlocks": blackfriday.EXTENSION_LAX_HTML_BLOCKS, + "spaceHeaders": blackfriday.EXTENSION_SPACE_HEADERS, + "hardLineBreak": blackfriday.EXTENSION_HARD_LINE_BREAK, + "tabSizeEight": blackfriday.EXTENSION_TAB_SIZE_EIGHT, + "footnotes": blackfriday.EXTENSION_FOOTNOTES, + "noEmptyLineBeforeBlock": blackfriday.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK, + "headerIds": blackfriday.EXTENSION_HEADER_IDS, + "titleblock": blackfriday.EXTENSION_TITLEBLOCK, + "autoHeaderIds": blackfriday.EXTENSION_AUTO_HEADER_IDS, + "backslashLineBreak": blackfriday.EXTENSION_BACKSLASH_LINE_BREAK, + "definitionLists": blackfriday.EXTENSION_DEFINITION_LISTS, + "joinLines": blackfriday.EXTENSION_JOIN_LINES, +} + +var ( + _ converter.DocumentInfo = (*blackfridayConverter)(nil) +) diff --git a/markup/blackfriday/convert_test.go b/markup/blackfriday/convert_test.go new file mode 100644 index 000000000..094edf35f --- /dev/null +++ b/markup/blackfriday/convert_test.go @@ -0,0 +1,194 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package blackfriday + +import ( + "testing" + + "github.com/spf13/viper" + + "github.com/gohugoio/hugo/markup/internal" + + "github.com/gohugoio/hugo/markup/converter" + + qt "github.com/frankban/quicktest" + "github.com/russross/blackfriday" +) + +func TestGetMarkdownExtensionsMasksAreRemovedFromExtensions(t *testing.T) { + c := qt.New(t) + b, err := internal.NewBlackfriday(converter.ProviderConfig{Cfg: viper.New()}) + c.Assert(err, qt.IsNil) + + b.Extensions = []string{"headerId"} + b.ExtensionsMask = []string{"noIntraEmphasis"} + + actualFlags := getMarkdownExtensions(b) + if actualFlags&blackfriday.EXTENSION_NO_INTRA_EMPHASIS == blackfriday.EXTENSION_NO_INTRA_EMPHASIS { + t.Errorf("Masked out flag {%v} found amongst returned extensions.", blackfriday.EXTENSION_NO_INTRA_EMPHASIS) + } +} + +func TestGetMarkdownExtensionsByDefaultAllExtensionsAreEnabled(t *testing.T) { + type data struct { + testFlag int + } + + c := qt.New(t) + b, err := internal.NewBlackfriday(converter.ProviderConfig{Cfg: viper.New()}) + c.Assert(err, qt.IsNil) + + b.Extensions = []string{""} + b.ExtensionsMask = []string{""} + allExtensions := []data{ + {blackfriday.EXTENSION_NO_INTRA_EMPHASIS}, + {blackfriday.EXTENSION_TABLES}, + {blackfriday.EXTENSION_FENCED_CODE}, + {blackfriday.EXTENSION_AUTOLINK}, + {blackfriday.EXTENSION_STRIKETHROUGH}, + // {blackfriday.EXTENSION_LAX_HTML_BLOCKS}, + {blackfriday.EXTENSION_SPACE_HEADERS}, + // {blackfriday.EXTENSION_HARD_LINE_BREAK}, + // {blackfriday.EXTENSION_TAB_SIZE_EIGHT}, + {blackfriday.EXTENSION_FOOTNOTES}, + // {blackfriday.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK}, + {blackfriday.EXTENSION_HEADER_IDS}, + // {blackfriday.EXTENSION_TITLEBLOCK}, + {blackfriday.EXTENSION_AUTO_HEADER_IDS}, + {blackfriday.EXTENSION_BACKSLASH_LINE_BREAK}, + {blackfriday.EXTENSION_DEFINITION_LISTS}, + } + + actualFlags := getMarkdownExtensions(b) + for _, e := range allExtensions { + if actualFlags&e.testFlag != e.testFlag { + t.Errorf("Flag %v was not found in the list of extensions.", e) + } + } +} + +func TestGetMarkdownExtensionsAddingFlagsThroughRenderingContext(t *testing.T) { + c := qt.New(t) + b, err := internal.NewBlackfriday(converter.ProviderConfig{Cfg: viper.New()}) + c.Assert(err, qt.IsNil) + + b.Extensions = []string{"definitionLists"} + b.ExtensionsMask = []string{""} + + actualFlags := getMarkdownExtensions(b) + if actualFlags&blackfriday.EXTENSION_DEFINITION_LISTS != blackfriday.EXTENSION_DEFINITION_LISTS { + t.Errorf("Masked out flag {%v} found amongst returned extensions.", blackfriday.EXTENSION_DEFINITION_LISTS) + } +} + +func TestGetFlags(t *testing.T) { + c := qt.New(t) + cfg := converter.ProviderConfig{Cfg: viper.New()} + b, err := internal.NewBlackfriday(cfg) + c.Assert(err, qt.IsNil) + flags := getFlags(false, b) + if flags&blackfriday.HTML_USE_XHTML != blackfriday.HTML_USE_XHTML { + t.Errorf("Test flag: %d was not found amongs set flags:%d; Result: %d", blackfriday.HTML_USE_XHTML, flags, flags&blackfriday.HTML_USE_XHTML) + } +} + +func TestGetAllFlags(t *testing.T) { + c := qt.New(t) + cfg := converter.ProviderConfig{Cfg: viper.New()} + b, err := internal.NewBlackfriday(cfg) + c.Assert(err, qt.IsNil) + + type data struct { + testFlag int + } + + allFlags := []data{ + {blackfriday.HTML_USE_XHTML}, + {blackfriday.HTML_FOOTNOTE_RETURN_LINKS}, + {blackfriday.HTML_USE_SMARTYPANTS}, + {blackfriday.HTML_SMARTYPANTS_QUOTES_NBSP}, + {blackfriday.HTML_SMARTYPANTS_ANGLED_QUOTES}, + {blackfriday.HTML_SMARTYPANTS_FRACTIONS}, + {blackfriday.HTML_HREF_TARGET_BLANK}, + {blackfriday.HTML_NOFOLLOW_LINKS}, + {blackfriday.HTML_NOREFERRER_LINKS}, + {blackfriday.HTML_SMARTYPANTS_DASHES}, + {blackfriday.HTML_SMARTYPANTS_LATEX_DASHES}, + } + + b.AngledQuotes = true + b.Fractions = true + b.HrefTargetBlank = true + b.NofollowLinks = true + b.NoreferrerLinks = true + b.LatexDashes = true + b.PlainIDAnchors = true + b.SmartDashes = true + b.Smartypants = true + b.SmartypantsQuotesNBSP = true + + actualFlags := getFlags(false, b) + + var expectedFlags int + //OR-ing flags together... + for _, d := range allFlags { + expectedFlags |= d.testFlag + } + if expectedFlags != actualFlags { + t.Errorf("Expected flags (%d) did not equal actual (%d) flags.", expectedFlags, actualFlags) + } +} + +func TestConvert(t *testing.T) { + c := qt.New(t) + p, err := Provider.New(converter.ProviderConfig{ + Cfg: viper.New(), + }) + c.Assert(err, qt.IsNil) + conv, err := p.New(converter.DocumentContext{}) + c.Assert(err, qt.IsNil) + b, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")}) + c.Assert(err, qt.IsNil) + c.Assert(string(b.Bytes()), qt.Equals, "

testContent

\n") +} + +func TestGetHTMLRendererAnchors(t *testing.T) { + c := qt.New(t) + p, err := Provider.New(converter.ProviderConfig{ + Cfg: viper.New(), + }) + c.Assert(err, qt.IsNil) + conv, err := p.New(converter.DocumentContext{ + DocumentID: "testid", + ConfigOverrides: map[string]interface{}{ + "plainIDAnchors": false, + "footnotes": true, + }, + }) + c.Assert(err, qt.IsNil) + b, err := conv.Convert(converter.RenderContext{Src: []byte(`# Header + +This is a footnote.[^1] And then some. + + +[^1]: Footnote text. + +`)}) + + c.Assert(err, qt.IsNil) + s := string(b.Bytes()) + c.Assert(s, qt.Contains, "

Header

") + c.Assert(s, qt.Contains, "This is a footnote.1") + c.Assert(s, qt.Contains, "[return]") +} diff --git a/markup/blackfriday/renderer.go b/markup/blackfriday/renderer.go new file mode 100644 index 000000000..9f4d44e02 --- /dev/null +++ b/markup/blackfriday/renderer.go @@ -0,0 +1,85 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package blackfriday + +import ( + "bytes" + "strings" + + "github.com/russross/blackfriday" +) + +// hugoHTMLRenderer wraps a blackfriday.Renderer, typically a blackfriday.Html +// adding some custom behaviour. +type hugoHTMLRenderer struct { + c *blackfridayConverter + blackfriday.Renderer +} + +// BlockCode renders a given text as a block of code. +// Pygments is used if it is setup to handle code fences. +func (r *hugoHTMLRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string) { + if r.c.pygmentsCodeFences && (lang != "" || r.c.pygmentsCodeFencesGuessSyntax) { + opts := r.c.pygmentsOptions + str := strings.Trim(string(text), "\n\r") + highlighted, _ := r.c.cfg.Highlight(str, lang, opts) + out.WriteString(highlighted) + } else { + r.Renderer.BlockCode(out, text, lang) + } +} + +// ListItem adds task list support to the Blackfriday renderer. +func (r *hugoHTMLRenderer) ListItem(out *bytes.Buffer, text []byte, flags int) { + if !r.c.bf.TaskLists { + r.Renderer.ListItem(out, text, flags) + return + } + + switch { + case bytes.HasPrefix(text, []byte("[ ] ")): + text = append([]byte(``)...) + + case bytes.HasPrefix(text, []byte("[x] ")) || bytes.HasPrefix(text, []byte("[X] ")): + text = append([]byte(``)...) + } + + r.Renderer.ListItem(out, text, flags) +} + +// List adds task list support to the Blackfriday renderer. +func (r *hugoHTMLRenderer) List(out *bytes.Buffer, text func() bool, flags int) { + if !r.c.bf.TaskLists { + r.Renderer.List(out, text, flags) + return + } + marker := out.Len() + r.Renderer.List(out, text, flags) + if out.Len() > marker { + list := out.Bytes()[marker:] + if bytes.Contains(list, []byte("task-list-item")) { + // Find the index of the first >, it might be 3 or 4 depending on whether + // there is a new line at the start, but this is safer than just hardcoding it. + closingBracketIndex := bytes.Index(list, []byte(">")) + // Rewrite the buffer from the marker + out.Truncate(marker) + // Safely assuming closingBracketIndex won't be -1 since there is a list + // May be either dl, ul or ol + list := append(list[:closingBracketIndex], append([]byte(` class="task-list"`), list[closingBracketIndex:]...)...) + out.Write(list) + } + } +} diff --git a/markup/converter/converter.go b/markup/converter/converter.go new file mode 100644 index 000000000..809efca8e --- /dev/null +++ b/markup/converter/converter.go @@ -0,0 +1,83 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package converter + +import ( + "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/config" + "github.com/spf13/afero" +) + +// ProviderConfig configures a new Provider. +type ProviderConfig struct { + Cfg config.Provider // Site config + ContentFs afero.Fs + Logger *loggers.Logger + Highlight func(code, lang, optsStr string) (string, error) +} + +// NewProvider creates converter providers. +type NewProvider interface { + New(cfg ProviderConfig) (Provider, error) +} + +// Provider creates converters. +type Provider interface { + New(ctx DocumentContext) (Converter, error) +} + +// NewConverter is an adapter that can be used as a ConverterProvider. +type NewConverter func(ctx DocumentContext) (Converter, error) + +// New creates a new Converter for the given ctx. +func (n NewConverter) New(ctx DocumentContext) (Converter, error) { + return n(ctx) +} + +// Converter wraps the Convert method that converts some markup into +// another format, e.g. Markdown to HTML. +type Converter interface { + Convert(ctx RenderContext) (Result, error) +} + +// Result represents the minimum returned from Convert. +type Result interface { + Bytes() []byte +} + +// DocumentInfo holds additional information provided by some converters. +type DocumentInfo interface { + AnchorSuffix() string +} + +// Bytes holds a byte slice and implements the Result interface. +type Bytes []byte + +// Bytes returns itself +func (b Bytes) Bytes() []byte { + return b +} + +// DocumentContext holds contextual information about the document to convert. +type DocumentContext struct { + DocumentID string + DocumentName string + ConfigOverrides map[string]interface{} +} + +// RenderContext holds contextual information about the content to render. +type RenderContext struct { + Src []byte + RenderTOC bool +} diff --git a/markup/internal/blackfriday.go b/markup/internal/blackfriday.go new file mode 100644 index 000000000..373df0c50 --- /dev/null +++ b/markup/internal/blackfriday.go @@ -0,0 +1,108 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package helpers implements general utility functions that work with +// and on content. The helper functions defined here lay down the +// foundation of how Hugo works with files and filepaths, and perform +// string operations on content. + +package internal + +import ( + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/markup/converter" + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" +) + +// BlackFriday holds configuration values for BlackFriday rendering. +// It is kept here because it's used in several packages. +type BlackFriday struct { + Smartypants bool + SmartypantsQuotesNBSP bool + AngledQuotes bool + Fractions bool + HrefTargetBlank bool + NofollowLinks bool + NoreferrerLinks bool + SmartDashes bool + LatexDashes bool + TaskLists bool + PlainIDAnchors bool + Extensions []string + ExtensionsMask []string + SkipHTML bool + + FootnoteAnchorPrefix string + FootnoteReturnLinkContents string +} + +func UpdateBlackFriday(old *BlackFriday, m map[string]interface{}) (*BlackFriday, error) { + // Create a copy so we can modify it. + bf := *old + if err := mapstructure.Decode(m, &bf); err != nil { + return nil, errors.WithMessage(err, "failed to decode rendering config") + } + return &bf, nil +} + +// NewBlackfriday creates a new Blackfriday filled with site config or some sane defaults. +func NewBlackfriday(cfg converter.ProviderConfig) (*BlackFriday, error) { + var siteConfig map[string]interface{} + if cfg.Cfg != nil { + siteConfig = cfg.Cfg.GetStringMap("blackfriday") + } + + defaultParam := map[string]interface{}{ + "smartypants": true, + "angledQuotes": false, + "smartypantsQuotesNBSP": false, + "fractions": true, + "hrefTargetBlank": false, + "nofollowLinks": false, + "noreferrerLinks": false, + "smartDashes": true, + "latexDashes": true, + "plainIDAnchors": true, + "taskLists": true, + "skipHTML": false, + } + + maps.ToLower(defaultParam) + + config := make(map[string]interface{}) + + for k, v := range defaultParam { + config[k] = v + } + + for k, v := range siteConfig { + config[k] = v + } + + combinedConfig := &BlackFriday{} + if err := mapstructure.Decode(config, combinedConfig); err != nil { + return nil, errors.Errorf("failed to decode Blackfriday config: %s", err) + } + + // TODO(bep) update/consolidate docs + if combinedConfig.FootnoteAnchorPrefix == "" { + combinedConfig.FootnoteAnchorPrefix = cfg.Cfg.GetString("footnoteAnchorPrefix") + } + + if combinedConfig.FootnoteReturnLinkContents == "" { + combinedConfig.FootnoteReturnLinkContents = cfg.Cfg.GetString("footnoteReturnLinkContents") + } + + return combinedConfig, nil +} diff --git a/markup/internal/external.go b/markup/internal/external.go new file mode 100644 index 000000000..2105e7cff --- /dev/null +++ b/markup/internal/external.go @@ -0,0 +1,52 @@ +package internal + +import ( + "bytes" + "os/exec" + "strings" + + "github.com/gohugoio/hugo/markup/converter" +) + +func ExternallyRenderContent( + cfg converter.ProviderConfig, + ctx converter.DocumentContext, + content []byte, path string, args []string) []byte { + + logger := cfg.Logger + cmd := exec.Command(path, args...) + cmd.Stdin = bytes.NewReader(content) + var out, cmderr bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &cmderr + err := cmd.Run() + // Most external helpers exit w/ non-zero exit code only if severe, i.e. + // halting errors occurred. -> log stderr output regardless of state of err + for _, item := range strings.Split(cmderr.String(), "\n") { + item := strings.TrimSpace(item) + if item != "" { + logger.ERROR.Printf("%s: %s", ctx.DocumentName, item) + } + } + if err != nil { + logger.ERROR.Printf("%s rendering %s: %v", path, ctx.DocumentName, err) + } + + return normalizeExternalHelperLineFeeds(out.Bytes()) +} + +// Strips carriage returns from third-party / external processes (useful for Windows) +func normalizeExternalHelperLineFeeds(content []byte) []byte { + return bytes.Replace(content, []byte("\r"), []byte(""), -1) +} + +func GetPythonExecPath() string { + path, err := exec.LookPath("python") + if err != nil { + path, err = exec.LookPath("python.exe") + if err != nil { + return "" + } + } + return path +} diff --git a/markup/markup.go b/markup/markup.go new file mode 100644 index 000000000..54193aba3 --- /dev/null +++ b/markup/markup.go @@ -0,0 +1,83 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package markup + +import ( + "strings" + + "github.com/gohugoio/hugo/markup/org" + + "github.com/gohugoio/hugo/markup/asciidoc" + "github.com/gohugoio/hugo/markup/blackfriday" + "github.com/gohugoio/hugo/markup/converter" + "github.com/gohugoio/hugo/markup/mmark" + "github.com/gohugoio/hugo/markup/pandoc" + "github.com/gohugoio/hugo/markup/rst" +) + +func NewConverterProvider(cfg converter.ProviderConfig) (ConverterProvider, error) { + converters := make(map[string]converter.Provider) + + add := func(p converter.NewProvider, aliases ...string) error { + c, err := p.New(cfg) + if err != nil { + return err + } + addConverter(converters, c, aliases...) + return nil + } + + if err := add(blackfriday.Provider, "md", "markdown", "blackfriday"); err != nil { + return nil, err + } + if err := add(mmark.Provider, "mmark"); err != nil { + return nil, err + } + if err := add(asciidoc.Provider, "asciidoc"); err != nil { + return nil, err + } + if err := add(rst.Provider, "rst"); err != nil { + return nil, err + } + if err := add(pandoc.Provider, "pandoc"); err != nil { + return nil, err + } + if err := add(org.Provider, "org"); err != nil { + return nil, err + } + + return &converterRegistry{converters: converters}, nil +} + +type ConverterProvider interface { + Get(name string) converter.Provider +} + +type converterRegistry struct { + // Maps name (md, markdown, blackfriday etc.) to a converter provider. + // Note that this is also used for aliasing, so the same converter + // may be registered multiple times. + // All names are lower case. + converters map[string]converter.Provider +} + +func (r *converterRegistry) Get(name string) converter.Provider { + return r.converters[strings.ToLower(name)] +} + +func addConverter(m map[string]converter.Provider, c converter.Provider, aliases ...string) { + for _, alias := range aliases { + m[alias] = c + } +} diff --git a/markup/markup_test.go b/markup/markup_test.go new file mode 100644 index 000000000..c4c1ee032 --- /dev/null +++ b/markup/markup_test.go @@ -0,0 +1,41 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package markup + +import ( + "testing" + + "github.com/spf13/viper" + + "github.com/gohugoio/hugo/markup/converter" + + qt "github.com/frankban/quicktest" +) + +func TestConverterRegistry(t *testing.T) { + c := qt.New(t) + + r, err := NewConverterProvider(converter.ProviderConfig{Cfg: viper.New()}) + + c.Assert(err, qt.IsNil) + + c.Assert(r.Get("foo"), qt.IsNil) + c.Assert(r.Get("markdown"), qt.Not(qt.IsNil)) + c.Assert(r.Get("mmark"), qt.Not(qt.IsNil)) + c.Assert(r.Get("asciidoc"), qt.Not(qt.IsNil)) + c.Assert(r.Get("rst"), qt.Not(qt.IsNil)) + c.Assert(r.Get("pandoc"), qt.Not(qt.IsNil)) + c.Assert(r.Get("org"), qt.Not(qt.IsNil)) + +} diff --git a/markup/mmark/convert.go b/markup/mmark/convert.go new file mode 100644 index 000000000..a0da346c1 --- /dev/null +++ b/markup/mmark/convert.go @@ -0,0 +1,143 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package mmark converts Markdown to HTML using MMark v1. +package mmark + +import ( + "github.com/gohugoio/hugo/markup/internal" + + "github.com/gohugoio/hugo/markup/converter" + "github.com/miekg/mmark" +) + +// Provider is the package entry point. +var Provider converter.NewProvider = provider{} + +type provider struct { +} + +func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) { + defaultBlackFriday, err := internal.NewBlackfriday(cfg) + if err != nil { + return nil, err + } + + defaultExtensions := getMmarkExtensions(defaultBlackFriday) + + var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) { + b := defaultBlackFriday + extensions := defaultExtensions + + if ctx.ConfigOverrides != nil { + var err error + b, err = internal.UpdateBlackFriday(b, ctx.ConfigOverrides) + if err != nil { + return nil, err + } + extensions = getMmarkExtensions(b) + } + + return &mmarkConverter{ + ctx: ctx, + b: b, + extensions: extensions, + cfg: cfg, + }, nil + } + + return n, nil + +} + +type mmarkConverter struct { + ctx converter.DocumentContext + extensions int + b *internal.BlackFriday + cfg converter.ProviderConfig +} + +func (c *mmarkConverter) Convert(ctx converter.RenderContext) (converter.Result, error) { + r := getHTMLRenderer(c.ctx, c.b, c.cfg) + return mmark.Parse(ctx.Src, r, c.extensions), nil +} + +func getHTMLRenderer( + ctx converter.DocumentContext, + cfg *internal.BlackFriday, + pcfg converter.ProviderConfig) mmark.Renderer { + + var ( + flags int + documentID string + ) + + documentID = ctx.DocumentID + + renderParameters := mmark.HtmlRendererParameters{ + FootnoteAnchorPrefix: cfg.FootnoteAnchorPrefix, + FootnoteReturnLinkContents: cfg.FootnoteReturnLinkContents, + } + + if documentID != "" && !cfg.PlainIDAnchors { + renderParameters.FootnoteAnchorPrefix = documentID + ":" + renderParameters.FootnoteAnchorPrefix + } + + htmlFlags := flags + htmlFlags |= mmark.HTML_FOOTNOTE_RETURN_LINKS + + return &mmarkRenderer{ + Config: cfg, + Cfg: pcfg.Cfg, + highlight: pcfg.Highlight, + Renderer: mmark.HtmlRendererWithParameters(htmlFlags, "", "", renderParameters), + } + +} + +func getMmarkExtensions(cfg *internal.BlackFriday) int { + flags := 0 + flags |= mmark.EXTENSION_TABLES + flags |= mmark.EXTENSION_FENCED_CODE + flags |= mmark.EXTENSION_AUTOLINK + flags |= mmark.EXTENSION_SPACE_HEADERS + flags |= mmark.EXTENSION_CITATION + flags |= mmark.EXTENSION_TITLEBLOCK_TOML + flags |= mmark.EXTENSION_HEADER_IDS + flags |= mmark.EXTENSION_AUTO_HEADER_IDS + flags |= mmark.EXTENSION_UNIQUE_HEADER_IDS + flags |= mmark.EXTENSION_FOOTNOTES + flags |= mmark.EXTENSION_SHORT_REF + flags |= mmark.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK + flags |= mmark.EXTENSION_INCLUDE + + for _, extension := range cfg.Extensions { + if flag, ok := mmarkExtensionMap[extension]; ok { + flags |= flag + } + } + return flags +} + +var mmarkExtensionMap = map[string]int{ + "tables": mmark.EXTENSION_TABLES, + "fencedCode": mmark.EXTENSION_FENCED_CODE, + "autolink": mmark.EXTENSION_AUTOLINK, + "laxHtmlBlocks": mmark.EXTENSION_LAX_HTML_BLOCKS, + "spaceHeaders": mmark.EXTENSION_SPACE_HEADERS, + "hardLineBreak": mmark.EXTENSION_HARD_LINE_BREAK, + "footnotes": mmark.EXTENSION_FOOTNOTES, + "noEmptyLineBeforeBlock": mmark.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK, + "headerIds": mmark.EXTENSION_HEADER_IDS, + "autoHeaderIds": mmark.EXTENSION_AUTO_HEADER_IDS, +} diff --git a/markup/mmark/convert_test.go b/markup/mmark/convert_test.go new file mode 100644 index 000000000..d015ee94c --- /dev/null +++ b/markup/mmark/convert_test.go @@ -0,0 +1,77 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mmark + +import ( + "testing" + + "github.com/spf13/viper" + + "github.com/gohugoio/hugo/common/loggers" + + "github.com/miekg/mmark" + + "github.com/gohugoio/hugo/markup/internal" + + "github.com/gohugoio/hugo/markup/converter" + + qt "github.com/frankban/quicktest" +) + +func TestGetMmarkExtensions(t *testing.T) { + c := qt.New(t) + b, err := internal.NewBlackfriday(converter.ProviderConfig{Cfg: viper.New()}) + c.Assert(err, qt.IsNil) + + //TODO: This is doing the same just with different marks... + type data struct { + testFlag int + } + + b.Extensions = []string{"tables"} + b.ExtensionsMask = []string{""} + allExtensions := []data{ + {mmark.EXTENSION_TABLES}, + {mmark.EXTENSION_FENCED_CODE}, + {mmark.EXTENSION_AUTOLINK}, + {mmark.EXTENSION_SPACE_HEADERS}, + {mmark.EXTENSION_CITATION}, + {mmark.EXTENSION_TITLEBLOCK_TOML}, + {mmark.EXTENSION_HEADER_IDS}, + {mmark.EXTENSION_AUTO_HEADER_IDS}, + {mmark.EXTENSION_UNIQUE_HEADER_IDS}, + {mmark.EXTENSION_FOOTNOTES}, + {mmark.EXTENSION_SHORT_REF}, + {mmark.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK}, + {mmark.EXTENSION_INCLUDE}, + } + + actualFlags := getMmarkExtensions(b) + for _, e := range allExtensions { + if actualFlags&e.testFlag != e.testFlag { + t.Errorf("Flag %v was not found in the list of extensions.", e) + } + } +} + +func TestConvert(t *testing.T) { + c := qt.New(t) + p, err := Provider.New(converter.ProviderConfig{Cfg: viper.New(), Logger: loggers.NewErrorLogger()}) + c.Assert(err, qt.IsNil) + conv, err := p.New(converter.DocumentContext{}) + c.Assert(err, qt.IsNil) + b, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")}) + c.Assert(err, qt.IsNil) + c.Assert(string(b.Bytes()), qt.Equals, "

testContent

\n") +} diff --git a/markup/mmark/renderer.go b/markup/mmark/renderer.go new file mode 100644 index 000000000..07fe71c95 --- /dev/null +++ b/markup/mmark/renderer.go @@ -0,0 +1,44 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mmark + +import ( + "bytes" + "strings" + + "github.com/miekg/mmark" + + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/markup/internal" +) + +// hugoHTMLRenderer wraps a blackfriday.Renderer, typically a blackfriday.Html +// adding some custom behaviour. +type mmarkRenderer struct { + Cfg config.Provider + Config *internal.BlackFriday + highlight func(code, lang, optsStr string) (string, error) + mmark.Renderer +} + +// BlockCode renders a given text as a block of code. +func (r *mmarkRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string, caption []byte, subfigure bool, callouts bool) { + if r.Cfg.GetBool("pygmentsCodeFences") && (lang != "" || r.Cfg.GetBool("pygmentsCodeFencesGuessSyntax")) { + str := strings.Trim(string(text), "\n\r") + highlighted, _ := r.highlight(str, lang, "") + out.WriteString(highlighted) + } else { + r.Renderer.BlockCode(out, text, lang, caption, subfigure, callouts) + } +} diff --git a/markup/org/convert.go b/markup/org/convert.go new file mode 100644 index 000000000..a951e6fe1 --- /dev/null +++ b/markup/org/convert.go @@ -0,0 +1,69 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package org converts Emacs Org-Mode to HTML. +package org + +import ( + "bytes" + + "github.com/gohugoio/hugo/markup/converter" + "github.com/niklasfasching/go-org/org" + "github.com/spf13/afero" +) + +// Provider is the package entry point. +var Provider converter.NewProvider = provide{} + +type provide struct { +} + +func (p provide) New(cfg converter.ProviderConfig) (converter.Provider, error) { + var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) { + return &orgConverter{ + ctx: ctx, + cfg: cfg, + }, nil + } + return n, nil +} + +type orgConverter struct { + ctx converter.DocumentContext + cfg converter.ProviderConfig +} + +func (c *orgConverter) Convert(ctx converter.RenderContext) (converter.Result, error) { + logger := c.cfg.Logger + config := org.New() + config.Log = logger.WARN + config.ReadFile = func(filename string) ([]byte, error) { + return afero.ReadFile(c.cfg.ContentFs, filename) + } + writer := org.NewHTMLWriter() + writer.HighlightCodeBlock = func(source, lang string) string { + highlightedSource, err := c.cfg.Highlight(source, lang, "") + if err != nil { + logger.ERROR.Printf("Could not highlight source as lang %s. Using raw source.", lang) + return source + } + return highlightedSource + } + + html, err := config.Parse(bytes.NewReader(ctx.Src), c.ctx.DocumentName).Write(writer) + if err != nil { + logger.ERROR.Printf("Could not render org: %s. Using unrendered content.", err) + return converter.Bytes(ctx.Src), nil + } + return converter.Bytes([]byte(html)), nil +} diff --git a/markup/org/convert_test.go b/markup/org/convert_test.go new file mode 100644 index 000000000..94fcdf836 --- /dev/null +++ b/markup/org/convert_test.go @@ -0,0 +1,35 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org + +import ( + "testing" + + "github.com/gohugoio/hugo/common/loggers" + + "github.com/gohugoio/hugo/markup/converter" + + qt "github.com/frankban/quicktest" +) + +func TestConvert(t *testing.T) { + c := qt.New(t) + p, err := Provider.New(converter.ProviderConfig{Logger: loggers.NewErrorLogger()}) + c.Assert(err, qt.IsNil) + conv, err := p.New(converter.DocumentContext{}) + c.Assert(err, qt.IsNil) + b, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")}) + c.Assert(err, qt.IsNil) + c.Assert(string(b.Bytes()), qt.Equals, "

\ntestContent\n

\n") +} diff --git a/markup/pandoc/convert.go b/markup/pandoc/convert.go new file mode 100644 index 000000000..4deab0b46 --- /dev/null +++ b/markup/pandoc/convert.go @@ -0,0 +1,76 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package pandoc converts content to HTML using Pandoc as an external helper. +package pandoc + +import ( + "os/exec" + + "github.com/gohugoio/hugo/markup/internal" + + "github.com/gohugoio/hugo/markup/converter" +) + +// Provider is the package entry point. +var Provider converter.NewProvider = provider{} + +type provider struct { +} + +func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) { + var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) { + return &pandocConverter{ + ctx: ctx, + cfg: cfg, + }, nil + } + return n, nil + +} + +type pandocConverter struct { + ctx converter.DocumentContext + cfg converter.ProviderConfig +} + +func (c *pandocConverter) Convert(ctx converter.RenderContext) (converter.Result, error) { + return converter.Bytes(c.getPandocContent(ctx.Src, c.ctx)), nil +} + +// getPandocContent calls pandoc as an external helper to convert pandoc markdown to HTML. +func (c *pandocConverter) getPandocContent(src []byte, ctx converter.DocumentContext) []byte { + logger := c.cfg.Logger + path := getPandocExecPath() + if path == "" { + logger.ERROR.Println("pandoc not found in $PATH: Please install.\n", + " Leaving pandoc content unrendered.") + return src + } + args := []string{"--mathjax"} + return internal.ExternallyRenderContent(c.cfg, ctx, src, path, args) +} + +func getPandocExecPath() string { + path, err := exec.LookPath("pandoc") + if err != nil { + return "" + } + + return path +} + +// Supports returns whether Pandoc is installed on this computer. +func Supports() bool { + return getPandocExecPath() != "" +} diff --git a/markup/pandoc/convert_test.go b/markup/pandoc/convert_test.go new file mode 100644 index 000000000..bd6ca19e6 --- /dev/null +++ b/markup/pandoc/convert_test.go @@ -0,0 +1,38 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pandoc + +import ( + "testing" + + "github.com/gohugoio/hugo/common/loggers" + + "github.com/gohugoio/hugo/markup/converter" + + qt "github.com/frankban/quicktest" +) + +func TestConvert(t *testing.T) { + if !Supports() { + t.Skip("pandoc not installed") + } + c := qt.New(t) + p, err := Provider.New(converter.ProviderConfig{Logger: loggers.NewErrorLogger()}) + c.Assert(err, qt.IsNil) + conv, err := p.New(converter.DocumentContext{}) + c.Assert(err, qt.IsNil) + b, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")}) + c.Assert(err, qt.IsNil) + c.Assert(string(b.Bytes()), qt.Equals, "

testContent

\n") +} diff --git a/markup/rst/convert.go b/markup/rst/convert.go new file mode 100644 index 000000000..e12e34f6d --- /dev/null +++ b/markup/rst/convert.go @@ -0,0 +1,109 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package rst converts content to HTML using the RST external helper. +package rst + +import ( + "bytes" + "os/exec" + "runtime" + + "github.com/gohugoio/hugo/markup/internal" + + "github.com/gohugoio/hugo/markup/converter" +) + +// Provider is the package entry point. +var Provider converter.NewProvider = provider{} + +type provider struct { +} + +func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) { + var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) { + return &rstConverter{ + ctx: ctx, + cfg: cfg, + }, nil + } + return n, nil + +} + +type rstConverter struct { + ctx converter.DocumentContext + cfg converter.ProviderConfig +} + +func (c *rstConverter) Convert(ctx converter.RenderContext) (converter.Result, error) { + return converter.Bytes(c.getRstContent(ctx.Src, c.ctx)), nil +} + +// getRstContent calls the Python script rst2html as an external helper +// to convert reStructuredText content to HTML. +func (c *rstConverter) getRstContent(src []byte, ctx converter.DocumentContext) []byte { + logger := c.cfg.Logger + path := getRstExecPath() + + if path == "" { + logger.ERROR.Println("rst2html / rst2html.py not found in $PATH: Please install.\n", + " Leaving reStructuredText content unrendered.") + return src + } + logger.INFO.Println("Rendering", ctx.DocumentName, "with", path, "...") + var result []byte + // certain *nix based OSs wrap executables in scripted launchers + // invoking binaries on these OSs via python interpreter causes SyntaxError + // invoke directly so that shebangs work as expected + // handle Windows manually because it doesn't do shebangs + if runtime.GOOS == "windows" { + python := internal.GetPythonExecPath() + args := []string{path, "--leave-comments", "--initial-header-level=2"} + result = internal.ExternallyRenderContent(c.cfg, ctx, src, python, args) + } else { + args := []string{"--leave-comments", "--initial-header-level=2"} + result = internal.ExternallyRenderContent(c.cfg, ctx, src, path, args) + } + // TODO(bep) check if rst2html has a body only option. + bodyStart := bytes.Index(result, []byte("\n")) + if bodyStart < 0 { + bodyStart = -7 //compensate for length + } + + bodyEnd := bytes.Index(result, []byte("\n")) + if bodyEnd < 0 || bodyEnd >= len(result) { + bodyEnd = len(result) - 1 + if bodyEnd < 0 { + bodyEnd = 0 + } + } + + return result[bodyStart+7 : bodyEnd] +} + +func getRstExecPath() string { + path, err := exec.LookPath("rst2html") + if err != nil { + path, err = exec.LookPath("rst2html.py") + if err != nil { + return "" + } + } + return path +} + +// Supports returns whether rst is installed on this computer. +func Supports() bool { + return getRstExecPath() != "" +} diff --git a/markup/rst/convert_test.go b/markup/rst/convert_test.go new file mode 100644 index 000000000..269d92caa --- /dev/null +++ b/markup/rst/convert_test.go @@ -0,0 +1,38 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rst + +import ( + "testing" + + "github.com/gohugoio/hugo/common/loggers" + + "github.com/gohugoio/hugo/markup/converter" + + qt "github.com/frankban/quicktest" +) + +func TestConvert(t *testing.T) { + if !Supports() { + t.Skip("rst not installed") + } + c := qt.New(t) + p, err := Provider.New(converter.ProviderConfig{Logger: loggers.NewErrorLogger()}) + c.Assert(err, qt.IsNil) + conv, err := p.New(converter.DocumentContext{}) + c.Assert(err, qt.IsNil) + b, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")}) + c.Assert(err, qt.IsNil) + c.Assert(string(b.Bytes()), qt.Equals, "
\n\n\n

testContent

\n
") +} -- cgit v1.2.3