Make .RenderString render shortcodes

Fixes #6703

10 files changed, 315 insertions, 111 deletions

diff --git a/common/text/transform.go b/common/text/transform.go
index 2b05b9b4f..de093af0d 100644
--- a/common/text/transform.go
+++ b/common/text/transform.go
@@ -64,11 +64,12 @@ func Puts(s string) string {
 
 // VisitLinesAfter calls the given function for each line, including newlines, in the given string.
 func VisitLinesAfter(s string, fn func(line string)) {
-	high := strings.Index(s, "\n")
+	high := strings.IndexRune(s, '\n')
 	for high != -1 {
 		fn(s[:high+1])
 		s = s[high+1:]
-		high = strings.Index(s, "\n")
+
+		high = strings.IndexRune(s, '\n')
 	}
 
 	if s != "" {
diff --git a/common/text/transform_test.go b/common/text/transform_test.go
index 10738aee7..41447715f 100644
--- a/common/text/transform_test.go
+++ b/common/text/transform_test.go
@@ -59,3 +59,18 @@ line 3`
 	c.Assert(collected, qt.DeepEquals, []string{"line 1\n", "line 2\n", "\n", "line 3"})
 
 }
+
+func BenchmarkVisitLinesAfter(b *testing.B) {
+	const lines = `line 1
+	line 2
+	
+	line 3`
+
+	for i := 0; i < b.N; i++ {
+		VisitLinesAfter(lines, func(s string) {
+
+		})
+
+	}
+
+}
diff --git a/hugolib/content_map_page.go b/hugolib/content_map_page.go
index a16e4720d..c6522809e 100644
--- a/hugolib/content_map_page.go
+++ b/hugolib/content_map_page.go
@@ -163,7 +163,7 @@ func (m *pageMap) newPageFromContentNode(n *contentNode, parentBucket *pagesMapB
 		},
 	}
 
-	ps.shortcodeState = newShortcodeHandler(ps, ps.s, nil)
+	ps.shortcodeState = newShortcodeHandler(ps, ps.s)
 
 	if err := ps.mapContent(parentBucket, metaProvider); err != nil {
 		return nil, ps.wrapError(err)
diff --git a/hugolib/content_render_hooks_test.go b/hugolib/content_render_hooks_test.go
index 33ebe1f41..dbfd46459 100644
--- a/hugolib/content_render_hooks_test.go
+++ b/hugolib/content_render_hooks_test.go
@@ -18,7 +18,6 @@ import (
 	"testing"
 
 	qt "github.com/frankban/quicktest"
-	"github.com/gohugoio/hugo/common/loggers"
 )
 
 func TestRenderHookEditNestedPartial(t *testing.T) {
@@ -428,72 +427,3 @@ Image:
 <p>html-image: image.jpg|Text: Hello<br> Goodbye|Plain: Hello GoodbyeEND</p>
 `)
 }
-
-func TestRenderString(t *testing.T) {
-	b := newTestSitesBuilder(t)
-
-	b.WithTemplates("index.html", `
-{{ $p := site.GetPage "p1.md" }}
-{{ $optBlock := dict "display" "block" }}
-{{ $optOrg := dict "markup" "org" }}
-RSTART:{{ "**Bold Markdown**" | $p.RenderString }}:REND
-RSTART:{{  "**Bold Block Markdown**" | $p.RenderString  $optBlock }}:REND
-RSTART:{{  "/italic org mode/" | $p.RenderString  $optOrg }}:REND
-RSTART:{{ "## Header2" | $p.RenderString }}:REND
-
-
-`, "_default/_markup/render-heading.html", "Hook Heading: {{ .Level }}")
-
-	b.WithContent("p1.md", `---
-title: "p1"
----
-`,
-	)
-
-	b.Build(BuildCfg{})
-
-	b.AssertFileContent("public/index.html", `
-RSTART:<strong>Bold Markdown</strong>:REND
-RSTART:<p><strong>Bold Block Markdown</strong></p>
-RSTART:<em>italic org mode</em>:REND
-RSTART:Hook Heading: 2:REND
-`)
-}
-
-// https://github.com/gohugoio/hugo/issues/6882
-func TestRenderStringOnListPage(t *testing.T) {
-	renderStringTempl := `
-{{ .RenderString "**Hello**" }}
-`
-	b := newTestSitesBuilder(t)
-	b.WithContent("mysection/p1.md", `FOO`)
-	b.WithTemplates(
-		"index.html", renderStringTempl,
-		"_default/list.html", renderStringTempl,
-		"_default/single.html", renderStringTempl,
-	)
-
-	b.Build(BuildCfg{})
-
-	for _, filename := range []string{
-		"index.html",
-		"mysection/index.html",
-		"categories/index.html",
-		"tags/index.html",
-		"mysection/p1/index.html",
-	} {
-		b.AssertFileContent("public/"+filename, `<strong>Hello</strong>`)
-	}
-}
-
-// Issue 9433
-func TestRenderStringOnPageNotBackedByAFile(t *testing.T) {
-	t.Parallel()
-	logger := loggers.NewWarningLogger()
-	b := newTestSitesBuilder(t).WithLogger(logger).WithConfigFile("toml", `
-disableKinds = ["page", "section", "taxonomy", "term"]	
-`)
-	b.WithTemplates("index.html", `{{ .RenderString "**Hello**" }}`).WithContent("p1.md", "")
-	b.BuildE(BuildCfg{})
-	b.Assert(int(logger.LogCounters().WarnCounter.Count()), qt.Equals, 0)
-}
diff --git a/hugolib/page.go b/hugolib/page.go
index 8bf13e320..e37b47300 100644
--- a/hugolib/page.go
+++ b/hugolib/page.go
@@ -336,7 +336,7 @@ func (p *pageState) HasShortcode(name string) bool {
 		return false
 	}
 
-	return p.shortcodeState.nameSet[name]
+	return p.shortcodeState.hasName(name)
 }
 
 func (p *pageState) Site() page.Site {
@@ -610,13 +610,30 @@ func (p *pageState) getContentConverter() converter.Converter {
 }
 
 func (p *pageState) mapContent(bucket *pagesMapBucket, meta *pageMeta) error {
-	s := p.shortcodeState
-
-	rn := &pageContentMap{
+	p.cmap = &pageContentMap{
 		items: make([]any, 0, 20),
 	}
 
-	iter := p.source.parsed.Iterator()
+	return p.mapContentForResult(
+		p.source.parsed,
+		p.shortcodeState,
+		p.cmap,
+		meta.markup,
+		func(m map[string]interface{}) error {
+			return meta.setMetadata(bucket, p, m)
+		},
+	)
+}
+
+func (p *pageState) mapContentForResult(
+	result pageparser.Result,
+	s *shortcodeHandler,
+	rn *pageContentMap,
+	markup string,
+	withFrontMatter func(map[string]any) error,
+) error {
+
+	iter := result.Iterator()
 
 	fail := func(err error, i pageparser.Item) error {
 		if fe, ok := err.(herrors.FileError); ok {
@@ -660,8 +677,10 @@ Loop:
 				}
 			}
 
-			if err := meta.setMetadata(bucket, p, m); err != nil {
-				return err
+			if withFrontMatter != nil {
+				if err := withFrontMatter(m); err != nil {
+					return err
+				}
 			}
 
 			frontMatterSet = true
@@ -697,7 +716,7 @@ Loop:
 			p.source.posBodyStart = posBody
 			p.source.hasSummaryDivider = true
 
-			if meta.markup != "html" {
+			if markup != "html" {
 				// The content will be rendered by Goldmark or similar,
 				// and we need to track the summary.
 				rn.AddReplacement(internalSummaryDividerPre, it)
@@ -720,7 +739,7 @@ Loop:
 			}
 
 			if currShortcode.name != "" {
-				s.nameSet[currShortcode.name] = true
+				s.addName(currShortcode.name)
 			}
 
 			if currShortcode.params == nil {
@@ -752,16 +771,14 @@ Loop:
 		}
 	}
 
-	if !frontMatterSet {
+	if !frontMatterSet && withFrontMatter != nil {
 		// Page content without front matter. Assign default front matter from
 		// cascades etc.
-		if err := meta.setMetadata(bucket, p, nil); err != nil {
+		if err := withFrontMatter(nil); err != nil {
 			return err
 		}
 	}
 
-	p.cmap = rn
-
 	return nil
 }
 
diff --git a/hugolib/page__content.go b/hugolib/page__content.go
index f6f4564eb..587188454 100644
--- a/hugolib/page__content.go
+++ b/hugolib/page__content.go
@@ -39,12 +39,12 @@ type pageContent struct {
 }
 
 // returns the content to be processed by Goldmark or similar.
-func (p pageContent) contentToRender(renderedShortcodes map[string]string) []byte {
-	source := p.source.parsed.Input()
+func (p pageContent) contentToRender(parsed pageparser.Result, pm *pageContentMap, renderedShortcodes map[string]string) []byte {
+	source := parsed.Input()
 
 	c := make([]byte, 0, len(source)+(len(source)/10))
 
-	for _, it := range p.cmap.items {
+	for _, it := range pm.items {
 		switch v := it.(type) {
 		case pageparser.Item:
 			c = append(c, source[v.Pos:v.Pos+len(v.Val)]...)
diff --git a/hugolib/page__per_output.go b/hugolib/page__per_output.go
index 3d0c6ad1d..2bf16dd9e 100644
--- a/hugolib/page__per_output.go
+++ b/hugolib/page__per_output.go
@@ -25,9 +25,11 @@ import (
 
 	"errors"
 
+	"github.com/gohugoio/hugo/common/herrors"
 	"github.com/gohugoio/hugo/common/text"
 	"github.com/gohugoio/hugo/common/types/hstring"
 	"github.com/gohugoio/hugo/identity"
+	"github.com/gohugoio/hugo/parser/pageparser"
 	"github.com/mitchellh/mapstructure"
 	"github.com/spf13/cast"
 
@@ -117,7 +119,7 @@ func newPageContentOutput(p *pageState, po *pageOutput) (*pageContentOutput, err
 			p.pageOutputTemplateVariationsState.Store(2)
 		}
 
-		cp.workContent = p.contentToRender(cp.contentPlaceholders)
+		cp.workContent = p.contentToRender(p.source.parsed, p.cmap, cp.contentPlaceholders)
 
 		isHTML := cp.p.m.markup == "html"
 
@@ -332,11 +334,12 @@ func (p *pageContentOutput) WordCount() int {
 }
 
 func (p *pageContentOutput) RenderString(args ...any) (template.HTML, error) {
+	defer herrors.Recover()
 	if len(args) < 1 || len(args) > 2 {
 		return "", errors.New("want 1 or 2 arguments")
 	}
 
-	var s string
+	var contentToRender string
 	opts := defaultRenderStringOpts
 	sidx := 1
 
@@ -353,16 +356,16 @@ func (p *pageContentOutput) RenderString(args ...any) (template.HTML, error) {
 		}
 	}
 
-	contentToRender := args[sidx]
+	contentToRenderv := args[sidx]
 
-	if _, ok := contentToRender.(hstring.RenderedString); ok {
+	if _, ok := contentToRenderv.(hstring.RenderedString); ok {
 		// This content is already rendered, this is potentially
 		// a infinite recursion.
 		return "", errors.New("text is already rendered, repeating it may cause infinite recursion")
 	}
 
 	var err error
-	s, err = cast.ToStringE(contentToRender)
+	contentToRender, err = cast.ToStringE(contentToRenderv)
 	if err != nil {
 		return "", err
 	}
@@ -381,20 +384,79 @@ func (p *pageContentOutput) RenderString(args ...any) (template.HTML, error) {
 		}
 	}
 
-	c, err := p.renderContentWithConverter(conv, []byte(s), false)
-	if err != nil {
-		return "", p.p.wrapError(err)
-	}
+	var rendered []byte
+
+	if strings.Contains(contentToRender, "{{") {
+		// Probably a shortcode.
+		parsed, err := pageparser.ParseMain(strings.NewReader(contentToRender), pageparser.Config{})
+		if err != nil {
+			return "", err
+		}
+		pm := &pageContentMap{
+			items: make([]any, 0, 20),
+		}
+		s := newShortcodeHandler(p.p, p.p.s)
+
+		if err := p.p.mapContentForResult(
+			parsed,
+			s,
+			pm,
+			opts.Markup,
+			nil,
+		); err != nil {
+			return "", err
+		}
+
+		placeholders, hasShortcodeVariants, err := s.renderShortcodesForPage(p.p, p.f)
+		if err != nil {
+			return "", err
+		}
+
+		if hasShortcodeVariants {
+			p.p.pageOutputTemplateVariationsState.Store(2)
+		}
+
+		b, err := p.renderContentWithConverter(conv, p.p.contentToRender(parsed, pm, placeholders), false)
+		if err != nil {
+			return "", p.p.wrapError(err)
+		}
+		rendered = b.Bytes()
 
-	b := c.Bytes()
+		if p.placeholdersEnabled {
+			// ToC was accessed via .Page.TableOfContents in the shortcode,
+			// at a time when the ToC wasn't ready.
+			if _, err := p.p.Content(); err != nil {
+				return "", err
+			}
+			placeholders[tocShortcodePlaceholder] = string(p.tableOfContents)
+		}
+
+		if pm.hasNonMarkdownShortcode || p.placeholdersEnabled {
+			rendered, err = replaceShortcodeTokens(rendered, placeholders)
+			if err != nil {
+				return "", err
+			}
+		}
+
+		// We need a consolidated view in $page.HasShortcode
+		p.p.shortcodeState.transferNames(s)
+
+	} else {
+		c, err := p.renderContentWithConverter(conv, []byte(contentToRender), false)
+		if err != nil {
+			return "", p.p.wrapError(err)
+		}
+
+		rendered = c.Bytes()
+	}
 
 	if opts.Display == "inline" {
 		// We may have to rethink this in the future when we get other
 		// renderers.
-		b = p.p.s.ContentSpec.TrimShortHTML(b)
+		rendered = p.p.s.ContentSpec.TrimShortHTML(rendered)
 	}
 
-	return template.HTML(string(b)), nil
+	return template.HTML(string(rendered)), nil
 }
 
 func (p *pageContentOutput) RenderWithTemplateInfo(info tpl.Info, layout ...string) (template.HTML, error) {
diff --git a/hugolib/renderstring_test.go b/hugolib/renderstring_test.go
new file mode 100644
index 000000000..d2f453c33
--- /dev/null
+++ b/hugolib/renderstring_test.go
@@ -0,0 +1,162 @@
+// Copyright 2022 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless requiredF by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugolib
+
+import (
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+	"github.com/gohugoio/hugo/common/loggers"
+)
+
+func TestRenderString(t *testing.T) {
+	b := newTestSitesBuilder(t)
+
+	b.WithTemplates("index.html", `
+{{ $p := site.GetPage "p1.md" }}
+{{ $optBlock := dict "display" "block" }}
+{{ $optOrg := dict "markup" "org" }}
+RSTART:{{ "**Bold Markdown**" | $p.RenderString }}:REND
+RSTART:{{  "**Bold Block Markdown**" | $p.RenderString  $optBlock }}:REND
+RSTART:{{  "/italic org mode/" | $p.RenderString  $optOrg }}:REND
+RSTART:{{ "## Header2" | $p.RenderString }}:REND
+
+
+`, "_default/_markup/render-heading.html", "Hook Heading: {{ .Level }}")
+
+	b.WithContent("p1.md", `---
+title: "p1"
+---
+`,
+	)
+
+	b.Build(BuildCfg{})
+
+	b.AssertFileContent("public/index.html", `
+RSTART:<strong>Bold Markdown</strong>:REND
+RSTART:<p><strong>Bold Block Markdown</strong></p>
+RSTART:<em>italic org mode</em>:REND
+RSTART:Hook Heading: 2:REND
+`)
+}
+
+// https://github.com/gohugoio/hugo/issues/6882
+func TestRenderStringOnListPage(t *testing.T) {
+	renderStringTempl := `
+{{ .RenderString "**Hello**" }}
+`
+	b := newTestSitesBuilder(t)
+	b.WithContent("mysection/p1.md", `FOO`)
+	b.WithTemplates(
+		"index.html", renderStringTempl,
+		"_default/list.html", renderStringTempl,
+		"_default/single.html", renderStringTempl,
+	)
+
+	b.Build(BuildCfg{})
+
+	for _, filename := range []string{
+		"index.html",
+		"mysection/index.html",
+		"categories/index.html",
+		"tags/index.html",
+		"mysection/p1/index.html",
+	} {
+		b.AssertFileContent("public/"+filename, `<strong>Hello</strong>`)
+	}
+}
+
+// Issue 9433
+func TestRenderStringOnPageNotBackedByAFile(t *testing.T) {
+	t.Parallel()
+	logger := loggers.NewWarningLogger()
+	b := newTestSitesBuilder(t).WithLogger(logger).WithConfigFile("toml", `
+disableKinds = ["page", "section", "taxonomy", "term"]	
+`)
+	b.WithTemplates("index.html", `{{ .RenderString "**Hello**" }}`).WithContent("p1.md", "")
+	b.BuildE(BuildCfg{})
+	b.Assert(int(logger.LogCounters().WarnCounter.Count()), qt.Equals, 0)
+}
+
+func TestRenderStringWithShortcode(t *testing.T) {
+	t.Parallel()
+
+	filesTemplate := `
+-- config.toml --
+title = "Hugo Rocks!"
+enableInlineShortcodes = true
+-- content/p1/index.md --
+---
+title: "P1"
+---
+## First
+-- layouts/shortcodes/mark1.md --
+{{ .Inner }}
+-- layouts/shortcodes/mark2.md --
+1. Item Mark2 1
+1. Item Mark2 2
+   1. Item Mark2 2-1
+1. Item Mark2 3
+-- layouts/shortcodes/myhthml.html --
+Title: {{ .Page.Title }}
+TableOfContents: {{ .Page.TableOfContents }}
+Page Type: {{ printf "%T" .Page }}
+-- layouts/_default/single.html --
+{{ .RenderString "Markdown: {{% mark2 %}}|HTML: {{< myhthml >}}|Inline: {{< foo.inline >}}{{ site.Title }}{{< /foo.inline >}}|" }}
+HasShortcode: mark2:{{ .HasShortcode "mark2" }}:true
+HasShortcode: foo:{{ .HasShortcode "foo" }}:false
+
+`
+
+	t.Run("Basic", func(t *testing.T) {
+
+		b := NewIntegrationTestBuilder(
+			IntegrationTestConfig{
+				T:           t,
+				TxtarString: filesTemplate,
+			},
+		).Build()
+
+		b.AssertFileContent("public/p1/index.html",
+			"<p>Markdown: 1. Item Mark2 1</p>\n<ol>\n<li>Item Mark2 2\n<ol>\n<li>Item Mark2 2-1</li>\n</ol>\n</li>\n<li>Item Mark2 3|",
+			"<a href=\"#first\">First</a>", // ToC
+			`
+HTML: Title: P1
+Inline: Hugo Rocks!
+HasShortcode: mark2:true:true
+HasShortcode: foo:false:false
+Page Type: *hugolib.pageForShortcode`,
+		)
+
+	})
+
+	t.Run("Edit shortcode", func(t *testing.T) {
+
+		b := NewIntegrationTestBuilder(
+			IntegrationTestConfig{
+				T:           t,
+				TxtarString: filesTemplate,
+				Running:     true,
+			},
+		).Build()
+
+		b.EditFiles("layouts/shortcodes/myhthml.html", "Edit shortcode").Build()
+
+		b.AssertFileContent("public/p1/index.html",
+			`Edit shortcode`,
+		)
+
+	})
+
+}
diff --git a/hugolib/shortcode.go b/hugolib/shortcode.go
index b822ecfe3..366875b88 100644
--- a/hugolib/shortcode.go
+++ b/hugolib/shortcode.go
@@ -250,13 +250,14 @@ type shortcodeHandler struct {
 	shortcodes []*shortcode
 
 	// All the shortcode names in this set.
-	nameSet map[string]bool
+	nameSet   map[string]bool
+	nameSetMu sync.RWMutex
 
 	// Configuration
 	enableInlineShortcodes bool
 }
 
-func newShortcodeHandler(p *pageState, s *Site, placeholderFunc func() string) *shortcodeHandler {
+func newShortcodeHandler(p *pageState, s *Site) *shortcodeHandler {
 	sh := &shortcodeHandler{
 		p:                      p,
 		s:                      s,
@@ -423,6 +424,28 @@ func (s *shortcodeHandler) hasShortcodes() bool {
 	return s != nil && len(s.shortcodes) > 0
 }
 
+func (s *shortcodeHandler) addName(name string) {
+	s.nameSetMu.Lock()
+	defer s.nameSetMu.Unlock()
+	s.nameSet[name] = true
+}
+
+func (s *shortcodeHandler) transferNames(in *shortcodeHandler) {
+	s.nameSetMu.Lock()
+	defer s.nameSetMu.Unlock()
+	for k := range in.nameSet {
+		s.nameSet[k] = true
+	}
+
+}
+
+func (s *shortcodeHandler) hasName(name string) bool {
+	s.nameSetMu.RLock()
+	defer s.nameSetMu.RUnlock()
+	_, ok := s.nameSet[name]
+	return ok
+}
+
 func (s *shortcodeHandler) renderShortcodesForPage(p *pageState, f output.Format) (map[string]string, bool, error) {
 	rendered := make(map[string]string)
 
@@ -503,7 +526,7 @@ Loop:
 				nested, err := s.extractShortcode(nestedOrdinal, nextLevel, pt)
 				nestedOrdinal++
 				if nested != nil && nested.name != "" {
-					s.nameSet[nested.name] = true
+					s.addName(nested.name)
 				}
 
 				if err == nil {
diff --git a/hugolib/shortcode_test.go b/hugolib/shortcode_test.go
index cafe76703..15c27a42e 100644
--- a/hugolib/shortcode_test.go
+++ b/hugolib/shortcode_test.go
@@ -107,15 +107,9 @@ title: "Shortcodes Galore!"
 			t.Parallel()
 			c := qt.New(t)
 
-			counter := 0
-			placeholderFunc := func() string {
-				counter++
-				return fmt.Sprintf("HAHA%s-%dHBHB", shortcodePlaceholderPrefix, counter)
-			}
-
 			p, err := pageparser.ParseMain(strings.NewReader(test.input), pageparser.Config{})
 			c.Assert(err, qt.IsNil)
-			handler := newShortcodeHandler(nil, s, placeholderFunc)
+			handler := newShortcodeHandler(nil, s)
 			iter := p.Iterator()
 
 			short, err := handler.extractShortcode(0, 0, iter)