Render the shortcodes as late as possible

This is needed to make shortcode users happy with the new multilanguage support,
but it will also solve many other related posts about "stuff not available in the shortcode".

We will have to revisit this re the handler chain at some point, but that will be easier
now as the integration test story has improved so much.

As part of this commit, the site-building tests in page_test.go is refreshed, they now
tests for all the rendering engines (when available), and all of them now uses the
same code-path as used in production.

Fixes #1229
Fixes #2323
Fixes ##1076

10 files changed, 610 insertions, 258 deletions

diff --git a/hugolib/handler_meta.go b/hugolib/handler_meta.go
index 0825ef1e6..36b0f5a2a 100644
--- a/hugolib/handler_meta.go
+++ b/hugolib/handler_meta.go
@@ -75,8 +75,6 @@ func (mh *MetaHandle) Convert(i interface{}, s *Site, results HandleResults) {
 		}
 
 		results <- h.PageConvert(p, s.Tmpl)
-		p.setSummary()
-		p.analyzePage()
 	}
 }
 
diff --git a/hugolib/handler_page.go b/hugolib/handler_page.go
index 73a972446..04af20ceb 100644
--- a/hugolib/handler_page.go
+++ b/hugolib/handler_page.go
@@ -14,10 +14,11 @@
 package hugolib
 
 import (
+	"bytes"
+
 	"github.com/spf13/hugo/helpers"
 	"github.com/spf13/hugo/source"
 	"github.com/spf13/hugo/tpl"
-	jww "github.com/spf13/jwalterweatherman"
 	"github.com/spf13/viper"
 )
 
@@ -67,18 +68,7 @@ type htmlHandler struct {
 func (h htmlHandler) Extensions() []string { return []string{"html", "htm"} }
 func (h htmlHandler) PageConvert(p *Page, t tpl.Template) HandledResult {
 	p.ProcessShortcodes(t)
-	var err error
-
-	if len(p.contentShortCodes) > 0 {
-		p.rawContent, err = replaceShortcodeTokens(p.rawContent, shortcodePlaceholderPrefix, p.contentShortCodes)
 
-		if err != nil {
-			jww.FATAL.Printf("Failed to replace short code tokens in %s:\n%s", p.BaseFileName(), err.Error())
-			return HandledResult{err: err}
-		}
-	}
-
-	p.Content = helpers.BytesToHTML(p.rawContent)
 	return HandledResult{err: nil}
 }
 
@@ -112,27 +102,22 @@ func (h mmarkHandler) PageConvert(p *Page, t tpl.Template) HandledResult {
 func commonConvert(p *Page, t tpl.Template) HandledResult {
 	p.ProcessShortcodes(t)
 
-	var err error
-
+	// TODO(bep) these page handlers need to be re-evaluated, as it is hard to
+	// process a page in isolation. See the new preRender func.
+	// TODO(bep) ml not so raw anymore, but do we need to keep it raw?
 	if viper.GetBool("EnableEmoji") {
 		p.rawContent = helpers.Emojify(p.rawContent)
 	}
 
-	renderedContent := p.renderContent(helpers.RemoveSummaryDivider(p.rawContent))
-
-	if len(p.contentShortCodes) > 0 {
-		renderedContent, err = replaceShortcodeTokens(renderedContent, shortcodePlaceholderPrefix, p.contentShortCodes)
-
-		if err != nil {
-			jww.FATAL.Printf("Failed to replace shortcode tokens in %s:\n%s", p.BaseFileName(), err.Error())
-			return HandledResult{err: err}
-		}
-	}
-
-	tmpContent, tmpTableOfContents := helpers.ExtractTOC(renderedContent)
+	// TODO(bep) ml we let the summary divider survive the rendering. Must check if
+	// it actually survives, replace it with something more robus, or maybe
+	// rethink this fragile concept.
+	//p.rawContent = p.renderContent(helpers.RemoveSummaryDivider(p.rawContent))
+	// We have to replace the <!--more--> with something that survives all the
+	// rendering engines.
+	// TODO(bep) inline replace
+	p.rawContent = bytes.Replace(p.rawContent, []byte(helpers.SummaryDivider), internalSummaryDivider, 1)
+	p.rawContent = p.renderContent(p.rawContent)
 
-	p.Content = helpers.BytesToHTML(tmpContent)
-	p.TableOfContents = helpers.BytesToHTML(tmpTableOfContents)
-	p.rendered = true
 	return HandledResult{err: nil}
 }
diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go
index 2dd1bb9be..76f9979fd 100644
--- a/hugolib/hugo_sites.go
+++ b/hugolib/hugo_sites.go
@@ -16,8 +16,11 @@ package hugolib
 import (
 	"errors"
 	"strings"
+	"sync"
 	"time"
 
+	"github.com/spf13/hugo/helpers"
+
 	"github.com/spf13/viper"
 
 	"github.com/fsnotify/fsnotify"
@@ -106,24 +109,27 @@ func (h HugoSites) Build(config BuildCfg) error {
 	}
 
 	for _, s := range h.Sites {
-
 		if err := s.PostProcess(); err != nil {
 			return err
 		}
+	}
+
+	if err := h.preRender(); err != nil {
+		return err
+	}
+
+	for _, s := range h.Sites {
 
 		if !config.skipRender {
 			if err := s.Render(); err != nil {
 				return err
 			}
 
+			if config.PrintStats {
+				s.Stats()
+			}
 		}
-
-		if config.PrintStats {
-			s.Stats()
-		}
-
 		// TODO(bep) ml lang in site.Info?
-		// TODO(bep) ml Page sorting?
 	}
 
 	if config.PrintStats {
@@ -153,22 +159,26 @@ func (h HugoSites) Rebuild(config BuildCfg, events ...fsnotify.Event) error {
 	// Assign pages to sites per translation.
 	h.setupTranslations(firstSite)
 
-	for _, s := range h.Sites {
-
-		if sourceChanged {
+	if sourceChanged {
+		for _, s := range h.Sites {
 			if err := s.PostProcess(); err != nil {
 				return err
 			}
 		}
+	}
 
-		if !config.skipRender {
+	if err := h.preRender(); err != nil {
+		return err
+	}
+
+	if !config.skipRender {
+		for _, s := range h.Sites {
 			if err := s.Render(); err != nil {
 				return err
 			}
-		}
-
-		if config.PrintStats {
-			s.Stats()
+			if config.PrintStats {
+				s.Stats()
+			}
 		}
 	}
 
@@ -219,6 +229,87 @@ func (s *HugoSites) setupTranslations(master *Site) {
 	}
 }
 
+// preRender performs build tasks that needs to be done as late as possible.
+// Shortcode handling is the main task in here.
+// TODO(bep) We need to look at the whole handler-chain construct witht he below in mind.
+func (h *HugoSites) preRender() error {
+	pageChan := make(chan *Page)
+
+	wg := &sync.WaitGroup{}
+
+	// We want all the pages, so just pick one.
+	s := h.Sites[0]
+
+	for i := 0; i < getGoMaxProcs()*4; i++ {
+		wg.Add(1)
+		go func(pages <-chan *Page, wg *sync.WaitGroup) {
+			defer wg.Done()
+			for p := range pages {
+				if err := handleShortcodes(p, s.Tmpl); err != nil {
+					jww.ERROR.Printf("Failed to handle shortcodes for page %s: %s", p.BaseFileName(), err)
+				}
+
+				if p.Markup == "markdown" {
+					tmpContent, tmpTableOfContents := helpers.ExtractTOC(p.rawContent)
+					p.TableOfContents = helpers.BytesToHTML(tmpTableOfContents)
+					p.rawContent = tmpContent
+				}
+
+				if p.Markup != "html" {
+
+					// Now we know enough to create a summary of the page and count some words
+					summaryContent, err := p.setUserDefinedSummaryIfProvided()
+
+					if err != nil {
+						jww.ERROR.Printf("Failed to set use defined summary: %s", err)
+					} else if summaryContent != nil {
+						p.rawContent = summaryContent.content
+					}
+
+					p.Content = helpers.BytesToHTML(p.rawContent)
+					p.rendered = true
+
+					if summaryContent == nil {
+						p.setAutoSummary()
+					}
+				}
+
+				//analyze for raw stats
+				p.analyzePage()
+			}
+		}(pageChan, wg)
+	}
+
+	for _, p := range s.AllPages {
+		pageChan <- p
+	}
+
+	close(pageChan)
+
+	wg.Wait()
+
+	return nil
+}
+
+func handleShortcodes(p *Page, t tpl.Template) error {
+	if len(p.contentShortCodes) > 0 {
+		jww.DEBUG.Printf("Replace %d shortcodes in %q", len(p.contentShortCodes), p.BaseFileName())
+		shortcodes, err := executeShortcodeFuncMap(p.contentShortCodes)
+
+		if err != nil {
+			return err
+		}
+
+		p.rawContent, err = replaceShortcodeTokens(p.rawContent, shortcodePlaceholderPrefix, shortcodes)
+
+		if err != nil {
+			jww.FATAL.Printf("Failed to replace short code tokens in %s:\n%s", p.BaseFileName(), err.Error())
+		}
+	}
+
+	return nil
+}
+
 func (s *Site) updateBuildStats(page *Page) {
 	if page.IsDraft() {
 		s.draftCount++
diff --git a/hugolib/hugo_sites_test.go b/hugolib/hugo_sites_test.go
index fc4801115..b63792a3b 100644
--- a/hugolib/hugo_sites_test.go
+++ b/hugolib/hugo_sites_test.go
@@ -37,6 +37,11 @@ func testCommonResetState() {
 	viper.Set("PublishDir", "public")
 	viper.Set("RSSUri", "rss")
 
+	viper.Set("Taxonomies", map[string]interface{}{
+		"tag":      "tags",
+		"category": "categories",
+	})
+
 	if err := hugofs.Source().Mkdir("content", 0755); err != nil {
 		panic("Content folder creation failed.")
 	}
diff --git a/hugolib/page.go b/hugolib/page.go
index 4248ff893..493e5b512 100644
--- a/hugolib/page.go
+++ b/hugolib/page.go
@@ -70,7 +70,7 @@ type Page struct {
 	linkTitle           string
 	frontmatter         []byte
 	rawContent          []byte
-	contentShortCodes   map[string]string // TODO(bep) this shouldn't be needed.
+	contentShortCodes   map[string]func() (string, error)
 	shortcodes          map[string]shortcode
 	plain               string // TODO should be []byte
 	plainWords          []string
@@ -212,46 +212,111 @@ func (p *Page) lineNumRawContentStart() int {
 	return bytes.Count(p.frontmatter, []byte("\n")) + 1
 }
 
-func (p *Page) setSummary() {
+var (
+	internalSummaryDivider            = []byte("HUGOMORE42")
+	internalSummaryDividerAndNewLines = []byte("HUGOMORE42\n\n")
+)
 
-	// at this point, p.rawContent contains placeholders for the short codes,
-	// rendered and ready in p.contentShortcodes
+// Returns the page as summary and main if a user defined split is provided.
+func (p *Page) setUserDefinedSummaryIfProvided() (*summaryContent, error) {
 
-	if bytes.Contains(p.rawContent, helpers.SummaryDivider) {
-		sections := bytes.Split(p.rawContent, helpers.SummaryDivider)
-		header := sections[0]
-		p.Truncated = true
-		if len(sections[1]) < 20 {
-			// only whitespace?
-			p.Truncated = len(bytes.Trim(sections[1], " \n\r")) > 0
-		}
+	sc := splitUserDefinedSummaryAndContent(p.Markup, p.rawContent)
 
-		// TODO(bep) consider doing this once only
-		renderedHeader := p.renderBytes(header)
-		if len(p.contentShortCodes) > 0 {
-			tmpContentWithTokensReplaced, err :=
-				replaceShortcodeTokens(renderedHeader, shortcodePlaceholderPrefix, p.contentShortCodes)
-			if err != nil {
-				jww.FATAL.Printf("Failed to replace short code tokens in Summary for %s:\n%s", p.BaseFileName(), err.Error())
-			} else {
-				renderedHeader = tmpContentWithTokensReplaced
-			}
-		}
-		p.Summary = helpers.BytesToHTML(renderedHeader)
-	} else {
-		// If hugo defines split:
-		// render, strip html, then split
-		var summary string
-		var truncated bool
-		if p.isCJKLanguage {
-			summary, truncated = helpers.TruncateWordsByRune(p.PlainWords(), helpers.SummaryLength)
-		} else {
-			summary, truncated = helpers.TruncateWordsToWholeSentence(p.PlainWords(), helpers.SummaryLength)
-		}
-		p.Summary = template.HTML(summary)
-		p.Truncated = truncated
+	if sc == nil {
+		// No divider found
+		return nil, nil
+	}
 
+	p.Truncated = true
+	if len(sc.content) < 20 {
+		// only whitespace?
+		p.Truncated = len(bytes.Trim(sc.content, " \n\r")) > 0
 	}
+
+	p.Summary = helpers.BytesToHTML(sc.summary)
+
+	return sc, nil
+}
+
+// Make this explicit so there is no doubt about what is what.
+type summaryContent struct {
+	summary               []byte
+	content               []byte
+	contentWithoutSummary []byte
+}
+
+func splitUserDefinedSummaryAndContent(markup string, c []byte) *summaryContent {
+	startDivider := bytes.Index(c, internalSummaryDivider)
+
+	if startDivider == -1 {
+		return nil
+	}
+
+	endDivider := startDivider + len(internalSummaryDivider)
+	endSummary := startDivider
+
+	var (
+		startMarkup []byte
+		endMarkup   []byte
+		addDiv      bool
+		divStart    = []byte("<div class=\"document\">")
+	)
+
+	switch markup {
+	default:
+		startMarkup = []byte("<p>")
+		endMarkup = []byte("</p>")
+	case "asciidoc":
+		startMarkup = []byte("<div class=\"paragraph\">")
+		endMarkup = []byte("</div>")
+	case "rst":
+		startMarkup = []byte("<p>")
+		endMarkup = []byte("</p>")
+		addDiv = true
+	}
+
+	// Find the closest end/start markup string to the divider
+	//firstStart := bytes.Index(c[:startDivider], startMarkup)
+	fromIdx := bytes.LastIndex(c[:startDivider], startMarkup)
+	fromStart := startDivider - fromIdx - len(startMarkup)
+	fromEnd := bytes.Index(c[endDivider:], endMarkup)
+
+	if fromEnd != -1 && fromEnd <= fromStart {
+		endSummary = startDivider + fromEnd + len(endMarkup)
+	} else if fromStart != -1 {
+		endSummary = startDivider - fromStart - len(startMarkup)
+	}
+
+	withoutDivider := bytes.TrimSpace(append(c[:startDivider], c[endDivider:]...))
+	contentWithoutSummary := bytes.TrimSpace(withoutDivider[endSummary:])
+	summary := bytes.TrimSpace(withoutDivider[:endSummary])
+
+	if addDiv {
+		// For the rst
+		summary = append(append([]byte(nil), summary...), []byte("</div>")...)
+		// TODO(bep) include the document class, maybe
+		contentWithoutSummary = append(divStart, contentWithoutSummary...)
+	}
+
+	return &summaryContent{
+		summary:               summary,
+		content:               withoutDivider,
+		contentWithoutSummary: contentWithoutSummary,
+	}
+}
+
+func (p *Page) setAutoSummary() error {
+	var summary string
+	var truncated bool
+	if p.isCJKLanguage {
+		summary, truncated = helpers.TruncateWordsByRune(p.PlainWords(), helpers.SummaryLength)
+	} else {
+		summary, truncated = helpers.TruncateWordsToWholeSentence(p.PlainWords(), helpers.SummaryLength)
+	}
+	p.Summary = template.HTML(summary)
+	p.Truncated = truncated
+
+	return nil
 }
 
 func (p *Page) renderBytes(content []byte) []byte {
@@ -972,27 +1037,6 @@ func (p *Page) ProcessShortcodes(t tpl.Template) {
 
 }
 
-// TODO(spf13): Remove this entirely
-// Here for backwards compatibility & testing. Only works in isolation
-func (p *Page) Convert() error {
-	var h Handler
-	if p.Markup != "" {
-		h = FindHandler(p.Markup)
-	} else {
-		h = FindHandler(p.File.Extension())
-	}
-	if h != nil {
-		h.PageConvert(p, tpl.T())
-	}
-
-	//// now we know enough to create a summary of the page and count some words
-	p.setSummary()
-	//analyze for raw stats
-	p.analyzePage()
-
-	return nil
-}
-
 func (p *Page) FullFilePath() string {
 	return filepath.Join(p.Dir(), p.LogicalName())
 }
diff --git a/hugolib/page_test.go b/hugolib/page_test.go
index 6fd797830..6927e6e82 100644
--- a/hugolib/page_test.go
+++ b/hugolib/page_test.go
@@ -27,6 +27,7 @@ import (
 	"github.com/spf13/hugo/helpers"
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 var emptyPage = ""
@@ -131,6 +132,15 @@ Summary Next Line
 <!--more-->
 Some more text
 `
+
+	simplePageWithSummaryDelimiterAndMarkdownThatCrossesBorder = `---
+title: Simple
+---
+The [best static site generator][hugo].[^1]
+<!--more-->
+[hugo]: http://gohugo.io/
+[^1]: Many people say so.
+`
 	simplePageWithShortcodeInSummary = `---
 title: Simple
 ---
@@ -485,21 +495,33 @@ func checkPageTitle(t *testing.T, page *Page, title string) {
 	}
 }
 
-func checkPageContent(t *testing.T, page *Page, content string) {
-	if page.Content != template.HTML(content) {
-		t.Fatalf("Page content is:\n%q\nExpected:\n%q", page.Content, content)
+func checkPageContent(t *testing.T, page *Page, content string, msg ...interface{}) {
+	a := normalizeContent(content)
+	b := normalizeContent(string(page.Content))
+	if a != b {
+		t.Fatalf("Page content is:\n%q\nExpected:\n%q (%q)", b, a, msg)
 	}
 }
 
+func normalizeContent(c string) string {
+	norm := strings.Replace(c, "\n", "", -1)
+	norm = strings.Replace(norm, "    ", " ", -1)
+	norm = strings.Replace(norm, "   ", " ", -1)
+	norm = strings.Replace(norm, "  ", " ", -1)
+	return strings.TrimSpace(norm)
+}
+
 func checkPageTOC(t *testing.T, page *Page, toc string) {
 	if page.TableOfContents != template.HTML(toc) {
 		t.Fatalf("Page TableOfContents is: %q.\nExpected %q", page.TableOfContents, toc)
 	}
 }
 
-func checkPageSummary(t *testing.T, page *Page, summary string) {
-	if page.Summary != template.HTML(summary) {
-		t.Fatalf("Page summary is: %q.\nExpected %q", page.Summary, summary)
+func checkPageSummary(t *testing.T, page *Page, summary string, msg ...interface{}) {
+	a := normalizeContent(string(page.Summary))
+	b := normalizeContent(summary)
+	if a != b {
+		t.Fatalf("Page summary is:\n%q.\nExpected\n%q (%q)", a, b, msg)
 	}
 }
 
@@ -534,147 +556,285 @@ func checkTruncation(t *testing.T, page *Page, shouldBe bool, msg string) {
 	}
 }
 
+func normalizeExpected(ext, str string) string {
+	str = normalizeContent(str)
+	switch ext {
+	default:
+		return str
+	case "html":
+		return strings.Trim(helpers.StripHTML(str), " ")
+	case "ad":
+		paragraphs := strings.Split(str, "</p>")
+		expected := ""
+		for _, para := range paragraphs {
+			if para == "" {
+				continue
+			}
+			expected += fmt.Sprintf("<div class=\"paragraph\">\n%s</p></div>\n", para)
+		}
+		return expected
+	case "rst":
+		return fmt.Sprintf("<div class=\"document\">\n\n\n%s</div>", str)
+	}
+}
+
+func testAllMarkdownEnginesForPage(t *testing.T,
+	assertFunc func(t *testing.T, ext string, p *Page), baseFilename, pageContent string) {
+
+	engines := []struct {
+		ext           string
+		shouldExecute func() bool
+	}{
+		{"ad", func() bool { return helpers.HasAsciidoc() }},
+		{"md", func() bool { return true }},
+		{"mmark", func() bool { return true }},
+		// TODO(bep) figure a way to include this without too much work.{"html", func() bool { return true }},
+		{"rst", func() bool { return helpers.HasRst() }},
+	}
+
+	for _, e := range engines {
+		if !e.shouldExecute() {
+			continue
+		}
+
+		filename := baseFilename + "." + e.ext
+
+		s := newSiteFromSources(filename, pageContent)
+
+		if err := buildSiteSkipRender(s); err != nil {
+			t.Fatalf("Failed to build site: %s", err)
+		}
+
+		require.Len(t, s.Pages, 1)
+
+		p := s.Pages[0]
+
+		assertFunc(t, e.ext, p)
+
+	}
+
+}
+
 func TestCreateNewPage(t *testing.T) {
-	p, _ := NewPage("simple.md")
-	_, err := p.ReadFrom(strings.NewReader(simplePage))
-	p.Convert()
 
-	if err != nil {
-		t.Fatalf("Unable to create a page with frontmatter and body content: %s", err)
+	assertFunc := func(t *testing.T, ext string, p *Page) {
+		assert.False(t, p.IsHome)
+		checkPageTitle(t, p, "Simple")
+		checkPageContent(t, p, normalizeExpected(ext, "<p>Simple Page</p>\n"))
+		checkPageSummary(t, p, "Simple Page")
+		checkPageType(t, p, "page")
+		checkPageLayout(t, p, "page/single.html", "_default/single.html", "theme/page/single.html", "theme/_default/single.html")
+		checkTruncation(t, p, false, "simple short page")
 	}
 
-	assert.False(t, p.IsHome)
-	checkPageTitle(t, p, "Simple")
-	checkPageContent(t, p, "<p>Simple Page</p>\n")
-	checkPageSummary(t, p, "Simple Page")
-	checkPageType(t, p, "page")
-	checkPageLayout(t, p, "page/single.html", "_default/single.html", "theme/page/single.html", "theme/_default/single.html")
-	checkTruncation(t, p, false, "simple short page")
+	testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePage)
+}
+
+func TestSplitSummaryAndContent(t *testing.T) {
+
+	for i, this := range []struct {
+		markup                        string
+		content                       string
+		expectedSummary               string
+		expectedContent               string
+		expectedContentWithoutSummary string
+	}{
+		{"markdown", `<p>Summary Same LineHUGOMORE42</p>
+
+<p>Some more text</p>`, "<p>Summary Same Line</p>", "<p>Summary Same Line</p>\n\n<p>Some more text</p>", "<p>Some more text</p>"},
+		{"asciidoc", `<div class="paragraph"><p>sn</p></div><div class="paragraph"><p>HUGOMORE42Some more text</p></div>`,
+			"<div class=\"paragraph\"><p>sn</p></div>",
+			"<div class=\"paragraph\"><p>sn</p></div><div class=\"paragraph\"><p>Some more text</p></div>",
+			"<div class=\"paragraph\"><p>Some more text</p></div>"},
+		{"rst",
+			"<div class=\"document\"><p>Summary Next Line</p><p>HUGOMORE42Some more text</p></div>",
+			"<div class=\"document\"><p>Summary Next Line</p></div>",
+			"<div class=\"document\"><p>Summary Next Line</p><p>Some more text</p></div>",
+			"<div class=\"document\"><p>Some more text</p></div>"},
+		{"markdown", "<p>a</p><p>b</p><p>HUGOMORE42c</p>", "<p>a</p><p>b</p>", "<p>a</p><p>b</p><p>c</p>", "<p>c</p>"},
+		{"markdown", "<p>a</p><p>b</p><p>cHUGOMORE42</p>", "<p>a</p><p>b</p><p>c</p>", "<p>a</p><p>b</p><p>c</p>", ""},
+		{"markdown", "<p>a</p><p>bHUGOMORE42</p><p>c</p>", "<p>a</p><p>b</p>", "<p>a</p><p>b</p><p>c</p>", "<p>c</p>"},
+		{"markdown", "<p>aHUGOMORE42</p><p>b</p><p>c</p>", "<p>a</p>", "<p>a</p><p>b</p><p>c</p>", "<p>b</p><p>c</p>"},
+	} {
+
+		sc := splitUserDefinedSummaryAndContent(this.markup, []byte(this.content))
+
+		require.NotNil(t, sc, fmt.Sprintf("[%d] Nil %s", i, this.markup))
+		require.Equal(t, this.expectedSummary, string(sc.summary), fmt.Sprintf("[%d] Summary markup %s", i, this.markup))
+		require.Equal(t, this.expectedContent, string(sc.content), fmt.Sprintf("[%d] Content markup %s", i, this.markup))
+		require.Equal(t, this.expectedContentWithoutSummary, string(sc.contentWithoutSummary), fmt.Sprintf("[%d] Content without summary, markup %s", i, this.markup))
+	}
+
+	if true {
+		return
+	}
+
+	ad := `<div class="paragraph"><p>sn</p></div>
+<div class="paragraph">
+<p>HUGOMORE42 
+Some more text</p>
+</div>
+`
+
+	md := `<p>Summary Same LineHUGOMORE42</p>
+
+<p>Some more text</p>`
+
+	sc := splitUserDefinedSummaryAndContent("markdown", []byte(md))
+
+	require.Equal(t, "adf", string(sc.summary))
+	require.Equal(t, "asdf", string(sc.content))
+
+	if true {
+		return
+	}
+	sc = splitUserDefinedSummaryAndContent("asciidoc", []byte(ad))
+	require.Equal(t, "<div class=\"paragraph\"><p>sn</p></div>", string(sc.summary))
+	require.Equal(t, "\n<div class=\"paragraph\">\n<p> \nSome more text</p>\n</div>\n", string(sc.summary))
+
 }
 
 func TestPageWithDelimiter(t *testing.T) {
-	p, _ := NewPage("simple.md")
-	_, err := p.ReadFrom(strings.NewReader(simplePageWithSummaryDelimiter))
-	p.Convert()
-	if err != nil {
-		t.Fatalf("Unable to create a page with frontmatter and body content: %s", err)
+
+	assertFunc := func(t *testing.T, ext string, p *Page) {
+		checkPageTitle(t, p, "Simple")
+		checkPageContent(t, p, normalizeExpected(ext, "<p>Summary Next Line</p>\n\n<p>Some more text</p>\n"), ext)
+		checkPageSummary(t, p, normalizeExpected(ext, "<p>Summary Next Line</p>"), ext)
+		checkPageType(t, p, "page")
+		checkPageLayout(t, p, "page/single.html", "_default/single.html", "theme/page/single.html", "theme/_default/single.html")
+		checkTruncation(t, p, true, "page with summary delimiter")
+	}
+
+	testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePageWithSummaryDelimiter)
+}
+
+// Issue #1076
+func TestPageWithDelimiterForMarkdownThatCrossesBorder(t *testing.T) {
+	s := newSiteFromSources("simple.md", simplePageWithSummaryDelimiterAndMarkdownThatCrossesBorder)
+
+	if err := buildSiteSkipRender(s); err != nil {
+		t.Fatalf("Failed to build site: %s", err)
+	}
+
+	require.Len(t, s.Pages, 1)
+
+	p := s.Pages[0]
+
+	if p.Summary != template.HTML("<p>The <a href=\"http://gohugo.io/\">best static site generator</a>.<sup class=\"footnote-ref\" id=\"fnref:1\"><a rel=\"footnote\" href=\"#fn:1\">1</a></sup>\n</p>") {
+		t.Fatalf("Got summary:\n%q", p.Summary)
+	}
+
+	if p.Content != template.HTML("<p>The <a href=\"http://gohugo.io/\">best static site generator</a>.<sup class=\"footnote-ref\" id=\"fnref:1\"><a rel=\"footnote\" href=\"#fn:1\">1</a></sup>\n</p>\n<div class=\"footnotes\">\n\n<hr />\n\n<ol>\n<li id=\"fn:1\">Many people say so.\n <a class=\"footnote-return\" href=\"#fnref:1\"><sup>[return]</sup></a></li>\n</ol>\n</div>") {
+		t.Fatalf("Got content:\n%q", p.Content)
 	}
-	checkPageTitle(t, p, "Simple")
-	checkPageContent(t, p, "<p>Summary Next Line</p>\n\n<p>Some more text</p>\n")
-	checkPageSummary(t, p, "<p>Summary Next Line</p>\n")
-	checkPageType(t, p, "page")
-	checkPageLayout(t, p, "page/single.html", "_default/single.html", "theme/page/single.html", "theme/_default/single.html")
-	checkTruncation(t, p, true, "page with summary delimiter")
 }
 
 func TestPageWithShortCodeInSummary(t *testing.T) {
-	s := new(Site)
-	s.prepTemplates(nil)
-	p, _ := NewPage("simple.md")
-	_, err := p.ReadFrom(strings.NewReader(simplePageWithShortcodeInSummary))
-	if err != nil {
-		t.Fatalf("Unable to create a page with frontmatter and body content: %s", err)
+
+	assertFunc := func(t *testing.T, ext string, p *Page) {
+		checkPageTitle(t, p, "Simple")
+		checkPageContent(t, p, normalizeExpected(ext, "<p>Summary Next Line. <figure > <img src=\"/not/real\" /> </figure>.\nMore text here.</p><p>Some more text</p>"), ext)
+		checkPageSummary(t, p, "Summary Next Line. . More text here. Some more text", ext)
+		checkPageType(t, p, "page")
+		checkPageLayout(t, p, "page/single.html", "_default/single.html", "theme/page/single.html", "theme/_default/single.html")
 	}
-	p.Convert()
 
-	checkPageTitle(t, p, "Simple")
-	checkPageContent(t, p, "<p>Summary Next Line. \n<figure >\n    \n        <img src=\"/not/real\" />\n    \n    \n</figure>\n.\nMore text here.</p>\n\n<p>Some more text</p>\n")
-	checkPageSummary(t, p, "Summary Next Line. . More text here. Some more text")
-	checkPageType(t, p, "page")
-	checkPageLayout(t, p, "page/single.html", "_default/single.html", "theme/page/single.html", "theme/_default/single.html")
+	testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePageWithShortcodeInSummary)
 }
 
 func TestPageWithEmbeddedScriptTag(t *testing.T) {
-	p, _ := NewPage("simple.md")
-	_, err := p.ReadFrom(strings.NewReader(simplePageWithEmbeddedScript))
-	p.Convert()
-	if err != nil {
-		t.Fatalf("Unable to create a page with frontmatter and body content: %s", err)
+
+	assertFunc := func(t *testing.T, ext string, p *Page) {
+		if ext == "ad" || ext == "rst" {
+			// TOD(bep)
+			return
+		}
+		checkPageContent(t, p, "<script type='text/javascript'>alert('the script tags are still there, right?');</script>\n", ext)
 	}
-	checkPageContent(t, p, "<script type='text/javascript'>alert('the script tags are still there, right?');</script>\n")
+
+	testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePageWithEmbeddedScript)
 }
 
 func TestPageWithAdditionalExtension(t *testing.T) {
-	p, _ := NewPage("simple.md")
-	_, err := p.ReadFrom(strings.NewReader(simplePageWithAdditionalExtension))
-	p.Convert()
-	if err != nil {
-		t.Fatalf("Unable to create a page with frontmatter and b