tpl/tplimpl: Handle late transformation of templates

Fixes #5865

7 files changed, 153 insertions, 65 deletions

diff --git a/tpl/template.go b/tpl/template.go
index 01f79c407..935771364 100644
--- a/tpl/template.go
+++ b/tpl/template.go
@@ -52,7 +52,7 @@ type TemplateHandler interface {
 
 	NewTextTemplate() TemplateParseFinder
 
-	MarkReady()
+	MarkReady() error
 	RebuildClone()
 }
 
diff --git a/tpl/tplimpl/ace.go b/tpl/tplimpl/ace.go
index 6fedcb583..bdbc71059 100644
--- a/tpl/tplimpl/ace.go
+++ b/tpl/tplimpl/ace.go
@@ -53,15 +53,15 @@ func (t *templateHandler) addAceTemplate(name, basePath, innerPath string, baseC
 
 	typ := resolveTemplateType(name)
 
-	info, err := applyTemplateTransformersToHMLTTemplate(typ, templ)
+	c, err := applyTemplateTransformersToHMLTTemplate(typ, templ)
 	if err != nil {
 		return err
 	}
 
 	if typ == templateShortcode {
-		t.addShortcodeVariant(name, info, templ)
+		t.addShortcodeVariant(name, c.Info, templ)
 	} else {
-		t.templateInfo[name] = info
+		t.templateInfo[name] = c.Info
 	}
 
 	return nil
diff --git a/tpl/tplimpl/template.go b/tpl/tplimpl/template.go
index 8fcaa8d64..c1fad3068 100644
--- a/tpl/tplimpl/template.go
+++ b/tpl/tplimpl/template.go
@@ -18,6 +18,7 @@ import (
 	"html/template"
 	"strings"
 	texttemplate "text/template"
+	"text/template/parse"
 
 	"github.com/gohugoio/hugo/hugofs"
 	"github.com/gohugoio/hugo/tpl/tplimpl/embedded"
@@ -51,7 +52,6 @@ var (
 	_ tpl.TemplateFinder        = (*textTemplates)(nil)
 	_ templateLoader            = (*htmlTemplates)(nil)
 	_ templateLoader            = (*textTemplates)(nil)
-	_ templateLoader            = (*templateHandler)(nil)
 	_ templateFuncsterTemplater = (*htmlTemplates)(nil)
 	_ templateFuncsterTemplater = (*textTemplates)(nil)
 )
@@ -66,7 +66,7 @@ type templateErr struct {
 
 type templateLoader interface {
 	handleMaster(name, overlayFilename, masterFilename string, onMissing func(filename string) (templateInfo, error)) error
-	addTemplate(name, tpl string) error
+	addTemplate(name, tpl string) (*templateContext, error)
 	addLateTemplate(name, tpl string) error
 }
 
@@ -329,6 +329,7 @@ func (t *templateHandler) clone(d *deps.Deps) *templateHandler {
 func newTemplateAdapter(deps *deps.Deps) *templateHandler {
 	common := &templatesCommon{
 		nameBaseTemplateName: make(map[string]string),
+		transformNotFound:    make(map[string]bool),
 	}
 
 	htmlT := &htmlTemplates{
@@ -364,6 +365,10 @@ type templatesCommon struct {
 
 	// Used to get proper filenames in errors
 	nameBaseTemplateName map[string]string
+
+	// Holds names of the templates not found during the first AST transformation
+	// pass.
+	transformNotFound map[string]bool
 }
 type htmlTemplates struct {
 	mu sync.RWMutex
@@ -491,37 +496,42 @@ func (t *templateHandler) LoadTemplates(prefix string) error {
 
 }
 
-func (t *htmlTemplates) addTemplateIn(tt *template.Template, name, tpl string) error {
+func (t *htmlTemplates) addTemplateIn(tt *template.Template, name, tpl string) (*templateContext, error) {
 	t.mu.Lock()
 	defer t.mu.Unlock()
 
 	templ, err := tt.New(name).Parse(tpl)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	typ := resolveTemplateType(name)
 
-	info, err := applyTemplateTransformersToHMLTTemplate(typ, templ)
+	c, err := applyTemplateTransformersToHMLTTemplate(typ, templ)
 	if err != nil {
-		return err
+		return nil, err
+	}
+
+	for k, _ := range c.notFound {
+		t.transformNotFound[k] = true
 	}
 
 	if typ == templateShortcode {
-		t.handler.addShortcodeVariant(name, info, templ)
+		t.handler.addShortcodeVariant(name, c.Info, templ)
 	} else {
-		t.handler.templateInfo[name] = info
+		t.handler.templateInfo[name] = c.Info
 	}
 
-	return nil
+	return c, nil
 }
 
-func (t *htmlTemplates) addTemplate(name, tpl string) error {
+func (t *htmlTemplates) addTemplate(name, tpl string) (*templateContext, error) {
 	return t.addTemplateIn(t.t, name, tpl)
 }
 
 func (t *htmlTemplates) addLateTemplate(name, tpl string) error {
-	return t.addTemplateIn(t.clone, name, tpl)
+	_, err := t.addTemplateIn(t.clone, name, tpl)
+	return err
 }
 
 type textTemplate struct {
@@ -556,41 +566,91 @@ func (t *textTemplate) parseIn(tt *texttemplate.Template, name, tpl string) (*te
 	return templ, nil
 }
 
-func (t *textTemplates) addTemplateIn(tt *texttemplate.Template, name, tpl string) error {
+func (t *textTemplates) addTemplateIn(tt *texttemplate.Template, name, tpl string) (*templateContext, error) {
 	name = strings.TrimPrefix(name, textTmplNamePrefix)
 	templ, err := t.parseIn(tt, name, tpl)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	typ := resolveTemplateType(name)
 
-	info, err := applyTemplateTransformersToTextTemplate(typ, templ)
+	c, err := applyTemplateTransformersToTextTemplate(typ, templ)
 	if err != nil {
-		return err
+		return nil, err
+	}
+
+	for k, _ := range c.notFound {
+		t.transformNotFound[k] = true
 	}
 
 	if typ == templateShortcode {
-		t.handler.addShortcodeVariant(name, info, templ)
+		t.handler.addShortcodeVariant(name, c.Info, templ)
 	} else {
-		t.handler.templateInfo[name] = info
+		t.handler.templateInfo[name] = c.Info
 	}
 
-	return nil
+	return c, nil
 }
 
-func (t *textTemplates) addTemplate(name, tpl string) error {
+func (t *textTemplates) addTemplate(name, tpl string) (*templateContext, error) {
 	return t.addTemplateIn(t.t, name, tpl)
 }
 
 func (t *textTemplates) addLateTemplate(name, tpl string) error {
-	return t.addTemplateIn(t.clone, name, tpl)
+	_, err := t.addTemplateIn(t.clone, name, tpl)
+	return err
 }
 
 func (t *templateHandler) addTemplate(name, tpl string) error {
 	return t.AddTemplate(name, tpl)
 }
 
+func (t *templateHandler) postTransform() error {
+	if len(t.html.transformNotFound) == 0 && len(t.text.transformNotFound) == 0 {
+		return nil
+	}
+
+	defer func() {
+		t.text.transformNotFound = make(map[string]bool)
+		t.html.transformNotFound = make(map[string]bool)
+	}()
+
+	for _, s := range []struct {
+		lookup            func(name string) *parse.Tree
+		transformNotFound map[string]bool
+	}{
+		// html templates
+		{func(name string) *parse.Tree {
+			templ := t.html.lookup(name)
+			if templ == nil {
+				return nil
+			}
+			return templ.Tree
+		}, t.html.transformNotFound},
+		// text templates
+		{func(name string) *parse.Tree {
+			templT := t.text.lookup(name)
+			if templT == nil {
+				return nil
+			}
+			return templT.Tree
+		}, t.text.transformNotFound},
+	} {
+		for name, _ := range s.transformNotFound {
+			templ := s.lookup(name)
+			if templ != nil {
+				_, err := applyTemplateTransformers(templateUndefined, templ, s.lookup)
+				if err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
 func (t *templateHandler) addLateTemplate(name, tpl string) error {
 	return t.AddLateTemplate(name, tpl)
 }
@@ -608,9 +668,11 @@ func (t *templateHandler) AddLateTemplate(name, tpl string) error {
 // AddTemplate parses and adds a template to the collection.
 // Templates with name prefixed with "_text" will be handled as plain
 // text templates.
+// TODO(bep) clean up these addTemplate variants
 func (t *templateHandler) AddTemplate(name, tpl string) error {
 	h := t.getTemplateHandler(name)
-	if err := h.addTemplate(name, tpl); err != nil {
+	_, err := h.addTemplate(name, tpl)
+	if err != nil {
 		return err
 	}
 	return nil
@@ -620,7 +682,11 @@ func (t *templateHandler) AddTemplate(name, tpl string) error {
 // after this is set.
 // TODO(bep) if this proves to be resource heavy, we could detect
 // earlier if we really need this, or make it lazy.
-func (t *templateHandler) MarkReady() {
+func (t *templateHandler) MarkReady() error {
+	if err := t.postTransform(); err != nil {
+		return err
+	}
+
 	if t.html.clone == nil {
 		t.html.clone = template.Must(t.html.t.Clone())
 		t.html.cloneClone = template.Must(t.html.clone.Clone())
@@ -629,6 +695,8 @@ func (t *templateHandler) MarkReady() {
 		t.text.clone = texttemplate.Must(t.text.t.Clone())
 		t.text.cloneClone = texttemplate.Must(t.text.clone.Clone())
 	}
+
+	return nil
 }
 
 // RebuildClone rebuilds the cloned templates. Used for live-reloads.
@@ -890,15 +958,15 @@ func (t *templateHandler) addTemplateFile(name, baseTemplatePath, path string) e
 
 		typ := resolveTemplateType(name)
 
-		info, err := applyTemplateTransformersToHMLTTemplate(typ, templ)
+		c, err := applyTemplateTransformersToHMLTTemplate(typ, templ)
 		if err != nil {
 			return err
 		}
 
 		if typ == templateShortcode {
-			t.addShortcodeVariant(templateName, info, templ)
+			t.addShortcodeVariant(templateName, c.Info, templ)
 		} else {
-			t.templateInfo[name] = info
+			t.templateInfo[name] = c.Info
 		}
 
 		return nil
diff --git a/tpl/tplimpl/templateProvider.go b/tpl/tplimpl/templateProvider.go
index 3a803f2da..932222344 100644
--- a/tpl/tplimpl/templateProvider.go
+++ b/tpl/tplimpl/templateProvider.go
@@ -46,9 +46,7 @@ func (*TemplateProvider) Update(deps *deps.Deps) error {
 
 	}
 
-	newTmpl.MarkReady()
-
-	return nil
+	return newTmpl.MarkReady()
 
 }
 
@@ -60,7 +58,6 @@ func (*TemplateProvider) Clone(d *deps.Deps) error {
 
 	d.Tmpl = clone
 
-	clone.MarkReady()
+	return clone.MarkReady()
 
-	return nil
 }
diff --git a/tpl/tplimpl/template_ast_transformers.go b/tpl/tplimpl/template_ast_transformers.go
index 57fafcd88..fb0728a63 100644
--- a/tpl/tplimpl/template_ast_transformers.go
+++ b/tpl/tplimpl/template_ast_transformers.go
@@ -50,6 +50,7 @@ const (
 type templateContext struct {
 	decl     decl
 	visited  map[string]bool
+	notFound map[string]bool
 	lookupFn func(name string) *parse.Tree
 
 	// The last error encountered.
@@ -72,7 +73,15 @@ func (c templateContext) getIfNotVisited(name string) *parse.Tree {
 		return nil
 	}
 	c.visited[name] = true
-	return c.lookupFn(name)
+	templ := c.lookupFn(name)
+	if templ == nil {
+		// This may be a inline template defined outside of this file
+		// and not yet parsed. Unusual, but it happens.
+		// Store the name to try again later.
+		c.notFound[name] = true
+	}
+
+	return templ
 }
 
 func newTemplateContext(lookupFn func(name string) *parse.Tree) *templateContext {
@@ -80,8 +89,8 @@ func newTemplateContext(lookupFn func(name string) *parse.Tree) *templateContext
 		Info:     tpl.Info{Config: tpl.DefaultConfig},
 		lookupFn: lookupFn,
 		decl:     make(map[string]string),
-		visited:  make(map[string]bool)}
-
+		visited:  make(map[string]bool),
+		notFound: make(map[string]bool)}
 }
 
 func createParseTreeLookup(templ *template.Template) func(nn string) *parse.Tree {
@@ -94,11 +103,11 @@ func createParseTreeLookup(templ *template.Template) func(nn string) *parse.Tree
 	}
 }
 
-func applyTemplateTransformersToHMLTTemplate(typ templateType, templ *template.Template) (tpl.Info, error) {
+func applyTemplateTransformersToHMLTTemplate(typ templateType, templ *template.Template) (*templateContext, error) {
 	return applyTemplateTransformers(typ, templ.Tree, createParseTreeLookup(templ))
 }
 
-func applyTemplateTransformersToTextTemplate(typ templateType, templ *texttemplate.Template) (tpl.Info, error) {
+func applyTemplateTransformersToTextTemplate(typ templateType, templ *texttemplate.Template) (*templateContext, error) {
 	return applyTemplateTransformers(typ, templ.Tree,
 		func(nn string) *parse.Tree {
 			tt := templ.Lookup(nn)
@@ -109,9 +118,9 @@ func applyTemplateTransformersToTextTemplate(typ templateType, templ *texttempla
 		})
 }
 
-func applyTemplateTransformers(typ templateType, templ *parse.Tree, lookupFn func(name string) *parse.Tree) (tpl.Info, error) {
+func applyTemplateTransformers(typ templateType, templ *parse.Tree, lookupFn func(name string) *parse.Tree) (*templateContext, error) {
 	if templ == nil {
-		return tpl.Info{}, errors.New("expected template, but none provided")
+		return nil, errors.New("expected template, but none provided")
 	}
 
 	c := newTemplateContext(lookupFn)
@@ -125,7 +134,7 @@ func applyTemplateTransformers(typ templateType, templ *parse.Tree, lookupFn fun
 		templ.Root = c.wrapInPartialReturnWrapper(templ.Root)
 	}
 
-	return c.Info, err
+	return c, err
 }
 
 const (
diff --git a/tpl/tplimpl/template_ast_transformers_test.go b/tpl/tplimpl/template_ast_transformers_test.go
index 9ed29d27f..be1efc522 100644
--- a/tpl/tplimpl/template_ast_transformers_test.go
+++ b/tpl/tplimpl/template_ast_transformers_test.go
@@ -26,10 +26,6 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-type handler interface {
-	addTemplate(name, tpl string) error
-}
-
 var (
 	testFuncs = map[string]interface{}{
 		"getif":  func(v interface{}) interface{} { return v },
@@ -413,7 +409,7 @@ func TestInsertIsZeroFunc(t *testing.T) {
 			"T":        &T{NonEmptyInterfaceTypedNil: (*T)(nil)},
 		}
 
-		templ = `
+		templ1 = `
 {{ if .True }}.True: TRUE{{ else }}.True: FALSE{{ end }}
 {{ if .TimeZero }}.TimeZero1: TRUE{{ else }}.TimeZero1: FALSE{{ end }}
 {{ if (.TimeZero) }}.TimeZero2: TRUE{{ else }}.TimeZero2: FALSE{{ end }}
@@ -423,31 +419,49 @@ func TestInsertIsZeroFunc(t *testing.T) {
 {{ template "mytemplate" . }}
 {{ if .T.NonEmptyInterfaceTypedNil }}.NonEmptyInterfaceTypedNil: TRUE{{ else }}.NonEmptyInterfaceTypedNil: FALSE{{ end }}
 
+{{ template "other-file-template" . }}
 
 {{ define "mytemplate" }}
 {{ if .TimeZero }}.TimeZero1: mytemplate: TRUE{{ else }}.TimeZero1: mytemplate: FALSE{{ end }}
 {{ end }}
 
 `
+
+		// https://github.com/gohugoio/hugo/issues/5865
+		templ2 = `{{ define "other-file-template" }}
+{{ if .TimeZero }}.TimeZero1: other-file-template: TRUE{{ else }}.TimeZero1: other-file-template: FALSE{{ end }}
+{{ end }}		
+`
 	)
 
 	d := newD(assert)
-	h := d.Tmpl.(handler)
-
-	assert.NoError(h.addTemplate("mytemplate.html", templ))
-
-	tt, _ := d.Tmpl.Lookup("mytemplate.html")
-	result, err := tt.(tpl.TemplateExecutor).ExecuteToString(ctx)
-	assert.NoError(err)
-
-	assert.Contains(result, ".True: TRUE")
-	assert.Contains(result, ".TimeZero1: FALSE")
-	assert.Contains(result, ".TimeZero2: FALSE")
-	assert.Contains(result, ".TimeZero3: TRUE")
-	assert.Contains(result, ".Now: TRUE")
-	assert.Contains(result, "TimeZero1 with: FALSE")
-	assert.Contains(result, ".TimeZero1: mytemplate: FALSE")
-	assert.Contains(result, ".NonEmptyInterfaceTypedNil: FALSE")
+	h := d.Tmpl.(tpl.TemplateHandler)
+
+	// HTML templates
+	assert.NoError(h.AddTemplate("mytemplate.html", templ1))
+	assert.NoError(h.AddTemplate("othertemplate.html", templ2))
+
+	// Text templates
+	assert.NoError(h.AddTemplate("_text/mytexttemplate.txt", templ1))
+	assert.NoError(h.AddTemplate("_text/myothertexttemplate.txt", templ2))
+
+	assert.NoError(h.MarkReady())
+
+	for _, name := range []string{"mytemplate.html", "mytexttemplate.txt"} {
+		tt, _ := d.Tmpl.Lookup(name)
+		result, err := tt.(tpl.TemplateExecutor).ExecuteToString(ctx)
+		assert.NoError(err)
+
+		assert.Contains(result, ".True: TRUE")
+		assert.Contains(result, ".TimeZero1: FALSE")
+		assert.Contains(result, ".TimeZero2: FALSE")
+		assert.Contains(result, ".TimeZero3: TRUE")
+		assert.Contains(result, ".Now: TRUE")
+		assert.Contains(result, "TimeZero1 with: FALSE")
+		assert.Contains(result, ".TimeZero1: mytemplate: FALSE")
+		assert.Contains(result, ".TimeZero1: other-file-template: FALSE")
+		assert.Contains(result, ".NonEmptyInterfaceTypedNil: FALSE")
+	}
 
 }
 
diff --git a/tpl/tplimpl/template_info_test.go b/tpl/tplimpl/template_info_test.go
index 0ebaa6da3..be9d7e2f1 100644
--- a/tpl/tplimpl/template_info_test.go
+++ b/tpl/tplimpl/template_info_test.go
@@ -24,9 +24,9 @@ import (
 func TestTemplateInfoShortcode(t *testing.T) {
 	assert := require.New(t)
 	d := newD(assert)
-	h := d.Tmpl.(handler)
+	h := d.Tmpl.(tpl.TemplateHandler)
 
-	assert.NoError(h.addTemplate("shortcodes/mytemplate.html", `
+	assert.NoError(h.AddTemplate("shortcodes/mytemplate.html", `
 {{ .Inner }}
 `))
 	tt, found, _ := d.Tmpl.LookupVariant("mytemplate", tpl.TemplateVariants{})