diff options
author | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2018-10-23 14:37:09 +0200 |
---|---|---|
committer | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2018-10-23 19:41:22 +0200 |
commit | 6636cf1bea77d20ef2a72a45fae59ac402fb133b (patch) | |
tree | 91c9435acd1a2139f8816abcd9b0d978ff2fa300 | |
parent | f669ef6bec25155d015b6ab231c53caef4fa5cdc (diff) |
Resolve error handling/parser related TODOs
See #5324
-rw-r--r-- | hugofs/rootmapping_fs_test.go | 2 | ||||
-rw-r--r-- | hugolib/hugo_sites_build_errors_test.go | 2 | ||||
-rw-r--r-- | hugolib/hugo_sites_build_test.go | 8 | ||||
-rw-r--r-- | hugolib/page.go | 8 | ||||
-rw-r--r-- | hugolib/page_content.go | 21 | ||||
-rw-r--r-- | hugolib/page_test.go | 12 | ||||
-rw-r--r-- | hugolib/site_render.go | 5 | ||||
-rw-r--r-- | hugolib/site_test.go | 3 | ||||
-rw-r--r-- | parser/pageparser/item.go | 5 | ||||
-rw-r--r-- | parser/pageparser/pagelexer.go | 56 | ||||
-rw-r--r-- | parser/pageparser/pageparser.go | 2 | ||||
-rw-r--r-- | parser/pageparser/pageparser_intro_test.go | 8 | ||||
-rw-r--r-- | tpl/data/data.go | 1 | ||||
-rw-r--r-- | tpl/template.go | 2 |
14 files changed, 67 insertions, 68 deletions
diff --git a/hugofs/rootmapping_fs_test.go b/hugofs/rootmapping_fs_test.go index d76e00d2e..e6a0301c9 100644 --- a/hugofs/rootmapping_fs_test.go +++ b/hugofs/rootmapping_fs_test.go @@ -50,7 +50,7 @@ func TestRootMappingFsDirnames(t *testing.T) { fif, err := rfs.Stat(filepath.Join("cf2", testfile)) assert.NoError(err) assert.Equal("myfile.txt", fif.Name()) - assert.Equal("f2t/myfile.txt", fif.(RealFilenameInfo).RealFilename()) + assert.Equal(filepath.FromSlash("f2t/myfile.txt"), fif.(RealFilenameInfo).RealFilename()) root, err := rfs.Open(filepathSeparator) assert.NoError(err) diff --git a/hugolib/hugo_sites_build_errors_test.go b/hugolib/hugo_sites_build_errors_test.go index 1e53eb3c4..8e913f061 100644 --- a/hugolib/hugo_sites_build_errors_test.go +++ b/hugolib/hugo_sites_build_errors_test.go @@ -191,7 +191,7 @@ func TestSiteBuildErrors(t *testing.T) { }, assertBuildError: func(a testSiteBuildErrorAsserter, err error) { assert.Error(err) - assert.Contains(err.Error(), `"content/mytoml.md": render of "page" failed: execute of template failed: panic in Execute`) + assert.Contains(err.Error(), `execute of template failed: panic in Execute`) }, }, } diff --git a/hugolib/hugo_sites_build_test.go b/hugolib/hugo_sites_build_test.go index 727cc6ed9..f1e317f59 100644 --- a/hugolib/hugo_sites_build_test.go +++ b/hugolib/hugo_sites_build_test.go @@ -631,12 +631,10 @@ func assertShouldNotBuild(t *testing.T, sites *HugoSites) { for _, p := range s.rawAllPages { // No HTML when not processed require.Equal(t, p.shouldBuild(), bytes.Contains(p.workContent, []byte("</")), p.BaseFileName()+": "+string(p.workContent)) - // TODO(bep) 2errors - /* - require.Equal(t, p.shouldBuild(), p.content() != "", fmt.Sprintf("%v:%v", p.content(), p.shouldBuild())) - require.Equal(t, p.shouldBuild(), p.content() != "", p.BaseFileName()) - */ + require.Equal(t, p.shouldBuild(), p.content() != "", fmt.Sprintf("%v:%v", p.content(), p.shouldBuild())) + + require.Equal(t, p.shouldBuild(), p.content() != "", p.BaseFileName()) } } diff --git a/hugolib/page.go b/hugolib/page.go index df6f88b01..f7149ad74 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -1685,9 +1685,13 @@ func (p *Page) shouldRenderTo(f output.Format) bool { return found } +// RawContent returns the un-rendered source content without +// any leading front matter. func (p *Page) RawContent() string { - // TODO(bep) 2errors - return string(p.source.parsed.Input()) + if p.source.posMainContent == -1 { + return "" + } + return string(p.source.parsed.Input()[p.source.posMainContent:]) } func (p *Page) FullFilePath() string { diff --git a/hugolib/page_content.go b/hugolib/page_content.go index be015253b..dab46411e 100644 --- a/hugolib/page_content.go +++ b/hugolib/page_content.go @@ -46,11 +46,11 @@ type rawPageContent struct { hasSummaryDivider bool // The AST of the parsed page. Contains information about: - // shortcBackup3odes, front matter, summary indicators. - // TODO(bep) 2errors add this to a new rawPagecContent struct - // with frontMatterItem (pos) etc. - // * RawContent, RawContentWithoutFrontMatter + // shortcodes, front matter, summary indicators. parsed pageparser.Result + + // Returns the position in bytes after any front matter. + posMainContent int } // TODO(bep) lazy consolidate @@ -58,6 +58,7 @@ func (p *Page) mapContent() error { p.shortcodeState = newShortcodeHandler(p) s := p.shortcodeState p.renderable = true + p.source.posMainContent = -1 result := bp.GetBuffer() defer bp.PutBuffer(result) @@ -81,8 +82,8 @@ Loop: case it.Type == pageparser.TypeIgnore: case it.Type == pageparser.TypeHTMLComment: // Ignore. This is only a leading Front matter comment. - case it.Type == pageparser.TypeHTMLDocument: - // This is HTML only. No shortcode, front matter etc. + case it.Type == pageparser.TypeHTMLStart: + // This is HTML without front matter. It can still have shortcodes. p.renderable = false result.Write(it.Val) case it.IsFrontMatter(): @@ -99,12 +100,17 @@ Loop: return err } + next := iter.Peek() + if !next.IsDone() { + p.source.posMainContent = next.Pos + } + if !p.shouldBuild() { // Nothing more to do. return nil } - case it.Type == pageparser.TypeLeadSummaryDivider, it.Type == pageparser.TypeSummaryDividerOrg: + case it.Type == pageparser.TypeLeadSummaryDivider: result.Write(internalSummaryDivider) p.source.hasSummaryDivider = true // Need to determine if the page is truncated. @@ -172,7 +178,6 @@ func (p *Page) parse(reader io.Reader) error { parsed: parseResult, } - // TODO(bep) 2errors p.lang = p.File.Lang() if p.s != nil && p.s.owner != nil { diff --git a/hugolib/page_test.go b/hugolib/page_test.go index ced7e78d8..4499a97e0 100644 --- a/hugolib/page_test.go +++ b/hugolib/page_test.go @@ -767,8 +767,7 @@ Simple Page With Some Date` } // Issue #2601 -// TODO(bep) 2errors -func _TestPageRawContent(t *testing.T) { +func TestPageRawContent(t *testing.T) { t.Parallel() cfg, fs := newTestCfg() @@ -784,7 +783,7 @@ title: Raw require.Len(t, s.RegularPages, 1) p := s.RegularPages[0] - require.Contains(t, p.RawContent(), "**Raw**") + require.Equal(t, p.RawContent(), "**Raw**") } @@ -1042,8 +1041,7 @@ func TestWordCountWithAllCJKRunesWithoutHasCJKLanguage(t *testing.T) { testAllMarkdownEnginesForPages(t, assertFunc, nil, simplePageWithAllCJKRunes) } -// TODO(bep) 2errors -func _TestWordCountWithAllCJKRunesHasCJKLanguage(t *testing.T) { +func TestWordCountWithAllCJKRunesHasCJKLanguage(t *testing.T) { t.Parallel() settings := map[string]interface{}{"hasCJKLanguage": true} @@ -1056,8 +1054,7 @@ func _TestWordCountWithAllCJKRunesHasCJKLanguage(t *testing.T) { testAllMarkdownEnginesForPages(t, assertFunc, settings, simplePageWithAllCJKRunes) } -// TODO(bep) 2errors -func _TestWordCountWithMainEnglishWithCJKRunes(t *testing.T) { +func TestWordCountWithMainEnglishWithCJKRunes(t *testing.T) { t.Parallel() settings := map[string]interface{}{"hasCJKLanguage": true} @@ -1164,7 +1161,6 @@ func TestShouldRenderContent(t *testing.T) { render bool }{ {contentNoFrontmatter, true}, - // TODO(bep) 2errors {invalidFrontmatterShortDelim, true}, {renderNoFrontmatter, false}, {contentWithCommentedFrontmatter, true}, {contentWithCommentedTextFrontmatter, true}, diff --git a/hugolib/site_render.go b/hugolib/site_render.go index 6583acd06..a8cdca255 100644 --- a/hugolib/site_render.go +++ b/hugolib/site_render.go @@ -134,8 +134,7 @@ func pageRenderer(s *Site, pages <-chan *Page, results chan<- error, wg *sync.Wa if shouldRender { if err := pageOutput.renderResources(); err != nil { - // TODO(bep) 2errors - s.Log.ERROR.Printf("Failed to render resources for page %q: %s", page, err) + s.SendError(page.errorf(err, "failed to render page resources")) continue } } @@ -147,7 +146,7 @@ func pageRenderer(s *Site, pages <-chan *Page, results chan<- error, wg *sync.Wa } else { layouts, err = s.layouts(pageOutput) if err != nil { - s.Log.ERROR.Printf("Failed to resolve layout output %q for page %q: %s", outFormat.Name, page, err) + s.Log.ERROR.Printf("Failed to resolve layout for output %q for page %q: %s", outFormat.Name, page, err) continue } } diff --git a/hugolib/site_test.go b/hugolib/site_test.go index 2142025cc..0fd3a397a 100644 --- a/hugolib/site_test.go +++ b/hugolib/site_test.go @@ -451,8 +451,7 @@ func doTestSectionNaming(t *testing.T, canonify, uglify, pluralize bool) { } -// TODO(bep) 2errors -func _TestSkipRender(t *testing.T) { +func TestSkipRender(t *testing.T) { t.Parallel() sources := [][2]string{ {filepath.FromSlash("sect/doc1.html"), "---\nmarkup: markdown\n---\n# title\nsome *content*"}, diff --git a/parser/pageparser/item.go b/parser/pageparser/item.go index 049db584a..0567bd8b9 100644 --- a/parser/pageparser/item.go +++ b/parser/pageparser/item.go @@ -103,10 +103,9 @@ const ( tEOF // page items - TypeHTMLDocument // document starting with < as first non-whitespace + TypeHTMLStart // document starting with < as first non-whitespace TypeHTMLComment // We ignore leading comments - TypeLeadSummaryDivider // <!--more--> - TypeSummaryDividerOrg // # more + TypeLeadSummaryDivider // <!--more-->, # more TypeFrontMatterYAML TypeFrontMatterTOML TypeFrontMatterJSON diff --git a/parser/pageparser/pagelexer.go b/parser/pageparser/pagelexer.go index e02475d42..ddf109b3d 100644 --- a/parser/pageparser/pagelexer.go +++ b/parser/pageparser/pagelexer.go @@ -48,6 +48,8 @@ type pageLexer struct { start int // item start position width int // width of last element + // The summary divider to look for. + summaryDivider []byte // Set when we have parsed any summary divider summaryDividerChecked bool @@ -69,7 +71,6 @@ func (l *pageLexer) Input() []byte { // note: the input position here is normally 0 (start), but // can be set if position of first shortcode is known -// TODO(bep) 2errors byte func newPageLexer(input []byte, inputPosition int, stateStart stateFunc) *pageLexer { lexer := &pageLexer{ input: input, @@ -117,7 +118,7 @@ var ( delimTOML = []byte("+++") delimYAML = []byte("---") delimOrg = []byte("#+") - htmlCOmmentStart = []byte("<!--") + htmlCommentStart = []byte("<!--") htmlCOmmentEnd = []byte("-->") ) @@ -195,17 +196,18 @@ func (l *pageLexer) consumeCRLF() bool { func lexMainSection(l *pageLexer) stateFunc { // Fast forward as far as possible. - var l1, l2, l3 int - if !l.summaryDividerChecked { - // TODO(bep) 2errors make the summary divider per type - l1 = l.index(summaryDivider) - l2 = l.index(summaryDividerOrg) - if l1 == -1 && l2 == -1 { + var l1, l2 int + + if !l.summaryDividerChecked && l.summaryDivider != nil { + l1 = l.index(l.summaryDivider) + if l1 == -1 { l.summaryDividerChecked = true } } - l3 = l.index(leftDelimSc) - skip := minPositiveIndex(l1, l2, l3) + + l2 = l.index(leftDelimSc) + skip := minPositiveIndex(l1, l2) + if skip > 0 { l.pos += skip } @@ -225,23 +227,14 @@ func lexMainSection(l *pageLexer) stateFunc { return lexShortcodeLeftDelim } - if !l.summaryDividerChecked { - if l.hasPrefix(summaryDivider) { + if !l.summaryDividerChecked && l.summaryDivider != nil { + if l.hasPrefix(l.summaryDivider) { if l.pos > l.start { l.emit(tText) } l.summaryDividerChecked = true - l.pos += len(summaryDivider) - //l.consumeCRLF() + l.pos += len(l.summaryDivider) l.emit(TypeLeadSummaryDivider) - } else if l.hasPrefix(summaryDividerOrg) { - if l.pos > l.start { - l.emit(tText) - } - l.summaryDividerChecked = true - l.pos += len(summaryDividerOrg) - //l.consumeCRLF() - l.emit(TypeSummaryDividerOrg) } } @@ -261,6 +254,8 @@ func (l *pageLexer) isShortCodeStart() bool { } func lexIntroSection(l *pageLexer) stateFunc { + l.summaryDivider = summaryDivider + LOOP: for { r := l.next() @@ -283,7 +278,7 @@ LOOP: // No front matter. if r == '<' { l.backup() - if l.hasPrefix(htmlCOmmentStart) { + if l.hasPrefix(htmlCommentStart) { right := l.index(htmlCOmmentEnd) if right == -1 { return l.errorf("starting HTML comment with no end") @@ -291,10 +286,14 @@ LOOP: l.pos += right + len(htmlCOmmentEnd) l.emit(TypeHTMLComment) } else { - // Not need to look further. Hugo treats this as plain HTML, - // no front matter, no shortcodes, no nothing. - l.pos = len(l.input) - l.emit(TypeHTMLDocument) + if l.pos > l.start { + l.emit(tText) + } + l.next() + // This is the start of a plain HTML document with no + // front matter. I still can contain shortcodes, so we + // have to keep looking. + l.emit(TypeHTMLStart) } } break LOOP @@ -365,10 +364,11 @@ func lexFrontMatterOrgMode(l *pageLexer) stateFunc { #+DESCRIPTION: Just another golang parser for org content! */ + l.summaryDivider = summaryDividerOrg + l.backup() if !l.hasPrefix(delimOrg) { - // TODO(bep) consider error return lexMainSection } diff --git a/parser/pageparser/pageparser.go b/parser/pageparser/pageparser.go index 6e75f195a..754397121 100644 --- a/parser/pageparser/pageparser.go +++ b/parser/pageparser/pageparser.go @@ -48,7 +48,7 @@ func Parse(r io.Reader) (Result, error) { } func parseMainSection(input []byte, from int) Result { - lexer := newPageLexer(input, from, lexMainSection) // TODO(bep) 2errors + lexer := newPageLexer(input, from, lexMainSection) lexer.run() return lexer } diff --git a/parser/pageparser/pageparser_intro_test.go b/parser/pageparser/pageparser_intro_test.go index 32de6dc44..ba4a2c84b 100644 --- a/parser/pageparser/pageparser_intro_test.go +++ b/parser/pageparser/pageparser_intro_test.go @@ -38,7 +38,7 @@ var ( tstFrontMatterJSON = nti(TypeFrontMatterJSON, tstJSON+"\r\n") tstSomeText = nti(tText, "\nSome text.\n") tstSummaryDivider = nti(TypeLeadSummaryDivider, "<!--more-->") - tstSummaryDividerOrg = nti(TypeSummaryDividerOrg, "# more") + tstHtmlStart = nti(TypeHTMLStart, "<") tstORG = ` #+TITLE: T1 @@ -54,8 +54,8 @@ var crLfReplacer = strings.NewReplacer("\r", "#", "\n", "$") var frontMatterTests = []lexerTest{ {"empty", "", []Item{tstEOF}}, {"Byte order mark", "\ufeff\nSome text.\n", []Item{nti(TypeIgnore, "\ufeff"), tstSomeText, tstEOF}}, - {"HTML Document", ` <html> `, []Item{nti(TypeHTMLDocument, " <html> "), tstEOF}}, - {"HTML Document 2", `<html><h1>Hugo Rocks</h1></html>`, []Item{nti(TypeHTMLDocument, "<html><h1>Hugo Rocks</h1></html>"), tstEOF}}, + {"HTML Document", ` <html> `, []Item{nti(tText, " "), tstHtmlStart, nti(tText, "html> "), tstEOF}}, + {"HTML Document with shortcode", `<html>{{< sc1 >}}</html>`, []Item{tstHtmlStart, nti(tText, "html>"), tstLeftNoMD, tstSC1, tstRightNoMD, nti(tText, "</html>"), tstEOF}}, {"No front matter", "\nSome text.\n", []Item{tstSomeText, tstEOF}}, {"YAML front matter", "---\nfoo: \"bar\"\n---\n\nSome text.\n", []Item{tstFrontMatterYAML, tstSomeText, tstEOF}}, {"YAML empty front matter", "---\n---\n\nSome text.\n", []Item{nti(TypeFrontMatterYAML, ""), tstSomeText, tstEOF}}, @@ -65,7 +65,7 @@ var frontMatterTests = []lexerTest{ {"TOML front matter", "+++\nfoo = \"bar\"\n+++\n\nSome text.\n", []Item{tstFrontMatterTOML, tstSomeText, tstEOF}}, {"JSON front matter", tstJSON + "\r\n\nSome text.\n", []Item{tstFrontMatterJSON, tstSomeText, tstEOF}}, {"ORG front matter", tstORG + "\nSome text.\n", []Item{tstFrontMatterORG, tstSomeText, tstEOF}}, - {"Summary divider ORG", tstORG + "\nSome text.\n# more\nSome text.\n", []Item{tstFrontMatterORG, tstSomeText, tstSummaryDividerOrg, tstSomeText, tstEOF}}, + {"Summary divider ORG", tstORG + "\nSome text.\n# more\nSome text.\n", []Item{tstFrontMatterORG, tstSomeText, nti(TypeLeadSummaryDivider, "# more"), tstSomeText, tstEOF}}, {"Summary divider", "+++\nfoo = \"bar\"\n+++\n\nSome text.\n<!--more-->\nSome text.\n", []Item{tstFrontMatterTOML, tstSomeText, tstSummaryDivider, tstSomeText, tstEOF}}, } diff --git a/tpl/data/data.go b/tpl/data/data.go index 03fd27606..8b3eb8292 100644 --- a/tpl/data/data.go +++ b/tpl/data/data.go @@ -120,7 +120,6 @@ func (ns *Namespace) GetJSON(urlParts ...string) (v interface{}, err error) { if err != nil { return nil, _errors.Wrapf(err, "failed to get getJSON resource %q", url) - return nil, nil } return } diff --git a/tpl/template.go b/tpl/template.go index 12a4607fb..968705493 100644 --- a/tpl/template.go +++ b/tpl/template.go @@ -179,7 +179,7 @@ func (t *TemplateAdapter) addFileContext(name string, inerr error) error { } return false } - // TODO(bep) 2errors text vs HTML + fe, ok := herrors.WithFileContext(ferr, realFilename, f, lineMatcher) if ok || !hasMaster { return fe |