diff options
author | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2022-07-07 16:11:47 +0200 |
---|---|---|
committer | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2022-07-09 16:03:11 +0200 |
commit | 223bf2800488ad5d38854bbb595d789bc35ebe32 (patch) | |
tree | 84b04f8f50b4450cf5f87943befe31fd9d7b8b90 /parser | |
parent | 72b0ccdb010fcdfeb3bb4a955d4fc04529816c0d (diff) |
parser/pageparser: Don't store the byte slices
On its own this change doesn't do any magic, but this is part of a bigger picture about making Hugo leaner in the
memory usage department.
Diffstat (limited to 'parser')
-rw-r--r-- | parser/pageparser/item.go | 71 | ||||
-rw-r--r-- | parser/pageparser/item_test.go | 27 | ||||
-rw-r--r-- | parser/pageparser/pagelexer.go | 54 | ||||
-rw-r--r-- | parser/pageparser/pageparser.go | 79 | ||||
-rw-r--r-- | parser/pageparser/pageparser_intro_test.go | 118 | ||||
-rw-r--r-- | parser/pageparser/pageparser_main_test.go | 22 | ||||
-rw-r--r-- | parser/pageparser/pageparser_shortcode_test.go | 140 | ||||
-rw-r--r-- | parser/pageparser/pageparser_test.go | 13 |
8 files changed, 354 insertions, 170 deletions
diff --git a/parser/pageparser/item.go b/parser/pageparser/item.go index 52546be41..2083be70a 100644 --- a/parser/pageparser/item.go +++ b/parser/pageparser/item.go @@ -22,21 +22,59 @@ import ( "github.com/yuin/goldmark/util" ) +type lowHigh struct { + Low int + High int +} + type Item struct { - Type ItemType - Pos int - Val []byte + Type ItemType + Err error + + // The common case is a single segment. + low int + high int + + // This is the uncommon case. + segments []lowHigh + + // Used for validation. + firstByte byte + isString bool } type Items []Item -func (i Item) ValStr() string { - return string(i.Val) +func (i Item) Pos() int { + if len(i.segments) > 0 { + return i.segments[0].Low + } + return i.low +} + +func (i Item) Val(source []byte) []byte { + if len(i.segments) == 0 { + return source[i.low:i.high] + } + + if len(i.segments) == 1 { + return source[i.segments[0].Low:i.segments[0].High] + } + + var b bytes.Buffer + for _, s := range i.segments { + b.Write(source[s.Low:s.High]) + } + return b.Bytes() +} + +func (i Item) ValStr(source []byte) string { + return string(i.Val(source)) } -func (i Item) ValTyped() any { - str := i.ValStr() +func (i Item) ValTyped(source []byte) any { + str := i.ValStr(source) if i.isString { // A quoted value that is a string even if it looks like a number etc. return str @@ -73,8 +111,8 @@ func (i Item) IsIndentation() bool { return i.Type == tIndentation } -func (i Item) IsNonWhitespace() bool { - return len(bytes.TrimSpace(i.Val)) > 0 +func (i Item) IsNonWhitespace(source []byte) bool { + return len(bytes.TrimSpace(i.Val(source))) > 0 } func (i Item) IsShortcodeName() bool { @@ -125,20 +163,21 @@ func (i Item) IsError() bool { return i.Type == tError } -func (i Item) String() string { +func (i Item) ToString(source []byte) string { + val := i.Val(source) switch { case i.Type == tEOF: return "EOF" case i.Type == tError: - return string(i.Val) + return string(val) case i.Type == tIndentation: - return fmt.Sprintf("%s:[%s]", i.Type, util.VisualizeSpaces(i.Val)) + return fmt.Sprintf("%s:[%s]", i.Type, util.VisualizeSpaces(val)) case i.Type > tKeywordMarker: - return fmt.Sprintf("<%s>", i.Val) - case len(i.Val) > 50: - return fmt.Sprintf("%v:%.20q...", i.Type, i.Val) + return fmt.Sprintf("<%s>", val) + case len(val) > 50: + return fmt.Sprintf("%v:%.20q...", i.Type, val) } - return fmt.Sprintf("%v:[%s]", i.Type, i.Val) + return fmt.Sprintf("%v:[%s]", i.Type, val) } type ItemType int diff --git a/parser/pageparser/item_test.go b/parser/pageparser/item_test.go index cd01202c6..db4cc127a 100644 --- a/parser/pageparser/item_test.go +++ b/parser/pageparser/item_test.go @@ -22,13 +22,22 @@ import ( func TestItemValTyped(t *testing.T) { c := qt.New(t) - c.Assert(Item{Val: []byte("3.14")}.ValTyped(), qt.Equals, float64(3.14)) - c.Assert(Item{Val: []byte(".14")}.ValTyped(), qt.Equals, float64(.14)) - c.Assert(Item{Val: []byte("314")}.ValTyped(), qt.Equals, 314) - c.Assert(Item{Val: []byte("314x")}.ValTyped(), qt.Equals, "314x") - c.Assert(Item{Val: []byte("314 ")}.ValTyped(), qt.Equals, "314 ") - c.Assert(Item{Val: []byte("314"), isString: true}.ValTyped(), qt.Equals, "314") - c.Assert(Item{Val: []byte("true")}.ValTyped(), qt.Equals, true) - c.Assert(Item{Val: []byte("false")}.ValTyped(), qt.Equals, false) - c.Assert(Item{Val: []byte("trues")}.ValTyped(), qt.Equals, "trues") + source := []byte("3.14") + c.Assert(Item{low: 0, high: len(source)}.ValTyped(source), qt.Equals, float64(3.14)) + source = []byte(".14") + c.Assert(Item{low: 0, high: len(source)}.ValTyped(source), qt.Equals, float64(0.14)) + source = []byte("314") + c.Assert(Item{low: 0, high: len(source)}.ValTyped(source), qt.Equals, 314) + source = []byte("314") + c.Assert(Item{low: 0, high: len(source), isString: true}.ValTyped(source), qt.Equals, "314") + source = []byte("314x") + c.Assert(Item{low: 0, high: len(source)}.ValTyped(source), qt.Equals, "314x") + source = []byte("314 ") + c.Assert(Item{low: 0, high: len(source)}.ValTyped(source), qt.Equals, "314 ") + source = []byte("true") + c.Assert(Item{low: 0, high: len(source)}.ValTyped(source), qt.Equals, true) + source = []byte("false") + c.Assert(Item{low: 0, high: len(source)}.ValTyped(source), qt.Equals, false) + source = []byte("trued") + } diff --git a/parser/pageparser/pagelexer.go b/parser/pageparser/pagelexer.go index 770f26eb9..a7e6b6cd4 100644 --- a/parser/pageparser/pagelexer.go +++ b/parser/pageparser/pagelexer.go @@ -54,7 +54,7 @@ type pageLexer struct { // Implement the Result interface func (l *pageLexer) Iterator() *Iterator { - return l.newIterator() + return NewIterator(l.items) } func (l *pageLexer) Input() []byte { @@ -85,10 +85,6 @@ func newPageLexer(input []byte, stateStart stateFunc, cfg Config) *pageLexer { return lexer } -func (l *pageLexer) newIterator() *Iterator { - return &Iterator{l: l, lastPos: -1} -} - // main loop func (l *pageLexer) run() *pageLexer { for l.state = l.stateStart; l.state != nil; { @@ -136,6 +132,13 @@ func (l *pageLexer) backup() { l.pos -= l.width } +func (l *pageLexer) append(item Item) { + if item.Pos() < len(l.input) { + item.firstByte = l.input[item.Pos()] + } + l.items = append(l.items, item) +} + // sends an item back to the client. func (l *pageLexer) emit(t ItemType) { defer func() { @@ -151,11 +154,11 @@ func (l *pageLexer) emit(t ItemType) { break } if i == l.start && b != '\n' { - l.items = append(l.items, Item{tIndentation, l.start, l.input[l.start:l.pos], false}) + l.append(Item{Type: tIndentation, low: l.start, high: l.pos}) return } else if b == '\n' && i < l.pos-1 { - l.items = append(l.items, Item{t, l.start, l.input[l.start : i+1], false}) - l.items = append(l.items, Item{tIndentation, i + 1, l.input[i+1 : l.pos], false}) + l.append(Item{Type: t, low: l.start, high: i + 1}) + l.append(Item{Type: tIndentation, low: i + 1, high: l.pos}) return } else if b == '\n' && i == l.pos-1 { break @@ -164,13 +167,13 @@ func (l *pageLexer) emit(t ItemType) { } } - l.items = append(l.items, Item{t, l.start, l.input[l.start:l.pos], false}) + l.append(Item{Type: t, low: l.start, high: l.pos}) } // sends a string item back to the client. func (l *pageLexer) emitString(t ItemType) { - l.items = append(l.items, Item{t, l.start, l.input[l.start:l.pos], true}) + l.append(Item{Type: t, low: l.start, high: l.pos, isString: true}) l.start = l.pos } @@ -180,14 +183,33 @@ func (l *pageLexer) isEOF() bool { // special case, do not send '\\' back to client func (l *pageLexer) ignoreEscapesAndEmit(t ItemType, isString bool) { - val := bytes.Map(func(r rune) rune { + i := l.start + k := i + + var segments []lowHigh + + for i < l.pos { + r, w := utf8.DecodeRune(l.input[i:l.pos]) if r == '\\' { - return -1 + if i > k { + segments = append(segments, lowHigh{k, i}) + } + l.append(Item{Type: TypeIgnore, low: i, high: i + w}) + k = i + w } - return r - }, l.input[l.start:l.pos]) - l.items = append(l.items, Item{t, l.start, val, isString}) + i += w + } + + if k < l.pos { + segments = append(segments, lowHigh{k, l.pos}) + } + + if len(segments) > 0 { + l.append(Item{Type: t, segments: segments}) + } + l.start = l.pos + } // gets the current value (for debugging and error handling) @@ -204,7 +226,7 @@ var lf = []byte("\n") // nil terminates the parser func (l *pageLexer) errorf(format string, args ...any) stateFunc { - l.items = append(l.items, Item{tError, l.start, []byte(fmt.Sprintf(format, args...)), true}) + l.append(Item{Type: tError, Err: fmt.Errorf(format, args...)}) return nil } diff --git a/parser/pageparser/pageparser.go b/parser/pageparser/pageparser.go index 67abefc30..0a9fc61af 100644 --- a/parser/pageparser/pageparser.go +++ b/parser/pageparser/pageparser.go @@ -15,6 +15,7 @@ package pageparser import ( "bytes" + "errors" "fmt" "io" "io/ioutil" @@ -33,9 +34,6 @@ type Result interface { var _ Result = (*pageLexer)(nil) // Parse parses the page in the given reader according to the given Config. -// TODO(bep) now that we have improved the "lazy order" init, it *may* be -// some potential saving in doing a buffered approach where the first pass does -// the frontmatter only. func Parse(r io.Reader, cfg Config) (Result, error) { return parseSection(r, cfg, lexIntroSection) } @@ -63,12 +61,12 @@ func ParseFrontMatterAndContent(r io.Reader) (ContentFrontMatter, error) { walkFn := func(item Item) bool { if frontMatterSource != nil { // The rest is content. - cf.Content = psr.Input()[item.Pos:] + cf.Content = psr.Input()[item.low:] // Done return false } else if item.IsFrontMatter() { cf.FrontMatterFormat = FormatFromFrontMatterType(item.Type) - frontMatterSource = item.Val + frontMatterSource = item.Val(psr.Input()) } return true } @@ -113,10 +111,15 @@ func parseBytes(b []byte, cfg Config, start stateFunc) (Result, error) { return lexer, nil } +// NewIterator creates a new Iterator. +func NewIterator(items Items) *Iterator { + return &Iterator{items: items, lastPos: -1} +} + // An Iterator has methods to iterate a parsed page with support going back // if needed. type Iterator struct { - l *pageLexer + items Items lastPos int // position of the last item returned by nextItem } @@ -126,19 +129,14 @@ func (t *Iterator) Next() Item { return t.Current() } -// Input returns the input source. -func (t *Iterator) Input() []byte { - return t.l.Input() -} - -var errIndexOutOfBounds = Item{tError, 0, []byte("no more tokens"), true} +var errIndexOutOfBounds = Item{Type: tError, Err: errors.New("no more tokens")} // Current will repeatably return the current item. func (t *Iterator) Current() Item { - if t.lastPos >= len(t.l.items) { + if t.lastPos >= len(t.items) { return errIndexOutOfBounds } - return t.l.items[t.lastPos] + return t.items[t.lastPos] } // backs up one token. @@ -163,14 +161,14 @@ func (t *Iterator) IsValueNext() bool { // look at, but do not consume, the next item // repeated, sequential calls will return the same item func (t *Iterator) Peek() Item { - return t.l.items[t.lastPos+1] + return t.items[t.lastPos+1] } // PeekWalk will feed the next items in the iterator to walkFn // until it returns false. func (t *Iterator) PeekWalk(walkFn func(item Item) bool) { - for i := t.lastPos + 1; i < len(t.l.items); i++ { - item := t.l.items[i] + for i := t.lastPos + 1; i < len(t.items); i++ { + item := t.items[i] if !walkFn(item) { break } @@ -190,6 +188,49 @@ func (t *Iterator) Consume(cnt int) { } // LineNumber returns the current line number. Used for logging. -func (t *Iterator) LineNumber() int { - return bytes.Count(t.l.input[:t.Current().Pos], lf) + 1 +func (t *Iterator) LineNumber(source []byte) int { + return bytes.Count(source[:t.Current().low], lf) + 1 +} + +// IsProbablySourceOfItems returns true if the given source looks like original +// source of the items. +// There may be some false positives, but that is highly unlikely and good enough +// for the planned purpose. +// It will also return false if the last item is not EOF (error situations) and +// true if both source and items are empty. +func IsProbablySourceOfItems(source []byte, items Items) bool { + if len(source) == 0 && len(items) == 0 { + return false + } + if len(items) == 0 { + return false + } + + last := items[len(items)-1] + if last.Type != tEOF { + return false + } + + if last.Pos() != len(source) { + return false + } + + for _, item := range items { + if item.Type == tError { + return false + } + if item.Type == tEOF { + return true + } + + if item.Pos() >= len(source) { + return false + } + + if item.firstByte != source[item.Pos()] { + return false + } + } + + return true } diff --git a/parser/pageparser/pageparser_intro_test.go b/parser/pageparser/pageparser_intro_test.go index 1b903d546..1b2d59ccc 100644 --- a/parser/pageparser/pageparser_intro_test.go +++ b/parser/pageparser/pageparser_intro_test.go @@ -15,19 +15,25 @@ package pageparser import ( "fmt" - "reflect" "strings" "testing" + + qt "github.com/frankban/quicktest" ) type lexerTest struct { name string input string - items []Item + items []typeText +} + +type typeText struct { + typ ItemType + text string } -func nti(tp ItemType, val string) Item { - return Item{tp, 0, []byte(val), false} +func nti(tp ItemType, val string) typeText { + return typeText{typ: tp, text: val} } var ( @@ -52,48 +58,79 @@ var crLfReplacer = strings.NewReplacer("\r", "#", "\n", "$") // TODO(bep) a way to toggle ORG mode vs the rest. var frontMatterTests = []lexerTest{ - {"empty", "", []Item{tstEOF}}, - {"Byte order mark", "\ufeff\nSome text.\n", []Item{nti(TypeIgnore, "\ufeff"), tstSomeText, tstEOF}}, - {"HTML Document", ` <html> `, []Item{nti(tError, "plain HTML documents not supported")}}, - {"HTML Document with shortcode", `<html>{{< sc1 >}}</html>`, []Item{nti(tError, "plain HTML documents not supported")}}, - {"No front matter", "\nSome text.\n", []Item{tstSomeText, tstEOF}}, - {"YAML front matter", "---\nfoo: \"bar\"\n---\n\nSome text.\n", []Item{tstFrontMatterYAML, tstSomeText, tstEOF}}, - {"YAML empty front matter", "---\n---\n\nSome text.\n", []Item{nti(TypeFrontMatterYAML, ""), tstSomeText, tstEOF}}, - {"YAML commented out front matter", "<!--\n---\nfoo: \"bar\"\n---\n-->\nSome text.\n", []Item{nti(TypeIgnore, "<!--\n"), tstFrontMatterYAML, nti(TypeIgnore, "-->"), tstSomeText, tstEOF}}, - {"YAML commented out front matter, no end", "<!--\n---\nfoo: \"bar\"\n---\nSome text.\n", []Item{nti(TypeIgnore, "<!--\n"), tstFrontMatterYAML, nti(tError, "starting HTML comment with no end")}}, + {"empty", "", []typeText{tstEOF}}, + {"Byte order mark", "\ufeff\nSome text.\n", []typeText{nti(TypeIgnore, "\ufeff"), tstSomeText, tstEOF}}, + {"HTML Document", ` <html> `, []typeText{nti(tError, "plain HTML documents not supported")}}, + {"HTML Document with shortcode", `<html>{{< sc1 >}}</html>`, []typeText{nti(tError, "plain HTML documents not supported")}}, + {"No front matter", "\nSome text.\n", []typeText{tstSomeText, tstEOF}}, + {"YAML front matter", "---\nfoo: \"bar\"\n---\n\nSome text.\n", []typeText{tstFrontMatterYAML, tstSomeText, tstEOF}}, + {"YAML empty front matter", "---\n---\n\nSome text.\n", []typeText{nti(TypeFrontMatterYAML, ""), tstSomeText, tstEOF}}, + {"YAML commented out front matter", "<!--\n---\nfoo: \"bar\"\n---\n-->\nSome text.\n", []typeText{nti(TypeIgnore, "<!--\n"), tstFrontMatterYAML, nti(TypeIgnore, "-->"), tstSomeText, tstEOF}}, + {"YAML commented out front matter, no end", "<!--\n---\nfoo: \"bar\"\n---\nSome text.\n", []typeText{nti(TypeIgnore, "<!--\n"), tstFrontMatterYAML, nti(tError, "starting HTML comment with no end")}}, // Note that we keep all bytes as they are, but we need to handle CRLF - {"YAML front matter CRLF", "---\r\nfoo: \"bar\"\r\n---\n\nSome text.\n", []Item{tstFrontMatterYAMLCRLF, tstSomeText, tstEOF}}, - {"TOML front matter", "+++\nfoo = \"bar\"\n+++\n\nSome text.\n", []Item{tstFrontMatterTOML, tstSomeText, tstEOF}}, - {"JSON front matter", tstJSON + "\r\n\nSome text.\n", []Item{tstFrontMatterJSON, tstSomeText, tstEOF}}, - {"ORG front matter", tstORG + "\nSome text.\n", []Item{tstFrontMatterORG, tstSomeText, tstEOF}}, - {"Summary divider ORG", tstORG + "\nSome text.\n# more\nSome text.\n", []Item{tstFrontMatterORG, tstSomeText, nti(TypeLeadSummaryDivider, "# more\n"), nti(tText, "Some text.\n"), tstEOF}}, - {"Summary divider", "+++\nfoo = \"bar\"\n+++\n\nSome text.\n<!--more-->\nSome text.\n", []Item{tstFrontMatterTOML, tstSomeText, tstSummaryDivider, nti(tText, "Some text.\n"), tstEOF}}, - {"Summary divider same line", "+++\nfoo = \"bar\"\n+++\n\nSome text.<!--more-->Some text.\n", []Item{tstFrontMatterTOML, nti(tText, "\nSome text."), nti(TypeLeadSummaryDivider, "<!--more-->"), nti(tText, "Some text.\n"), tstEOF}}, + {"YAML front matter CRLF", "---\r\nfoo: \"bar\"\r\n---\n\nSome text.\n", []typeText{tstFrontMatterYAMLCRLF, tstSomeText, tstEOF}}, + {"TOML front matter", "+++\nfoo = \"bar\"\n+++\n\nSome text.\n", []typeText{tstFrontMatterTOML, tstSomeText, tstEOF}}, + {"JSON front matter", tstJSON + "\r\n\nSome text.\n", []typeText{tstFrontMatterJSON, tstSomeText, tstEOF}}, + {"ORG front matter", tstORG + "\nSome text.\n", []typeText{tstFrontMatterORG, tstSomeText, tstEOF}}, + {"Summary divider ORG", tstORG + "\nSome text.\n# more\nSome text.\n", []typeText{tstFrontMatterORG, tstSomeText, nti(TypeLeadSummaryDivider, "# more\n"), nti(tText, "Some text.\n"), tstEOF}}, + {"Summary divider", "+++\nfoo = \"bar\"\n+++\n\nSome text.\n<!--more-->\nSome text.\n", []typeText{tstFrontMatterTOML, tstSomeText, tstSummaryDivider, nti(tText, "Some text.\n"), tstEOF}}, + {"Summary divider same line", "+++\nfoo = \"bar\"\n+++\n\nSome text.<!--more-->Some text.\n", []typeText{tstFrontMatterTOML, nti(tText, "\nSome text."), nti(TypeLeadSummaryDivider, "<!--more-->"), nti(tText, "Some text.\n"), tstEOF}}, // https://github.com/gohugoio/hugo/issues/5402 - {"Summary and shortcode, no space", "+++\nfoo = \"bar\"\n+++\n\nSome text.\n<!--more-->{{< sc1 >}}\nSome text.\n", []Item{tstFrontMatterTOML, tstSomeText, nti(TypeLeadSummaryDivider, "<!--more-->"), tstLeftNoMD, tstSC1, tstRightNoMD, tstSomeText, tstEOF}}, + {"Summary and shortcode, no space", "+++\nfoo = \"bar\"\n+++\n\nSome text.\n<!--more-->{{< sc1 >}}\nSome text.\n", []typeText{tstFrontMatterTOML, tstSomeText, nti(TypeLeadSummaryDivider, "<!--more-->"), tstLeftNoMD, tstSC1, tstRightNoMD, tstSomeText, tstEOF}}, // https://github.com/gohugoio/hugo/issues/5464 - {"Summary and shortcode only", "+++\nfoo = \"bar\"\n+++\n{{< sc1 >}}\n<!--more-->\n{{< sc2 >}}", []Item{tstFrontMatterTOML, tstLeftNoMD, tstSC1, tstRightNoMD, tstNewline, tstSummaryDivider, tstLeftNoMD, tstSC2, tstRightNoMD, tstEOF}}, + {"Summary and shortcode only", "+++\nfoo = \"bar\"\n+++\n{{< sc1 >}}\n<!--more-->\n{{< sc2 >}}", []typeText{tstFrontMatterTOML, tstLeftNoMD, tstSC1, tstRightNoMD, tstNewline, tstSummaryDivider, tstLeftNoMD, tstSC2, tstRightNoMD, tstEOF}}, } func TestFrontMatter(t *testing.T) { t.Parallel() + c := qt.New(t) for i, test := range frontMatterTests { items := collect([]byte(test.input), false, lexIntroSection) - if !equal(items, test.items) { - got := crLfReplacer.Replace(fmt.Sprint(items)) - expected := crLfReplacer.Replace(fmt.Sprint(test.items)) - t.Errorf("[%d] %s: got\n\t%v\nexpected\n\t%v", i, test.name, got, expected) + if !equal(test.input, items, test.items) { + got := itemsToString(items, []byte(test.input)) + expected := testItemsToString(test.items) + c.Assert(got, qt.Equals, expected, qt.Commentf("Test %d: %s", i, test.name)) + } + } +} + +func itemsToString(items []Item, source []byte) string { + var sb strings.Builder + for i, item := range items { + var s string + if item.Err != nil { + s = item.Err.Error() + } else { + s = string(item.Val(source)) + } + sb.WriteString(fmt.Sprintf("%s: %s\n", item.Type, s)) + + if i < len(items)-1 { + sb.WriteString("\n") } } + return crLfReplacer.Replace(sb.String()) +} + +func testItemsToString(items []typeText) string { + var sb strings.Builder + for i, item := range items { + sb.WriteString(fmt.Sprintf("%s: %s\n", item.typ, item.text)) + + if i < len(items)-1 { + sb.WriteString("\n") + } + } + return crLfReplacer.Replace(sb.String()) } func collectWithConfig(input []byte, skipFrontMatter bool, stateStart stateFunc, cfg Config) (items []Item) { l := newPageLexer(input, stateStart, cfg) l.run() - t := l.newIterator() + iter := NewIterator(l.items) for { - item := t.Next() + item := iter.Next() items = append(items, item) if item.Type == tEOF || item.Type == tError { break @@ -108,19 +145,34 @@ func collect(input []byte, skipFrontMatter bool, stateStart stateFunc) (items [] return collectWithConfig(input, skipFrontMatter, stateStart, cfg) } +func collectStringMain(input string) []Item { + return collect([]byte(input), true, lexMainSection) +} + // no positional checking, for now ... -func equal(i1, i2 []Item) bool { - if len(i1) != len(i2) { +func equal(source string, got []Item, expect []typeText) bool { + if len(got) != len(expect) { return false } - for k := range i1 { - if i1[k].Type != i2[k].Type { + sourceb := []byte(source) + for k := range got { + g := got[k] + e := expect[k] + if g.Type != e.typ { return false } - if !reflect.DeepEqual(i1[k].Val, i2[k].Val) { + var s string + if g.Err != nil { + s = g.Err.Error() + } else { + s = string(g.Val(sourceb)) + } + + if s != e.text { return false } + } return true } diff --git a/parser/pageparser/pageparser_main_test.go b/parser/pageparser/pageparser_main_test.go index 8fed2bffa..4e3fe8e84 100644 --- a/parser/pageparser/pageparser_main_test.go +++ b/parser/pageparser/pageparser_main_test.go @@ -14,27 +14,29 @@ package pageparser import ( - "fmt" "testing" + + qt "github.com/frankban/quicktest" ) func TestMain(t *testing.T) { t.Parallel() + c := qt.New(t) mainTests := []lexerTest{ - {"emoji #1", "Some text with :emoji:", []Item{nti(tText, "Some text with "), nti(TypeEmoji, ":emoji:"), tstEOF}}, - {"emoji #2", "Some text with :emoji: and some text.", []Item{nti(tText, "Some text with "), nti(TypeEmoji, ":emoji:"), nti(tText, " and some text."), tstEOF}}, - {"looks like an emoji #1", "Some text and then :emoji", []Item{nti(tText, "Some text and then "), nti(tText, ":"), nti(tText, "emoji"), tstEOF}}, - {"looks like an emoji #2", "Some text and then ::", []Item{nti(tText, "Some text and then "), nti(tText, ":"), nti(tText, ":"), tstEOF}}, - {"looks like an emoji #3", ":Some :text", []Item{nti(tText, ":"), nti(tText, "Some "), nti(tText, ":"), nti(tText, "text"), tstEOF}}, + {"emoji #1", "Some text with :emoji:", []typeText{nti(tText, "Some text with "), nti(TypeEmoji, ":emoji:"), tstEOF}}, + {"emoji #2", "Some text with :emoji: and some text.", []typeText{nti(tText, "Some text with "), nti(TypeEmoji, ":emoji:"), nti(tText, " and some text."), tstEOF}}, + {"looks like an emoji #1", "Some text and then :emoji", []typeText{nti(tText, "Some text and then "), nti(tText, ":"), nti(tText, "emoji"), tstEOF}}, + {"looks like an emoji #2", "Some text and then ::", []typeText{nti(tText, "Some text and then "), nti(tText, ":"), nti(tText, ":"), tstEOF}}, + {"looks like an emoji #3", ":Some :text", []typeText{nti(tText, ":"), nti(tText, "Some "), nti(tText, ":"), nti(tText, "text"), tstEOF}}, } for i, test := range mainTests { items := collectWithConfig([]byte(test.input), false, lexMainSection, Config{EnableEmoji: true}) - if !equal(items, test.items) { - got := crLfReplacer.Replace(fmt.Sprint(items)) - expected := crLfReplacer.Replace(fmt.Sprint(test.items)) - t.Errorf("[%d] %s: got\n\t%v\nexpected\n\t%v", i, test.name, got, expected) + if !equal(test.input, items, test.items) { + got := itemsToString(items, []byte(test.input)) + expected := testItemsToString(test.items) + c.Assert(got, qt.Equals, expected, qt.Commentf("Test %d: %s", i, test.name)) } } } diff --git a/parser/pageparser/pageparser_shortcode_test.go b/parser/pageparser/pageparser_shortcode_test.go index ce1297573..a95d55ef3 100644 --- a/parser/pageparser/pageparser_shortcode_test.go +++ b/parser/pageparser/pageparser_shortcode_test.go @@ -15,6 +15,8 @@ package pageparser import ( "testing" + + qt "github.com/frankban/quicktest" ) var ( @@ -38,27 +40,28 @@ var ( tstParamFloat = nti(tScParam, "3.14") tstVal = nti(tScParamVal, "Hello World") tstText = nti(tText, "Hello World") + tstIgnoreEscape = nti(TypeIgnore, "\\") ) var shortCodeLexerTests = []lexerTest{ - {"empty", "", []Item{tstEOF}}, - {"spaces", " \t\n", []Item{nti(tText, " \t\n"), tstEOF}}, - {"text", `to be or not`, []Item{nti(tText, "to be or not"), tstEOF}}, - {"no markup", `{{< sc1 >}}`, []Item{tstLeftNoMD, tstSC1, tstRightNoMD, tstEOF}}, - {"with EOL", "{{< sc1 \n >}}", []Item{tstLeftNoMD, tstSC1, tstRightNoMD, tstEOF}}, + {"empty", "", []typeText{tstEOF}}, + {"spaces", " \t\n", []typeText{nti(tText, " \t\n"), tstEOF}}, + {"text", `to be or not`, []typeText{nti(tText, "to be or not"), tstEOF}}, + {"no markup", `{{< sc1 >}}`, []typeText{tstLeftNoMD, tstSC1, tstRightNoMD, tstEOF}}, + {"with EOL", "{{< sc1 \n >}}", []typeText{tstLeftNoMD, tstSC1, tstRightNoMD, tstEOF}}, - {"forward slash inside name", `{{< sc/sub >}}`, []Item{tstLeftNoMD, tstSCSlash, tstRightNoMD, tstEOF}}, + {"forward slash inside name", `{{< sc/sub >}}`, []typeText{tstLeftNoMD, tstSCSlash, tstRightNoMD, tstEOF}}, - {"simple with markup", `{{% sc1 %}}`, []Item{tstLeftMD, tstSC1, tstRightMD, tstEOF}}, - {"with spaces", `{{< sc1 >}}`, []Item{tstLeftNoMD, tstSC1, tstRightNoMD, tstEOF}}, - {"indented on new line", "Hello\n {{% sc1 %}}", []Item{nti(tText, "Hello\n"), nti(tIndentation, " "), tstLeftMD, tstSC1, tstRightMD, tstEOF}}, - {"indented on new line tab", "Hello\n\t{{% sc1 %}}", []Item{nti(tText, "Hello\n"), nti(tIndentation, "\t"), tstLeftMD, tstSC1, tstRightMD, tstEOF}}, - {"indented on first line", " {{% sc1 %}}", []Item{nti(tIndentation, " "), tstLeftMD, tstSC1, tstRightMD, tstEOF}}, - {"mismatched rightDelim", `{{< sc1 %}}`, []Item{ + {"simple with markup", `{{% sc1 %}}`, []typeText{tstLeftMD, tstSC1, tstRightMD, tstEOF}}, + {"with spaces", `{{< sc1 >}}`, []typeText{tstLeftNoMD, tstSC1, tstRightNoMD, tstEOF}}, + {"indented on new line", "Hello\n {{% sc1 %}}", []typeText{nti(tText, "Hello\n"), nti(tIndentation, " "), tstLeftMD, tstSC1, tstRightMD, tstEOF}}, + {"indented on new line tab", "Hello\n\t{{% sc1 %}}", []typeText{nti(tText, "Hello\n"), nti(tIndentation, "\t"), tstLeftMD, tstSC1, tstRightMD, tstEOF}}, + {"indented on first line", " {{% sc1 %}}", []typeText{nti(tIndentation, " "), tstLeftMD, tstSC1, tstRightMD, tstEOF}}, + {"mismatched rightDelim", `{{< sc1 %}}`, []typeText{ tstLeftNoMD, tstSC1, nti(tError, "unrecognized character in shortcode action: U+0025 '%'. Note: Parameters with non-alphanumeric args must be quoted"), }}, - {"inner, markup", `{{% sc1 %}} inner {{% /sc1 %}}`, []Item{ + {"inner, markup", `{{% sc1 %}} inner {{% /sc1 %}}`, []typeText{ tstLeftMD, tstSC1, tstRightMD, @@ -69,79 +72,79 @@ var shortCodeLexerTests = []lexerTest{ tstRightMD, tstEOF, }}, - {"close, but no open", `{{< /sc1 >}}`, []Item{ + {"close, but no open", `{{< /sc1 >}}`, []typeText{ tstLeftNoMD, nti(tError, "got closing shortcode, but none is open"), }}, - {"close wrong", `{{< sc1 >}}{{< /another >}}`, []Item{ + {"close wrong", `{{< sc1 >}}{{< /another >}}`, []typeText{ |