diff options
author | Noah Campbell <noahcampbell@gmail.com> | 2013-08-05 07:53:58 -0700 |
---|---|---|
committer | Noah Campbell <noahcampbell@gmail.com> | 2013-08-09 17:36:32 -0700 |
commit | 085ce15f7c041ba71772a0cd6eed4f06b6b0ca34 (patch) | |
tree | e3c0b6c8a48b60374ef4cff2fade9116f257f921 | |
parent | 274d324c8bb818a76ffc5c35afcc33f4cf9eb5c3 (diff) |
Adding ability to read from io.Reader
This allows for testing without relying on the file system. Parsing algorithm to not read the entire file into memory.
-rw-r--r-- | hugolib/page.go | 230 | ||||
-rw-r--r-- | hugolib/page_test.go | 124 | ||||
-rw-r--r-- | hugolib/path_seperators_test.go | 20 |
3 files changed, 279 insertions, 95 deletions
diff --git a/hugolib/page.go b/hugolib/page.go index b6a8498f7..11ca82ddb 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -14,12 +14,15 @@ package hugolib import ( + "bufio" "bytes" "encoding/json" + "errors" "fmt" "github.com/BurntSushi/toml" "github.com/theplant/blackfriday" "html/template" + "io" "io/ioutil" "launchpad.net/goyaml" "os" @@ -28,6 +31,7 @@ import ( "sort" "strings" "time" + "unicode" ) var _ = filepath.Base("") @@ -126,6 +130,22 @@ func (page *Page) Layout(l ...string) string { return strings.ToLower(page.Type()) + "/" + layout + ".html" } +func ReadFrom(buf io.Reader, name string) (page *Page, err error) { + if len(name) == 0 { + return nil, errors.New("Zero length page name") + } + + p := initializePage(name) + + if err = p.parse(buf); err != nil { + return + } + + p.analyzePage() + + return &p, nil +} + // TODO should return errors as well // TODO new page should return just a page // TODO initalize separately... load from reader (file, or []byte) @@ -133,7 +153,6 @@ func NewPage(filename string) *Page { p := initializePage(filename) if err := p.buildPageFromFile(); err != nil { fmt.Println(err) - os.Exit(1) } p.analyzePage() @@ -146,49 +165,6 @@ func (p *Page) analyzePage() { p.FuzzyWordCount = int((p.WordCount+100)/100) * 100 } -// TODO //rewrite to use byte methods instead -func (page *Page) parseYamlMetaData(data []byte) ([]string, error) { - var err error - - datum, lines := splitPageContent(data, "---", "---") - d, err := page.handleYamlMetaData([]byte(strings.Join(datum, "\n"))) - - if err != nil { - return lines, err - } - - err = page.handleMetaData(d) - return lines, err -} - -func (page *Page) parseTomlMetaData(data []byte) ([]string, error) { - var err error - - datum, lines := splitPageContent(data, "+++", "+++") - d, err := page.handleTomlMetaData([]byte(strings.Join(datum, "\n"))) - - if err != nil { - return lines, err - } - - err = page.handleMetaData(d) - return lines, err -} - -func (page *Page) parseJsonMetaData(data []byte) ([]string, error) { - var err error - - datum, lines := splitPageContent(data, "{", "}") - d, err := page.handleJsonMetaData([]byte(strings.Join(datum, "\n"))) - - if err != nil { - return lines, err - } - - err = page.handleMetaData(d) - return lines, err -} - func splitPageContent(data []byte, start string, end string) ([]string, []string) { lines := strings.Split(string(data), "\n") datum := lines[0:] @@ -211,18 +187,6 @@ func splitPageContent(data []byte, start string, end string) ([]string, []string break } } - } else { // Start token & end token are the same - for i, line := range lines { - if found == 1 && strings.HasPrefix(line, end) { - datum = lines[1:i] - lines = lines[i+1:] - break - } - - if found == 0 && strings.HasPrefix(line, start) { - found = 1 - } - } } return datum, lines } @@ -272,7 +236,7 @@ func (page *Page) handleJsonMetaData(datum []byte) (interface{}, error) { return f, nil } -func (page *Page) handleMetaData(f interface{}) error { +func (page *Page) update(f interface{}) error { m := f.(map[string]interface{}) for k, v := range m { @@ -304,7 +268,6 @@ func (page *Page) handleMetaData(f interface{}) error { page.Status = interfaceToString(v) default: // If not one of the explicit values, store in Params - //fmt.Println(strings.ToLower(k)) switch vv := v.(type) { case string: // handle string values page.Params[strings.ToLower(k)] = vv @@ -340,25 +303,106 @@ func (page *Page) GetParam(key string) interface{} { return nil } -func (page *Page) Err(message string) { - fmt.Println(page.FileName + " : " + message) +// TODO return error on last line instead of nil +func (page *Page) parseFrontMatter(data *bufio.Reader) (err error) { + + if err = checkEmpty(data); err != nil { + return err + } + + var mark rune + if mark, err = chompWhitespace(data); err != nil { + return err + } + + f := page.detectFrontMatter(mark) + if f == nil { + return errors.New("unable to match beginning front matter delimiter") + } + + if found, err := beginFrontMatter(data, f); err != nil || !found { + return errors.New("unable to match beginning front matter delimiter") + } + + var frontmatter = new(bytes.Buffer) + for { + line, _, err := data.ReadLine() + if err != nil { + if err == io.EOF { + return errors.New("unable to match ending front matter delimiter") + } + return err + } + if bytes.Equal(line, f.markend) { + break + } + frontmatter.Write(line) + frontmatter.Write([]byte{'\n'}) + } + + metadata, err := f.parse(frontmatter.Bytes()) + if err != nil { + return err + } + + if err = page.update(metadata); err != nil { + return err + } + + return } -// TODO return error on last line instead of nil -func (page *Page) parseFileHeading(data []byte) ([]string, error) { - if len(data) == 0 { - page.Err("Empty File, skipping") - } else { - switch data[0] { - case '{': - return page.parseJsonMetaData(data) - case '-': - return page.parseYamlMetaData(data) - case '+': - return page.parseTomlMetaData(data) +func checkEmpty(data *bufio.Reader) (err error) { + if _, _, err = data.ReadRune(); err != nil { + return errors.New("unable to locate front matter") + } + if err = data.UnreadRune(); err != nil { + return errors.New("unable to unread first charactor in page buffer.") + } + return +} + +type frontmatterType struct { + markstart, markend []byte + parse func([]byte) (interface{}, error) +} + +func (page *Page) detectFrontMatter(mark rune) (f *frontmatterType) { + switch mark { + case '-': + return &frontmatterType{[]byte{'-', '-', '-'}, []byte{'-', '-', '-'}, page.handleYamlMetaData} + case '+': + return &frontmatterType{[]byte{'+', '+', '+'}, []byte{'+', '+', '+'}, page.handleTomlMetaData} + case '{': + return &frontmatterType{[]byte{'{'}, []byte{'}'}, page.handleJsonMetaData} + default: + return nil + } +} + +func beginFrontMatter(data *bufio.Reader, f *frontmatterType) (bool, error) { + peek := make([]byte, 3) + _, err := data.Read(peek) + if err != nil { + return false, err + } + return bytes.Equal(peek, f.markstart), nil +} + +func chompWhitespace(data *bufio.Reader) (r rune, err error) { + for { + r, _, err = data.ReadRune() + if err != nil { + return } + if unicode.IsSpace(r) { + continue + } + if err := data.UnreadRune(); err != nil { + return r, errors.New("unable to unread first charactor in front matter.") + } + return r, nil } - return nil, nil } func (p *Page) Render(layout ...string) template.HTML { @@ -378,46 +422,50 @@ func (p *Page) ExecuteTemplate(layout string) *bytes.Buffer { return buffer } -func (page *Page) readFile() []byte { - var data, err = ioutil.ReadFile(page.FileName) +func (page *Page) readFile() (data []byte, err error) { + data, err = ioutil.ReadFile(page.FileName) if err != nil { - PrintErr("Error Reading: " + page.FileName) - return nil + return nil, err } - return data + return data, nil } func (page *Page) buildPageFromFile() error { - data := page.readFile() + f, err := os.Open(page.FileName) + if err != nil { + return err + } + return page.parse(bufio.NewReader(f)) +} + +func (page *Page) parse(reader io.Reader) error { + data := bufio.NewReader(reader) - content, err := page.parseFileHeading(data) + err := page.parseFrontMatter(data) if err != nil { return err } switch page.Markup { case "md": - page.convertMarkdown(content) + page.convertMarkdown(data) case "rst": - page.convertRestructuredText(content) + page.convertRestructuredText(data) } return nil } -func (page *Page) convertMarkdown(lines []string) { - - page.RawMarkdown = strings.Join(lines, "\n") - content := string(blackfriday.MarkdownCommon([]byte(page.RawMarkdown))) +func (page *Page) convertMarkdown(lines io.Reader) { + b := new(bytes.Buffer) + b.ReadFrom(lines) + content := string(blackfriday.MarkdownCommon(b.Bytes())) page.Content = template.HTML(content) page.Summary = template.HTML(TruncateWordsToWholeSentence(StripHTML(StripShortcodes(content)), summaryLength)) } -func (page *Page) convertRestructuredText(lines []string) { - - page.RawMarkdown = strings.Join(lines, "\n") - +func (page *Page) convertRestructuredText(lines io.Reader) { cmd := exec.Command("rst2html.py") - cmd.Stdin = strings.NewReader(page.RawMarkdown) + cmd.Stdin = lines var out bytes.Buffer cmd.Stdout = &out if err := cmd.Run(); err != nil { diff --git a/hugolib/page_test.go b/hugolib/page_test.go new file mode 100644 index 000000000..a31688cc9 --- /dev/null +++ b/hugolib/page_test.go @@ -0,0 +1,124 @@ +package hugolib + +import ( + "html/template" + "io" + "strings" + "testing" +) + +var EMPTY_PAGE = "" + +var SIMPLE_PAGE = `--- +title: Simple +--- +Simple Page +` + +var INVALID_FRONT_MATTER_MISSING = `This is a test` + +var INVALID_FRONT_MATTER_SHORT_DELIM = ` +-- +title: Short delim start +--- +Short Delim +` + +var INVALID_FRONT_MATTER_SHORT_DELIM_ENDING = ` +--- +title: Short delim ending +-- +Short Delim +` + +var INVALID_FRONT_MATTER_LEADING_WS = ` + + --- +title: Leading WS +--- +Leading +` + +func checkError(t *testing.T, err error, expected string) { + if err == nil { + t.Fatalf("err is nil") + } + if err.Error() != expected { + t.Errorf("err.Error() returned: '%s'. Expected: '%s'", err.Error(), expected) + } +} + +func TestDegenerateEmptyPageZeroLengthName(t *testing.T) { + _, err := ReadFrom(strings.NewReader(EMPTY_PAGE), "") + if err == nil { + t.Fatalf("A zero length page name must return an error") + } + + checkError(t, err, "Zero length page name") +} + +func TestDegenerateEmptyPage(t *testing.T) { + _, err := ReadFrom(strings.NewReader(EMPTY_PAGE), "test") + if err == nil { + t.Fatalf("Expected ReadFrom to return an error when an empty buffer is passed.") + } + + checkError(t, err, "unable to locate front matter") +} + +func checkPageTitle(t *testing.T, page *Page, title string) { + if page.Title != title { + t.Fatalf("Page title is: %s. Expected %s", page.Title, title) + } +} + +func checkPageContent(t *testing.T, page *Page, content string) { + if page.Content != template.HTML(content) { + t.Fatalf("Page content is: %s. Expected %s", page.Content, content) + } +} + +func checkPageType(t *testing.T, page *Page, pageType string) { + if page.Type() != pageType { + t.Fatalf("Page type is: %s. Expected: %s", page.Type(), pageType) + } +} + +func checkPageLayout(t *testing.T, page *Page, layout string) { + if page.Layout() != layout { + t.Fatalf("Page layout is: %s. Expected: %s", page.Layout(), layout) + } +} + +func TestCreateNewPage(t *testing.T) { + p, err := ReadFrom(strings.NewReader(SIMPLE_PAGE), "simple") + if err != nil { + t.Fatalf("Unable to create a page with frontmatter and body content: %s", err) + } + checkPageTitle(t, p, "Simple") + checkPageContent(t, p, "<p>Simple Page</p>\n") + checkPageType(t, p, "page") + checkPageLayout(t, p, "page/single.html") +} + +func TestDegenerateInvalidFrontMatterShortDelim(t *testing.T) { + var tests = []struct { + r io.Reader + err string + }{ + {strings.NewReader(INVALID_FRONT_MATTER_SHORT_DELIM), "unable to match beginning front matter delimiter"}, + {strings.NewReader(INVALID_FRONT_MATTER_SHORT_DELIM_ENDING), "unable to match ending front matter delimiter"}, + {strings.NewReader(INVALID_FRONT_MATTER_MISSING), "unable to match beginning front matter delimiter"}, + } + for _, test := range tests { + _, err := ReadFrom(test.r, "invalid/front/matter/short/delim") + checkError(t, err, test.err) + } +} + +func TestDegenerateInvalidFrontMatterLeadingWhitespace(t *testing.T) { + _, err := ReadFrom(strings.NewReader(INVALID_FRONT_MATTER_LEADING_WS), "invalid/front/matter/leading/ws") + if err != nil { + t.Fatalf("Unable to parse front matter given leading whitespace: %s", err) + } +} diff --git a/hugolib/path_seperators_test.go b/hugolib/path_seperators_test.go index e86d5a216..f1d3a8687 100644 --- a/hugolib/path_seperators_test.go +++ b/hugolib/path_seperators_test.go @@ -2,17 +2,27 @@ package hugolib import ( "path/filepath" + "strings" "testing" ) +var SIMPLE_PAGE_YAML = `--- +contenttype: "" +--- +Sample Text +` + func TestDegenerateMissingFolderInPageFilename(t *testing.T) { - p := NewPage(filepath.Join("foobar")) + p, err := ReadFrom(strings.NewReader(SIMPLE_PAGE_YAML), filepath.Join("foobar")) + if err != nil { + t.Fatalf("Error in ReadFrom") + } if p.Section != "" { t.Fatalf("No section should be set for a file path: foobar") } } -func TestCreateNewPage(t *testing.T) { +func TestNewPageWithFilePath(t *testing.T) { toCheck := []map[string]string{ {"input": filepath.Join("sub", "foobar.html"), "expect": "sub"}, {"input": filepath.Join("content", "sub", "foobar.html"), "expect": "sub"}, @@ -20,14 +30,16 @@ func TestCreateNewPage(t *testing.T) { } for _, el := range toCheck { - p := NewPage(el["input"]) + p, err := ReadFrom(strings.NewReader(SIMPLE_PAGE_YAML), el["input"]) + if err != nil { + t.Fatalf("Reading from SIMPLE_PAGE_YAML resulted in an error: %s", err) + } if p.Section != el["expect"] { t.Fatalf("Section not set to %s for page %s. Got: %s", el["expect"], el["input"], p.Section) } } } - func TestSettingOutFileOnPageContainsCorrectSlashes(t *testing.T) { s := NewSite(&Config{}) p := NewPage(filepath.Join("sub", "foobar")) |