diff options
35 files changed, 1260 insertions, 987 deletions
diff --git a/commands/hugo.go b/commands/hugo.go index 959006557..9ad46b3bf 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -49,7 +49,7 @@ import ( // Hugo represents the Hugo sites to build. This variable is exported as it // is used by at least one external library (the Hugo caddy plugin). We should // provide a cleaner external API, but until then, this is it. -var Hugo hugolib.HugoSites +var Hugo *hugolib.HugoSites // Reset resets Hugo ready for a new full build. This is mainly only useful // for benchmark testing etc. via the CLI commands. @@ -715,11 +715,11 @@ func getDirList() []string { func buildSites(watching ...bool) (err error) { fmt.Println("Started building sites ...") w := len(watching) > 0 && watching[0] - return Hugo.Build(w, true) + return Hugo.Build(hugolib.BuildCfg{Watching: w, PrintStats: true}) } func rebuildSites(events []fsnotify.Event) error { - return Hugo.Rebuild(events, true) + return Hugo.Rebuild(hugolib.BuildCfg{PrintStats: true}, events...) } // NewWatcher creates a new watcher to watch filesystem events. diff --git a/commands/list.go b/commands/list.go index bc5bb557a..f47b4820c 100644 --- a/commands/list.go +++ b/commands/list.go @@ -53,7 +53,7 @@ var listDraftsCmd = &cobra.Command{ site := &hugolib.Site{} - if err := site.Process(); err != nil { + if err := site.PreProcess(hugolib.BuildCfg{}); err != nil { return newSystemError("Error Processing Source Content", err) } @@ -84,7 +84,7 @@ posted in the future.`, site := &hugolib.Site{} - if err := site.Process(); err != nil { + if err := site.PreProcess(hugolib.BuildCfg{}); err != nil { return newSystemError("Error Processing Source Content", err) } @@ -115,7 +115,7 @@ expired.`, site := &hugolib.Site{} - if err := site.Process(); err != nil { + if err := site.PreProcess(hugolib.BuildCfg{}); err != nil { return newSystemError("Error Processing Source Content", err) } diff --git a/commands/multilingual.go b/commands/multilingual.go index 7c43d15bc..4d0f6e107 100644 --- a/commands/multilingual.go +++ b/commands/multilingual.go @@ -11,30 +11,31 @@ import ( "github.com/spf13/viper" ) -func readMultilingualConfiguration() (hugolib.HugoSites, error) { - h := make(hugolib.HugoSites, 0) +func readMultilingualConfiguration() (*hugolib.HugoSites, error) { + sites := make([]*hugolib.Site, 0) multilingual := viper.GetStringMap("Multilingual") if len(multilingual) == 0 { // TODO(bep) multilingo langConfigsList = append(langConfigsList, hugolib.NewLanguage("en")) - h = append(h, hugolib.NewSite(hugolib.NewLanguage("en"))) - return h, nil + sites = append(sites, hugolib.NewSite(hugolib.NewLanguage("en"))) } - var err error + if len(multilingual) > 0 { + var err error - langConfigsList, err := toSortedLanguages(multilingual) + languages, err := toSortedLanguages(multilingual) - if err != nil { - return nil, fmt.Errorf("Failed to parse multilingual config: %s", err) - } + if err != nil { + return nil, fmt.Errorf("Failed to parse multilingual config: %s", err) + } + + for _, lang := range languages { + sites = append(sites, hugolib.NewSite(lang)) + } - for _, lang := range langConfigsList { - s := hugolib.NewSite(lang) - s.SetMultilingualConfig(lang, langConfigsList) - h = append(h, s) } - return h, nil + return hugolib.NewHugoSites(sites...) + } func toSortedLanguages(l map[string]interface{}) (hugolib.Languages, error) { diff --git a/helpers/url.go b/helpers/url.go index 927e3c87c..085f9e9fa 100644 --- a/helpers/url.go +++ b/helpers/url.go @@ -169,6 +169,17 @@ func AbsURL(path string) string { return MakePermalink(baseURL, path).String() } +// IsAbsURL determines whether the given path points to an absolute URL. +// TODO(bep) ml tests +func IsAbsURL(path string) bool { + url, err := url.Parse(path) + if err != nil { + return false + } + + return url.IsAbs() || strings.HasPrefix(path, "//") +} + // RelURL creates a URL relative to the BaseURL root. // Note: The result URL will not include the context root if canonifyURLs is enabled. func RelURL(path string) string { diff --git a/hugolib/embedded_shortcodes_test.go b/hugolib/embedded_shortcodes_test.go index 18f807fbf..e668ff4c8 100644 --- a/hugolib/embedded_shortcodes_test.go +++ b/hugolib/embedded_shortcodes_test.go @@ -56,8 +56,8 @@ func doTestShortcodeCrossrefs(t *testing.T, relative bool) { templ := tpl.New() p, _ := pageFromString(simplePageWithURL, path) p.Node.Site = &SiteInfo{ - AllPages: &(Pages{p}), - BaseURL: template.URL(helpers.SanitizeURLKeepTrailingSlash(baseURL)), + rawAllPages: &(Pages{p}), + BaseURL: template.URL(helpers.SanitizeURLKeepTrailingSlash(baseURL)), } output, err := HandleShortcodes(in, p, templ) @@ -72,8 +72,7 @@ func doTestShortcodeCrossrefs(t *testing.T, relative bool) { } func TestShortcodeHighlight(t *testing.T) { - viper.Reset() - defer viper.Reset() + testCommonResetState() if !helpers.HasPygments() { t.Skip("Skip test as Pygments is not installed") diff --git a/hugolib/handler_test.go b/hugolib/handler_test.go index a84d528cb..fce29df44 100644 --- a/hugolib/handler_test.go +++ b/hugolib/handler_test.go @@ -25,8 +25,7 @@ import ( ) func TestDefaultHandler(t *testing.T) { - viper.Reset() - defer viper.Reset() + testCommonResetState() hugofs.InitMemFs() sources := []source.ByteSource{ @@ -45,33 +44,30 @@ func TestDefaultHandler(t *testing.T) { viper.Set("verbose", true) s := &Site{ - Source: &source.InMemorySource{ByteSource: sources}, - targets: targetList{page: &target.PagePub{UglyURLs: true}}, - Lang: NewLanguage("en"), + Source: &source.InMemorySource{ByteSource: sources}, + targets: targetList{page: &target.PagePub{UglyURLs: true, PublishDir: "public"}}, + Language: NewLanguage("en"), } - s.initializeSiteInfo() - - s.prepTemplates( + if err := buildAndRenderSite(s, "_default/single.html", "{{.Content}}", "head", "<head><script src=\"script.js\"></script></head>", - "head_abs", "<head><script src=\"/script.js\"></script></head>") - - // From site_test.go - createAndRenderPages(t, s) + "head_abs", "<head><script src=\"/script.js\"></script></head>"); err != nil { + t.Fatalf("Failed to render site: %s", err) + } tests := []struct { doc string expected string }{ - {filepath.FromSlash("sect/doc1.html"), "\n\n<h1 id=\"title\">title</h1>\n\n<p>some <em>content</em></p>\n"}, - {filepath.FromSlash("sect/doc2.html"), "<!doctype html><html><body>more content</body></html>"}, - {filepath.FromSlash("sect/doc3.html"), "\n\n<h1 id=\"doc3\">doc3</h1>\n\n<p><em>some</em> content</p>\n"}, - {filepath.FromSlash("sect/doc3/img1.png"), string([]byte("‰PNG ��� IHDR����������:~›U��� IDATWcø��ZMoñ����IEND®B`‚"))}, - {filepath.FromSlash("sect/img2.gif"), string([]byte("GIF89a��€��ÿÿÿ���,�������D�;"))}, - {filepath.FromSlash("sect/img2.spf"), string([]byte("****FAKE-FILETYPE****"))}, - {filepath.FromSlash("doc7.html"), "<html><body>doc7 content</body></html>"}, - {filepath.FromSlash("sect/doc8.html"), "\n\n<h1 id=\"title\">title</h1>\n\n<p>some <em>content</em></p>\n"}, + {filepath.FromSlash("public/sect/doc1.html"), "\n\n<h1 id=\"title\">title</h1>\n\n<p>some <em>content</em></p>\n"}, + {filepath.FromSlash("public/sect/doc2.html"), "<!doctype html><html><body>more content</body></html>"}, + {filepath.FromSlash("public/sect/doc3.html"), "\n\n<h1 id=\"doc3\">doc3</h1>\n\n<p><em>some</em> content</p>\n"}, + {filepath.FromSlash("public/sect/doc3/img1.png"), string([]byte("‰PNG ��� IHDR����������:~›U��� IDATWcø��ZMoñ����IEND®B`‚"))}, + {filepath.FromSlash("public/sect/img2.gif"), string([]byte("GIF89a��€��ÿÿÿ���,�������D�;"))}, + {filepath.FromSlash("public/sect/img2.spf"), string([]byte("****FAKE-FILETYPE****"))}, + {filepath.FromSlash("public/doc7.html"), "<html><body>doc7 content</body></html>"}, + {filepath.FromSlash("public/sect/doc8.html"), "\n\n<h1 id=\"title\">title</h1>\n\n<p>some <em>content</em></p>\n"}, } for _, test := range tests { diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index dd8d3e5d2..2dd1bb9be 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -14,42 +14,119 @@ package hugolib import ( + "errors" + "strings" "time" - "github.com/fsnotify/fsnotify" + "github.com/spf13/viper" + "github.com/fsnotify/fsnotify" + "github.com/spf13/hugo/source" + "github.com/spf13/hugo/tpl" jww "github.com/spf13/jwalterweatherman" ) // HugoSites represents the sites to build. Each site represents a language. -type HugoSites []*Site +type HugoSites struct { + Sites []*Site + + Multilingual *Multilingual +} + +func NewHugoSites(sites ...*Site) (*HugoSites, error) { + languages := make(Languages, len(sites)) + for i, s := range sites { + if s.Language == nil { + return nil, errors.New("Missing language for site") + } + languages[i] = s.Language + } + defaultLang := viper.GetString("DefaultContentLanguage") + if defaultLang == "" { + defaultLang = "en" + } + langConfig := &Multilingual{Languages: languages, DefaultLang: NewLanguage(defaultLang)} + + return &HugoSites{Multilingual: langConfig, Sites: sites}, nil +} // Reset resets the sites, making it ready for a full rebuild. // TODO(bep) multilingo func (h HugoSites) Reset() { - for i, s := range h { - h[i] = s.Reset() + for i, s := range h.Sites { + h.Sites[i] = s.Reset() } } +type BuildCfg struct { + // Whether we are in watch (server) mode + Watching bool + // Print build stats at the end of a build + PrintStats bool + // Skip rendering. Useful for testing. + skipRender bool + // Use this to add templates to use for rendering. + // Useful for testing. + withTemplate func(templ tpl.Template) error +} + // Build builds all sites. -func (h HugoSites) Build(watching, printStats bool) error { +func (h HugoSites) Build(config BuildCfg) error { + + if h.Sites == nil || len(h.Sites) == 0 { + return errors.New("No site(s) to build") + } + t0 := time.Now() - for _, site := range h { - t1 := time.Now() + // We should probably refactor the Site and pull up most of the logic from there to here, + // but that seems like a daunting task. + // So for now, if there are more than one site (language), + // we pre-process the first one, then configure all the sites based on that. + firstSite := h.Sites[0] + + for _, s := range h.Sites { + // TODO(bep) ml + s.Multilingual = h.Multilingual + s.RunMode.Watching = config.Watching + } + + if err := firstSite.PreProcess(config); err != nil { + return err + } - site.RunMode.Watching = watching + h.setupTranslations(firstSite) - if err := site.Build(); err != nil { + if len(h.Sites) > 1 { + // Initialize the rest + for _, site := range h.Sites[1:] { + site.Tmpl = firstSite.Tmpl + site.initializeSiteInfo() + } + } + + for _, s := range h.Sites { + + if err := s.PostProcess(); err != nil { return err } - if printStats { - site.Stats(t1) + + if !config.skipRender { + if err := s.Render(); err != nil { + return err + } + + } + + if config.PrintStats { + s.Stats() } + + // TODO(bep) ml lang in site.Info? + // TODO(bep) ml Page sorting? } - if printStats { + if config.PrintStats { jww.FEEDBACK.Printf("total in %v ms\n", int(1000*time.Since(t0).Seconds())) } @@ -58,25 +135,159 @@ func (h HugoSites) Build(watching, printStats bool) error { } // Rebuild rebuilds all sites. -func (h HugoSites) Rebuild(events []fsnotify.Event, printStats bool) error { +func (h HugoSites) Rebuild(config BuildCfg, events ...fsnotify.Event) error { t0 := time.Now() - for _, site := range h { - t1 := time.Now() + firstSite := h.Sites[0] - if err := site.ReBuild(events); err != nil { - return err + for _, s := range h.Sites { + s.resetBuildState() + } + + sourceChanged, err := firstSite.ReBuild(events) + + if err != nil { + return err + } + + // Assign pages to sites per translation. + h.setupTranslations(firstSite) + + for _, s := range h.Sites { + + if sourceChanged { + if err := s.PostProcess(); err != nil { + return err + } } - if printStats { - site.Stats(t1) + if !config.skipRender { + if err := s.Render(); err != nil { + return err + } + } + + if config.PrintStats { + s.Stats() } } - if printStats { + if config.PrintStats { jww.FEEDBACK.Printf("total in %v ms\n", int(1000*time.Since(t0).Seconds())) } return nil } + +func (s *HugoSites) setupTranslations(master *Site) { + + for _, p := range master.rawAllPages { + if p.Lang() == "" { + panic("Page language missing: " + p.Title) + } + + shouldBuild := p.shouldBuild() + + for i, site := range s.Sites { + if strings.HasPrefix(site.Language.Lang, p.Lang()) { + site.updateBuildStats(p) + if shouldBuild { + site.Pages = append(site.Pages, p) + p.Site = &site.Info + } + } + + if !shouldBuild { + continue + } + + if i == 0 { + site.AllPages = append(site.AllPages, p) + } + } + + for i := 1; i < len(s.Sites); i++ { + s.Sites[i].AllPages = s.Sites[0].AllPages + } + } + + if len(s.Sites) > 1 { + pages := s.Sites[0].AllPages + allTranslations := pagesToTranslationsMap(s.Multilingual, pages) + assignTranslationsToPages(allTranslations, pages) + } +} + +func (s *Site) updateBuildStats(page *Page) { + if page.IsDraft() { + s.draftCount++ + } + + if page.IsFuture() { + s.futureCount++ + } + + if page.IsExpired() { + s.expiredCount++ + } +} + +// Convenience func used in tests to build a single site/language excluding render phase. +func buildSiteSkipRender(s *Site, additionalTemplates ...string) error { + return doBuildSite(s, false, additionalTemplates...) +} + +// Convenience func used in tests to build a single site/language including render phase. +func buildAndRenderSite(s *Site, additionalTemplates ...string) error { + return doBuildSite(s, true, additionalTemplates...) +} + +// Convenience func used in tests to build a single site/language. +func doBuildSite(s *Site, render bool, additionalTemplates ...string) error { + sites, err := NewHugoSites(s) + if err != nil { + return err + } + + addTemplates := func(templ tpl.Template) error { + for i := 0; i < len(additionalTemplates); i += 2 { + err := templ.AddTemplate(additionalTemplates[i], additionalTemplates[i+1]) + if err != nil { + return err + } + } + return nil + } + + config := BuildCfg{skipRender: !render, withTemplate: addTemplates} + return sites.Build(config) +} + +// Convenience func used in tests. +func newHugoSitesFromSourceAndLanguages(input []source.ByteSource, languages Languages) (*HugoSites, error) { + if len(languages) == 0 { + panic("Must provide at least one language") + } + first := &Site{ + Source: &source.InMemorySource{ByteSource: input}, + Language: languages[0], + } + if len(languages) == 1 { + return NewHugoSites(first) + } + + sites := make([]*Site, len(languages)) + sites[0] = first + for i := 1; i < len(languages); i++ { + sites[i] = &Site{Language: languages[i]} + } + + return NewHugoSites(sites...) + +} + +// Convenience func used in tests. +func newHugoSitesFromLanguages(languages Languages) (*HugoSites, error) { + return newHugoSitesFromSourceAndLanguages(nil, languages) +} diff --git a/hugolib/hugo_sites_test.go b/hugolib/hugo_sites_test.go new file mode 100644 index 000000000..fc4801115 --- /dev/null +++ b/hugolib/hugo_sites_test.go @@ -0,0 +1,522 @@ +package hugolib + +import ( + "fmt" + "strings" + "testing" + + "path/filepath" + + "os" + + "github.com/fsnotify/fsnotify" + "github.com/spf13/afero" + "github.com/spf13/hugo/helpers" + "github.com/spf13/hugo/hugofs" + "github.com/spf13/hugo/source" + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + + jww "github.com/spf13/jwalterweatherman" +) + +func init() { + testCommonResetState() + jww.SetStdoutThreshold(jww.LevelError) + +} + +func testCommonResetState() { + hugofs.InitMemFs() + viper.Reset() + viper.Set("ContentDir", "content") + viper.Set("DataDir", "data") + viper.Set("I18nDir", "i18n") + viper.Set("themesDir", "themes") + viper.Set("LayoutDir", "layouts") + viper.Set("PublishDir", "public") + viper.Set("RSSUri", "rss") + + if err := hugofs.Source().Mkdir("content", 0755); err != nil { + panic("Content folder creation failed.") + } + +} + +func _TestMultiSites(t *testing.T) { + + sites := createMultiTestSites(t) + + err := sites.Build(BuildCfg{skipRender: true}) + + if err != nil { + t.Fatalf("Failed to build sites: %s", err) + } + + enSite := sites.Sites[0] + + assert.Equal(t, "en", enSite.Language.Lang) + + if len(enSite.Pages) != 3 { + t.Fatal("Expected 3 english pages") + } + assert.Len(t, enSite.Source.Files(), 6, "should have 6 source files") + assert.Len(t, enSite.AllPages, 6, "should have 6 total pages (including translations)") + + doc1en := enSite.Pages[0] + permalink, err := doc1en.Permalink() + assert.NoError(t, err, "permalink call failed") + assert.Equal(t, "http://example.com/blog/en/sect/doc1-slug/", permalink, "invalid doc1.en permalink") + assert.Len(t, doc1en.Translations(), 1, "doc1-en should have one translation, excluding itself") + + doc2 := enSite.Pages[1] + permalink, err = doc2.Permalink() + assert.NoError(t, err, "permalink call failed") + assert.Equal(t, "http://example.com/blog/en/sect/doc2/", permalink, "invalid doc2 permalink") + + doc3 := enSite.Pages[2] + permalink, err = doc3.Permalink() + assert.NoError(t, err, "permalink call failed") + assert.Equal(t, "http://example.com/blog/superbob", permalink, "invalid doc3 permalink") + + // TODO(bep) multilingo. Check this case. This has url set in frontmatter, but we must split into lang folders + // The assertion below was missing the /en prefix. + assert.Equal(t, "/en/superbob", doc3.URL(), "invalid url, was specified on doc3 TODO(bep)") + + assert.Equal(t, doc2.Next, doc3, "doc3 should follow doc2, in .Next") + + doc1fr := doc1en.Translations()[0] + permalink, err = doc1fr.Permalink() + assert.NoError(t, err, "permalink call failed") + assert.Equal(t, "http://example.com/blog/fr/sect/doc1/", permalink, "invalid doc1fr permalink") + + assert.Equal(t, doc1en.Translations()[0], doc1fr, "doc1-en should have doc1-fr as translation") + assert.Equal(t, doc1fr.Translations()[0], doc1en, "doc1-fr should have doc1-en as translation") + assert.Equal(t, "fr", doc1fr.Language().Lang) + + doc4 := enSite.AllPages[4] + permalink, err = doc4.Permalink() + assert.NoError(t, err, "permalink call failed") + assert.Equal(t, "http://example.com/blog/fr/sect/doc4/", permalink, "invalid doc4 permalink") + assert.Len(t, doc4.Translations(), 0, "found translations for doc4") + + doc5 := enSite.AllPages[5] + permalink, err = doc5.Permalink() + assert.NoError(t, err, "permalink call failed") + assert.Equal(t, "http://example.com/blog/fr/somewhere/else/doc5", permalink, "invalid doc5 permalink") + + // Taxonomies and their URLs + assert.Len(t, enSite.Taxonomies, 1, "should have 1 taxonomy") + tags := enSite.Taxonomies["tags"] + assert.Len(t, tags, 2, "should have 2 different tags") + assert.Equal(t, tags["tag1"][0].Page, doc1en, "first tag1 page should be doc1") + + frSite := sites.Sites[1] + + assert.Equal(t, "fr", frSite.Language.Lang) + assert.Len(t, frSite.Pages, 3, "should have 3 pages") + assert.Len(t, frSite.AllPages, 6, "should have 6 total pages (including translations)") + + for _, frenchPage := range frSite.Pages { + assert.Equal(t, "fr", frenchPage.Lang()) + } + +} + +func TestMultiSitesRebuild(t *testing.T) { + + sites := createMultiTestSites(t) + cfg := BuildCfg{} + + err := sites.Build(cfg) + + if err != nil { + t.Fatalf("Failed to build sites: %s", err) + } + + _, err = hugofs.Destination().Open("public/en/sect/doc2/index.html") + + if err != nil { + t.Fatalf("Unable to locate file") + } + + enSite := sites.Sites[0] + frSite := sites.Sites[1] + + assert.Len(t, enSite.Pages, 3) + assert.Len(t, |