diff options
author | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2020-02-25 21:40:02 +0100 |
---|---|---|
committer | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2020-04-07 21:59:20 +0200 |
commit | 2f721f8ec69c52202815cd1b543ca4bf535c0901 (patch) | |
tree | cae7d1ee9ce867a4bffc70e94513f09e2aebe162 /hugolib | |
parent | 8568928aa8e82a6bd7de4555c3703d8835fbd25b (diff) |
Add basic "post resource publish support"
Fixes #7146
Diffstat (limited to 'hugolib')
-rw-r--r-- | hugolib/filesystems/basefs.go | 2 | ||||
-rw-r--r-- | hugolib/hugo_sites_build.go | 101 | ||||
-rw-r--r-- | hugolib/page__common.go | 3 | ||||
-rw-r--r-- | hugolib/page__new.go | 3 | ||||
-rw-r--r-- | hugolib/resource_chain_test.go | 78 |
5 files changed, 183 insertions, 4 deletions
diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go index 47d6d11f5..57a95a037 100644 --- a/hugolib/filesystems/basefs.go +++ b/hugolib/filesystems/basefs.go @@ -345,7 +345,7 @@ func NewBase(p *paths.Paths, logger *loggers.Logger, options ...func(*BaseFs) er logger = loggers.NewWarningLogger() } - publishFs := afero.NewBasePathFs(fs.Destination, p.AbsPublishDir) + publishFs := hugofs.NewBaseFileDecorator(afero.NewBasePathFs(fs.Destination, p.AbsPublishDir)) b := &BaseFs{ PublishFs: publishFs, diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index 15eca4bb3..6a65605fc 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -17,7 +17,17 @@ import ( "bytes" "context" "fmt" + "os" "runtime/trace" + "strings" + + "github.com/gohugoio/hugo/common/para" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/resources/postpub" + + "github.com/spf13/afero" + + "github.com/gohugoio/hugo/resources/resource" "github.com/gohugoio/hugo/output" @@ -138,6 +148,10 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { } } + if err := h.postProcess(); err != nil { + h.SendError(err) + } + if h.Metrics != nil { var b bytes.Buffer h.Metrics.WriteMetrics(&b) @@ -321,3 +335,90 @@ func (h *HugoSites) render(config *BuildCfg) error { return nil } + +func (h *HugoSites) postProcess() error { + var toPostProcess []resource.OriginProvider + for _, s := range h.Sites { + for _, v := range s.ResourceSpec.PostProcessResources { + toPostProcess = append(toPostProcess, v) + } + } + + if len(toPostProcess) == 0 { + return nil + } + + workers := para.New(config.GetNumWorkerMultiplier()) + g, _ := workers.Start(context.Background()) + + handleFile := func(filename string) error { + + content, err := afero.ReadFile(h.BaseFs.PublishFs, filename) + if err != nil { + return err + } + + k := 0 + changed := false + + for { + l := bytes.Index(content[k:], []byte(postpub.PostProcessPrefix)) + if l == -1 { + break + } + m := bytes.Index(content[k+l:], []byte(postpub.PostProcessSuffix)) + len(postpub.PostProcessSuffix) + + low, high := k+l, k+l+m + + field := content[low:high] + + forward := l + m + + for i, r := range toPostProcess { + if r == nil { + panic(fmt.Sprintf("resource %d to post process is nil", i+1)) + } + v, ok := r.GetFieldString(string(field)) + if ok { + content = append(content[:low], append([]byte(v), content[high:]...)...) + changed = true + forward = len(v) + break + } + } + + k += forward + } + + if changed { + return afero.WriteFile(h.BaseFs.PublishFs, filename, content, 0666) + } + + return nil + + } + + _ = afero.Walk(h.BaseFs.PublishFs, "", func(path string, info os.FileInfo, err error) error { + if info == nil || info.IsDir() { + return nil + } + + if !strings.HasSuffix(path, "html") { + return nil + } + + g.Run(func() error { + return handleFile(path) + }) + + return nil + }) + + // Prepare for a new build. + for _, s := range h.Sites { + s.ResourceSpec.PostProcessResources = make(map[string]postpub.PostPublishedResource) + } + + return g.Wait() + +} diff --git a/hugolib/page__common.go b/hugolib/page__common.go index be6bb090b..d1c7ba866 100644 --- a/hugolib/page__common.go +++ b/hugolib/page__common.go @@ -86,7 +86,8 @@ type pageCommon struct { resource.ResourceDataProvider resource.ResourceMetaProvider resource.ResourceParamsProvider - resource.ResourceTypesProvider + resource.ResourceTypeProvider + resource.MediaTypeProvider resource.TranslationKeyProvider compare.Eqer diff --git a/hugolib/page__new.go b/hugolib/page__new.go index 938c13d7c..9ec089f27 100644 --- a/hugolib/page__new.go +++ b/hugolib/page__new.go @@ -49,7 +49,8 @@ func newPageBase(metaProvider *pageMeta) (*pageState, error) { PageMetaProvider: metaProvider, RelatedKeywordsProvider: metaProvider, OutputFormatsProvider: page.NopPage, - ResourceTypesProvider: pageTypesProvider, + ResourceTypeProvider: pageTypesProvider, + MediaTypeProvider: pageTypesProvider, RefProvider: page.NopPage, ShortcodeInfoProvider: page.NopPage, LanguageProvider: s, diff --git a/hugolib/resource_chain_test.go b/hugolib/resource_chain_test.go index 8bca6c7b5..0d0c9203a 100644 --- a/hugolib/resource_chain_test.go +++ b/hugolib/resource_chain_test.go @@ -14,13 +14,16 @@ package hugolib import ( + "fmt" "io" + "math/rand" "os" "os/exec" "path/filepath" "runtime" "strings" "testing" + "time" "github.com/gohugoio/hugo/common/herrors" @@ -352,6 +355,80 @@ Edited content. } } +func TestResourceChainPostProcess(t *testing.T) { + t.Parallel() + + rnd := rand.New(rand.NewSource(time.Now().UnixNano())) + + b := newTestSitesBuilder(t) + b.WithContent("page1.md", "---\ntitle: Page1\n---") + b.WithContent("page2.md", "---\ntitle: Page2\n---") + + b.WithTemplates( + "_default/single.html", `{{ $hello := "<h1> Hello World! </h1>" | resources.FromString "hello.html" | minify | fingerprint "md5" | resources.PostProcess }} +HELLO: {{ $hello.RelPermalink }} +`, + "index.html", `Start. +{{ $hello := "<h1> Hello World! </h1>" | resources.FromString "hello.html" | minify | fingerprint "md5" | resources.PostProcess }} + +HELLO: {{ $hello.RelPermalink }}|Integrity: {{ $hello.Data.Integrity }}|MediaType: {{ $hello.MediaType.Type }} +HELLO2: Name: {{ $hello.Name }}|Content: {{ $hello.Content }}|Title: {{ $hello.Title }}|ResourceType: {{ $hello.ResourceType }} + +`+strings.Repeat("a b", rnd.Intn(10)+1)+` + + +End.`) + + b.Running() + b.Build(BuildCfg{}) + b.AssertFileContent("public/index.html", + `Start. +HELLO: /hello.min.a2d1cb24f24b322a7dad520414c523e9.html|Integrity: md5-otHLJPJLMip9rVIEFMUj6Q==|MediaType: text/html +HELLO2: Name: hello.html|Content: <h1>Hello World!</h1>|Title: hello.html|ResourceType: html +End.`) + + b.AssertFileContent("public/page1/index.html", `HELLO: /hello.min.a2d1cb24f24b322a7dad520414c523e9.html`) + b.AssertFileContent("public/page2/index.html", `HELLO: /hello.min.a2d1cb24f24b322a7dad520414c523e9.html`) + +} + +func BenchmarkResourceChainPostProcess(b *testing.B) { + + for i := 0; i < b.N; i++ { + b.StopTimer() + s := newTestSitesBuilder(b) + for i := 0; i < 300; i++ { + s.WithContent(fmt.Sprintf("page%d.md", i+1), "---\ntitle: Page\n---") + } + s.WithTemplates("_default/single.html", `Start. +Some text. + + +{{ $hello1 := "<h1> Hello World 2! </h1>" | resources.FromString "hello.html" | minify | fingerprint "md5" | resources.PostProcess }} +{{ $hello2 := "<h1> Hello World 2! </h1>" | resources.FromString (printf "%s.html" .Path) | minify | fingerprint "md5" | resources.PostProcess }} + +Some more text. + +HELLO: {{ $hello1.RelPermalink }}|Integrity: {{ $hello1.Data.Integrity }}|MediaType: {{ $hello1.MediaType.Type }} + +Some more text. + +HELLO2: Name: {{ $hello2.Name }}|Content: {{ $hello2.Content }}|Title: {{ $hello2.Title }}|ResourceType: {{ $hello2.ResourceType }} + +Some more text. + +HELLO2_2: Name: {{ $hello2.Name }}|Content: {{ $hello2.Content }}|Title: {{ $hello2.Title }}|ResourceType: {{ $hello2.ResourceType }} + +End. +`) + + b.StartTimer() + s.Build(BuildCfg{}) + + } + +} + func TestResourceChains(t *testing.T) { t.Parallel() @@ -769,7 +846,6 @@ func TestResourceChainPostCSS(t *testing.T) { } if runtime.GOOS == "windows" { - // TODO(bep) t.Skip("skip npm test on Windows") } |