summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>2024-02-02 16:00:48 +0100
committerBjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>2024-02-02 18:46:23 +0100
commit28733248983c1afc1e4e15dfc30fcf4c442e6ca4 (patch)
treef8680096a715dd88c5ad3f8de7d8f6fdde54faa8
parentd0788b96ae74eb2d48a75c7147e7c6a5457977da (diff)
Misc resource fixes/improvements
* Add --pprof flag to server to enable profile debugging. * Don't cache the resource content, it seem to eat memory on bigger sites. * Keep --printMemoryUsag running in server Fixes #11974
-rw-r--r--commands/commandeer.go10
-rw-r--r--commands/hugobuilder.go20
-rw-r--r--commands/server.go14
-rw-r--r--hugolib/hugo_sites_build.go2
-rw-r--r--hugolib/rebuild_test.go4
-rw-r--r--resources/resource.go55
-rw-r--r--resources/resource_spec.go18
7 files changed, 58 insertions, 65 deletions
diff --git a/commands/commandeer.go b/commands/commandeer.go
index 1aac08c42..0052d91b4 100644
--- a/commands/commandeer.go
+++ b/commands/commandeer.go
@@ -22,6 +22,7 @@ import (
"os"
"os/signal"
"path/filepath"
+ "runtime"
"strings"
"sync"
"sync/atomic"
@@ -341,8 +342,12 @@ func (r *rootCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, args
if r.buildWatch {
defer r.timeTrack(time.Now(), "Built")
}
- err := b.build()
- return err
+ close, err := b.build()
+ if err != nil {
+ return err
+ }
+ close()
+ return nil
}()
if err != nil {
return err
@@ -411,6 +416,7 @@ func (r *rootCommand) PreRun(cd, runner *simplecobra.Commandeer) error {
MaxEntries: 1,
OnEvict: func(key int32, value *hugolib.HugoSites) {
value.Close()
+ runtime.GC()
},
})
diff --git a/commands/hugobuilder.go b/commands/hugobuilder.go
index 41f42ae6d..ddc92129c 100644
--- a/commands/hugobuilder.go
+++ b/commands/hugobuilder.go
@@ -361,34 +361,32 @@ func (c *hugoBuilder) newWatcher(pollIntervalStr string, dirList ...string) (*wa
return watcher, nil
}
-func (c *hugoBuilder) build() error {
+func (c *hugoBuilder) build() (func(), error) {
stopProfiling, err := c.initProfiling()
if err != nil {
- return err
+ return nil, err
}
- defer func() {
- if stopProfiling != nil {
- stopProfiling()
- }
- }()
-
if err := c.fullBuild(false); err != nil {
- return err
+ return nil, err
}
if !c.r.quiet {
c.r.Println()
h, err := c.hugo()
if err != nil {
- return err
+ return nil, err
}
h.PrintProcessingStats(os.Stdout)
c.r.Println()
}
- return nil
+ return func() {
+ if stopProfiling != nil {
+ stopProfiling()
+ }
+ }, nil
}
func (c *hugoBuilder) buildSites(noBuildLock bool) (err error) {
diff --git a/commands/server.go b/commands/server.go
index 97cf405b7..574f5c340 100644
--- a/commands/server.go
+++ b/commands/server.go
@@ -25,6 +25,7 @@ import (
"io"
"net"
"net/http"
+ _ "net/http/pprof"
"net/url"
"os"
"os/signal"
@@ -451,6 +452,7 @@ type serverCommand struct {
tlsCertFile string
tlsKeyFile string
tlsAuto bool
+ pprof bool
serverPort int
liveReloadPort int
serverWatch bool
@@ -465,6 +467,11 @@ func (c *serverCommand) Name() string {
}
func (c *serverCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, args []string) error {
+ if c.pprof {
+ go func() {
+ http.ListenAndServe("localhost:8080", nil)
+ }()
+ }
// Watch runs its own server as part of the routine
if c.serverWatch {
@@ -487,15 +494,19 @@ func (c *serverCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, arg
}
+ var close func()
err := func() error {
defer c.r.timeTrack(time.Now(), "Built")
- err := c.build()
+ var err error
+ close, err = c.build()
return err
}()
if err != nil {
return err
}
+ defer close()
+
return c.serve()
}
@@ -520,6 +531,7 @@ of a second, you will be able to save and see your changes nearly instantly.`
cmd.Flags().StringVarP(&c.tlsCertFile, "tlsCertFile", "", "", "path to TLS certificate file")
cmd.Flags().StringVarP(&c.tlsKeyFile, "tlsKeyFile", "", "", "path to TLS key file")
cmd.Flags().BoolVar(&c.tlsAuto, "tlsAuto", false, "generate and use locally-trusted certificates.")
+ cmd.Flags().BoolVar(&c.pprof, "pprof", false, "enable the pprof server (port 8080)")
cmd.Flags().BoolVarP(&c.serverWatch, "watch", "w", true, "watch filesystem for changes and recreate as needed")
cmd.Flags().BoolVar(&c.noHTTPCache, "noHTTPCache", false, "prevent HTTP caching")
cmd.Flags().BoolVarP(&c.serverAppend, "appendPort", "", true, "append port to baseURL")
diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go
index 257949334..a15e15504 100644
--- a/hugolib/hugo_sites_build.go
+++ b/hugolib/hugo_sites_build.go
@@ -720,9 +720,11 @@ func (h *HugoSites) processPartial(ctx context.Context, l logg.LevelLogger, conf
h.pageTrees.treeTaxonomyEntries.DeletePrefix("")
if delete {
+
_, ok := h.pageTrees.treePages.LongestPrefixAll(pathInfo.Base())
if ok {
h.pageTrees.treePages.DeleteAll(pathInfo.Base())
+ h.pageTrees.resourceTrees.DeleteAll(pathInfo.Base())
if pathInfo.IsBundle() {
// Assume directory removed.
h.pageTrees.treePages.DeletePrefixAll(pathInfo.Base() + "/")
diff --git a/hugolib/rebuild_test.go b/hugolib/rebuild_test.go
index d3ac5665d..74b04fe01 100644
--- a/hugolib/rebuild_test.go
+++ b/hugolib/rebuild_test.go
@@ -101,10 +101,10 @@ func TestRebuildEditTextFileInBranchBundle(t *testing.T) {
func TestRebuildRenameTextFileInLeafBundle(t *testing.T) {
b := TestRunning(t, rebuildFilesSimple)
- b.AssertFileContent("public/mysection/mysectionbundle/index.html", "My Section Bundle Text 2 Content.")
+ b.AssertFileContent("public/mysection/mysectionbundle/index.html", "My Section Bundle Text 2 Content.", "Len Resources: 2|")
b.RenameFile("content/mysection/mysectionbundle/mysectionbundletext.txt", "content/mysection/mysectionbundle/mysectionbundletext2.txt").Build()
- b.AssertFileContent("public/mysection/mysectionbundle/index.html", "mysectionbundletext2", "My Section Bundle Text 2 Content.")
+ b.AssertFileContent("public/mysection/mysectionbundle/index.html", "mysectionbundletext2", "My Section Bundle Text 2 Content.", "Len Resources: 2|")
b.AssertRenderCountPage(3)
b.AssertRenderCountContent(3)
}
diff --git a/resources/resource.go b/resources/resource.go
index e78dd12cb..6bef1b275 100644
--- a/resources/resource.go
+++ b/resources/resource.go
@@ -343,7 +343,7 @@ func GetTestInfoForResource(r resource.Resource) GenericResourceTestInfo {
// genericResource represents a generic linkable resource.
type genericResource struct {
- *resourceContent
+ publishInit *sync.Once
sd ResourceSourceDescriptor
paths internal.ResourcePaths
@@ -412,11 +412,18 @@ func (l *genericResource) cloneTo(targetPath string) resource.Resource {
}
func (l *genericResource) Content(context.Context) (any, error) {
- if err := l.initContent(); err != nil {
- return nil, err
+ r, err := l.ReadSeekCloser()
+ if err != nil {
+ return "", err
}
+ defer r.Close()
- return l.content, nil
+ var b []byte
+ b, err = io.ReadAll(r)
+ if err != nil {
+ return "", err
+ }
+ return string(b), nil
}
func (r *genericResource) Err() resource.ResourceError {
@@ -527,28 +534,6 @@ func (l *genericResource) Title() string {
return l.title
}
-func (l *genericResource) initContent() error {
- var err error
- l.contentInit.Do(func() {
- var r hugio.ReadSeekCloser
- r, err = l.ReadSeekCloser()
- if err != nil {
- return
- }
- defer r.Close()
-
- var b []byte
- b, err = io.ReadAll(r)
- if err != nil {
- return
- }
-
- l.content = string(b)
- })
-
- return err
-}
-
func (l *genericResource) getSpec() *Spec {
return l.spec
}
@@ -588,12 +573,9 @@ func (rc *genericResource) cloneWithUpdates(u *transformationUpdate) (baseResour
r := rc.clone()
if u.content != nil {
- r.contentInit.Do(func() {
- r.content = *u.content
- r.sd.OpenReadSeekCloser = func() (hugio.ReadSeekCloser, error) {
- return hugio.NewReadSeekerNoOpCloserFromString(r.content), nil
- }
- })
+ r.sd.OpenReadSeekCloser = func() (hugio.ReadSeekCloser, error) {
+ return hugio.NewReadSeekerNoOpCloserFromString(*u.content), nil
+ }
}
r.sd.MediaType = u.mediaType
@@ -620,7 +602,7 @@ func (rc *genericResource) cloneWithUpdates(u *transformationUpdate) (baseResour
}
func (l genericResource) clone() *genericResource {
- l.resourceContent = &resourceContent{}
+ l.publishInit = &sync.Once{}
return &l
}
@@ -633,13 +615,6 @@ type targetPather interface {
TargetPath() string
}
-type resourceContent struct {
- content string
- contentInit sync.Once
-
- publishInit sync.Once
-}
-
type resourceHash struct {
value string
size int64
diff --git a/resources/resource_spec.go b/resources/resource_spec.go
index 66f56d147..5fdcdd831 100644
--- a/resources/resource_spec.go
+++ b/resources/resource_spec.go
@@ -163,15 +163,15 @@ func (r *Spec) NewResource(rd ResourceSourceDescriptor) (resource.Resource, erro
}
gr := &genericResource{
- Staler: &AtomicStaler{},
- h: &resourceHash{},
- paths: rp,
- spec: r,
- sd: rd,
- params: make(map[string]any),
- name: rd.Name,
- title: rd.Name,
- resourceContent: &resourceContent{},
+ Staler: &AtomicStaler{},
+ h: &resourceHash{},
+ publishInit: &sync.Once{},
+ paths: rp,
+ spec: r,
+ sd: rd,
+ params: make(map[string]any),
+ name: rd.Name,
+ title: rd.Name,
}
if rd.MediaType.MainType == "image" {