From 7285e74090852b5d52f25e577850fa75f4aa8573 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Sun, 24 Dec 2023 19:11:05 +0100 Subject: all: Rework page store, add a dynacache, improve partial rebuilds, and some general spring cleaning There are some breaking changes in this commit, see #11455. Closes #11455 Closes #11549 This fixes a set of bugs (see issue list) and it is also paying some technical debt accumulated over the years. We now build with Staticcheck enabled in the CI build. The performance should be about the same as before for regular sized Hugo sites, but it should perform and scale much better to larger data sets, as objects that uses lots of memory (e.g. rendered Markdown, big JSON files read into maps with transform.Unmarshal etc.) will now get automatically garbage collected if needed. Performance on partial rebuilds when running the server in fast render mode should be the same, but the change detection should be much more accurate. A list of the notable new features: * A new dependency tracker that covers (almost) all of Hugo's API and is used to do fine grained partial rebuilds when running the server. * A new and simpler tree document store which allows fast lookups and prefix-walking in all dimensions (e.g. language) concurrently. * You can now configure an upper memory limit allowing for much larger data sets and/or running on lower specced PCs. We have lifted the "no resources in sub folders" restriction for branch bundles (e.g. sections). Memory Limit * Hugos will, by default, set aside a quarter of the total system memory, but you can set this via the OS environment variable HUGO_MEMORYLIMIT (in gigabytes). This is backed by a partitioned LRU cache used throughout Hugo. A cache that gets dynamically resized in low memory situations, allowing Go's Garbage Collector to free the memory. New Dependency Tracker: Hugo has had a rule based coarse grained approach to server rebuilds that has worked mostly pretty well, but there have been some surprises (e.g. stale content). This is now revamped with a new dependency tracker that can quickly calculate the delta given a changed resource (e.g. a content file, template, JS file etc.). This handles transitive relations, e.g. $page -> js.Build -> JS import, or $page1.Content -> render hook -> site.GetPage -> $page2.Title, or $page1.Content -> shortcode -> partial -> site.RegularPages -> $page2.Content -> shortcode ..., and should also handle changes to aggregated values (e.g. site.Lastmod) effectively. This covers all of Hugo's API with 2 known exceptions (a list that may not be fully exhaustive): Changes to files loaded with template func os.ReadFile may not be handled correctly. We recommend loading resources with resources.Get Changes to Hugo objects (e.g. Page) passed in the template context to lang.Translate may not be detected correctly. We recommend having simple i18n templates without too much data context passed in other than simple types such as strings and numbers. Note that the cachebuster configuration (when A changes then rebuild B) works well with the above, but we recommend that you revise that configuration, as it in most situations should not be needed. One example where it is still needed is with TailwindCSS and using changes to hugo_stats.json to trigger new CSS rebuilds. Document Store: Previously, a little simplified, we split the document store (where we store pages and resources) in a tree per language. This worked pretty well, but the structure made some operations harder than they needed to be. We have now restructured it into one Radix tree for all languages. Internally the language is considered to be a dimension of that tree, and the tree can be viewed in all dimensions concurrently. This makes some operations re. language simpler (e.g. finding translations is just a slice range), but the idea is that it should also be relatively inexpensive to add more dimensions if needed (e.g. role). Fixes #10169 Fixes #10364 Fixes #10482 Fixes #10630 Fixes #10656 Fixes #10694 Fixes #10918 Fixes #11262 Fixes #11439 Fixes #11453 Fixes #11457 Fixes #11466 Fixes #11540 Fixes #11551 Fixes #11556 Fixes #11654 Fixes #11661 Fixes #11663 Fixes #11664 Fixes #11669 Fixes #11671 Fixes #11807 Fixes #11808 Fixes #11809 Fixes #11815 Fixes #11840 Fixes #11853 Fixes #11860 Fixes #11883 Fixes #11904 Fixes #7388 Fixes #7425 Fixes #7436 Fixes #7544 Fixes #7882 Fixes #7960 Fixes #8255 Fixes #8307 Fixes #8863 Fixes #8927 Fixes #9192 Fixes #9324 --- common/constants/constants.go | 21 ++ common/hcontext/context.go | 46 +++ common/herrors/error_locator.go | 3 +- common/herrors/error_locator_test.go | 2 +- common/herrors/errors.go | 21 +- common/herrors/errors_test.go | 3 +- common/herrors/file_error.go | 22 +- common/herrors/file_error_test.go | 6 +- common/hreflect/helpers.go | 41 ++- common/hreflect/helpers_test.go | 37 +++ common/hstrings/strings.go | 7 +- common/hstrings/strings_test.go | 4 +- common/htime/integration_test.go | 2 +- common/hugio/copy.go | 9 +- common/hugio/hasBytesWriter.go | 2 +- common/hugio/hasBytesWriter_test.go | 2 +- common/hugio/readers.go | 20 ++ common/hugo/hugo.go | 9 +- common/loggers/handlerdefault.go | 6 +- common/loggers/handlersmisc.go | 23 +- common/loggers/handlerterminal.go | 4 +- common/loggers/logger.go | 57 +++- common/loggers/logger_test.go | 2 +- common/loggers/loggerglobal.go | 2 +- common/maps/cache.go | 90 ++++++ common/maps/maps.go | 17 +- common/maps/params.go | 10 +- common/paths/path.go | 222 ++++++++++---- common/paths/path_test.go | 114 ++++--- common/paths/pathparser.go | 494 +++++++++++++++++++++++++++++++ common/paths/pathparser_test.go | 351 ++++++++++++++++++++++ common/paths/paths_integration_test.go | 80 +++++ common/paths/pathtype_string.go | 27 ++ common/paths/url.go | 10 + common/predicate/predicate.go | 72 +++++ common/predicate/predicate_test.go | 83 ++++++ common/rungroup/rungroup.go | 93 ++++++ common/rungroup/rungroup_test.go | 44 +++ common/terminal/colors.go | 2 +- common/types/css/csstypes.go | 2 +- common/types/evictingqueue.go | 15 +- common/types/hstring/stringtypes.go | 2 +- common/types/hstring/stringtypes_test.go | 2 +- common/types/types.go | 13 + common/urls/baseURL.go | 28 +- common/urls/baseURL_test.go | 28 +- 46 files changed, 1957 insertions(+), 193 deletions(-) create mode 100644 common/hcontext/context.go create mode 100644 common/maps/cache.go create mode 100644 common/paths/pathparser.go create mode 100644 common/paths/pathparser_test.go create mode 100644 common/paths/paths_integration_test.go create mode 100644 common/paths/pathtype_string.go create mode 100644 common/predicate/predicate.go create mode 100644 common/predicate/predicate_test.go create mode 100644 common/rungroup/rungroup.go create mode 100644 common/rungroup/rungroup_test.go (limited to 'common') diff --git a/common/constants/constants.go b/common/constants/constants.go index 6afb9e283..e4f5a63a2 100644 --- a/common/constants/constants.go +++ b/common/constants/constants.go @@ -20,3 +20,24 @@ const ( ErrRemoteGetJSON = "error-remote-getjson" ErrRemoteGetCSV = "error-remote-getcsv" ) + +// Field/method names with special meaning. +const ( + FieldRelPermalink = "RelPermalink" + FieldPermalink = "Permalink" +) + +// IsFieldRelOrPermalink returns whether the given name is a RelPermalink or Permalink. +func IsFieldRelOrPermalink(name string) bool { + return name == FieldRelPermalink || name == FieldPermalink +} + +// Resource transformations. +const ( + ResourceTransformationFingerprint = "fingerprint" +) + +// IsResourceTransformationLinkChange returns whether the given name is a resource transformation that changes the permalink based on the content. +func IsResourceTransformationPermalinkHash(name string) bool { + return name == ResourceTransformationFingerprint +} diff --git a/common/hcontext/context.go b/common/hcontext/context.go new file mode 100644 index 000000000..9524ef284 --- /dev/null +++ b/common/hcontext/context.go @@ -0,0 +1,46 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcontext + +import "context" + +// ContextDispatcher is a generic interface for setting and getting values from a context. +type ContextDispatcher[T any] interface { + Set(ctx context.Context, value T) context.Context + Get(ctx context.Context) T +} + +// NewContextDispatcher creates a new ContextDispatcher with the given key. +func NewContextDispatcher[T any, R comparable](key R) ContextDispatcher[T] { + return keyInContext[T, R]{ + id: key, + } +} + +type keyInContext[T any, R comparable] struct { + zero T + id R +} + +func (f keyInContext[T, R]) Get(ctx context.Context) T { + v := ctx.Value(f.id) + if v == nil { + return f.zero + } + return v.(T) +} + +func (f keyInContext[T, R]) Set(ctx context.Context, value T) context.Context { + return context.WithValue(ctx, f.id, value) +} diff --git a/common/herrors/error_locator.go b/common/herrors/error_locator.go index b880fe045..1ece0cca4 100644 --- a/common/herrors/error_locator.go +++ b/common/herrors/error_locator.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -74,7 +74,6 @@ func ContainsMatcher(text string) func(m LineMatcher) int { // ErrorContext contains contextual information about an error. This will // typically be the lines surrounding some problem in a file. type ErrorContext struct { - // If a match will contain the matched line and up to 2 lines before and after. // Will be empty if no match. Lines []string diff --git a/common/herrors/error_locator_test.go b/common/herrors/error_locator_test.go index 6135657d8..62f15213d 100644 --- a/common/herrors/error_locator_test.go +++ b/common/herrors/error_locator_test.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/common/herrors/errors.go b/common/herrors/errors.go index 8e62b2c99..59739a86a 100644 --- a/common/herrors/errors.go +++ b/common/herrors/errors.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ import ( "os" "runtime" "runtime/debug" + "time" ) // PrintStackTrace prints the current stacktrace to w. @@ -47,6 +48,24 @@ func Recover(args ...any) { } } +// IsTimeoutError returns true if the given error is or contains a TimeoutError. +func IsTimeoutError(err error) bool { + return errors.Is(err, &TimeoutError{}) +} + +type TimeoutError struct { + Duration time.Duration +} + +func (e *TimeoutError) Error() string { + return fmt.Sprintf("timeout after %s", e.Duration) +} + +func (e *TimeoutError) Is(target error) bool { + _, ok := target.(*TimeoutError) + return ok +} + // IsFeatureNotAvailableError returns true if the given error is or contains a FeatureNotAvailableError. func IsFeatureNotAvailableError(err error) bool { return errors.Is(err, &FeatureNotAvailableError{}) diff --git a/common/herrors/errors_test.go b/common/herrors/errors_test.go index 223782e23..2f53a1e89 100644 --- a/common/herrors/errors_test.go +++ b/common/herrors/errors_test.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -42,5 +42,4 @@ func TestIsFeatureNotAvailableError(t *testing.T) { c.Assert(IsFeatureNotAvailableError(ErrFeatureNotAvailable), qt.Equals, true) c.Assert(IsFeatureNotAvailableError(&FeatureNotAvailableError{}), qt.Equals, true) c.Assert(IsFeatureNotAvailableError(errors.New("asdf")), qt.Equals, false) - } diff --git a/common/herrors/file_error.go b/common/herrors/file_error.go index f8bcecd34..32a6f0081 100644 --- a/common/herrors/file_error.go +++ b/common/herrors/file_error.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,13 +15,13 @@ package herrors import ( "encoding/json" - - godartsassv1 "github.com/bep/godartsass" - + "errors" "fmt" "io" "path/filepath" + godartsassv1 "github.com/bep/godartsass" + "github.com/bep/godartsass/v2" "github.com/bep/golibsass/libsass/libsasserrors" "github.com/gohugoio/hugo/common/paths" @@ -29,8 +29,6 @@ import ( "github.com/pelletier/go-toml/v2" "github.com/spf13/afero" "github.com/tdewolff/parse/v2" - - "errors" ) // FileError represents an error when handling a file: Parsing a config file, @@ -48,6 +46,9 @@ type FileError interface { // UpdateContent updates the error with a new ErrorContext from the content of the file. UpdateContent(r io.Reader, linematcher LineMatcherFn) FileError + + // SetFilename sets the filename of the error. + SetFilename(filename string) FileError } // Unwrapper can unwrap errors created with fmt.Errorf. @@ -60,6 +61,11 @@ var ( _ Unwrapper = (*fileError)(nil) ) +func (fe *fileError) SetFilename(filename string) FileError { + fe.position.Filename = filename + return fe +} + func (fe *fileError) UpdatePosition(pos text.Position) FileError { oldFilename := fe.Position().Filename if pos.Filename != "" && fe.fileType == "" { @@ -115,7 +121,6 @@ func (fe *fileError) UpdateContent(r io.Reader, linematcher LineMatcherFn) FileE } return fe - } type fileError struct { @@ -181,7 +186,6 @@ func NewFileErrorFromName(err error, name string) FileError { } return &fileError{cause: err, fileType: fileType, position: pos} - } // NewFileErrorFromPos will use the filename and line number from pos to create a new FileError, wrapping err. @@ -192,7 +196,6 @@ func NewFileErrorFromPos(err error, pos text.Position) FileError { _, fileType = paths.FileAndExtNoDelimiter(filepath.Clean(pos.Filename)) } return &fileError{cause: err, fileType: fileType, position: pos} - } func NewFileErrorFromFileInErr(err error, fs afero.Fs, linematcher LineMatcherFn) FileError { @@ -249,7 +252,6 @@ func openFile(filename string, fs afero.Fs) (afero.File, string, error) { }); ok { realFilename = s.Filename() } - } f, err2 := fs.Open(filename) diff --git a/common/herrors/file_error_test.go b/common/herrors/file_error_test.go index 0b260a255..7aca08405 100644 --- a/common/herrors/file_error_test.go +++ b/common/herrors/file_error_test.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,12 +14,11 @@ package herrors import ( + "errors" "fmt" "strings" "testing" - "errors" - "github.com/gohugoio/hugo/common/text" qt "github.com/frankban/quicktest" @@ -48,7 +47,6 @@ func TestNewFileError(t *testing.T) { c.Assert(errorContext.Lines, qt.DeepEquals, []string{"line 30", "line 31", "line 32", "line 33", "line 34"}) c.Assert(errorContext.LinesPos, qt.Equals, 2) c.Assert(errorContext.ChromaLexer, qt.Equals, "go-html-template") - } func TestNewFileErrorExtractFromMessage(t *testing.T) { diff --git a/common/hreflect/helpers.go b/common/hreflect/helpers.go index 17afbf912..b5a8bacc9 100644 --- a/common/hreflect/helpers.go +++ b/common/hreflect/helpers.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // Some functions in this file (see comments) is based on the Go source code, // copyright The Go Authors and governed by a BSD-style license. // @@ -23,6 +23,7 @@ import ( "time" "github.com/gohugoio/hugo/common/htime" + "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/common/types" ) @@ -188,6 +189,20 @@ func IsTime(tp reflect.Type) bool { return false } +// IsValid returns whether v is not nil and a valid value. +func IsValid(v reflect.Value) bool { + if !v.IsValid() { + return false + } + + switch v.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: + return !v.IsNil() + } + + return true +} + // AsTime returns v as a time.Time if possible. // The given location is only used if the value implements AsTimeProvider (e.g. go-toml local). // A zero Time and false is returned if this isn't possible. @@ -217,7 +232,7 @@ func CallMethodByName(cxt context.Context, name string, v reflect.Value) []refle panic("not supported") } first := tp.In(0) - if first.Implements(ContextInterface) { + if IsContextType(first) { args = append(args, reflect.ValueOf(cxt)) } } @@ -236,4 +251,24 @@ func indirectInterface(v reflect.Value) reflect.Value { return v.Elem() } -var ContextInterface = reflect.TypeOf((*context.Context)(nil)).Elem() +var contextInterface = reflect.TypeOf((*context.Context)(nil)).Elem() + +var isContextCache = maps.NewCache[reflect.Type, bool]() + +type k string + +var contextTypeValue = reflect.TypeOf(context.WithValue(context.Background(), k("key"), 32)) + +// IsContextType returns whether tp is a context.Context type. +func IsContextType(tp reflect.Type) bool { + if tp == contextTypeValue { + return true + } + if tp == contextInterface { + return true + } + + return isContextCache.GetOrCreate(tp, func() bool { + return tp.Implements(contextInterface) + }) +} diff --git a/common/hreflect/helpers_test.go b/common/hreflect/helpers_test.go index d16b9b9b3..27b774337 100644 --- a/common/hreflect/helpers_test.go +++ b/common/hreflect/helpers_test.go @@ -14,6 +14,7 @@ package hreflect import ( + "context" "reflect" "testing" "time" @@ -40,6 +41,42 @@ func TestGetMethodByName(t *testing.T) { c.Assert(GetMethodIndexByName(tp, "Foo"), qt.Equals, -1) } +func TestIsContextType(t *testing.T) { + c := qt.New(t) + type k string + ctx := context.Background() + valueCtx := context.WithValue(ctx, k("key"), 32) + c.Assert(IsContextType(reflect.TypeOf(ctx)), qt.IsTrue) + c.Assert(IsContextType(reflect.TypeOf(valueCtx)), qt.IsTrue) +} + +func BenchmarkIsContextType(b *testing.B) { + type k string + b.Run("value", func(b *testing.B) { + ctx := context.Background() + ctxs := make([]reflect.Type, b.N) + for i := 0; i < b.N; i++ { + ctxs[i] = reflect.TypeOf(context.WithValue(ctx, k("key"), i)) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if !IsContextType(ctxs[i]) { + b.Fatal("not context") + } + } + }) + + b.Run("background", func(b *testing.B) { + var ctxt reflect.Type = reflect.TypeOf(context.Background()) + for i := 0; i < b.N; i++ { + if !IsContextType(ctxt) { + b.Fatal("not context") + } + } + }) +} + func BenchmarkIsTruthFul(b *testing.B) { v := reflect.ValueOf("Hugo") diff --git a/common/hstrings/strings.go b/common/hstrings/strings.go index 88df97607..d9426ab5d 100644 --- a/common/hstrings/strings.go +++ b/common/hstrings/strings.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -122,3 +122,8 @@ func InSlicEqualFold(arr []string, el string) bool { } return false } + +type Tuple struct { + First string + Second string +} diff --git a/common/hstrings/strings_test.go b/common/hstrings/strings_test.go index 85068bdf9..d8e9e204a 100644 --- a/common/hstrings/strings_test.go +++ b/common/hstrings/strings_test.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -33,7 +33,6 @@ func TestStringEqualFold(t *testing.T) { c.Assert(StringEqualFold(s1).EqualFold("b"), qt.Equals, false) c.Assert(StringEqualFold(s1).Eq(s2), qt.Equals, true) c.Assert(StringEqualFold(s1).Eq("b"), qt.Equals, false) - } func TestGetOrCompileRegexp(t *testing.T) { @@ -42,7 +41,6 @@ func TestGetOrCompileRegexp(t *testing.T) { re, err := GetOrCompileRegexp(`\d+`) c.Assert(err, qt.IsNil) c.Assert(re.MatchString("123"), qt.Equals, true) - } func BenchmarkGetOrCompileRegexp(b *testing.B) { diff --git a/common/htime/integration_test.go b/common/htime/integration_test.go index e72c216d9..983fff1f7 100644 --- a/common/htime/integration_test.go +++ b/common/htime/integration_test.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/common/hugio/copy.go b/common/hugio/copy.go index 8dbadc48c..31d679dfc 100644 --- a/common/hugio/copy.go +++ b/common/hugio/copy.go @@ -16,6 +16,7 @@ package hugio import ( "fmt" "io" + iofs "io/fs" "path/filepath" "github.com/spf13/afero" @@ -60,12 +61,16 @@ func CopyDir(fs afero.Fs, from, to string, shouldCopy func(filename string) bool return fmt.Errorf("%q is not a directory", from) } - err = fs.MkdirAll(to, 0777) // before umask + err = fs.MkdirAll(to, 0o777) // before umask if err != nil { return err } - entries, _ := afero.ReadDir(fs, from) + d, err := fs.Open(from) + if err != nil { + return err + } + entries, _ := d.(iofs.ReadDirFile).ReadDir(-1) for _, entry := range entries { fromFilename := filepath.Join(from, entry.Name()) toFilename := filepath.Join(to, entry.Name()) diff --git a/common/hugio/hasBytesWriter.go b/common/hugio/hasBytesWriter.go index 7b7d7a5d7..5148c82f9 100644 --- a/common/hugio/hasBytesWriter.go +++ b/common/hugio/hasBytesWriter.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/common/hugio/hasBytesWriter_test.go b/common/hugio/hasBytesWriter_test.go index b1b8011d5..af53fa5dd 100644 --- a/common/hugio/hasBytesWriter_test.go +++ b/common/hugio/hasBytesWriter_test.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/common/hugio/readers.go b/common/hugio/readers.go index 60bd97992..feb1b1412 100644 --- a/common/hugio/readers.go +++ b/common/hugio/readers.go @@ -14,6 +14,7 @@ package hugio import ( + "bytes" "io" "strings" ) @@ -57,3 +58,22 @@ func NewReadSeekerNoOpCloser(r ReadSeeker) ReadSeekerNoOpCloser { func NewReadSeekerNoOpCloserFromString(content string) ReadSeekerNoOpCloser { return ReadSeekerNoOpCloser{strings.NewReader(content)} } + +// NewReadSeekerNoOpCloserFromString uses strings.NewReader to create a new ReadSeekerNoOpCloser +// from the given bytes slice. +func NewReadSeekerNoOpCloserFromBytes(content []byte) ReadSeekerNoOpCloser { + return ReadSeekerNoOpCloser{bytes.NewReader(content)} +} + +// NewReadSeekCloser creates a new ReadSeekCloser from the given ReadSeeker. +// The ReadSeeker will be seeked to the beginning before returned. +func NewOpenReadSeekCloser(r ReadSeekCloser) OpenReadSeekCloser { + return func() (ReadSeekCloser, error) { + r.Seek(0, io.SeekStart) + return r, nil + } +} + +// OpenReadSeekCloser allows setting some other way (than reading from a filesystem) +// to open or create a ReadSeekCloser. +type OpenReadSeekCloser func() (ReadSeekCloser, error) diff --git a/common/hugo/hugo.go b/common/hugo/hugo.go index 67d52f6c8..be43e2a38 100644 --- a/common/hugo/hugo.go +++ b/common/hugo/hugo.go @@ -35,6 +35,8 @@ import ( "github.com/spf13/afero" + iofs "io/fs" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/hugofs" ) @@ -159,7 +161,12 @@ func GetExecEnviron(workDir string, cfg config.AllProvider, fs afero.Fs) []strin config.SetEnvVars(&env, "HUGO_PUBLISHDIR", filepath.Join(workDir, cfg.BaseConfig().PublishDir)) if fs != nil { - fis, err := afero.ReadDir(fs, files.FolderJSConfig) + var fis []iofs.DirEntry + d, err := fs.Open(files.FolderJSConfig) + if err == nil { + fis, err = d.(iofs.ReadDirFile).ReadDir(-1) + } + if err == nil { for _, fi := range fis { key := fmt.Sprintf("HUGO_FILE_%s", strings.ReplaceAll(strings.ToUpper(fi.Name()), ".", "_")) diff --git a/common/loggers/handlerdefault.go b/common/loggers/handlerdefault.go index bb48895bc..bc3c7eec2 100644 --- a/common/loggers/handlerdefault.go +++ b/common/loggers/handlerdefault.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // Some functions in this file (see comments) is based on the Go source code, // copyright The Go Authors and governed by a BSD-style license. // @@ -27,10 +27,9 @@ import ( "github.com/fatih/color" ) -var bold = color.New(color.Bold) - // levelColor mapping. var levelColor = [...]*color.Color{ + logg.LevelTrace: color.New(color.FgWhite), logg.LevelDebug: color.New(color.FgWhite), logg.LevelInfo: color.New(color.FgBlue), logg.LevelWarn: color.New(color.FgYellow), @@ -39,6 +38,7 @@ var levelColor = [...]*color.Color{ // levelString mapping. var levelString = [...]string{ + logg.LevelTrace: "TRACE", logg.LevelDebug: "DEBUG", logg.LevelInfo: "INFO ", logg.LevelWarn: "WARN ", diff --git a/common/loggers/handlersmisc.go b/common/loggers/handlersmisc.go index 5c9d6c091..55bf8b940 100644 --- a/common/loggers/handlersmisc.go +++ b/common/loggers/handlersmisc.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // Some functions in this file (see comments) is based on the Go source code, // copyright The Go Authors and governed by a BSD-style license. // @@ -69,7 +69,7 @@ func (h *logLevelCounter) HandleLog(e *logg.Entry) error { return nil } -var stopError = fmt.Errorf("stop") +var errStop = fmt.Errorf("stop") type logOnceHandler struct { threshold logg.Level @@ -87,7 +87,7 @@ func (h *logOnceHandler) HandleLog(e *logg.Entry) error { defer h.mu.Unlock() hash := identity.HashUint64(e.Level, e.Message, e.Fields) if h.seen[hash] { - return stopError + return errStop } h.seen[hash] = true return nil @@ -107,7 +107,7 @@ type stopHandler struct { func (h *stopHandler) HandleLog(e *logg.Entry) error { for _, handler := range h.handlers { if err := handler.HandleLog(e); err != nil { - if err == stopError { + if err == errStop { return nil } return err @@ -124,26 +124,13 @@ func (h *suppressStatementsHandler) HandleLog(e *logg.Entry) error { for _, field := range e.Fields { if field.Name == FieldNameStatementID { if h.statements[field.Value.(string)] { - return stopError + return errStop } } } return nil } -// replacer creates a new log handler that does string replacement in log messages. -func replacer(repl *strings.Replacer) logg.Handler { - return logg.HandlerFunc(func(e *logg.Entry) error { - e.Message = repl.Replace(e.Message) - for i, field := range e.Fields { - if s, ok := field.Value.(string); ok { - e.Fields[i].Value = repl.Replace(s) - } - } - return nil - }) -} - // whiteSpaceTrimmer creates a new log handler that trims whitespace from log messages and string fields. func whiteSpaceTrimmer() logg.Handler { return logg.HandlerFunc(func(e *logg.Entry) error { diff --git a/common/loggers/handlerterminal.go b/common/loggers/handlerterminal.go index e3d377bbf..53f6e41da 100644 --- a/common/loggers/handlerterminal.go +++ b/common/loggers/handlerterminal.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // Some functions in this file (see comments) is based on the Go source code, // copyright The Go Authors and governed by a BSD-style license. // @@ -81,7 +81,7 @@ func (h *noColoursHandler) HandleLog(e *logg.Entry) error { if strings.HasPrefix(field.Name, reservedFieldNamePrefix) { continue } - fmt.Fprintf(w, " %s %q", field.Name, field.Value) + fmt.Fprintf(w, " %s %v", field.Name, field.Value) } fmt.Fprintln(w) diff --git a/common/loggers/logger.go b/common/loggers/logger.go index bc64ae0e5..c4d81fb83 100644 --- a/common/loggers/logger.go +++ b/common/loggers/logger.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // Some functions in this file (see comments) is based on the Go source code, // copyright The Go Authors and governed by a BSD-style license. // @@ -68,11 +68,24 @@ func New(opts Options) Logger { errorsw := &strings.Builder{} logCounters := newLogLevelCounter() handlers := []logg.Handler{ - whiteSpaceTrimmer(), - logHandler, logCounters, } + if opts.Level == logg.LevelTrace { + // Trace is used during development only, and it's useful to + // only see the trace messages. + handlers = append(handlers, + logg.HandlerFunc(func(e *logg.Entry) error { + if e.Level != logg.LevelTrace { + return logg.ErrStopLogEntry + } + return nil + }), + ) + } + + handlers = append(handlers, whiteSpaceTrimmer(), logHandler) + if opts.HandlerPost != nil { var hookHandler logg.HandlerFunc = func(e *logg.Entry) error { opts.HandlerPost(e) @@ -127,6 +140,7 @@ func New(opts Options) Logger { out: opts.Stdout, level: opts.Level, logger: logger, + tracel: l.WithLevel(logg.LevelTrace), debugl: l.WithLevel(logg.LevelDebug), infol: l.WithLevel(logg.LevelInfo), warnl: l.WithLevel(logg.LevelWarn), @@ -145,11 +159,22 @@ func NewDefault() Logger { return New(opts) } +func NewTrace() Logger { + opts := Options{ + DistinctLevel: logg.LevelWarn, + Level: logg.LevelTrace, + Stdout: os.Stdout, + Stderr: os.Stdout, + } + return New(opts) +} + func LevelLoggerToWriter(l logg.LevelLogger) io.Writer { return logWriter{l: l} } type Logger interface { + Debug() logg.LevelLogger Debugf(format string, v ...any) Debugln(v ...any) Error() logg.LevelLogger @@ -174,6 +199,7 @@ type Logger interface { Warnf(format string, v ...any) Warnln(v ...any) Deprecatef(fail bool, format string, v ...any) + Trace(s logg.StringFunc) } type logAdapter struct { @@ -183,12 +209,17 @@ type logAdapter struct { out io.Writer level logg.Level logger logg.Logger + tracel logg.LevelLogger debugl logg.LevelLogger infol logg.LevelLogger warnl logg.LevelLogger errorl logg.LevelLogger } +func (l *logAdapter) Debug() logg.LevelLogger { + return l.debugl +} + func (l *logAdapter) Debugf(format string, v ...any) { l.debugl.Logf(format, v...) } @@ -294,6 +325,10 @@ func (l *logAdapter) Errorsf(id, format string, v ...any) { l.errorl.WithField(FieldNameStatementID, id).Logf(format, v...) } +func (l *logAdapter) Trace(s logg.StringFunc) { + l.tracel.Log(s) +} + func (l *logAdapter) sprint(v ...any) string { return strings.TrimRight(fmt.Sprintln(v...), "\n") } @@ -315,3 +350,19 @@ func (w logWriter) Write(p []byte) (n int, err error) { w.l.Log(logg.String(string(p))) return len(p), nil } + +func TimeTrackf(l logg.LevelLogger, start time.Time, fields logg.Fields, format string, a ...any) { + elapsed := time.Since(start) + if fields != nil { + l = l.WithFields(fields) + } + l.WithField("duration", elapsed).Logf(format, a...) +} + +func TimeTrackfn(fn func() (logg.LevelLogger, error)) error { + start := time.Now() + l, err := fn() + elapsed := time.Since(start) + l.WithField("duration", elapsed).Logf("") + return err +} diff --git a/common/loggers/logger_test.go b/common/loggers/logger_test.go index 6f589aafe..dcf94b123 100644 --- a/common/loggers/logger_test.go +++ b/common/loggers/logger_test.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // Some functions in this file (see comments) is based on the Go source code, // copyright The Go Authors and governed by a BSD-style license. // diff --git a/common/loggers/loggerglobal.go b/common/loggers/loggerglobal.go index 6fd474a69..c3e2970d0 100644 --- a/common/loggers/loggerglobal.go +++ b/common/loggers/loggerglobal.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // Some functions in this file (see comments) is based on the Go source code, // copyright The Go Authors and governed by a BSD-style license. // diff --git a/common/maps/cache.go b/common/maps/cache.go new file mode 100644 index 000000000..7e23a2662 --- /dev/null +++ b/common/maps/cache.go @@ -0,0 +1,90 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package maps + +import "sync" + +// Cache is a simple thread safe cache backed by a map. +type Cache[K comparable, T any] struct { + m map[K]T + sync.RWMutex +} + +// NewCache creates a new Cache. +func NewCache[K comparable, T any]() *Cache[K, T] { + return &Cache[K, T]{m: make(map[K]T)} +} + +// Delete deletes the given key from the cache. +func (c *Cache[K, T]) Get(key K) (T, bool) { + c.RLock() + v, found := c.m[key] + c.RUnlock() + return v, found +} + +// GetOrCreate gets the value for the given key if it exists, or creates it if not. +func (c *Cache[K, T]) GetOrCreate(key K, create func() T) T { + c.RLock() + v, found := c.m[key] + c.RUnlock() + if found { + return v + } + c.Lock() + defer c.Unlock() + v, found = c.m[key] + if found { + return v + } + v = create() + c.m[key] = v + return v +} + +// Set sets the given key to the given value. +func (c *Cache[K, T]) Set(key K, value T) { + c.Lock() + c.m[key] = value + c.Unlock() +} + +// SliceCache is a simple thread safe cache backed by a map. +type SliceCache[T any] struct { + m map[string][]T + sync.RWMutex +} + +func NewSliceCache[T any]() *SliceCache[T] { + return &SliceCache[T]{m: make(map[string][]T)} +} + +func (c *SliceCache[T]) Get(key string) ([]T, bool) { + c.RLock() + v, found := c.m[key] + c.RUnlock() + return v, found +} + +func (c *SliceCache[T]) Append(key string, values ...T) { + c.Lock() + c.m[key] = append(c.m[key], values...) + c.Unlock() +} + +func (c *SliceCache[T]) Reset() { + c.Lock() + c.m = make(map[string][]T) + c.Unlock() +} diff --git a/common/maps/maps.go b/common/maps/maps.go index f0fd3d5ce..2686baad6 100644 --- a/common/maps/maps.go +++ b/common/maps/maps.go @@ -29,7 +29,7 @@ func ToStringMapE(in any) (map[string]any, error) { case Params: return vv, nil case map[string]string: - var m = map[string]any{} + m := map[string]any{} for k, v := range vv { m[k] = v } @@ -192,21 +192,20 @@ func (KeyRenamer) keyPath(k1, k2 string) string { } func (r KeyRenamer) renamePath(parentKeyPath string, m map[string]any) { - for key, val := range m { - keyPath := r.keyPath(parentKeyPath, key) - switch val.(type) { + for k, v := range m { + keyPath := r.keyPath(parentKeyPath, k) + switch vv := v.(type) { case map[any]any: - val = cast.ToStringMap(val) - r.renamePath(keyPath, val.(map[string]any)) + r.renamePath(keyPath, cast.ToStringMap(vv)) case map[string]any: - r.renamePath(keyPath, val.(map[string]any)) + r.renamePath(keyPath, vv) } newKey := r.getNewKey(keyPath) if newKey != "" { - delete(m, key) - m[newKey] = val + delete(m, k) + m[newKey] = v } } } diff --git a/common/maps/params.go b/common/maps/params.go index d94d16f9d..a8cbba555 100644 --- a/common/maps/params.go +++ b/common/maps/params.go @@ -61,7 +61,7 @@ func SetParams(dst, src Params) { // IsZero returns true if p is considered empty. func (p Params) IsZero() bool { - if p == nil || len(p) == 0 { + if len(p) == 0 { return true } @@ -74,7 +74,6 @@ func (p Params) IsZero() bool { } return false - } // MergeParamsWithStrategy transfers values from src to dst for new keys using the merge strategy given. @@ -93,7 +92,7 @@ func MergeParams(dst, src Params) { func (p Params) merge(ps ParamsMergeStrategy, pp Params) { ns, found := p.GetMergeStrategy() - var ms = ns + ms := ns if !found && ps != "" { ms = ps } @@ -248,7 +247,7 @@ const ( // CleanConfigStringMapString removes any processing instructions from m, // m will never be modified. func CleanConfigStringMapString(m map[string]string) map[string]string { - if m == nil || len(m) == 0 { + if len(m) == 0 { return m } if _, found := m[MergeStrategyKey]; !found { @@ -267,7 +266,7 @@ func CleanConfigStringMapString(m map[string]string) map[string]string { // CleanConfigStringMap is the same as CleanConfigStringMapString but for // map[string]any. func CleanConfigStringMap(m map[string]any) map[string]any { - if m == nil || len(m) == 0 { + if len(m) == 0 { return m } if _, found := m[MergeStrategyKey]; !found { @@ -291,7 +290,6 @@ func CleanConfigStringMap(m map[string]any) map[string]any { } return m2 - } func toMergeStrategy(v any) ParamsMergeStrategy { diff --git a/common/paths/path.go b/common/paths/path.go index 5d211c5e0..da99b16ac 100644 --- a/common/paths/path.go +++ b/common/paths/path.go @@ -16,14 +16,18 @@ package paths import ( "errors" "fmt" + "net/url" "path" "path/filepath" - "regexp" "strings" + "unicode" ) // FilePathSeparator as defined by os.Separator. -const FilePathSeparator = string(filepath.Separator) +const ( + FilePathSeparator = string(filepath.Separator) + slash = "/" +) // filepathPathBridge is a bridge for common functionality in filepath vs path type filepathPathBridge interface { @@ -72,6 +76,30 @@ func AbsPathify(workingDir, inPath string) string { return filepath.Join(workingDir, inPath) } +// AddTrailingSlash adds a trailing Unix styled slash (/) if not already +// there. +func AddTrailingSlash(path string) string { + if !strings.HasSuffix(path, "/") { + path += "/" + } + return path +} + +// AddLeadingSlash adds a leading Unix styled slash (/) if not already +// there. +func AddLeadingSlash(path string) string { + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + return path +} + +// AddTrailingAndLeadingSlash adds a leading and trailing Unix styled slash (/) if not already +// there. +func AddLeadingAndTrailingSlash(path string) string { + return AddTrailingSlash(AddLeadingSlash(path)) +} + // MakeTitle converts the path given to a suitable title, trimming whitespace // and replacing hyphens with whitespace. func MakeTitle(inpath string) string { @@ -94,43 +122,6 @@ func makePathRelative(inPath string, possibleDirectories ...string) (string, err return inPath, errors.New("can't extract relative path, unknown prefix") } -// Should be good enough for Hugo. -var isFileRe = regexp.MustCompile(`.*\..{1,6}$`) - -// GetDottedRelativePath expects a relative path starting after the content directory. -// It returns a relative path with dots ("..") navigating up the path structure. -func GetDottedRelativePath(inPath string) string { - inPath = path.Clean(filepath.ToSlash(inPath)) - - if inPath == "." { - return "./" - } - - if !isFileRe.MatchString(inPath) && !strings.HasSuffix(inPath, "/") { - inPath += "/" - } - - if !strings.HasPrefix(inPath, "/") { - inPath = "/" + inPath - } - - dir, _ := filepath.Split(inPath) - - sectionCount := strings.Count(dir, "/") - - if sectionCount == 0 || dir == "/" { - return "./" - } - - var dottedPath string - - for i := 1; i < sectionCount; i++ { - dottedPath += "../" - } - - return dottedPath -} - // ExtNoDelimiter takes a path and returns the extension, excluding the delimiter, i.e. "md". func ExtNoDelimiter(in string) string { return strings.TrimPrefix(Ext(in), ".") @@ -167,12 +158,6 @@ func Filename(in string) (name string) { return } -// PathNoExt takes a path, strips out the extension, -// and returns the name of the file. -func PathNoExt(in string) string { - return strings.TrimSuffix(in, path.Ext(in)) -} - // FileAndExt returns the filename and any extension of a file path as // two separate strings. // @@ -252,16 +237,125 @@ func prettifyPath(in string, b filepathPathBridge) string { return b.Join(b.Dir(in), name, "index"+ext) } -type NamedSlice struct { - Name string - Slice []string +// CommonDir returns the common directory of the given paths. +func CommonDir(path1, path2 string) string { + if path1 == "" || path2 == "" { + return "" + } + + p1 := strings.Split(path1, "/") + p2 := strings.Split(path2, "/") + + var common []string + + for i := 0; i < len(p1) && i < len(p2); i++ { + if p1[i] == p2[i] { + common = append(common, p1[i]) + } else { + break + } + } + + return strings.Join(common, "/") +} + +// Sanitize sanitizes string to be used in Hugo's file paths and URLs, allowing only +// a predefined set of special Unicode characters. +// +// Spaces will be replaced with a single hyphen. +// +// This function is the core function used to normalize paths in Hugo. +// +// Note that this is the first common step for URL/path sanitation, +// the final URL/path may end up looking differently if the user has stricter rules defined (e.g. removePathAccents=true). +func Sanitize(s string) string { + var willChange bool + for i, r := range s { + willChange = !isAllowedPathCharacter(s, i, r) + if willChange { + break + } + } + + if !willChange { + // Prevent allocation when nothing changes. + return s + } + + target := make([]rune, 0, len(s)) + var ( + prependHyphen bool + wasHyphen bool + ) + + for i, r := range s { + isAllowed := isAllowedPathCharacter(s, i, r) + + if isAllowed { + // track explicit hyphen in input; no need to add a new hyphen if + // we just saw one. + wasHyphen = r == '-' + + if prependHyphen { + // if currently have a hyphen, don't prepend an extra one + if !wasHyphen { + target = append(target, '-') + } + prependHyphen = false + } + target = append(target, r) + } else if len(target) > 0 && !wasHyphen && unicode.IsSpace(r) { + prependHyphen = true + } + } + + return string(target) +} + +func isAllowedPathCharacter(s string, i int, r rune) bool { + if r == ' ' { + return false + } + // Check for the most likely first (faster). + isAllowed := unicode.IsLetter(r) || unicode.IsDigit(r) + isAllowed = isAllowed || r == '.' || r == '/' || r == '\\' || r == '_' || r == '#' || r == '+' || r == '~' || r == '-' || r == '@' + isAllowed = isAllowed || unicode.IsMark(r) + isAllowed = isAllowed || (r == '%' && i+2 < len(s) && ishex(s[i+1]) && ishex(s[i+2])) + return isAllowed } -func (n NamedSlice) String() string { - if len(n.Slice) == 0 { - return n.Name +// From https://golang.org/src/net/url/url.go +func ishex(c byte) bool { + switch { + case '0' <= c && c <= '9': + return true + case 'a' <= c && c <= 'f': + return true + case 'A' <= c && c <= 'F': + return true } - return fmt.Sprintf("%s%s{%s}", n.Name, FilePathSeparator, strings.Join(n.Slice, ",")) + return false +} + +var slashFunc = func(r rune) bool { + return r == '/' +} + +// Dir behaves like path.Dir without the path.Clean step. +// +// The returned path ends in a slash only if it is the root "/". +func Dir(s string) string { + dir, _ := path.Split(s) + if len(dir) > 1 && dir[len(dir)-1] == '/' { + return dir[:len(dir)-1] + } + return dir +} + +// FieldsSlash cuts s into fields separated with '/'. +func FieldsSlash(s string) []string { + f := strings.FieldsFunc(s, slashFunc) + return f } // DirFile holds the result from path.Split. @@ -274,3 +368,27 @@ type DirFile struct { func (df DirFile) String() string { return fmt.Sprintf("%s|%s", df.Dir, df.File) } + +// PathEscape escapes unicode letters in pth. +// Use URLEscape to escape full URLs including scheme, query etc. +// This is slightly faster for the common case. +// Note, there is a url.PathEscape function, but that also +// escapes /. +func PathEscape(pth string) string { + u, err := url.Parse(pth) + if err != nil { + panic(err) + } + return u.EscapedPath() +} + +// ToSlashTrimLeading is just a filepath.ToSlash with an added / prefix trimmer. +func ToSlashTrimLeading(s string) string { + return strings.TrimPrefix(filepath.ToSlash(s), "/") +} + +// ToSlashPreserveLeading converts the path given to a forward slash separated path +// and preserves the leading slash if present trimming any trailing slash. +func ToSlashPreserveLeading(s string) string { + return "/" + strings.Trim(filepath.ToSlash(s), "/") +} diff --git a/common/paths/path_test.go b/common/paths/path_test.go index 2400f16ab..3605bfc43 100644 --- a/common/paths/path_test.go +++ b/common/paths/path_test.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -75,44 +75,6 @@ func TestMakePathRelative(t *testing.T) { } } -func TestGetDottedRelativePath(t *testing.T) { - // on Windows this will receive both kinds, both country and western ... - for _, f := range []func(string) string{filepath.FromSlash, func(s string) string { return s }} { - doTestGetDottedRelativePath(f, t) - } -} - -func doTestGetDottedRelativePath(urlFixer func(string) string, t *testing.T) { - type test struct { - input, expected string - } - data := []test{ - {"", "./"}, - {urlFixer("/"), "./"}, - {urlFixer("post"), "../"}, - {urlFixer("/post"), "../"}, - {urlFixer("post/"), "../"}, - {urlFixer("tags/foo.html"), "../"}, - {urlFixer("/tags/foo.html"), "../"}, - {urlFixer("/post/"), "../"}, - {urlFixer("////post/////"), "../"}, - {urlFixer("/foo/bar/index.html"), "../../"}, - {urlFixer("/foo/bar/foo/"), "../../../"}, - {urlFixer("/foo/bar/foo"), "../../../"}, - {urlFixer("foo/bar/foo/"), "../../../"}, - {urlFixer("foo/bar/foo/bar"), "../../../../"}, - {"404.html", "./"}, - {"404.xml", "./"}, - {"/404.html", "./"}, - } - for i, d := range data { - output := GetDottedRelativePath(d.input) - if d.expected != output { - t.Errorf("Test %d failed. Expected %q got %q", i, d.expected, output) - } - } -} - func TestMakeTitle(t *testing.T) { type test struct { input, expected string @@ -226,3 +188,77 @@ func TestFileAndExt(t *testing.T) { } } } + +func TestSanitize(t *testing.T) { + c := qt.New(t) + tests := []struct { + input string + expected string + }{ + {" Foo bar ", "Foo-bar"}, + {"Foo.Bar/foo_Bar-Foo", "Foo.Bar/foo_Bar-Foo"}, + {"fOO,bar:foobAR", "fOObarfoobAR"}, + {"FOo/BaR.html", "FOo/BaR.html"}, + {"FOo/Ba---R.html", "FOo/Ba---R.html"}, /// See #10104 + {"FOo/Ba R.html", "FOo/Ba-R.html"}, + {"трям/трям", "трям/трям"}, + {"은행", "은행"}, + {"Банковский кассир", "Банковский-кассир"}, + // Issue #1488 + {"संस्कृत", "संस्कृत"}, + {"a%C3%B1ame", "a%C3%B1ame"}, // Issue #1292 + {"this+is+a+test", "this+is+a+test"}, // Issue #1290 + {"~foo", "~foo"}, // Issue #2177 + + } + + for _, test := range tests { + c.Assert(Sanitize(test.input), qt.Equals, test.expected) + } +} + +func BenchmarkSanitize(b *testing.B) { + const ( + allAlowedPath = "foo/bar" + spacePath = "foo bar" + ) + + // This should not allocate any memory. + b.Run("All allowed", func(b *testing.B) { + for i := 0; i < b.N; i++ { + got := Sanitize(allAlowedPath) + if got != allAlowedPath { + b.Fatal(got) + } + } + }) + + // This will allocate some memory. + b.Run("Spaces", func(b *testing.B) { + for i := 0; i < b.N; i++ { + got := Sanitize(spacePath) + if got != "foo-bar" { + b.Fatal(got) + } + } + }) +} + +func TestDir(t *testing.T) { + c := qt.New(t) + c.Assert(Dir("/a/b/c/d"), qt.Equals, "/a/b/c") + c.Assert(Dir("/a"), qt.Equals, "/") + c.Assert(Dir("/"), qt.Equals, "/") + c.Assert(Dir(""), qt.Equals, "") +} + +func TestFieldsSlash(t *testing.T) { + c := qt.New(t) + + c.Assert(FieldsSlash("a/b/c"), qt.DeepEquals, []string{"a", "b", "c"}) + c.Assert(FieldsSlash("/a/b/c"), qt.DeepEquals, []string{"a", "b", "c"}) + c.Assert(FieldsSlash("/a/b/c/"), qt.DeepEquals, []string{"a", "b", "c"}) + c.Assert(FieldsSlash("a/b/c/"), qt.DeepEquals, []string{"a", "b", "c"}) + c.Assert(FieldsSlash("/"), qt.DeepEquals, []string{}) + c.Assert(FieldsSlash(""), qt.DeepEquals, []string{}) +} diff --git a/common/paths/pathparser.go b/common/paths/pathparser.go new file mode 100644 index 000000000..842d9307b --- /dev/null +++ b/common/paths/pathparser.go @@ -0,0 +1,494 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package paths + +import ( + "path" + "path/filepath" + "runtime" + "strings" + + "github.com/gohugoio/hugo/common/types" + "github.com/gohugoio/hugo/hugofs/files" +) + +var defaultPathParser PathParser + +// PathParser parses a path into a Path. +type PathParser struct { + // Maps the language code to its index in the languages/sites slice. + LanguageIndex map[string]int +} + +// Parse parses component c with path s into Path using the default path parser. +func Parse(c, s string) *Path { + return defaultPathParser.Parse(c, s) +} + +// NormalizePathString returns a normalized path string using the very basic Hugo rules. +func NormalizePathStringBasic(s string) string { + // All lower case. + s = strings.ToLower(s) + + // Replace spaces with hyphens. + s = strings.ReplaceAll(s, " ", "-") + + return s +} + +// Parse parses component c with path s into Path using Hugo's content path rules. +func (parser PathParser) Parse(c, s string) *Path { + p, err := parser.parse(c, s) + if err != nil { + panic(err) + } + return p +} + +func (pp *PathParser) parse(component, s string) (*Path, error) { + ss := NormalizePathStringBasic(s) + + p, err := pp.doParse(component, ss) + if err != nil { + return nil, err + } + + if s != ss { + var err error + // Preserve the original case for titles etc. + p.unnormalized, err = pp.doParse(component, s) + + if err != nil { + return nil, err + } + } else { + p.unnormalized = p + } + + return p, nil +} + +func (pp *PathParser) doParse(component, s string) (*Path, error) { + p := &Path{ + component: component, + posContainerLow: -1, + posContainerHigh: -1, + posSectionHigh: -1, + posIdentifierLanguage: -1, + } + + hasLang := pp.LanguageIndex != nil + hasLang = hasLang && (component == files.ComponentFolderContent || component == files.ComponentFolderLayouts) + + if runtime.GOOS == "windows" { + s = path.Clean(filepath.ToSlash(s)) + if s == "." { + s = "" + } + } + + if s == "" { + s = "/" + } + + // Leading slash, no trailing slash. + if !strings.HasPrefix(s, "/") { + s = "/" + s + } + + if s != "/" && s[len(s)-1] == '/' { + s = s[:len(s)-1] + } + + p.s = s + slashCount := 0 + + for i := len(s) - 1; i >= 0; i-- { + c := s[i] + + switch c { + case '.': + if p.posContainerHigh == -1 { + var high int + if len(p.identifiers) > 0 { + high = p.identifiers[len(p.identifiers)-1].Low - 1 + } else { + high = len(p.s) + } + id := types.LowHigh{Low: i + 1, High: high} + if len(p.identifiers) == 0 { + p.identifiers = append(p.identifiers, id) + } else if len(p.identifiers) == 1 { + // Check for a valid language. + s := p.s[id.Low:id.High] + + if hasLang { + if _, found := pp.LanguageIndex[s]; found { + p.posIdentifierLanguage = 1 + p.identifiers = append(p.identifiers, id) + } + } + } + } + case '/': + slashCount++ + if p.posContainerHigh == -1 { + p.posContainerHigh = i + 1 + } else if p.posContainerLow == -1 { + p.posContainerLow = i + 1 + } + if i > 0 { + p.posSectionHigh = i + } + } + } + + isContentComponent := p.component == files.ComponentFolderContent || p.component == files.ComponentFolderArchetypes + isContent := isContentComponent && files.IsContentExt(p.Ext()) + + if isContent { + id := p.identifiers[len(p.identifiers)-1] + b := p.s[p.posContainerHigh : id.Low-1] + switch b { + case "index": + p.bundleType = PathTypeLeaf + case "_index": + p.bundleType = PathTypeBranch + default: + p.bundleType = PathTypeContentSingle + } + + if slashCount == 2 && p.IsLeafBundle() { + p.posSectionHigh = 0 + } + } + + return p, nil +} + +func ModifyPathBundleTypeResource(p *Path) { + if p.IsContent() { + p.bundleType = PathTypeContentResource + } else { + p.bundleType = PathTypeFile + } +} + +type PathType int + +const ( + // A generic resource, e.g. a JSON file. + PathTypeFile PathType = iota + + // All below are content files. + // A resource of a content type with front matter. + PathTypeContentResource + + // E.g. /blog/my-post.md + PathTypeContentSingle + + // All bewlow are bundled content files. + + // Leaf bundles, e.g. /blog/my-post/index.md + PathTypeLeaf + + // Branch bundles, e.g. /blog/_index.md + PathTypeBranch +) + +type Path struct { + s string + + posContainerLow int + posContainerHigh int + posSectionHigh int + + component string + bundleType PathType + + identifiers []types.LowHigh + + posIdentifierLanguage int + + trimLeadingSlash bool + + unnormalized *Path +} + +// TrimLeadingSlash returns a copy of the Path with the leading slash removed. +func (p Path) TrimLeadingSlash() *Path { + p.trimLeadingSlash = true + return &p +} + +func (p *Path) norm(s string) string { + if p.trimLeadingSlash { + s = strings.TrimPrefix(s, "/") + } + return s +} + +// IdentifierBase satifies identity.Identity. +func (p *Path) IdentifierBase() string { + return p.Base()[1:] +} + +// Component returns the component for this path (e.g. "content"). +func (p *Path) Component() string { + return p.component +} + +// Container returns the base name of the container directory for this path. +func (p *Path) Container() string { + if p.posContainerLow == -1 { + return "" + } + return p.norm(p.s[p.posContainerLow : p.posContainerHigh-1]) +} + +// ContainerDir returns the container directory for this path. +// For content bundles this will be the parent directory. +func (p *Path) ContainerDir() string { + if p.posContainerLow == -1 || !p.IsBundle() { + return p.Dir() + } + return p.norm(p.s[:p.posContainerLow-1]) +} + +// Section returns the first path element (section). +func (p *Path) Section() string { + if p.posSectionHigh <= 0 { + return "" + } + return p.norm(p.s[1:p.posSectionHigh]) +} + +// IsContent returns true if the path is a content file (e.g. mypost.md). +// Note that this will also return true for content files in a bundle. +func (p *Path) IsContent() bool { + return p.BundleType() >= PathTypeContentResource +} + +// isContentPage returns true if the path is a content file (e.g. mypost.md), +// but nof if inside a leaf bundle. +func (p *Path) isContentPage() bool { + return p.BundleType() >= PathTypeContentSingle +} + +// Name returns the last element of path. +func (p *Path) Name() string { + if p.posContainerHigh > 0 { + return p.s[p.posContainerHigh:] + } + return p.s +} + +// Name returns the last element of path withhout any extension. +func (p *Path) NameNoExt() string { + if i := p.identifierIndex(0); i != -1 { + return p.s[p.posContainerHigh : p.identifiers[i].Low-1] + } + return p.s[p.posContainerHigh:] +} + +// Name returns the last element of path withhout any language identifier. +func (p *Path) NameNoLang() string { + i := p.identifierIndex(p.posIdentifierLanguage) + if i == -1 { + return p.Name() + } + + return p.s[p.posContainerHigh:p.identifiers[i].Low-1] + p.s[p.identifiers[i].High:] +} + +// BaseNameNoIdentifier returns the logcical base name for a resource without any idenifier (e.g. no extension). +// For bundles this will be the containing directory's name, e.g. "blog". +func (p *Path) BaseNameNoIdentifier() string { + if p.IsBundle() { + return p.Container() + } + return p.NameNoIdentifier() +} + +// NameNoIdentifier returns the last element of path withhout any identifier (e.g. no extension). +func (p *Path) NameNoIdentifier() string { + if len(p.identifiers) > 0 { + return p.s[p.posContainerHigh : p.identifiers[len(p.identifiers)-1].Low-1] + } + return p.s[p.posContainerHigh:] +} + +// Dir returns all but the last element of path, typically the path's directory. +func (p *Path) Dir() (d string) { + if p.posContainerHigh > 0 { + d = p.s[:p.posContainerHigh-1] + } + if d == "" { + d = "/" + } + d = p.norm(d) + return +} + +// Path returns the full path. +func (p *Path) Path() (d string) { + return p.norm(p.s) +} + +// Unmormalized returns the Path with the original case preserved. +func (p *Path) Unmormalized() *Path { + return p.unnormalized +} + +// PathNoLang returns the Path but with any language identifier removed. +func (p *Path) PathNoLang() string { + return p.base(true, false) +} + +// PathNoIdentifier returns the Path but with any identifier (ext, lang) removed. +func (p *Path) PathNoIdentifier() string { + return p.base(false, false) +} + +// PathRel returns the path relativeto the given owner. +func (p *Path) PathRel(owner *Path) string { + ob := owner.Base() + if !strings.HasSuffix(ob, "/") { + ob += "/" + } + return strings.TrimPrefix(p.Path(), ob) +} + +// BaseRel returns the base path relative to the given owner. +func (p *Path) BaseRel(owner *Path) string { + ob := owner.Base() + if ob == "/" { + ob = "" + } + return p.Base()[len(ob)+1:] +} + +// For content files, Base returns the path without any identifiers (extension, language code etc.). +// Any 'index' as the last path element is ignored. +// +// For other files (Resources), any extension is kept. +func (p *Path) Base() string { + return p.base(!p.isContentPage(), p.IsBundle()) +} + +// BaseNoLeadingSlash returns the base path without the leading slash. +func (p *Path) BaseNoLeadingSlash() string { + return p.Base()[1:] +} + +func (p *Path) base(preserveExt, isBundle bool) string { + if len(p.identifiers) == 0 { + return p.norm(p.s) + } + + if preserveExt && len(p.identifiers) == 1 { + // Preserve extension. + return p.norm(p.s) + } + + id := p.identifiers[len(p.identifiers)-1] + high := id.Low - 1 + + if isBundle { + high = p.posContainerHigh - 1 + } + + if high == 0 { + high++ + } + + if !preserveExt { + return p.norm(p.s[:high]) + } + + // For txt files etc. we want to preserve the extension. + id = p.identifiers[0] + + return p.norm(p.s[:high] + p.s[id.Low-1:id.High]) +} + +func (p *Path) Ext() string { + return p.identifierAsString(0) +} + +func (p *Path) Lang() string { + return p.identifierAsString(1) +} + +func (p *Path) Identifier(i int) string { + return p.identifierAsString(i) +} + +func (p *Path) Identifiers() []string { + ids := make([]string, len(p.identifiers)) + for i, id := range p.identifiers { + ids[i] = p.s[id.Low:id.High] + } + return ids +} + +func (p *Path) IsHTML() bool { + return files.IsHTML(p.Ext()) +} + +func (p *Path) BundleType() PathType { + return p.bundleType +} + +func (p *Path) IsBundle() bool { + return p.bundleType >= PathTypeLeaf +} + +func (p *Path) IsBranchBundle() bool { + return p.bundleType == PathTypeBranch +} + +func (p *Path) IsLeafBundle() bool { + return p.bundleType == PathTypeLeaf +} + +func (p *Path) identifierAsString(i int) string { + i = p.identifierIndex(i) + if i == -1 { + return "" + } + + id := p.identifiers[i] + return p.s[id.Low:id.High] +} + +func (p *Path) identifierIndex(i int) int { + if i < 0 || i >= len(p.identifiers) { + return -1 + } + return i +} + +// HasExt returns true if the Unix styled path has an extension. +func HasExt(p string) bool { + for i := len(p) - 1; i >= 0; i-- { + if p[i] == '.' { + return true + } + if p[i] == '/' { + return false + } + } + return false +} diff --git a/common/paths/pathparser_test.go b/common/paths/pathparser_test.go new file mode 100644 index 000000000..3546b6605 --- /dev/null +++ b/common/paths/pathparser_test.go @@ -0,0 +1,351 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package paths + +import ( + "path/filepath" + "testing" + + "github.com/gohugoio/hugo/hugofs/files" + + qt "github.com/frankban/quicktest" +) + +var testParser = &PathParser{ + LanguageIndex: map[string]int{ + "no": 0, + "en": 1, + }, +} + +func TestParse(t *testing.T) { + c := qt.New(t) + + tests := []struct { + name string + path string + assert func(c *qt.C, p *Path) + }{ + { + "Basic text file", + "/a/b.txt", + func(c *qt.C, p *Path) { + c.Assert(p.Name(), qt.Equals, "b.txt") + c.Assert(p.Base(), qt.Equals, "/a/b.txt") + c.Assert(p.Container(), qt.Equals, "a") + c.Assert(p.Dir(), qt.Equals, "/a") + c.Assert(p.Ext(), qt.Equals, "txt") + c.Assert(p.IsContent(), qt.IsFalse) + }, + }, + { + "Basic text file, upper case", + "/A/B.txt", + func(c *qt.C, p *Path) { + c.Assert(p.Name(), qt.Equals, "b.txt") + c.Assert(p.NameNoExt(), qt.Equals, "b") + c.Assert(p.NameNoIdentifier(), qt.Equals, "b") + c.Assert(p.BaseNameNoIdentifier(), qt.Equals, "b") + c.Assert(p.Base(), qt.Equals, "/a/b.txt") + c.Assert(p.Ext(), qt.Equals, "txt") + }, + },