diff options
author | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2019-01-02 12:33:26 +0100 |
---|---|---|
committer | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2019-03-23 18:51:22 +0100 |
commit | 597e418cb02883418f2cebb41400e8e61413f651 (patch) | |
tree | 177ad9c540b2583b6dab138c9f0490d28989c7f7 /lazy | |
parent | 44f5c1c14cb1f42cc5f01739c289e9cfc83602af (diff) |
Make Page an interface
The main motivation of this commit is to add a `page.Page` interface to replace the very file-oriented `hugolib.Page` struct.
This is all a preparation step for issue #5074, "pages from other data sources".
But this also fixes a set of annoying limitations, especially related to custom output formats, and shortcodes.
Most notable changes:
* The inner content of shortcodes using the `{{%` as the outer-most delimiter will now be sent to the content renderer, e.g. Blackfriday.
This means that any markdown will partake in the global ToC and footnote context etc.
* The Custom Output formats are now "fully virtualized". This removes many of the current limitations.
* The taxonomy list type now has a reference to the `Page` object.
This improves the taxonomy template `.Title` situation and make common template constructs much simpler.
See #5074
Fixes #5763
Fixes #5758
Fixes #5090
Fixes #5204
Fixes #4695
Fixes #5607
Fixes #5707
Fixes #5719
Fixes #3113
Fixes #5706
Fixes #5767
Fixes #5723
Fixes #5769
Fixes #5770
Fixes #5771
Fixes #5759
Fixes #5776
Fixes #5777
Fixes #5778
Diffstat (limited to 'lazy')
-rw-r--r-- | lazy/init.go | 199 | ||||
-rw-r--r-- | lazy/init_test.go | 150 | ||||
-rw-r--r-- | lazy/once.go | 69 |
3 files changed, 418 insertions, 0 deletions
diff --git a/lazy/init.go b/lazy/init.go new file mode 100644 index 000000000..5c1bee609 --- /dev/null +++ b/lazy/init.go @@ -0,0 +1,199 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lazy + +import ( + "context" + "sync" + "time" + + "github.com/pkg/errors" +) + +// New creates a new empty Init. +func New() *Init { + return &Init{} +} + +// Init holds a graph of lazily initialized dependencies. +type Init struct { + mu sync.Mutex + + prev *Init + children []*Init + + init onceMore + out interface{} + err error + f func() (interface{}, error) +} + +// Add adds a func as a new child dependency. +func (ini *Init) Add(initFn func() (interface{}, error)) *Init { + if ini == nil { + ini = New() + } + return ini.add(false, initFn) +} + +// AddWithTimeout is same as Add, but with a timeout that aborts initialization. +func (ini *Init) AddWithTimeout(timeout time.Duration, f func(ctx context.Context) (interface{}, error)) *Init { + return ini.Add(func() (interface{}, error) { + return ini.withTimeout(timeout, f) + }) +} + +// Branch creates a new dependency branch based on an existing and adds +// the given dependency as a child. +func (ini *Init) Branch(initFn func() (interface{}, error)) *Init { + if ini == nil { + ini = New() + } + return ini.add(true, initFn) +} + +// BranchdWithTimeout is same as Branch, but with a timeout. +func (ini *Init) BranchdWithTimeout(timeout time.Duration, f func(ctx context.Context) (interface{}, error)) *Init { + return ini.Branch(func() (interface{}, error) { + return ini.withTimeout(timeout, f) + }) +} + +// Do initializes the entire dependency graph. +func (ini *Init) Do() (interface{}, error) { + if ini == nil { + panic("init is nil") + } + + ini.init.Do(func() { + var ( + dependencies []*Init + children []*Init + ) + + prev := ini.prev + for prev != nil { + if prev.shouldInitialize() { + dependencies = append(dependencies, prev) + } + prev = prev.prev + } + + for _, child := range ini.children { + if child.shouldInitialize() { + children = append(children, child) + } + } + + for _, dep := range dependencies { + _, err := dep.Do() + if err != nil { + ini.err = err + return + } + } + + if ini.f != nil { + ini.out, ini.err = ini.f() + } + + for _, dep := range children { + _, err := dep.Do() + if err != nil { + ini.err = err + return + } + } + + }) + + var counter time.Duration + for !ini.init.Done() { + counter += 10 + if counter > 600000000 { + panic("BUG: timed out in lazy init") + } + time.Sleep(counter * time.Microsecond) + } + + return ini.out, ini.err +} + +func (ini *Init) shouldInitialize() bool { + return !(ini == nil || ini.init.Done() || ini.init.InProgress()) +} + +// Reset resets the current and all its dependencies. +func (ini *Init) Reset() { + mu := ini.init.ResetWithLock() + defer mu.Unlock() + for _, d := range ini.children { + d.Reset() + } +} + +func (ini *Init) add(branch bool, initFn func() (interface{}, error)) *Init { + ini.mu.Lock() + defer ini.mu.Unlock() + + if !branch { + ini.checkDone() + } + + init := &Init{ + f: initFn, + prev: ini, + } + + if !branch { + ini.children = append(ini.children, init) + } + + return init +} + +func (ini *Init) checkDone() { + if ini.init.Done() { + panic("init cannot be added to after it has run") + } +} + +func (ini *Init) withTimeout(timeout time.Duration, f func(ctx context.Context) (interface{}, error)) (interface{}, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + c := make(chan verr, 1) + + go func() { + v, err := f(ctx) + select { + case <-ctx.Done(): + return + default: + c <- verr{v: v, err: err} + } + }() + + select { + case <-ctx.Done(): + return nil, errors.New("timed out initializing value. This is most likely a circular loop in a shortcode") + case ve := <-c: + return ve.v, ve.err + } + +} + +type verr struct { + v interface{} + err error +} diff --git a/lazy/init_test.go b/lazy/init_test.go new file mode 100644 index 000000000..bcb57acb3 --- /dev/null +++ b/lazy/init_test.go @@ -0,0 +1,150 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lazy + +import ( + "context" + "errors" + "math/rand" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestInit(t *testing.T) { + assert := require.New(t) + + var result string + + bigOrSmall := func() int { + if rand.Intn(10) < 3 { + return 10000 + rand.Intn(100000) + } + return 1 + rand.Intn(50) + } + + f1 := func(name string) func() (interface{}, error) { + return func() (interface{}, error) { + result += name + "|" + size := bigOrSmall() + _ = strings.Repeat("Hugo Rocks! ", size) + return name, nil + } + } + + f2 := func() func() (interface{}, error) { + return func() (interface{}, error) { + size := bigOrSmall() + _ = strings.Repeat("Hugo Rocks! ", size) + return size, nil + } + } + + root := New() + + root.Add(f1("root(1)")) + root.Add(f1("root(2)")) + + branch1 := root.Branch(f1("branch_1")) + branch1.Add(f1("branch_1_1")) + branch1_2 := branch1.Add(f1("branch_1_2")) + branch1_2_1 := branch1_2.Add(f1("branch_1_2_1")) + + var wg sync.WaitGroup + + // Add some concurrency and randomness to verify thread safety and + // init order. + for i := 0; i < 100; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + var err error + if rand.Intn(10) < 5 { + _, err = root.Do() + assert.NoError(err) + } + + // Add a new branch on the fly. + if rand.Intn(10) > 5 { + branch := branch1_2.Branch(f2()) + init := branch.Add(f2()) + _, err = init.Do() + assert.NoError(err) + } else { + _, err = branch1_2_1.Do() + assert.NoError(err) + } + _, err = branch1_2.Do() + assert.NoError(err) + + }(i) + + wg.Wait() + + assert.Equal("root(1)|root(2)|branch_1|branch_1_1|branch_1_2|branch_1_2_1|", result) + + } + +} + +func TestInitAddWithTimeout(t *testing.T) { + assert := require.New(t) + + init := New().AddWithTimeout(100*time.Millisecond, func(ctx context.Context) (interface{}, error) { + return nil, nil + }) + + _, err := init.Do() + + assert.NoError(err) +} + +func TestInitAddWithTimeoutTimeout(t *testing.T) { + assert := require.New(t) + + init := New().AddWithTimeout(100*time.Millisecond, func(ctx context.Context) (interface{}, error) { + time.Sleep(500 * time.Millisecond) + select { + case <-ctx.Done(): + return nil, nil + default: + } + t.Fatal("slept") + return nil, nil + }) + + _, err := init.Do() + + assert.Error(err) + + assert.Contains(err.Error(), "timed out") + + time.Sleep(1 * time.Second) + +} + +func TestInitAddWithTimeoutError(t *testing.T) { + assert := require.New(t) + + init := New().AddWithTimeout(100*time.Millisecond, func(ctx context.Context) (interface{}, error) { + return nil, errors.New("failed") + }) + + _, err := init.Do() + + assert.Error(err) +} diff --git a/lazy/once.go b/lazy/once.go new file mode 100644 index 000000000..c434bfa0b --- /dev/null +++ b/lazy/once.go @@ -0,0 +1,69 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lazy + +import ( + "sync" + "sync/atomic" +) + +// onceMore is similar to sync.Once. +// +// Additional features are: +// * it can be reset, so the action can be repeated if needed +// * it has methods to check if it's done or in progress +// +type onceMore struct { + mu sync.Mutex + lock uint32 + done uint32 +} + +func (t *onceMore) Do(f func()) { + if atomic.LoadUint32(&t.done) == 1 { + return + } + + // f may call this Do and we would get a deadlock. + locked := atomic.CompareAndSwapUint32(&t.lock, 0, 1) + if !locked { + return + } + defer atomic.StoreUint32(&t.lock, 0) + + t.mu.Lock() + defer t.mu.Unlock() + + // Double check + if t.done == 1 { + return + } + defer atomic.StoreUint32(&t.done, 1) + f() + +} + +func (t *onceMore) InProgress() bool { + return atomic.LoadUint32(&t.lock) == 1 +} + +func (t *onceMore) Done() bool { + return atomic.LoadUint32(&t.done) == 1 +} + +func (t *onceMore) ResetWithLock() *sync.Mutex { + t.mu.Lock() + defer atomic.StoreUint32(&t.done, 0) + return &t.mu +} |