diff options
author | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2024-06-23 12:49:10 +0200 |
---|---|---|
committer | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2024-06-25 15:48:02 +0200 |
commit | e1317dd32281dc5ce670e34165dc7780c8f5892b (patch) | |
tree | 986f45feec6d2590b859697f7498f7f9a3cdcc1e | |
parent | eddcd2bac6bfd3cc0ac1a3b38bf8c4ae452ea23b (diff) |
Add css.TailwindCSS
Closes #12618
Closes #12620
20 files changed, 644 insertions, 285 deletions
diff --git a/commands/hugobuilder.go b/commands/hugobuilder.go index 84608ca37..1a4a61264 100644 --- a/commands/hugobuilder.go +++ b/commands/hugobuilder.go @@ -920,7 +920,7 @@ func (c *hugoBuilder) handleEvents(watcher *watcher.Batcher, if len(otherChanges) > 0 { livereload.ForceRefresh() // Allow some time for the live reload script to get reconnected. - time.Sleep(100 * time.Millisecond) + time.Sleep(200 * time.Millisecond) } for _, ev := range cssChanges { diff --git a/common/hexec/exec.go b/common/hexec/exec.go index 49291354d..4f23d20f5 100644 --- a/common/hexec/exec.go +++ b/common/hexec/exec.go @@ -21,8 +21,10 @@ import ( "io" "os" "os/exec" + "path/filepath" "regexp" "strings" + "sync" "github.com/cli/safeexec" "github.com/gohugoio/hugo/config" @@ -84,7 +86,7 @@ var WithEnviron = func(env []string) func(c *commandeer) { } // New creates a new Exec using the provided security config. -func New(cfg security.Config) *Exec { +func New(cfg security.Config, workingDir string) *Exec { var baseEnviron []string for _, v := range os.Environ() { k, _ := config.SplitEnvVar(v) @@ -95,6 +97,7 @@ func New(cfg security.Config) *Exec { return &Exec{ sc: cfg, + workingDir: workingDir, baseEnviron: baseEnviron, } } @@ -119,15 +122,23 @@ func SafeCommand(name string, arg ...string) (*exec.Cmd, error) { // Exec enforces a security policy for commands run via os/exec. type Exec struct { - sc security.Config + sc security.Config + workingDir string // os.Environ filtered by the Exec.OsEnviron whitelist filter. baseEnviron []string + + npxInit sync.Once + npxAvailable bool +} + +func (e *Exec) New(name string, arg ...any) (Runner, error) { + return e.new(name, "", arg...) } // New will fail if name is not allowed according to the configured security policy. // Else a configured Runner will be returned ready to be Run. -func (e *Exec) New(name string, arg ...any) (Runner, error) { +func (e *Exec) new(name string, fullyQualifiedName string, arg ...any) (Runner, error) { if err := e.sc.CheckAllowedExec(name); err != nil { return nil, err } @@ -136,27 +147,51 @@ func (e *Exec) New(name string, arg ...any) (Runner, error) { copy(env, e.baseEnviron) cm := &commandeer{ - name: name, - env: env, + name: name, + fullyQualifiedName: fullyQualifiedName, + env: env, } return cm.command(arg...) } -// Npx will try to run npx, and if that fails, it will -// try to run the binary directly. +// Npx will in order: +// 1. Try fo find the binary in the WORKINGDIR/node_modules/.bin directory. +// 2. If not found, and npx is available, run npx --no-install <name> <args>. +// 3. Fall back to the PATH. func (e *Exec) Npx(name string, arg ...any) (Runner, error) { - r, err := e.npx(name, arg...) + // npx is slow, so first try the common case. + nodeBinFilename := filepath.Join(e.workingDir, nodeModulesBinPath, name) + _, err := safeexec.LookPath(nodeBinFilename) if err == nil { - return r, nil + return e.new(name, nodeBinFilename, arg...) + } + e.checkNpx() + if e.npxAvailable { + r, err := e.npx(name, arg...) + if err == nil { + return r, nil + } } return e.New(name, arg...) } +const ( + npxNoInstall = "--no-install" + npxBinary = "npx" + nodeModulesBinPath = "node_modules/.bin" +) + +func (e *Exec) checkNpx() { + e.npxInit.Do(func() { + e.npxAvailable = InPath(npxBinary) + }) +} + // npx is a convenience method to create a Runner running npx --no-install <name> <args. func (e *Exec) npx(name string, arg ...any) (Runner, error) { - arg = append(arg[:0], append([]any{"--no-install", name}, arg[0:]...)...) - return e.New("npx", arg...) + arg = append(arg[:0], append([]any{npxNoInstall, name}, arg[0:]...)...) + return e.New(npxBinary, arg...) } // Sec returns the security policies this Exec is configured with. @@ -165,11 +200,12 @@ func (e *Exec) Sec() security.Config { } type NotFoundError struct { - name string + name string + method string } func (e *NotFoundError) Error() string { - return fmt.Sprintf("binary with name %q not found", e.name) + return fmt.Sprintf("binary with name %q not found %s", e.name, e.method) } // Runner wraps a *os.Cmd. @@ -192,8 +228,14 @@ func (c *cmdWrapper) Run() error { if err == nil { return nil } + name := c.name + method := "in PATH" + if name == npxBinary { + name = c.c.Args[2] + method = "using npx" + } if notFoundRe.MatchString(c.outerr.String()) { - return &NotFoundError{name: c.name} + return &NotFoundError{name: name, method: method} } return fmt.Errorf("failed to execute binary %q with args %v: %s", c.name, c.c.Args[1:], c.outerr.String()) } @@ -209,8 +251,9 @@ type commandeer struct { dir string ctx context.Context - name string - env []string + name string + fullyQualifiedName string + env []string } func (c *commandeer) command(arg ...any) (*cmdWrapper, error) { @@ -230,10 +273,17 @@ func (c *commandeer) command(arg ...any) (*cmdWrapper, error) { } } - bin, err := safeexec.LookPath(c.name) - if err != nil { - return nil, &NotFoundError{ - name: c.name, + var bin string + if c.fullyQualifiedName != "" { + bin = c.fullyQualifiedName + } else { + var err error + bin, err = safeexec.LookPath(c.name) + if err != nil { + return nil, &NotFoundError{ + name: c.name, + method: "in PATH", + } } } diff --git a/config/allconfig/load.go b/config/allconfig/load.go index 117b8e89c..84419cb2e 100644 --- a/config/allconfig/load.go +++ b/config/allconfig/load.go @@ -467,7 +467,7 @@ func (l *configLoader) loadModules(configs *Configs) (modules.ModulesConfig, *mo ignoreVendor, _ = hglob.GetGlob(hglob.NormalizePath(s)) } - ex := hexec.New(conf.Security) + ex := hexec.New(conf.Security, workingDir) hook := func(m *modules.ModulesConfig) error { for _, tc := range m.AllModules { diff --git a/config/security/securityConfig.go b/config/security/securityConfig.go index be9e901f1..46a5d0a23 100644 --- a/config/security/securityConfig.go +++ b/config/security/securityConfig.go @@ -39,6 +39,7 @@ var DefaultConfig = Config{ "^go$", // for Go Modules "^npx$", // used by all Node tools (Babel, PostCSS). "^postcss$", + "^tailwindcss$", ), // These have been tested to work with Hugo's external programs // on Windows, Linux and MacOS. diff --git a/config/security/securityConfig_test.go b/config/security/securityConfig_test.go index 57e615a48..87a243012 100644 --- a/config/security/securityConfig_test.go +++ b/config/security/securityConfig_test.go @@ -135,7 +135,7 @@ func TestToTOML(t *testing.T) { got := DefaultConfig.ToTOML() c.Assert(got, qt.Equals, - "[security]\n enableInlineShortcodes = false\n\n [security.exec]\n allow = ['^(dart-)?sass(-embedded)?$', '^go$', '^npx$', '^postcss$']\n osEnv = ['(?i)^((HTTPS?|NO)_PROXY|PATH(EXT)?|APPDATA|TE?MP|TERM|GO\\w+|(XDG_CONFIG_)?HOME|USERPROFILE|SSH_AUTH_SOCK|DISPLAY|LANG|SYSTEMDRIVE)$']\n\n [security.funcs]\n getenv = ['^HUGO_', '^CI$']\n\n [security.http]\n methods = ['(?i)GET|POST']\n urls = ['.*']", + "[security]\n enableInlineShortcodes = false\n\n [security.exec]\n allow = ['^(dart-)?sass(-embedded)?$', '^go$', '^npx$', '^postcss$', '^tailwindcss$']\n osEnv = ['(?i)^((HTTPS?|NO)_PROXY|PATH(EXT)?|APPDATA|TE?MP|TERM|GO\\w+|(XDG_CONFIG_)?HOME|USERPROFILE|SSH_AUTH_SOCK|DISPLAY|LANG|SYSTEMDRIVE)$']\n\n [security.funcs]\n getenv = ['^HUGO_', '^CI$']\n\n [security.http]\n methods = ['(?i)GET|POST']\n urls = ['.*']", ) } diff --git a/deps/deps.go b/deps/deps.go index 4805af1aa..0d0f283c2 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -163,7 +163,7 @@ func (d *Deps) Init() error { } if d.ExecHelper == nil { - d.ExecHelper = hexec.New(d.Conf.GetConfigSection("security").(security.Config)) + d.ExecHelper = hexec.New(d.Conf.GetConfigSection("security").(security.Config), d.Conf.WorkingDir()) } if d.MemCache == nil { diff --git a/hugolib/integrationtest_builder.go b/hugolib/integrationtest_builder.go index 758fc4ec9..551b807db 100644 --- a/hugolib/integrationtest_builder.go +++ b/hugolib/integrationtest_builder.go @@ -659,7 +659,7 @@ func (s *IntegrationTestBuilder) initBuilder() error { sc := security.DefaultConfig sc.Exec.Allow, err = security.NewWhitelist("npm") s.Assert(err, qt.IsNil) - ex := hexec.New(sc) + ex := hexec.New(sc, s.Cfg.WorkingDir) command, err := ex.New("npm", "install") s.Assert(err, qt.IsNil) s.Assert(command.Run(), qt.IsNil) diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go index dab693623..746f4c26e 100644 --- a/hugolib/testhelpers_test.go +++ b/hugolib/testhelpers_test.go @@ -834,7 +834,7 @@ func (s *sitesBuilder) NpmInstall() hexec.Runner { var err error sc.Exec.Allow, err = security.NewWhitelist("npm") s.Assert(err, qt.IsNil) - ex := hexec.New(sc) + ex := hexec.New(sc, s.workingDir) command, err := ex.New("npm", "install") s.Assert(err, qt.IsNil) return command diff --git a/markup/asciidocext/convert_test.go b/markup/asciidocext/convert_test.go index 18c38a621..b3f63b4d8 100644 --- a/markup/asciidocext/convert_test.go +++ b/markup/asciidocext/convert_test.go @@ -313,7 +313,7 @@ allow = ['asciidoctor'] converter.ProviderConfig{ Logger: loggers.NewDefault(), Conf: conf, - Exec: hexec.New(securityConfig), + Exec: hexec.New(securityConfig, ""), }, ) c.Assert(err, qt.IsNil) diff --git a/markup/pandoc/convert_test.go b/markup/pandoc/convert_test.go index dec30c410..8beedc115 100644 --- a/markup/pandoc/convert_test.go +++ b/markup/pandoc/convert_test.go @@ -34,7 +34,7 @@ func TestConvert(t *testing.T) { var err error sc.Exec.Allow, err = security.NewWhitelist("pandoc") c.Assert(err, qt.IsNil) - p, err := Provider.New(converter.ProviderConfig{Exec: hexec.New(sc), Logger: loggers.NewDefault()}) + p, err := Provider.New(converter.ProviderConfig{Exec: hexec.New(sc, ""), Logger: loggers.NewDefault()}) c.Assert(err, qt.IsNil) conv, err := p.New(converter.DocumentContext{}) c.Assert(err, qt.IsNil) diff --git a/markup/rst/convert_test.go b/markup/rst/convert_test.go index 1897e650f..182858263 100644 --- a/markup/rst/convert_test.go +++ b/markup/rst/convert_test.go @@ -36,7 +36,7 @@ func TestConvert(t *testing.T) { p, err := Provider.New( converter.ProviderConfig{ Logger: loggers.NewDefault(), - Exec: hexec.New(sc), + Exec: hexec.New(sc, ""), }) c.Assert(err, qt.IsNil) conv, err := p.New(converter.DocumentContext{}) diff --git a/modules/client_test.go b/modules/client_test.go index d727c4586..0ee7e0dbc 100644 --- a/modules/client_test.go +++ b/modules/client_test.go @@ -61,7 +61,7 @@ github.com/gohugoio/hugoTestModules1_darwin/modh2_2@v1.4.0 github.com/gohugoio/h WorkingDir: workingDir, ThemesDir: themesDir, PublishDir: publishDir, - Exec: hexec.New(security.DefaultConfig), + Exec: hexec.New(security.DefaultConfig, ""), } withConfig(&ccfg) diff --git a/resources/resource_transformers/cssjs/inline_imports.go b/resources/resource_transformers/cssjs/inline_imports.go new file mode 100644 index 000000000..98e3292cd --- /dev/null +++ b/resources/resource_transformers/cssjs/inline_imports.go @@ -0,0 +1,247 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cssjs + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "path" + "path/filepath" + "regexp" + "strconv" + "strings" + + "github.com/gohugoio/hugo/common/herrors" + "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/common/text" + "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/identity" + "github.com/spf13/afero" +) + +const importIdentifier = "@import" + +var ( + cssSyntaxErrorRe = regexp.MustCompile(`> (\d+) \|`) + shouldImportRe = regexp.MustCompile(`^@import ["'](.*?)["'];?\s*(/\*.*\*/)?$`) +) + +type fileOffset struct { + Filename string + Offset int +} + +type importResolver struct { + r io.Reader + inPath string + opts InlineImports + + contentSeen map[string]bool + dependencyManager identity.Manager + linemap map[int]fileOffset + fs afero.Fs + logger loggers.Logger +} + +func newImportResolver(r io.Reader, inPath string, opts InlineImports, fs afero.Fs, logger loggers.Logger, dependencyManager identity.Manager) *importResolver { + return &importResolver{ + r: r, + dependencyManager: dependencyManager, + inPath: inPath, + fs: fs, logger: logger, + linemap: make(map[int]fileOffset), contentSeen: make(map[string]bool), + opts: opts, + } +} + +func (imp *importResolver) contentHash(filename string) ([]byte, string) { + b, err := afero.ReadFile(imp.fs, filename) + if err != nil { + return nil, "" + } + h := sha256.New() + h.Write(b) + return b, hex.EncodeToString(h.Sum(nil)) +} + +func (imp *importResolver) importRecursive( + lineNum int, + content string, + inPath string, +) (int, string, error) { + basePath := path.Dir(inPath) + + var replacements []string + lines := strings.Split(content, "\n") + + trackLine := func(i, offset int, line string) { + // TODO(bep) this is not very efficient. + imp.linemap[i+lineNum] = fileOffset{Filename: inPath, Offset: offset} + } + + i := 0 + for offset, line := range lines { + i++ + lineTrimmed := strings.TrimSpace(line) + column := strings.Index(line, lineTrimmed) + line = lineTrimmed + + if !imp.shouldImport(line) { + trackLine(i, offset, line) + } else { + path := strings.Trim(strings.TrimPrefix(line, importIdentifier), " \"';") + filename := filepath.Join(basePath, path) + imp.dependencyManager.AddIdentity(identity.CleanStringIdentity(filename)) + importContent, hash := imp.contentHash(filename) + + if importContent == nil { + if imp.opts.SkipInlineImportsNotFound { + trackLine(i, offset, line) + continue + } + pos := text.Position{ + Filename: inPath, + LineNumber: offset + 1, + ColumnNumber: column + 1, + } + return 0, "", herrors.NewFileErrorFromFileInPos(fmt.Errorf("failed to resolve CSS @import \"%s\"", filename), pos, imp.fs, nil) + } + + i-- + + if imp.contentSeen[hash] { + i++ + // Just replace the line with an empty string. + replacements = append(replacements, []string{line, ""}...) + trackLine(i, offset, "IMPORT") + continue + } + + imp.contentSeen[hash] = true + + // Handle recursive imports. + l, nested, err := imp.importRecursive(i+lineNum, string(importContent), filepath.ToSlash(filename)) + if err != nil { + return 0, "", err + } + + trackLine(i, offset, line) + + i += l + + importContent = []byte(nested) + + replacements = append(replacements, []string{line, string(importContent)}...) + } + } + + if len(replacements) > 0 { + repl := strings.NewReplacer(replacements...) + content = repl.Replace(content) + } + + return i, content, nil +} + +func (imp *importResolver) resolve() (io.Reader, error) { + content, err := io.ReadAll(imp.r) + if err != nil { + return nil, err + } + + contents := string(content) + + _, newContent, err := imp.importRecursive(0, contents, imp.inPath) + if err != nil { + return nil, err + } + + return strings.NewReader(newContent), nil +} + +// See https://www.w3schools.com/cssref/pr_import_rule.asp +// We currently only support simple file imports, no urls, no media queries. +// So this is OK: +// +// @import "navigation.css"; +// +// This is not: +// +// @import url("navigation.css"); +// @import "mobstyle.css" screen and (max-width: 768px); +func (imp *importResolver) shouldImport(s string) bool { + if !strings.HasPrefix(s, importIdentifier) { + return false + } + if strings.Contains(s, "url(") { + return false + } + + m := shouldImportRe.FindStringSubmatch(s) + if m == nil { + return false + } + + if len(m) != 3 { + return false + } + + if tailwindImportExclude(m[1]) { + return false + } + + return true +} + +func (imp *importResolver) toFileError(output string) error { + inErr := errors.New(output) + + match := cssSyntaxErrorRe.FindStringSubmatch(output) + if match == nil { + return inErr + } + + lineNum, err := strconv.Atoi(match[1]) + if err != nil { + return inErr + } + + file, ok := imp.linemap[lineNum] + if !ok { + return inErr + } + + fi, err := imp.fs.Stat(file.Filename) + if err != nil { + return inErr + } + + meta := fi.(hugofs.FileMetaInfo).Meta() + realFilename := meta.Filename + f, err := meta.Open() + if err != nil { + return inErr + } + defer f.Close() + + ferr := herrors.NewFileErrorFromName(inErr, realFilename) + pos := ferr.Position() + pos.LineNumber = file.Offset + 1 + return ferr.UpdatePosition(pos).UpdateContent(f, nil) + + // return herrors.NewFileErrorFromFile(inErr, file.Filename, realFilename, hugofs.Os, herrors.SimpleLineMatcher) +} diff --git a/resources/resource_transformers/postcss/postcss_test.go b/resources/resource_transformers/cssjs/inline_imports_test.go index 1edaaaaf5..9bcb7f9a3 100644 --- a/resources/resource_transformers/postcss/postcss_test.go +++ b/resources/resource_transformers/cssjs/inline_imports_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package postcss +package cssjs import ( "regexp" @@ -32,14 +32,14 @@ import ( // Issue 6166 func TestDecodeOptions(t *testing.T) { c := qt.New(t) - opts1, err := decodeOptions(map[string]any{ + opts1, err := decodePostCSSOptions(map[string]any{ "no-map": true, }) c.Assert(err, qt.IsNil) c.Assert(opts1.NoMap, qt.Equals, true) - opts2, err := decodeOptions(map[string]any{ + opts2, err := decodePostCSSOptions(map[string]any{ "noMap": true, }) @@ -67,6 +67,16 @@ func TestShouldImport(t *testing.T) { } } +func TestShouldImportExcludes(t *testing.T) { + c := qt.New(t) + var imp *importResolver + + c.Assert(imp.shouldImport(`@import "navigation.css";`), qt.Equals, true) + c.Assert(imp.shouldImport(`@import "tailwindcss";`), qt.Equals, false) + c.Assert(imp.shouldImport(`@import "tailwindcss.css";`), qt.Equals, true) + c.Assert(imp.shouldImport(`@import "tailwindcss/preflight";`), qt.Equals, false) +} + func TestImportResolver(t *testing.T) { c := qt.New(t) fs := afero.NewMemMapFs() @@ -95,7 +105,7 @@ LOCAL_STYLE imp := newImportResolver( mainStyles, "styles.css", - Options{}, + InlineImports{}, fs, loggers.NewDefault(), identity.NopManager, ) @@ -153,7 +163,7 @@ LOCAL_STYLE imp := newImportResolver( strings.NewReader(mainStyles), "styles.css", - Options{}, + InlineImports{}, fs, logger, identity.NopManager, ) diff --git a/resources/resource_transformers/postcss/postcss.go b/resources/resource_transformers/cssjs/postcss.go index 9015e120d..1a9e01142 100644 --- a/resources/resource_transformers/postcss/postcss.go +++ b/resources/resource_transformers/cssjs/postcss.go @@ -11,32 +11,23 @@ // See the License for the specific language governing permissions and // limitations under the License. -package postcss +// Package cssjs provides resource transformations backed by some popular JS based frameworks. +package cssjs import ( "bytes" - "crypto/sha256" - "encoding/hex" - "errors" "fmt" "io" - "path" "path/filepath" - "regexp" - "strconv" "strings" "github.com/gohugoio/hugo/common/collections" "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/common/loggers" - "github.com/gohugoio/hugo/common/text" - "github.com/gohugoio/hugo/hugofs" - "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/common/hugo" "github.com/gohugoio/hugo/resources/internal" - "github.com/spf13/afero" "github.com/spf13/cast" "github.com/mitchellh/mapstructure" @@ -46,19 +37,12 @@ import ( "github.com/gohugoio/hugo/resources/resource" ) -const importIdentifier = "@import" - -var ( - cssSyntaxErrorRe = regexp.MustCompile(`> (\d+) \|`) - shouldImportRe = regexp.MustCompile(`^@import ["'].*["'];?\s*(/\*.*\*/)?$`) -) - -// New creates a new Client with the given specification. -func New(rs *resources.Spec) *Client { - return &Client{rs: rs} +// NewPostCSSClient creates a new PostCSSClient with the given specification. +func NewPostCSSClient(rs *resources.Spec) *PostCSSClient { + return &PostCSSClient{rs: rs} } -func decodeOptions(m map[string]any) (opts Options, err error) { +func decodePostCSSOptions(m map[string]any) (opts PostCSSOptions, err error) { if m == nil { return } @@ -74,23 +5 |