diff options
author | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2019-05-03 09:16:58 +0200 |
---|---|---|
committer | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2019-07-24 09:35:53 +0200 |
commit | 9f5a92078a3f388b52d597b5a59af5c933a112d2 (patch) | |
tree | 0b2b07e5b3a3f21877bc5585a4bdd76306a09dde /modules | |
parent | 47953148b6121441d0147c960a99829c53b5a5ba (diff) |
Add Hugo Modules
This commit implements Hugo Modules.
This is a broad subject, but some keywords include:
* A new `module` configuration section where you can import almost anything. You can configure both your own file mounts nd the file mounts of the modules you import. This is the new recommended way of configuring what you earlier put in `configDir`, `staticDir` etc. And it also allows you to mount folders in non-Hugo-projects, e.g. the `SCSS` folder in the Bootstrap GitHub project.
* A module consists of a set of mounts to the standard 7 component types in Hugo: `static`, `content`, `layouts`, `data`, `assets`, `i18n`, and `archetypes`. Yes, Theme Components can now include content, which should be very useful, especially in bigger multilingual projects.
* Modules not in your local file cache will be downloaded automatically and even "hot replaced" while the server is running.
* Hugo Modules supports and encourages semver versioned modules, and uses the minimal version selection algorithm to resolve versions.
* A new set of CLI commands are provided to manage all of this: `hugo mod init`, `hugo mod get`, `hugo mod graph`, `hugo mod tidy`, and `hugo mod vendor`.
All of the above is backed by Go Modules.
Fixes #5973
Fixes #5996
Fixes #6010
Fixes #5911
Fixes #5940
Fixes #6074
Fixes #6082
Fixes #6092
Diffstat (limited to 'modules')
-rw-r--r-- | modules/client.go | 570 | ||||
-rw-r--r-- | modules/client_test.go | 117 | ||||
-rw-r--r-- | modules/collect.go | 574 | ||||
-rw-r--r-- | modules/collect_test.go | 38 | ||||
-rw-r--r-- | modules/config.go | 335 | ||||
-rw-r--r-- | modules/config_test.go | 132 | ||||
-rw-r--r-- | modules/module.go | 196 |
7 files changed, 1962 insertions, 0 deletions
diff --git a/modules/client.go b/modules/client.go new file mode 100644 index 000000000..ac09721dc --- /dev/null +++ b/modules/client.go @@ -0,0 +1,570 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package modules + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + + "github.com/gohugoio/hugo/hugofs/files" + + "github.com/gohugoio/hugo/common/loggers" + + "strings" + "time" + + "github.com/gohugoio/hugo/config" + + "github.com/rogpeppe/go-internal/module" + + "github.com/gohugoio/hugo/common/hugio" + + "github.com/pkg/errors" + "github.com/spf13/afero" +) + +var ( + fileSeparator = string(os.PathSeparator) +) + +const ( + goBinaryStatusOK goBinaryStatus = iota + goBinaryStatusNotFound + goBinaryStatusTooOld +) + +// The "vendor" dir is reserved for Go Modules. +const vendord = "_vendor" + +const ( + goModFilename = "go.mod" + goSumFilename = "go.sum" +) + +// NewClient creates a new Client that can be used to manage the Hugo Components +// in a given workingDir. +// The Client will resolve the dependencies recursively, but needs the top +// level imports to start out. +func NewClient(cfg ClientConfig) *Client { + fs := cfg.Fs + + n := filepath.Join(cfg.WorkingDir, goModFilename) + goModEnabled, _ := afero.Exists(fs, n) + var goModFilename string + if goModEnabled { + goModFilename = n + } + + env := os.Environ() + mcfg := cfg.ModuleConfig + + config.SetEnvVars(&env, + "PWD", cfg.WorkingDir, + "GOPROXY", mcfg.Proxy, + "GOPRIVATE", mcfg.Private, + "GONOPROXY", mcfg.NoProxy) + + if cfg.CacheDir != "" { + // Module cache stored below $GOPATH/pkg + config.SetEnvVars(&env, "GOPATH", cfg.CacheDir) + + } + + logger := cfg.Logger + if logger == nil { + logger = loggers.NewWarningLogger() + } + + return &Client{ + fs: fs, + ignoreVendor: cfg.IgnoreVendor, + workingDir: cfg.WorkingDir, + themesDir: cfg.ThemesDir, + logger: logger, + moduleConfig: mcfg, + environ: env, + GoModulesFilename: goModFilename} +} + +// Client contains most of the API provided by this package. +type Client struct { + fs afero.Fs + logger *loggers.Logger + + // Ignore any _vendor directory. + ignoreVendor bool + + // Absolute path to the project dir. + workingDir string + + // Absolute path to the project's themes dir. + themesDir string + + // The top level module config + moduleConfig Config + + // Environment variables used in "go get" etc. + environ []string + + // Set when Go modules are initialized in the current repo, that is: + // a go.mod file exists. + GoModulesFilename string + + // Set if we get a exec.ErrNotFound when running Go, which is most likely + // due to being run on a system without Go installed. We record it here + // so we can give an instructional error at the end if module/theme + // resolution fails. + goBinaryStatus goBinaryStatus +} + +// Graph writes a module dependenchy graph to the given writer. +func (c *Client) Graph(w io.Writer) error { + mc, coll := c.collect(true) + if coll.err != nil { + return coll.err + } + for _, module := range mc.AllModules { + if module.Owner() == nil { + continue + } + + prefix := "" + if module.Disabled() { + prefix = "DISABLED " + } + dep := pathVersion(module.Owner()) + " " + pathVersion(module) + if replace := module.Replace(); replace != nil { + if replace.Version() != "" { + dep += " => " + pathVersion(replace) + } else { + // Local dir. + dep += " => " + replace.Dir() + } + + } + fmt.Fprintln(w, prefix+dep) + } + + return nil +} + +// Tidy can be used to remove unused dependencies from go.mod and go.sum. +func (c *Client) Tidy() error { + tc, coll := c.collect(false) + if coll.err != nil { + return coll.err + } + + if coll.skipTidy { + return nil + } + + return c.tidy(tc.AllModules, false) +} + +// Vendor writes all the module dependencies to a _vendor folder. +// +// Unlike Go, we support it for any level. +// +// We, by default, use the /_vendor folder first, if found. To disable, +// run with +// hugo --ignoreVendor +// +// Given a module tree, Hugo will pick the first module for a given path, +// meaning that if the top-level module is vendored, that will be the full +// set of dependencies. +func (c *Client) Vendor() error { + vendorDir := filepath.Join(c.workingDir, vendord) + if err := c.rmVendorDir(vendorDir); err != nil { + return err + } + + // Write the modules list to modules.txt. + // + // On the form: + // + // # github.com/alecthomas/chroma v0.6.3 + // + // This is how "go mod vendor" does it. Go also lists + // the packages below it, but that is currently not applicable to us. + // + var modulesContent bytes.Buffer + + tc, coll := c.collect(true) + if coll.err != nil { + return coll.err + } + + for _, t := range tc.AllModules { + if t.Owner() == nil { + // This is the project. + continue + } + // We respect the --ignoreVendor flag even for the vendor command. + if !t.IsGoMod() && !t.Vendor() { + // We currently do not vendor components living in the + // theme directory, see https://github.com/gohugoio/hugo/issues/5993 + continue + } + + fmt.Fprintln(&modulesContent, "# "+t.Path()+" "+t.Version()) + + dir := t.Dir() + + for _, mount := range t.Mounts() { + if err := hugio.CopyDir(c.fs, filepath.Join(dir, mount.Source), filepath.Join(vendorDir, t.Path(), mount.Source), nil); err != nil { + return errors.Wrap(err, "failed to copy module to vendor dir") + } + } + + // Include the resource cache if present. + resourcesDir := filepath.Join(dir, files.FolderResources) + _, err := c.fs.Stat(resourcesDir) + if err == nil { + if err := hugio.CopyDir(c.fs, resourcesDir, filepath.Join(vendorDir, t.Path(), files.FolderResources), nil); err != nil { + return errors.Wrap(err, "failed to copy resources to vendor dir") + } + } + + // Also include any theme.toml or config.* files in the root. + configFiles, _ := afero.Glob(c.fs, filepath.Join(dir, "config.*")) + configFiles = append(configFiles, filepath.Join(dir, "theme.toml")) + for _, configFile := range configFiles { + if err := hugio.CopyFile(c.fs, configFile, filepath.Join(vendorDir, t.Path(), filepath.Base(configFile))); err != nil { + if !os.IsNotExist(err) { + return err + } + } + } + } + + if modulesContent.Len() > 0 { + if err := afero.WriteFile(c.fs, filepath.Join(vendorDir, vendorModulesFilename), modulesContent.Bytes(), 0666); err != nil { + return err + } + } + + return nil +} + +// Get runs "go get" with the supplied arguments. +func (c *Client) Get(args ...string) error { + if err := c.runGo(context.Background(), os.Stdout, append([]string{"get"}, args...)...); err != nil { + errors.Wrapf(err, "failed to get %q", args) + } + return nil +} + +// Init initializes this as a Go Module with the given path. +// If path is empty, Go will try to guess. +// If this succeeds, this project will be marked as Go Module. +func (c *Client) Init(path string) error { + err := c.runGo(context.Background(), os.Stdout, "mod", "init", path) + if err != nil { + return errors.Wrap(err, "failed to init modules") + } + + c.GoModulesFilename = filepath.Join(c.workingDir, goModFilename) + + return nil +} + +func (c *Client) isProbablyModule(path string) bool { + return module.CheckPath(path) == nil +} + +func (c *Client) listGoMods() (goModules, error) { + if c.GoModulesFilename == "" { + return nil, nil + } + + out := ioutil.Discard + err := c.runGo(context.Background(), out, "mod", "download") + if err != nil { + return nil, errors.Wrap(err, "failed to download modules") + } + + b := &bytes.Buffer{} + err = c.runGo(context.Background(), b, "list", "-m", "-json", "all") + if err != nil { + return nil, errors.Wrap(err, "failed to list modules") + } + + var modules goModules + + dec := json.NewDecoder(b) + for { + m := &goModule{} + if err := dec.Decode(m); err != nil { + if err == io.EOF { + break + } + return nil, errors.Wrap(err, "failed to decode modules list") + } + + modules = append(modules, m) + } + + return modules, err + +} + +func (c *Client) rewriteGoMod(name string, isGoMod map[string]bool) error { + data, err := c.rewriteGoModRewrite(name, isGoMod) + if err != nil { + return err + } + if data != nil { + if err := afero.WriteFile(c.fs, filepath.Join(c.workingDir, name), data, 0666); err != nil { + return err + } + } + + return nil +} + +func (c *Client) rewriteGoModRewrite(name string, isGoMod map[string]bool) ([]byte, error) { + if name == goModFilename && c.GoModulesFilename == "" { + // Already checked. + return nil, nil + } + + modlineSplitter := getModlineSplitter(name == goModFilename) + + b := &bytes.Buffer{} + f, err := c.fs.Open(filepath.Join(c.workingDir, name)) + if err != nil { + if os.IsNotExist(err) { + // It's been deleted. + return nil, nil + } + return nil, err + } + defer f.Close() + + scanner := bufio.NewScanner(f) + var dirty bool + + for scanner.Scan() { + line := scanner.Text() + var doWrite bool + + if parts := modlineSplitter(line); parts != nil { + modname, modver := parts[0], parts[1] + modver = strings.TrimSuffix(modver, "/"+goModFilename) + modnameVer := modname + " " + modver + doWrite = isGoMod[modnameVer] + } else { + doWrite = true + } + + if doWrite { + fmt.Fprintln(b, line) + } else { + dirty = true + } + } + + if !dirty { + // Nothing changed + return nil, nil + } + + return b.Bytes(), nil + +} + +func (c *Client) rmVendorDir(vendorDir string) error { + modulestxt := filepath.Join(vendorDir, vendorModulesFilename) + + if _, err := c.fs.Stat(vendorDir); err != nil { + return nil + } + + _, err := c.fs.Stat(modulestxt) + if err != nil { + // If we have a _vendor dir without modules.txt it sounds like + // a _vendor dir created by others. + return errors.New("found _vendor dir without modules.txt, skip delete") + } + + return c.fs.RemoveAll(vendorDir) +} + +func (c *Client) runGo( + ctx context.Context, + stdout io.Writer, + args ...string) error { + + if c.goBinaryStatus != 0 { + return nil + } + + stderr := new(bytes.Buffer) + cmd := exec.CommandContext(ctx, "go", args...) + + cmd.Env = c.environ + cmd.Dir = c.workingDir + cmd.Stdout = stdout + cmd.Stderr = io.MultiWriter(stderr, os.Stderr) + + if err := cmd.Run(); err != nil { + if ee, ok := err.(*exec.Error); ok && ee.Err == exec.ErrNotFound { + c.goBinaryStatus = goBinaryStatusNotFound + return nil + } + + _, ok := err.(*exec.ExitError) + if !ok { + return errors.Errorf("failed to execute 'go %v': %s %T", args, err, err) + } + + // Too old Go version + if strings.Contains(stderr.String(), "flag provided but not defined") { + c.goBinaryStatus = goBinaryStatusTooOld + return nil + } + + return errors.Errorf("go command failed: %s", stderr) + + } + + return nil +} + +func (c *Client) tidy(mods Modules, goModOnly bool) error { + isGoMod := make(map[string]bool) + for _, m := range mods { + if m.Owner() == nil { + continue + } + if m.IsGoMod() { + // Matching the format in go.mod + pathVer := m.Path() + " " + m.Version() + isGoMod[pathVer] = true + } + } + + if err := c.rewriteGoMod(goModFilename, isGoMod); err != nil { + return err + } + + if goModOnly { + return nil + } + + if err := c.rewriteGoMod(goSumFilename, isGoMod); err != nil { + return err + } + + return nil +} + +// ClientConfig configures the module Client. +type ClientConfig struct { + Fs afero.Fs + Logger *loggers.Logger + IgnoreVendor bool + WorkingDir string + ThemesDir string // Absolute directory path + CacheDir string // Module cache + ModuleConfig Config +} + +type goBinaryStatus int + +type goModule struct { + Path string // module path + Version string // module version + Versions []string // available module versions (with -versions) + Replace *goModule // replaced by this module + Time *time.Time // time version was created + Update *goModule // available update, if any (with -u) + Main bool // is this the main module? + Indirect bool // is this module only an indirect dependency of main module? + Dir string // directory holding files for this module, if any + GoMod string // path to go.mod file for this module, if any + Error *goModuleError // error loading module +} + +type goModuleError struct { + Err string // the error itself +} + +type goModules []*goModule + +func (modules goModules) GetByPath(p string) *goModule { + if modules == nil { + return nil + } + + for _, m := range modules { + if strings.EqualFold(p, m.Path) { + return m + } + } + + return nil +} + +func (modules goModules) GetMain() *goModule { + for _, m := range modules { + if m.Main { + return m + } + } + + return nil +} + +func getModlineSplitter(isGoMod bool) func(line string) []string { + if isGoMod { + return func(line string) []string { + if strings.HasPrefix(line, "require (") { + return nil + } + if !strings.HasPrefix(line, "require") && !strings.HasPrefix(line, "\t") { + return nil + } + line = strings.TrimPrefix(line, "require") + line = strings.TrimSpace(line) + line = strings.TrimSuffix(line, "// indirect") + + return strings.Fields(line) + } + } + + return func(line string) []string { + return strings.Fields(line) + } +} + +func pathVersion(m Module) string { + versionStr := m.Version() + if m.Vendor() { + versionStr += "+vendor" + } + if versionStr == "" { + return m.Path() + } + return fmt.Sprintf("%s@%s", m.Path(), versionStr) +} diff --git a/modules/client_test.go b/modules/client_test.go new file mode 100644 index 000000000..d8301514d --- /dev/null +++ b/modules/client_test.go @@ -0,0 +1,117 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package modules + +import ( + "bytes" + "testing" + + "github.com/gohugoio/hugo/common/hugo" + + "github.com/gohugoio/hugo/htesting" + + "github.com/gohugoio/hugo/hugofs" + + "github.com/stretchr/testify/require" +) + +func TestClient(t *testing.T) { + if hugo.GoMinorVersion() < 12 { + // https://github.com/golang/go/issues/26794 + // There were some concurrent issues with Go modules in < Go 12. + t.Skip("skip this for Go <= 1.11 due to a bug in Go's stdlib") + } + + t.Parallel() + + modName := "hugo-modules-basic-test" + modPath := "github.com/gohugoio/tests/" + modName + modConfig := DefaultModuleConfig + modConfig.Imports = []Import{Import{Path: "github.com/gohugoio/hugoTestModules1_darwin/modh2_2"}} + + assert := require.New(t) + + workingDir, clean, err := htesting.CreateTempDir(hugofs.Os, modName) + assert.NoError(err) + defer clean() + + client := NewClient(ClientConfig{ + Fs: hugofs.Os, + WorkingDir: workingDir, + ModuleConfig: modConfig, + }) + + // Test Init + assert.NoError(client.Init(modPath)) + + // Test Collect + mc, err := client.Collect() + assert.NoError(err) + assert.Equal(4, len(mc.AllModules)) + for _, m := range mc.AllModules { + assert.NotNil(m) + } + + // Test Graph + var graphb bytes.Buffer + assert.NoError(client.Graph(&graphb)) + + expect := `github.com/gohugoio/tests/hugo-modules-basic-test github.com/gohugoio/hugoTestModules1_darwin/modh2_2@v1.4.0 +github.com/gohugoio/hugoTestModules1_darwin/modh2_2@v1.4.0 github.com/gohugoio/hugoTestModules1_darwin/modh2_2_1v@v1.3.0 +github.com/gohugoio/hugoTestModules1_darwin/modh2_2@v1.4.0 github.com/gohugoio/hugoTestModules1_darwin/modh2_2_2@v1.3.0 +` + + assert.Equal(expect, graphb.String()) + + // Test Vendor + assert.NoError(client.Vendor()) + graphb.Reset() + assert.NoError(client.Graph(&graphb)) + expectVendored := `github.com/gohugoio/tests/hugo-modules-basic-test github.com/gohugoio/hugoTestModules1_darwin/modh2_2@v1.4.0+vendor +github.com/gohugoio/tests/hugo-modules-basic-test github.com/gohugoio/hugoTestModules1_darwin/modh2_2_1v@v1.3.0+vendor +github.com/gohugoio/tests/hugo-modules-basic-test github.com/gohugoio/hugoTestModules1_darwin/modh2_2_2@v1.3.0+vendor +` + assert.Equal(expectVendored, graphb.String()) + + // Test the ignoreVendor setting + clientIgnoreVendor := NewClient(ClientConfig{ + Fs: hugofs.Os, + WorkingDir: workingDir, + ModuleConfig: modConfig, + IgnoreVendor: true, + }) + + graphb.Reset() + assert.NoError(clientIgnoreVendor.Graph(&graphb)) + assert.Equal(expect, graphb.String()) + + // Test Tidy + assert.NoError(client.Tidy()) + +} + +func TestGetModlineSplitter(t *testing.T) { + + assert := require.New(t) + + gomodSplitter := getModlineSplitter(true) + + assert.Equal([]string{"github.com/BurntSushi/toml", "v0.3.1"}, gomodSplitter("\tgithub.com/BurntSushi/toml v0.3.1")) + assert.Equal([]string{"github.com/cpuguy83/go-md2man", "v1.0.8"}, gomodSplitter("\tgithub.com/cpuguy83/go-md2man v1.0.8 // indirect")) + assert.Nil(gomodSplitter("require (")) + + gosumSplitter := getModlineSplitter(false) + assert.Equal([]string{"github.com/BurntSushi/toml", "v0.3.1"}, gosumSplitter("github.com/BurntSushi/toml v0.3.1")) + +} diff --git a/modules/collect.go b/modules/collect.go new file mode 100644 index 000000000..f57b4d04b --- /dev/null +++ b/modules/collect.go @@ -0,0 +1,574 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package modules + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/spf13/cast" + + "github.com/gohugoio/hugo/common/maps" + + "github.com/gohugoio/hugo/common/hugo" + "github.com/gohugoio/hugo/parser/metadecoders" + + "github.com/gohugoio/hugo/hugofs/files" + + "github.com/rogpeppe/go-internal/module" + + "github.com/pkg/errors" + + "github.com/gohugoio/hugo/config" + "github.com/spf13/afero" +) + +var ErrNotExist = errors.New("module does not exist") + +const vendorModulesFilename = "modules.txt" + +// IsNotExist returns whether an error means that a module could not be found. +func IsNotExist(err error) bool { + return errors.Cause(err) == ErrNotExist +} + +// CreateProjectModule creates modules from the given config. +// This is used in tests only. +func CreateProjectModule(cfg config.Provider) (Module, error) { + workingDir := cfg.GetString("workingDir") + var modConfig Config + + mod := createProjectModule(nil, workingDir, modConfig) + if err := ApplyProjectConfigDefaults(cfg, mod); err != nil { + return nil, err + } + + return mod, nil +} + +func (h *Client) Collect() (ModulesConfig, error) { + mc, coll := h.collect(true) + return mc, coll.err + +} + +func (h *Client) collect(tidy bool) (ModulesConfig, *collector) { + c := &collector{ + Client: h, + } + + c.collect() + if c.err != nil { + return ModulesConfig{}, c + } + + if !c.skipTidy && tidy { + if err := h.tidy(c.modules, true); err != nil { + c.err = err + return ModulesConfig{}, c + } + } + + // TODO(bep) consider --ignoreVendor vs removing from go.mod + var activeMods Modules + for _, mod := range c.modules { + if !mod.Config().HugoVersion.IsValid() { + h.logger.WARN.Printf(`Module %q is not compatible with this Hugo version; run "hugo mod graph" for more information.`, mod.Path()) + } + if !mod.Disabled() { + activeMods = append(activeMods, mod) + } + } + + return ModulesConfig{ + AllModules: c.modules, + ActiveModules: activeMods, + GoModulesFilename: c.GoModulesFilename, + }, c + +} + +type ModulesConfig struct { + // All modules, including any disabled. + AllModules Modules + + // All active modules. + ActiveModules Modules + + // Set if this is a Go modules enabled project. + GoModulesFilename string +} + +type collected struct { + // Pick the first and prevent circular loops. + seen map[string]bool + + // Maps module path to a _vendor dir. These values are fetched from + // _vendor/modules.txt, and the first (top-most) will win. + vendored map[string]vendoredModule + + // Set if a Go modules enabled project. + gomods goModules + + // Ordered list of collected modules, including Go Modules and theme + // components stored below /themes. + modules Modules +} + +// Collects and creates a module tree. +type collector struct { + *Client + + // Store away any non-fatal error and return at the end. + err error + + // Set to disable any Tidy operation in the end. + skipTidy bool + + *collected +} + +func (c *collector) initModules() error { + c.collected = &collected{ + seen: make(map[string]bool), + vendored: make(map[string]vendoredModule), + } + + // We may fail later if we don't find the mods. + return c.loadModules() +} + +func (c *collector) isSeen(path string) bool { + key := pathKey(path) + if c.seen[key] { + return true + } + c.seen[key] = true + return false +} + +func (c *collector) getVendoredDir(path string) (vendoredModule, bool) { + v, found := c.vendored[path] + return v, found +} + +func (c *collector) add(owner *moduleAdapter, moduleImport Import, disabled bool) (*moduleAdapter, error) { + var ( + mod *goModule + moduleDir string + version string + vendored bool + ) + + modulePath := moduleImport.Path + var realOwner Module = owner + + if !c.ignoreVendor { + if err := c.collectModulesTXT(owner); err != nil { + return nil, err + } + + // Try _vendor first. + var vm vendoredModule + vm, vendored = c.getVendoredDir(modulePath) + if vendored { + moduleDir = vm.Dir + realOwner = vm.Owner + version = vm.Version + + if owner.projectMod { + // We want to keep the go.mod intact with the versions and all. + c.skipTidy = true + } + + } + } + + if moduleDir == "" { + mod = c.gomods.GetByPath(modulePath) + if mod != nil { + moduleDir = mod.Dir + } + + if moduleDir == "" { + + if c.GoModulesFilename != "" && c.isProbablyModule(modulePath) { + // Try to "go get" it and reload the module configuration. + if err := c.Get(modulePath); err != nil { + return nil, err + } + if err := c.loadModules(); err != nil { + return nil, err + } + + mod = c.gomods.GetByPath(modulePath) + if mod != nil { + moduleDir = mod.Dir + } + } + + // Fall back to /themes/<mymodule> + if moduleDir == "" { + moduleDir = filepath.Join(c.themesDir, modulePath) + + if found, _ := afero.Exists(c.fs, moduleDir); !found { + c.err = c.wrapModuleNotFound(errors.E |