diff options
Diffstat (limited to 'commands')
-rw-r--r-- | commands/benchmark.go | 9 | ||||
-rw-r--r-- | commands/commandeer.go | 34 | ||||
-rw-r--r-- | commands/convert.go | 135 | ||||
-rw-r--r-- | commands/hugo.go | 462 | ||||
-rw-r--r-- | commands/import_jekyll.go | 1 | ||||
-rw-r--r-- | commands/list.go | 42 | ||||
-rw-r--r-- | commands/list_config.go | 2 | ||||
-rw-r--r-- | commands/new.go | 35 | ||||
-rw-r--r-- | commands/server.go | 151 | ||||
-rw-r--r-- | commands/static_syncer.go | 16 | ||||
-rw-r--r-- | commands/undraft.go | 4 |
11 files changed, 554 insertions, 337 deletions
diff --git a/commands/benchmark.go b/commands/benchmark.go index 51f2be876..77790e24a 100644 --- a/commands/benchmark.go +++ b/commands/benchmark.go @@ -48,12 +48,7 @@ func init() { } func benchmark(cmd *cobra.Command, args []string) error { - cfg, err := InitializeConfig(benchmarkCmd) - if err != nil { - return err - } - - c, err := newCommandeer(cfg) + c, err := InitializeConfig(false, nil, benchmarkCmd) if err != nil { return err } @@ -84,7 +79,7 @@ func benchmark(cmd *cobra.Command, args []string) error { t := time.Now() for i := 0; i < benchmarkTimes; i++ { - if err = c.resetAndBuildSites(false); err != nil { + if err = c.resetAndBuildSites(); err != nil { return err } } diff --git a/commands/commandeer.go b/commands/commandeer.go index f538ba619..a69ce2084 100644 --- a/commands/commandeer.go +++ b/commands/commandeer.go @@ -18,6 +18,7 @@ import ( "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" + src "github.com/gohugoio/hugo/source" ) type commandeer struct { @@ -25,7 +26,10 @@ type commandeer struct { pathSpec *helpers.PathSpec visitedURLs *types.EvictingStringQueue + staticDirsConfig []*src.Dirs + serverPorts []int + languages helpers.Languages configured bool } @@ -44,10 +48,6 @@ func (c *commandeer) PathSpec() *helpers.PathSpec { return c.pathSpec } -func (c *commandeer) languages() helpers.Languages { - return c.Cfg.Get("languagesSorted").(helpers.Languages) -} - func (c *commandeer) initFs(fs *hugofs.Fs) error { c.DepsCfg.Fs = fs ps, err := helpers.NewPathSpec(fs, c.Cfg) @@ -55,18 +55,26 @@ func (c *commandeer) initFs(fs *hugofs.Fs) error { return err } c.pathSpec = ps + + dirsConfig, err := c.createStaticDirsConfig() + if err != nil { + return err + } + c.staticDirsConfig = dirsConfig + return nil } -func newCommandeer(cfg *deps.DepsCfg) (*commandeer, error) { - l := cfg.Language - if l == nil { - l = helpers.NewDefaultLanguage(cfg.Cfg) - } - ps, err := helpers.NewPathSpec(cfg.Fs, l) - if err != nil { - return nil, err +func newCommandeer(cfg *deps.DepsCfg, running bool) (*commandeer, error) { + cfg.Running = running + + var languages helpers.Languages + + if l, ok := cfg.Cfg.Get("languagesSorted").(helpers.Languages); ok { + languages = l } - return &commandeer{DepsCfg: cfg, pathSpec: ps, visitedURLs: types.NewEvictingStringQueue(10)}, nil + c := &commandeer{DepsCfg: cfg, languages: languages, visitedURLs: types.NewEvictingStringQueue(10)} + + return c, nil } diff --git a/commands/convert.go b/commands/convert.go index 298ff6019..f63f8522f 100644 --- a/commands/convert.go +++ b/commands/convert.go @@ -14,12 +14,15 @@ package commands import ( - "errors" "fmt" - "path/filepath" "time" + src "github.com/gohugoio/hugo/source" + "github.com/gohugoio/hugo/hugolib" + + "path/filepath" + "github.com/gohugoio/hugo/parser" "github.com/spf13/cast" "github.com/spf13/cobra" @@ -78,81 +81,103 @@ func init() { } func convertContents(mark rune) error { - cfg, err := InitializeConfig() + if outputDir == "" && !unsafe { + return newUserError("Unsafe operation not allowed, use --unsafe or set a different output path") + } + + c, err := InitializeConfig(false, nil) if err != nil { return err } - h, err := hugolib.NewHugoSites(*cfg) + h, err := hugolib.NewHugoSites(*c.DepsCfg) if err != nil { return err } - site := h.Sites[0] - - if err = site.Initialise(); err != nil { + if err := h.Build(hugolib.BuildCfg{SkipRender: true}); err != nil { return err } - if site.Source == nil { - panic("site.Source not set") - } - if len(site.Source.Files()) < 1 { - return errors.New("No source files found") - } + site := h.Sites[0] - contentDir := site.PathSpec.AbsPathify(site.Cfg.GetString("contentDir")) - site.Log.FEEDBACK.Println("processing", len(site.Source.Files()), "content files") - for _, file := range site.Source.Files() { - site.Log.INFO.Println("Attempting to convert", file.LogicalName()) - page, err := site.NewPage(file.LogicalName()) - if err != nil { + site.Log.FEEDBACK.Println("processing", len(site.AllPages), "content files") + for _, p := range site.AllPages { + if err := convertAndSavePage(p, site, mark); err != nil { return err } + } + return nil +} - psr, err := parser.ReadFrom(file.Contents) - if err != nil { - site.Log.ERROR.Println("Error processing file:", file.Path()) - return err - } - metadata, err := psr.Metadata() - if err != nil { - site.Log.ERROR.Println("Error processing file:", file.Path()) +func convertAndSavePage(p *hugolib.Page, site *hugolib.Site, mark rune) error { + // The resources are not in .Site.AllPages. + for _, r := range p.Resources.ByType("page") { + if err := convertAndSavePage(r.(*hugolib.Page), site, mark); err != nil { return err } + } - // better handling of dates in formats that don't have support for them - if mark == parser.FormatToLeadRune("json") || mark == parser.FormatToLeadRune("yaml") || mark == parser.FormatToLeadRune("toml") { - newMetadata := cast.ToStringMap(metadata) - for k, v := range newMetadata { - switch vv := v.(type) { - case time.Time: - newMetadata[k] = vv.Format(time.RFC3339) - } - } - metadata = newMetadata - } + if p.Filename() == "" { + // No content file. + return nil + } - page.SetDir(filepath.Join(contentDir, file.Dir())) - page.SetSourceContent(psr.Content()) - if err = page.SetSourceMetaData(metadata, mark); err != nil { - site.Log.ERROR.Printf("Failed to set source metadata for file %q: %s. For more info see For more info see https://github.com/gohugoio/hugo/issues/2458", page.FullFilePath(), err) - continue - } + site.Log.INFO.Println("Attempting to convert", p.LogicalName()) + newPage, err := site.NewPage(p.LogicalName()) + if err != nil { + return err + } - if outputDir != "" { - if err = page.SaveSourceAs(filepath.Join(outputDir, page.FullFilePath())); err != nil { - return fmt.Errorf("Failed to save file %q: %s", page.FullFilePath(), err) - } - } else { - if unsafe { - if err = page.SaveSource(); err != nil { - return fmt.Errorf("Failed to save file %q: %s", page.FullFilePath(), err) - } - } else { - site.Log.FEEDBACK.Println("Unsafe operation not allowed, use --unsafe or set a different output path") + f, _ := p.File.(src.ReadableFile) + file, err := f.Open() + if err != nil { + site.Log.ERROR.Println("Error reading file:", p.Path()) + file.Close() + return nil + } + + psr, err := parser.ReadFrom(file) + if err != nil { + site.Log.ERROR.Println("Error processing file:", p.Path()) + file.Close() + return err + } + + file.Close() + + metadata, err := psr.Metadata() + if err != nil { + site.Log.ERROR.Println("Error processing file:", p.Path()) + return err + } + + // better handling of dates in formats that don't have support for them + if mark == parser.FormatToLeadRune("json") || mark == parser.FormatToLeadRune("yaml") || mark == parser.FormatToLeadRune("toml") { + newMetadata := cast.ToStringMap(metadata) + for k, v := range newMetadata { + switch vv := v.(type) { + case time.Time: + newMetadata[k] = vv.Format(time.RFC3339) } } + metadata = newMetadata + } + + newPage.SetSourceContent(psr.Content()) + if err = newPage.SetSourceMetaData(metadata, mark); err != nil { + site.Log.ERROR.Printf("Failed to set source metadata for file %q: %s. For more info see For more info see https://github.com/gohugoio/hugo/issues/2458", newPage.FullFilePath(), err) + return nil } + + newFilename := p.Filename() + if outputDir != "" { + newFilename = filepath.Join(outputDir, p.Dir(), newPage.LogicalName()) + } + + if err = newPage.SaveSourceAs(newFilename); err != nil { + return fmt.Errorf("Failed to save file %q: %s", newFilename, err) + } + return nil } diff --git a/commands/hugo.go b/commands/hugo.go index 7b50d0bb3..200a5e1c3 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -18,6 +18,10 @@ package commands import ( "fmt" "io/ioutil" + "sort" + "sync/atomic" + + "golang.org/x/sync/errgroup" "github.com/gohugoio/hugo/hugofs" @@ -58,6 +62,13 @@ import ( // provide a cleaner external API, but until then, this is it. var Hugo *hugolib.HugoSites +const ( + ansiEsc = "\u001B" + clearLine = "\r\033[K" + hideCursor = ansiEsc + "[?25l" + showCursor = ansiEsc + "[?25h" +) + // Reset resets Hugo ready for a new full build. This is mainly only useful // for benchmark testing etc. via the CLI commands. func Reset() error { @@ -116,18 +127,20 @@ built with love by spf13 and friends in Go. Complete documentation is available at http://gohugo.io/.`, RunE: func(cmd *cobra.Command, args []string) error { - cfg, err := InitializeConfig() - if err != nil { - return err + + cfgInit := func(c *commandeer) error { + if buildWatch { + c.Set("disableLiveReload", true) + } + return nil } - c, err := newCommandeer(cfg) + c, err := InitializeConfig(buildWatch, cfgInit) if err != nil { return err } if buildWatch { - cfg.Cfg.Set("disableLiveReload", true) c.watchConfig() } @@ -149,6 +162,7 @@ var ( ) var ( + gc bool baseURL string cacheDir string contentDir string @@ -201,6 +215,7 @@ func AddCommands() { genCmd.AddCommand(genmanCmd) genCmd.AddCommand(createGenDocsHelper().cmd) genCmd.AddCommand(createGenChromaStyles().cmd) + } // initHugoBuilderFlags initializes all common flags, typically used by the @@ -240,6 +255,7 @@ func initHugoBuildCommonFlags(cmd *cobra.Command) { cmd.Flags().Bool("canonifyURLs", false, "if true, all relative URLs will be canonicalized using baseURL") cmd.Flags().StringVarP(&baseURL, "baseURL", "b", "", "hostname (and path) to the root, e.g. http://spf13.com/") cmd.Flags().Bool("enableGitInfo", false, "add Git revision, date and author info to the pages") + cmd.Flags().BoolVar(&gc, "gc", false, "enable to run some cleanup tasks (remove unused cache files) after the build") cmd.Flags().BoolVar(&nitro.AnalysisOn, "stepAnalysis", false, "display memory and timing of different steps of the program") cmd.Flags().Bool("templateMetrics", false, "display metrics about template executions") @@ -285,7 +301,7 @@ func init() { } // InitializeConfig initializes a config file with sensible default configuration flags. -func InitializeConfig(subCmdVs ...*cobra.Command) (*deps.DepsCfg, error) { +func InitializeConfig(running bool, doWithCommandeer func(c *commandeer) error, subCmdVs ...*cobra.Command) (*commandeer, error) { var cfg *deps.DepsCfg = &deps.DepsCfg{} @@ -294,13 +310,13 @@ func InitializeConfig(subCmdVs ...*cobra.Command) (*deps.DepsCfg, error) { config, err := hugolib.LoadConfig(osFs, source, cfgFile) if err != nil { - return cfg, err + return nil, err } // Init file systems. This may be changed at a later point. cfg.Cfg = config - c, err := newCommandeer(cfg) + c, err := newCommandeer(cfg, running) if err != nil { return nil, err } @@ -309,23 +325,29 @@ func InitializeConfig(subCmdVs ...*cobra.Command) (*deps.DepsCfg, error) { c.initializeFlags(cmdV) } + if baseURL != "" { + config.Set("baseURL", baseURL) + } + + if doWithCommandeer != nil { + if err := doWithCommandeer(c); err != nil { + return nil, err + } + } + if len(disableKinds) > 0 { c.Set("disableKinds", disableKinds) } logger, err := createLogger(cfg.Cfg) if err != nil { - return cfg, err + return nil, err } cfg.Logger = logger config.Set("logI18nWarnings", logI18nWarnings) - if baseURL != "" { - config.Set("baseURL", baseURL) - } - if !config.GetBool("relativeURLs") && config.GetString("baseURL") == "" { cfg.Logger.ERROR.Println("No 'baseURL' set in configuration or as a flag. Features like page menus will not work without one.") } @@ -350,17 +372,6 @@ func InitializeConfig(subCmdVs ...*cobra.Command) (*deps.DepsCfg, error) { } config.Set("workingDir", dir) - fs := hugofs.NewFrom(osFs, config) - - // Hugo writes the output to memory instead of the disk. - // This is only used for benchmark testing. Cause the content is only visible - // in memory. - if renderToMemory { - fs.Destination = new(afero.MemMapFs) - // Rendering to memoryFS, publish to Root regardless of publishDir. - c.Set("publishDir", "/") - } - if contentDir != "" { config.Set("contentDir", contentDir) } @@ -373,6 +384,17 @@ func InitializeConfig(subCmdVs ...*cobra.Command) (*deps.DepsCfg, error) { config.Set("cacheDir", cacheDir) } + fs := hugofs.NewFrom(osFs, config) + + // Hugo writes the output to memory instead of the disk. + // This is only used for benchmark testing. Cause the content is only visible + // in memory. + if config.GetBool("renderToMemory") { + fs.Destination = new(afero.MemMapFs) + // Rendering to memoryFS, publish to Root regardless of publishDir. + config.Set("publishDir", "/") + } + cacheDir = config.GetString("cacheDir") if cacheDir != "" { if helpers.FilePathSeparator != cacheDir[len(cacheDir)-1:] { @@ -397,7 +419,7 @@ func InitializeConfig(subCmdVs ...*cobra.Command) (*deps.DepsCfg, error) { themeDir := c.PathSpec().GetThemeDir() if themeDir != "" { if _, err := cfg.Fs.Source.Stat(themeDir); os.IsNotExist(err) { - return cfg, newSystemError("Unable to find theme Directory:", themeDir) + return nil, newSystemError("Unable to find theme Directory:", themeDir) } } @@ -408,7 +430,7 @@ func InitializeConfig(subCmdVs ...*cobra.Command) (*deps.DepsCfg, error) { helpers.CurrentHugoVersion.ReleaseVersion(), minVersion) } - return cfg, nil + return c, nil } @@ -482,17 +504,17 @@ func (c *commandeer) initializeFlags(cmd *cobra.Command) { "templateMetricsHints", } - // Remove these in Hugo 0.23. + // Remove these in Hugo 0.33. if cmd.Flags().Changed("disable404") { - helpers.Deprecated("command line", "--disable404", "Use --disableKinds=404", false) + helpers.Deprecated("command line", "--disable404", "Use --disableKinds=404", true) } if cmd.Flags().Changed("disableRSS") { - helpers.Deprecated("command line", "--disableRSS", "Use --disableKinds=RSS", false) + helpers.Deprecated("command line", "--disableRSS", "Use --disableKinds=RSS", true) } if cmd.Flags().Changed("disableSitemap") { - helpers.Deprecated("command line", "--disableSitemap", "Use --disableKinds=sitemap", false) + helpers.Deprecated("command line", "--disableSitemap", "Use --disableKinds=sitemap", true) } for _, key := range persFlagKeys { @@ -525,16 +547,71 @@ func (c *commandeer) watchConfig() { }) } -func (c *commandeer) build(watches ...bool) error { - if err := c.copyStatic(); err != nil { - return fmt.Errorf("Error copying static files: %s", err) +func (c *commandeer) fullBuild(watches ...bool) error { + var ( + g errgroup.Group + langCount map[string]uint64 + ) + + if !quiet { + fmt.Print(hideCursor + "Building sites … ") + defer func() { + fmt.Print(showCursor + clearLine) + }() } - watch := false - if len(watches) > 0 && watches[0] { - watch = true + + g.Go(func() error { + cnt, err := c.copyStatic() + if err != nil { + return fmt.Errorf("Error copying static files: %s", err) + } + langCount = cnt + return nil + }) + + g.Go(func() error { + if err := c.buildSites(); err != nil { + return fmt.Errorf("Error building site: %s", err) + } + + return nil + }) + + if err := g.Wait(); err != nil { + return err } - if err := c.buildSites(buildWatch || watch); err != nil { - return fmt.Errorf("Error building site: %s", err) + + for _, s := range Hugo.Sites { + s.ProcessingStats.Static = langCount[s.Language.Lang] + } + + if gc { + count, err := Hugo.GC() + if err != nil { + return err + } + for _, s := range Hugo.Sites { + // We have no way of knowing what site the garbage belonged to. + s.ProcessingStats.Cleaned = uint64(count) + } + } + + return nil + +} + +func (c *commandeer) build(watches ...bool) error { + defer c.timeTrack(time.Now(), "Total") + + if err := c.fullBuild(watches...); err != nil { + return err + } + + // TODO(bep) Feedback? + if !quiet { + fmt.Println() + Hugo.PrintProcessingStats(os.Stdout) + fmt.Println() } if buildWatch { @@ -550,62 +627,101 @@ func (c *commandeer) build(watches ...bool) error { return nil } -func (c *commandeer) copyStatic() error { +func (c *commandeer) copyStatic() (map[string]uint64, error) { return c.doWithPublishDirs(c.copyStaticTo) } -func (c *commandeer) doWithPublishDirs(f func(dirs *src.Dirs, publishDir string) error) error { - publishDir := c.PathSpec().AbsPathify(c.Cfg.GetString("publishDir")) + helpers.FilePathSeparator - // If root, remove the second '/' - if publishDir == "//" { - publishDir = helpers.FilePathSeparator - } - - languages := c.languages() +func (c *commandeer) createStaticDirsConfig() ([]*src.Dirs, error) { + var dirsConfig []*src.Dirs - if !languages.IsMultihost() { + if !c.languages.IsMultihost() { dirs, err := src.NewDirs(c.Fs, c.Cfg, c.DepsCfg.Logger) if err != nil { - return err + return nil, err + } + dirsConfig = append(dirsConfig, dirs) + } else { + for _, l := range c.languages { + dirs, err := src.NewDirs(c.Fs, l, c.DepsCfg.Logger) + if err != nil { + return nil, err + } + dirsConfig = append(dirsConfig, dirs) } - return f(dirs, publishDir) } - for _, l := range languages { - dir := filepath.Join(publishDir, l.Lang) - dirs, err := src.NewDirs(c.Fs, l, c.DepsCfg.Logger) + return dirsConfig, nil + +} + +func (c *commandeer) doWithPublishDirs(f func(dirs *src.Dirs, publishDir string) (uint64, error)) (map[string]uint64, error) { + + langCount := make(map[string]uint64) + + for _, dirs := range c.staticDirsConfig { + + cnt, err := f(dirs, c.pathSpec.PublishDir) if err != nil { - return err + return langCount, err } - if err := f(dirs, dir); err != nil { - return err + + if dirs.Language == nil { + // Not multihost + for _, l := range c.languages { + langCount[l.Lang] = cnt + } + } else { + langCount[dirs.Language.Lang] = cnt } + } - return nil + return langCount, nil +} + +type countingStatFs struct { + afero.Fs + statCounter uint64 } -func (c *commandeer) copyStaticTo(dirs *src.Dirs, publishDir string) error { +func (fs *countingStatFs) Stat(name string) (os.FileInfo, error) { + f, err := fs.Fs.Stat(name) + if err == nil { + if !f.IsDir() { + atomic.AddUint64(&fs.statCounter, 1) + } + } + return f, err +} + +func (c *commandeer) copyStaticTo(dirs *src.Dirs, publishDir string) (uint64, error) { // If root, remove the second '/' if publishDir == "//" { publishDir = helpers.FilePathSeparator } + if dirs.Language != nil { + // Multihost setup. + publishDir = filepath.Join(publishDir, dirs.Language.Lang) + } + staticSourceFs, err := dirs.CreateStaticFs() if err != nil { - return err + return 0, err } if staticSourceFs == nil { c.Logger.WARN.Println("No static directories found to sync") - return nil + return 0, nil } + fs := &countingStatFs{Fs: staticSourceFs} + syncer := fsync.NewSyncer() syncer.NoTimes = c.Cfg.GetBool("noTimes") syncer.NoChmod = c.Cfg.GetBool("noChmod") - syncer.SrcFs = staticSourceFs + syncer.SrcFs = fs syncer.DestFs = c.Fs.Destination // Now that we are using a unionFs for the static directories // We can effectively clean the publishDir on initial sync @@ -622,12 +738,30 @@ func (c *commandeer) copyStaticTo(dirs *src.Dirs, publishDir string) error { // because we are using a baseFs (to get the union right). // set sync src to root - return syncer.Sync(publishDir, helpers.FilePathSeparator) + err = syncer.Sync(publishDir, helpers.FilePathSeparator) + if err != nil { + return 0, err + } + + // Sync runs Stat 3 times for every source file (which sounds much) + numFiles := fs.statCounter / 3 + + return numFiles, err +} + +func (c *commandeer) timeTrack(start time.Time, name string) { + elapsed := time.Since(start) + c.Logger.FEEDBACK.Printf("%s in %v ms", name, int(1000*elapsed.Seconds())) } // getDirList provides NewWatcher() with a list of directories to watch for changes. func (c *commandeer) getDirList() ([]string, error) { var a []string + + // To handle nested symlinked content dirs + var seen = make(map[string]bool) + var nested []string + dataDir := c.PathSpec().AbsPathify(c.Cfg.GetString("dataDir")) i18nDir := c.PathSpec().AbsPathify(c.Cfg.GetString("i18nDir")) staticSyncer, err := newStaticSyncer(c) @@ -638,86 +772,121 @@ func (c *commandeer) getDirList() ([]string, error) { layoutDir := c.PathSpec().GetLayoutDirPath() staticDirs := staticSyncer.d.AbsStaticDirs - walker := func(path string, fi os.FileInfo, err error) error { - if err != nil { - if path == dataDir && os.IsNotExist(err) { - c.Logger.WARN.Println("Skip dataDir:", err) - return nil - } + newWalker := func(allowSymbolicDirs bool) func(path string, fi os.FileInfo, err error) error { + return func(path string, fi os.FileInfo, err error) error { + if err != nil { + if path == dataDir && os.IsNotExist(err) { + c.Logger.WARN.Println("Skip dataDir:", err) + return nil + } - if path == i18nDir && os.IsNotExist(err) { - c.Logger.WARN.Println("Skip i18nDir:", err) - return nil - } + if path == i18nDir && os.IsNotExist(err) { + c.Logger.WARN.Println("Skip i18nDir:", err) + return nil + } - if path == layoutDir && os.IsNotExist(err) { - c.Logger.WARN.Println("Skip layoutDir:", err) - return nil - } + if path == layoutDir && os.IsNotExist(err) { + c.Logger.WARN.Println("Skip layoutDir:", err) + return nil + } - if os.IsNotExist(err) { - for _, staticDir := range staticDirs { - if path == staticDir && os.IsNotExist(err) { - c.Logger.WARN.Println("Skip staticDir:", err) + if os.IsNotExist(err) { + for _, staticDir := range staticDirs { + if path == staticDir && os.IsNotExist(err) { + c.Logger.WARN.Println("Skip staticDir:", err) + } } + // Ignore. + return nil } - // Ignore. - return nil - } - - c.Logger.ERROR.Println("Walker: ", err) - return nil - } - // Skip .git directories. - // Related to https://github.com/gohugoio/hugo/issues/3468. - if fi.Name() == ".git" { - return nil - } - - if fi.Mode()&os.ModeSymlink == os.ModeSymlink { - link, err := filepath.EvalSymlinks(path) - if err != nil { - c.Logger.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", path, err) + c.Logger.ERROR.Println("Walker: ", err) return nil } - linkfi, err := c.Fs.Source.Stat(link) - if err != nil { - c.Logger.ERROR.Printf("Cannot stat '%s', error was: %s", link, err) + + // Skip .git directories. + // Related to https://github.com/gohugoio/hugo/issues/3468. + if fi.Name() == ".git" { return nil } - if !linkfi.Mode().IsRegular() { - c.Logger.ERROR.Printf("Symbolic links for directories not supported, skipping '%s'", path) + + if fi.Mode()&os.ModeSymlink == os.ModeSymlink { + link, err := filepath.EvalSymlinks(path) + if err != nil { + c.Logger.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", path, err) + return nil + } + linkfi, err := helpers.LstatIfOs(c.Fs.Source, link) + if err != nil { + c.Logger.ERROR.Printf("Cannot stat %q: %s", link, err) + return nil + } + if !allowSymbolicDirs && !linkfi.Mode().IsRegular() { + c.Logger.ERROR.Printf("Symbolic links for directories not supported, skipping %q", path) + return nil + } + + if allowSymbolicDirs && linkfi.IsDir() { + // afero.Walk will not walk symbolic links, so wee need to do it. + if !seen[path] { + seen[path] = true + nested = append(nested, path) + } + return nil + } + + fi = linkfi } - return nil - } - if fi.IsDir() { - if fi.Name() == ".git" || - fi.Name() == "node_modules" || fi.Name() == "bower_components" { - return filepath.SkipDir + if fi.IsDir() { + if fi.Name() == ".git" || + fi.Name() == "node_modules" || fi.Name() == "bower_components" { + return filepath.SkipDir + } + a = append(a, path) } - a = append(a, path) + return nil } - return nil } + symLinkWalker := newWalker(true) + regularWalker := newWalker(false) + // SymbolicWalk will log anny ERRORs - _ = helpers.SymbolicWalk(c.Fs.Source, dataDir, walker) - _ = helpers.SymbolicWalk(c.Fs.Source, c.PathSpec().AbsPathify(c.Cfg.GetString("contentDir")), walker) - _ = helpers.SymbolicWalk(c.Fs.Source, i18nDir, walker) - _ = helpers.SymbolicWalk(c.Fs.Source, layoutDir, walker) + _ = helpers.SymbolicWalk(c.Fs.Source, dataDir, regularWalker) + _ = helpers.SymbolicWalk(c.Fs.Source, c.PathSpec().AbsPathify(c.Cfg.GetString("contentDir")), symLinkWalker) + _ = helpers.SymbolicWalk(c.Fs.Source, i18nDir, regularWalker) + _ = helpers.SymbolicWalk(c.Fs.Source, layoutDir, regularWalker) for _, staticDir := range staticDirs { - _ = helpers.SymbolicWalk(c.Fs.Source, staticDir, walker) + _ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker) } if c.PathSpec().ThemeSet() { themesDir := c.PathSpec().GetThemeDir() - _ = helpers.SymbolicWalk(c.Fs.Source, filepath.Join(themesDir, "layouts"), walker) - _ = helpers.SymbolicWalk(c.Fs.Source, filepath.Join(themesDir, "i18n"), walker) - _ = helpers.SymbolicWalk(c.Fs.Source, filepath.Join(themesDir, " |