summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--commands/hugo.go7
-rw-r--r--docs/content/commands/hugo.md1
-rw-r--r--docs/content/extras/robots-txt.md34
-rw-r--r--docs/content/extras/urls.md2
-rw-r--r--docs/content/overview/configuration.md2
-rw-r--r--hugolib/robotstxt_test.go67
-rw-r--r--hugolib/site.go26
-rw-r--r--hugolib/sitemap_test.go4
-rw-r--r--tpl/template_embedded.go2
9 files changed, 143 insertions, 2 deletions
diff --git a/commands/hugo.go b/commands/hugo.go
index 4d3f9bab6..3b2e172c6 100644
--- a/commands/hugo.go
+++ b/commands/hugo.go
@@ -112,7 +112,7 @@ Complete documentation is available at http://gohugo.io/.`,
var hugoCmdV *cobra.Command
// Flags that are to be added to commands.
-var BuildWatch, IgnoreCache, Draft, Future, UglyURLs, CanonifyURLs, Verbose, Logging, VerboseLog, DisableRSS, DisableSitemap, PluralizeListTitles, PreserveTaxonomyNames, NoTimes, ForceSync bool
+var BuildWatch, IgnoreCache, Draft, Future, UglyURLs, CanonifyURLs, Verbose, Logging, VerboseLog, DisableRSS, DisableSitemap, DisableRobotsTXT, PluralizeListTitles, PreserveTaxonomyNames, NoTimes, ForceSync bool
var Source, CacheDir, Destination, Theme, BaseURL, CfgFile, LogFile, Editor string
// Execute adds all child commands to the root command HugoCmd and sets flags appropriately.
@@ -159,6 +159,7 @@ func initCoreCommonFlags(cmd *cobra.Command) {
cmd.Flags().BoolVarP(&Future, "buildFuture", "F", false, "include content with publishdate in the future")
cmd.Flags().BoolVar(&DisableRSS, "disableRSS", false, "Do not build RSS files")
cmd.Flags().BoolVar(&DisableSitemap, "disableSitemap", false, "Do not build Sitemap file")
+ cmd.Flags().BoolVar(&DisableRobotsTXT, "disableRobotsTXT", false, "Do not build Robots TXT file")
cmd.Flags().StringVarP(&Source, "source", "s", "", "filesystem path to read files relative from")
cmd.Flags().StringVarP(&CacheDir, "cacheDir", "", "", "filesystem path to cache directory. Defaults: $TMPDIR/hugo_cache/")
cmd.Flags().BoolVarP(&IgnoreCache, "ignoreCache", "", false, "Ignores the cache directory for reading but still writes to it")
@@ -204,6 +205,7 @@ func LoadDefaultSettings() {
viper.SetDefault("MetaDataFormat", "toml")
viper.SetDefault("DisableRSS", false)
viper.SetDefault("DisableSitemap", false)
+ viper.SetDefault("DisableRobotsTXT", false)
viper.SetDefault("ContentDir", "content")
viper.SetDefault("LayoutDir", "layouts")
viper.SetDefault("StaticDir", "static")
@@ -294,6 +296,9 @@ func InitializeConfig(subCmdVs ...*cobra.Command) error {
if cmdV.Flags().Lookup("disableSitemap").Changed {
viper.Set("DisableSitemap", DisableSitemap)
}
+ if cmdV.Flags().Lookup("disableRobotsTXT").Changed {
+ viper.Set("DisableRobotsTXT", DisableRobotsTXT)
+ }
if cmdV.Flags().Lookup("pluralizeListTitles").Changed {
viper.Set("PluralizeListTitles", PluralizeListTitles)
}
diff --git a/docs/content/commands/hugo.md b/docs/content/commands/hugo.md
index d36cb0bab..190f67a14 100644
--- a/docs/content/commands/hugo.md
+++ b/docs/content/commands/hugo.md
@@ -34,6 +34,7 @@ hugo
-d, --destination="": filesystem path to write files to
--disableRSS[=false]: Do not build RSS files
--disableSitemap[=false]: Do not build Sitemap file
+ --disableRobotsTXT[=false]: Do not build robots TXT file
--editor="": edit new content with this editor, if provided
--ignoreCache[=false]: Ignores the cache directory for reading but still writes to it
--log[=false]: Enable Logging
diff --git a/docs/content/extras/robots-txt.md b/docs/content/extras/robots-txt.md
new file mode 100644
index 000000000..336999d1c
--- /dev/null
+++ b/docs/content/extras/robots-txt.md
@@ -0,0 +1,34 @@
+---
+date: 2013-07-09
+menu:
+ main:
+ parent: extras
+next: /community/mailing-list
+prev: /extras/urls
+title: Table of Contents
+weight: 120
+---
+
+Hugo can generated customized [robots.txt](http://www.robotstxt.org/) in the
+[same way than any other template]({{< ref "templates/go-templates.md" >}}).
+
+By default it generates a robots.txt which allows everything, it looks exactly
+
+ User-agent: *
+
+To disable it just set `disableRobotsTXT` option to false in the [command line]({{< ref "commands/hugo.md" >}}) or [configuration file]({{< ref "overview/configuration.md" >}}).
+
+Hugo will use the template `robots.txt` following the list starting with the one with more priority
+
+* /layouts/robots.txt
+* /themes/`THEME`/layout/robots.txt
+
+An example of a robots.txt layout is:
+
+ User-agent: *
+
+ {{range .Data.Pages}}
+ Disallow: {{.RelPermalink}}{{end}}
+
+
+This template disallows and all the pages of the site creating one `Disallow` entry for each one.
diff --git a/docs/content/extras/urls.md b/docs/content/extras/urls.md
index 497caa0fe..fb53b8f13 100644
--- a/docs/content/extras/urls.md
+++ b/docs/content/extras/urls.md
@@ -5,7 +5,7 @@ date: 2014-01-03
menu:
main:
parent: extras
-next: /community/mailing-list
+next: /extras/robots-txt
notoc: true
prev: /extras/toc
title: URLs
diff --git a/docs/content/overview/configuration.md b/docs/content/overview/configuration.md
index 89a2642c5..d5db7520e 100644
--- a/docs/content/overview/configuration.md
+++ b/docs/content/overview/configuration.md
@@ -96,6 +96,8 @@ Following is a list of Hugo-defined variables that you can configure and their c
disableRSS: false
# Do not build Sitemap file
disableSitemap: false
+ # Do not build robots.txt file
+ disableRobotsTXT: false
# edit new content with this editor, if provided
editor: ""
footnoteAnchorPrefix: ""
diff --git a/hugolib/robotstxt_test.go b/hugolib/robotstxt_test.go
new file mode 100644
index 000000000..c964c0231
--- /dev/null
+++ b/hugolib/robotstxt_test.go
@@ -0,0 +1,67 @@
+package hugolib
+
+import (
+ "bytes"
+ "testing"
+
+ "github.com/spf13/afero"
+ "github.com/spf13/hugo/helpers"
+ "github.com/spf13/hugo/hugofs"
+ "github.com/spf13/hugo/source"
+ "github.com/spf13/viper"
+)
+
+const ROBOTSTXT_TEMPLATE = `User-agent: Googlebot
+ {{ range .Data.Pages }}
+ Disallow: {{.RelPermalink}}
+ {{ end }}
+`
+
+func TestRobotsTXTOutput(t *testing.T) {
+ viper.Reset()
+ defer viper.Reset()
+
+ hugofs.DestinationFS = new(afero.MemMapFs)
+
+ viper.Set("baseurl", "http://auth/bub/")
+
+ s := &Site{
+ Source: &source.InMemorySource{ByteSource: WEIGHTED_SOURCES},
+ }
+
+ s.initializeSiteInfo()
+
+ s.prepTemplates()
+ s.addTemplate("robots.txt", ROBOTSTXT_TEMPLATE)
+
+ if err := s.CreatePages(); err != nil {
+ t.Fatalf("Unable to create pages: %s", err)
+ }
+
+ if err := s.BuildSiteMeta(); err != nil {
+ t.Fatalf("Unable to build site metadata: %s", err)
+ }
+
+ if err := s.RenderHomePage(); err != nil {
+ t.Fatalf("Unable to RenderHomePage: %s", err)
+ }
+
+ if err := s.RenderSitemap(); err != nil {
+ t.Fatalf("Unable to RenderSitemap: %s", err)
+ }
+
+ if err := s.RenderRobotsTXT(); err != nil {
+ t.Fatalf("Unable to RenderRobotsTXT :%s", err)
+ }
+
+ robotsFile, err := hugofs.DestinationFS.Open("robots.txt")
+
+ if err != nil {
+ t.Fatalf("Unable to locate: robots.txt")
+ }
+
+ robots := helpers.ReaderToBytes(robotsFile)
+ if !bytes.HasPrefix(robots, []byte("User-agent: Googlebot")) {
+ t.Errorf("Robots file should start with 'User-agentL Googlebot'. %s", robots)
+ }
+}
diff --git a/hugolib/site.go b/hugolib/site.go
index 648ba6454..957e8ed4c 100644
--- a/hugolib/site.go
+++ b/hugolib/site.go
@@ -583,6 +583,12 @@ func (s *Site) Render() (err error) {
return
}
s.timerStep("render and write Sitemap")
+
+ if err = s.RenderRobotsTXT(); err != nil {
+ return
+ }
+ s.timerStep("render and write robots.txt")
+
return
}
@@ -1561,6 +1567,26 @@ func (s *Site) RenderSitemap() error {
return nil
}
+func (s *Site) RenderRobotsTXT() error {
+ if viper.GetBool("DisableRobotsTXT") {
+ return nil
+ }
+
+ n := s.NewNode()
+ n.Data["Pages"] = s.Pages
+
+ rLayouts := []string{"robots.txt", "_default/robots.txt", "_internal/_default/robots.txt"}
+ outBuffer := bp.GetBuffer()
+ defer bp.PutBuffer(outBuffer)
+ err := s.render("robots", n, outBuffer, s.appendThemeTemplates(rLayouts)...)
+
+ if err == nil {
+ err = s.WriteDestFile("robots.txt", outBuffer)
+ }
+
+ return err
+}
+
func (s *Site) Stats() {
jww.FEEDBACK.Println(s.draftStats())
jww.FEEDBACK.Println(s.futureStats())
diff --git a/hugolib/sitemap_test.go b/hugolib/sitemap_test.go
index 7bc16b482..13bc92a18 100644
--- a/hugolib/sitemap_test.go
+++ b/hugolib/sitemap_test.go
@@ -68,6 +68,10 @@ func TestSitemapOutput(t *testing.T) {
t.Fatalf("Unable to RenderSitemap: %s", err)
}
+ if err := s.RenderRobotsTXT(); err != nil {
+ t.Fatalf("Unable to RenderRobotsTXT :%s", err)
+ }
+
sitemapFile, err := hugofs.DestinationFS.Open("sitemap.xml")
if err != nil {
diff --git a/tpl/template_embedded.go b/tpl/template_embedded.go
index a2fde4b10..14364f6db 100644
--- a/tpl/template_embedded.go
+++ b/tpl/template_embedded.go
@@ -242,4 +242,6 @@ ga('send', 'pageview');
</script>
<script async src='//www.google-analytics.com/analytics.js'></script>
{{ end }}`)
+
+ t.AddInternalTemplate("_default", "robots.txt", "User-agent: *")
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-22 04:43:31 +0000