:sparkles: Implement Page bundling and image handling

This commit is not the smallest in Hugo's history.

Some hightlights include:

* Page bundles (for complete articles, keeping images and content together etc.).
* Bundled images can be processed in as many versions/sizes as you need with the three methods `Resize`, `Fill` and `Fit`.
* Processed images are cached inside `resources/_gen/images` (default) in your project.
* Symbolic links (both files and dirs) are now allowed anywhere inside /content
* A new table based build summary
* The "Total in nn ms" now reports the total including the handling of the files inside /static. So if it now reports more than you're used to, it is just **more real** and probably faster than before (see below).

A site building  benchmark run compared to `v0.31.1` shows that this should be slightly faster and use less memory:

```bash
▶ ./benchSite.sh "TOML,num_langs=.*,num_root_sections=5,num_pages=(500|1000),tags_per_page=5,shortcodes,render"

benchmark                                                                                                         old ns/op     new ns/op     delta
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      101785785     78067944      -23.30%
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     185481057     149159919     -19.58%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      103149918     85679409      -16.94%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     203515478     169208775     -16.86%

benchmark                                                                                                         old allocs     new allocs     delta
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      532464         391539         -26.47%
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     1056549        772702         -26.87%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      555974         406630         -26.86%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     1086545        789922         -27.30%

benchmark                                                                                                         old bytes     new bytes     delta
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      53243246      43598155      -18.12%
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     105811617     86087116      -18.64%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      54558852      44545097      -18.35%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     106903858     86978413      -18.64%
```

Fixes #3651
Closes #3158
Fixes #1014
Closes #2021
Fixes #1240
Updates #3757

8 files changed, 202 insertions, 85 deletions

diff --git a/helpers/content.go b/helpers/content.go
index ca93d7d9e..f2cfc9b0f 100644
--- a/helpers/content.go
+++ b/helpers/content.go
@@ -47,6 +47,10 @@ type ContentSpec struct {
 	// SummaryLength is the length of the summary that Hugo extracts from a content.
 	summaryLength int
 
+	BuildFuture  bool
+	BuildExpired bool
+	BuildDrafts  bool
+
 	Highlight            func(code, lang, optsStr string) (string, error)
 	defatultPygmentsOpts map[string]string
 
@@ -62,6 +66,9 @@ func NewContentSpec(cfg config.Provider) (*ContentSpec, error) {
 		footnoteAnchorPrefix:       cfg.GetString("footnoteAnchorPrefix"),
 		footnoteReturnLinkContents: cfg.GetString("footnoteReturnLinkContents"),
 		summaryLength:              cfg.GetInt("summaryLength"),
+		BuildFuture:                cfg.GetBool("buildFuture"),
+		BuildExpired:               cfg.GetBool("buildExpired"),
+		BuildDrafts:                cfg.GetBool("buildDrafts"),
 
 		cfg: cfg,
 	}
diff --git a/helpers/content_test.go b/helpers/content_test.go
index c10ad881b..2f8884c55 100644
--- a/helpers/content_test.go
+++ b/helpers/content_test.go
@@ -19,9 +19,12 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/spf13/viper"
+
 	"github.com/miekg/mmark"
 	"github.com/russross/blackfriday"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 const tstHTMLContent = "<!DOCTYPE html><html><head><script src=\"http://two/foobar.js\"></script></head><body><nav><ul><li hugo-nav=\"section_0\"></li><li hugo-nav=\"section_1\"></li></ul></nav><article>content <a href=\"http://two/foobar\">foobar</a>. Follow up</article><p>This is some text.<br>And some more.</p></body></html>"
@@ -73,6 +76,25 @@ func TestBytesToHTML(t *testing.T) {
 	assert.Equal(t, template.HTML("dobedobedo"), BytesToHTML([]byte("dobedobedo")))
 }
 
+func TestNewContentSpec(t *testing.T) {
+	cfg := viper.New()
+	assert := require.New(t)
+
+	cfg.Set("summaryLength", 32)
+	cfg.Set("buildFuture", true)
+	cfg.Set("buildExpired", true)
+	cfg.Set("buildDrafts", true)
+
+	spec, err := NewContentSpec(cfg)
+
+	assert.NoError(err)
+	assert.Equal(32, spec.summaryLength)
+	assert.True(spec.BuildFuture)
+	assert.True(spec.BuildExpired)
+	assert.True(spec.BuildDrafts)
+
+}
+
 var benchmarkTruncateString = strings.Repeat("This is a sentence about nothing.", 20)
 
 func BenchmarkTestTruncateWordsToWholeSentence(b *testing.B) {
diff --git a/helpers/general.go b/helpers/general.go
index 4517a3232..4bb4eb584 100644
--- a/helpers/general.go
+++ b/helpers/general.go
@@ -365,8 +365,8 @@ func SliceToLower(s []string) []string {
 	return l
 }
 
-// Md5String takes a string and returns its MD5 hash.
-func Md5String(f string) string {
+// MD5String takes a string and returns its MD5 hash.
+func MD5String(f string) string {
 	h := md5.New()
 	h.Write([]byte(f))
 	return hex.EncodeToString(h.Sum([]byte{}))
diff --git a/helpers/language.go b/helpers/language.go
index d9f3f69ed..3cb388abb 100644
--- a/helpers/language.go
+++ b/helpers/language.go
@@ -53,7 +53,16 @@ func (l *Language) String() string {
 
 // NewLanguage creates a new language.
 func NewLanguage(lang string, cfg config.Provider) *Language {
-	return &Language{Lang: lang, Cfg: cfg, params: make(map[string]interface{})}
+	params := make(map[string]interface{})
+	// Merge with global config.
+	globalParams := cfg.GetStringMap("params")
+	for k, v := range globalParams {
+		if _, ok := params[k]; !ok {
+			params[k] = v
+		}
+	}
+	l := &Language{Lang: lang, Cfg: cfg, params: params}
+	return l
 }
 
 // NewDefaultLanguage creates the default language for a config.Provider.
@@ -88,17 +97,6 @@ func (l Languages) Swap(i, j int)      { l[i], l[j] = l[j], l[i] }
 
 // Params retunrs language-specific params merged with the global params.
 func (l *Language) Params() map[string]interface{} {
-	l.paramsInit.Do(func() {
-		// Merge with global config.
-		// TODO(bep) consider making this part of a constructor func.
-
-		globalParams := l.Cfg.GetStringMap("params")
-		for k, v := range globalParams {
-			if _, ok := l.params[k]; !ok {
-				l.params[k] = v
-			}
-		}
-	})
 	return l.params
 }
 
diff --git a/helpers/path.go b/helpers/path.go
index 57f02da68..44d53d018 100644
--- a/helpers/path.go
+++ b/helpers/path.go
@@ -277,6 +277,12 @@ func Ext(in string) string {
 	return ext
 }
 
+// FileAndExt takes a path and returns the file and extension separated,
+// the extension including the delmiter, i.e. ".md".
+func FileAndExt(in string) (string, string) {
+	return fileAndExt(in, fpb)
+}
+
 // Filename takes a path, strips out the extension,
 // and returns the name of the file.
 func Filename(in string) (name string) {
@@ -348,40 +354,6 @@ func GetRelativePath(path, base string) (final string, err error) {
 	return name, nil
 }
 
-// GuessSection returns the section given a source path.
-// A section is the part between the root slash and the second slash
-// or before the first slash.
-func GuessSection(in string) string {
-	parts := strings.Split(in, FilePathSeparator)
-	// This will include an empty entry before and after paths with leading and trailing slashes
-	// eg... /sect/one/ -> ["", "sect", "one", ""]
-
-	// Needs to have at least a value and a slash
-	if len(parts) < 2 {
-		return ""
-	}
-
-	// If it doesn't have a leading slash and value and file or trailing slash, then return ""
-	if parts[0] == "" && len(parts) < 3 {
-		return ""
-	}
-
-	// strip leading slash
-	if parts[0] == "" {
-		parts = parts[1:]
-	}
-
-	// if first directory is "content", return second directory
-	if parts[0] == "content" {
-		if len(parts) > 2 {
-			return parts[1]
-		}
-		return ""
-	}
-
-	return parts[0]
-}
-
 // PathPrep prepares the path using the uglify setting to create paths on
 // either the form /section/name/index.html or /section/name.html.
 func PathPrep(ugly bool, in string) string {
@@ -504,7 +476,7 @@ func SymbolicWalk(fs afero.Fs, root string, walker filepath.WalkFunc) error {
 }
 
 func getRealFileInfo(fs afero.Fs, path string) (os.FileInfo, string, error) {
-	fileInfo, err := lstatIfOs(fs, path)
+	fileInfo, err := LstatIfOs(fs, path)
 	realPath := path
 
 	if err != nil {
@@ -516,7 +488,7 @@ func getRealFileInfo(fs afero.Fs, path string) (os.FileInfo, string, error) {
 		if err != nil {
 			return nil, "", fmt.Errorf("Cannot read symbolic link '%s', error was: %s", path, err)
 		}
-		fileInfo, err = lstatIfOs(fs, link)
+		fileInfo, err = LstatIfOs(fs, link)
 		if err != nil {
 			return nil, "", fmt.Errorf("Cannot stat '%s', error was: %s", link, err)
 		}
@@ -539,7 +511,7 @@ func GetRealPath(fs afero.Fs, path string) (string, error) {
 
 // Code copied from Afero's path.go
 // if the filesystem is OsFs use Lstat, else use fs.Stat
-func lstatIfOs(fs afero.Fs, path string) (info os.FileInfo, err error) {
+func LstatIfOs(fs afero.Fs, path string) (info os.FileInfo, err error) {
 	_, ok := fs.(*afero.OsFs)
 	if ok {
 		info, err = os.Lstat(path)
diff --git a/helpers/path_test.go b/helpers/path_test.go
index 8d895d762..bb5125034 100644
--- a/helpers/path_test.go
+++ b/helpers/path_test.go
@@ -638,40 +638,6 @@ func TestFileAndExt(t *testing.T) {
 
 }
 
-func TestGuessSection(t *testing.T) {
-	type test struct {
-		input, expected string
-	}
-
-	data := []test{
-		{"/", ""},
-		{"", ""},
-		{"/content", ""},
-		{"content/", ""},
-		{"/content/", ""}, // /content/ is a special case. It will never be the section
-		{"/blog", ""},
-		{"/blog/", "blog"},
-		{"blog", ""},
-		{"content/blog", ""},
-		{"/content/blog/", "blog"},
-		{"/content/blog", ""}, // Lack of trailing slash indicates 'blog' is not a directory.
-		{"content/blog/", "blog"},
-		{"/contents/myblog/", "contents"},
-		{"/contents/yourblog", "contents"},
-		{"/contents/ourblog/", "contents"},
-		{"/content/myblog/", "myblog"},
-		{"/content/yourblog", ""},
-		{"/content/ourblog/", "ourblog"},
-	}
-
-	for i, d := range data {
-		expected := GuessSection(filepath.FromSlash(d.input))
-		if d.expected != expected {
-			t.Errorf("Test %d failed. Expected %q got %q", i, d.expected, expected)
-		}
-	}
-}
-
 func TestPathPrep(t *testing.T) {
 
 }
diff --git a/helpers/pathspec.go b/helpers/pathspec.go
index 164d242a0..d5a2c92e8 100644
--- a/helpers/pathspec.go
+++ b/helpers/pathspec.go
@@ -15,6 +15,7 @@ package helpers
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/hugofs"
@@ -31,6 +32,7 @@ type PathSpec struct {
 	canonifyURLs       bool
 
 	language *Language
+	//StatsCounter *siteSta
 
 	// pagination path handling
 	paginatePath string
@@ -38,10 +40,12 @@ type PathSpec struct {
 	theme string
 
 	// Directories
+	contentDir string
 	themesDir  string
 	layoutDir  string
 	workingDir string
 	staticDirs []string
+	PublishDir string
 
 	// The PathSpec looks up its config settings in both the current language
 	// and then in the global Viper config.
@@ -52,6 +56,8 @@ type PathSpec struct {
 	defaultContentLanguage         string
 	multilingual                   bool
 
+	ProcessingStats *ProcessingStats
+
 	// The file systems to use
 	Fs *hugofs.Fs
 
@@ -79,6 +85,11 @@ func NewPathSpec(fs *hugofs.Fs, cfg config.Provider) (*PathSpec, error) {
 		staticDirs = append(staticDirs, getStringOrStringSlice(cfg, "staticDir", i)...)
 	}
 
+	var lang string
+	if l, ok := cfg.(*Language); ok {
+		lang = l.Lang
+	}
+
 	ps := &PathSpec{
 		Fs:                             fs,
 		Cfg:                            cfg,
@@ -91,13 +102,23 @@ func NewPathSpec(fs *hugofs.Fs, cfg config.Provider) (*PathSpec, error) {
 		defaultContentLanguage:         cfg.GetString("defaultContentLanguage"),
 		paginatePath:                   cfg.GetString("paginatePath"),
 		BaseURL:                        baseURL,
+		contentDir:                     cfg.GetString("contentDir"),
 		themesDir:                      cfg.GetString("themesDir"),
 		layoutDir:                      cfg.GetString("layoutDir"),
 		workingDir:                     cfg.GetString("workingDir"),
 		staticDirs:                     staticDirs,
 		theme:                          cfg.GetString("theme"),
+		ProcessingStats:                NewProcessingStats(lang),
 	}
 
+	publishDir := ps.AbsPathify(cfg.GetString("publishDir")) + FilePathSeparator
+	// If root, remove the second '/'
+	if publishDir == "//" {
+		publishDir = FilePathSeparator
+	}
+
+	ps.PublishDir = publishDir
+
 	if language, ok := cfg.(*Language); ok {
 		ps.language = language
 	}
@@ -129,6 +150,11 @@ func (p *PathSpec) PaginatePath() string {
 	return p.paginatePath
 }
 
+// ContentDir returns the configured workingDir.
+func (p *PathSpec) ContentDir() string {
+	return p.contentDir
+}
+
 // WorkingDir returns the configured workingDir.
 func (p *PathSpec) WorkingDir() string {
 	return p.workingDir
@@ -153,3 +179,13 @@ func (p *PathSpec) Theme() string {
 func (p *PathSpec) ThemesDir() string {
 	return p.themesDir
 }
+
+// PermalinkForBaseURL creates a permalink from the given link and baseURL.
+func (p *PathSpec) PermalinkForBaseURL(link, baseURL string) string {
+	link = strings.TrimPrefix(link, "/")
+	if !strings.HasSuffix(baseURL, "/") {
+		baseURL += "/"
+	}
+	return baseURL + link
+
+}
diff --git a/helpers/processing_stats.go b/helpers/processing_stats.go
new file mode 100644
index 000000000..36dfcda69
--- /dev/null
+++ b/helpers/processing_stats.go
@@ -0,0 +1,116 @@
+// Copyright 2017 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package helpers
+
+import (
+	"io"
+	"strconv"
+	"sync/atomic"
+
+	"github.com/olekukonko/tablewriter"
+)
+
+type ProcessingStats struct {
+	Name string
+
+	Pages           uint64
+	PaginatorPages  uint64
+	Static          uint64
+	ProcessedImages uint64
+	Files           uint64
+	Aliases         uint64
+	Sitemaps        uint64
+	Cleaned         uint64
+}
+
+type processingStatsTitleVal struct {
+	name string
+	val  uint64
+}
+
+func (s *ProcessingStats) toVals() []processingStatsTitleVal {
+	return []processingStatsTitleVal{
+		processingStatsTitleVal{"Pages", s.Pages},
+		processingStatsTitleVal{"Paginator pages", s.PaginatorPages},
+		processingStatsTitleVal{"Non-page files", s.Files},
+		processingStatsTitleVal{"Static files", s.Static},
+		processingStatsTitleVal{"Processed images", s.ProcessedImages},
+		processingStatsTitleVal{"Aliases", s.Aliases},
+		processingStatsTitleVal{"Sitemaps", s.Sitemaps},
+		processingStatsTitleVal{"Cleaned", s.Cleaned},
+	}
+}
+
+func NewProcessingStats(name string) *ProcessingStats {
+	return &ProcessingStats{Name: name}
+}
+
+func (s *ProcessingStats) Incr(counter *uint64) {
+	atomic.AddUint64(counter, 1)
+}
+
+func (s *ProcessingStats) Add(counter *uint64, amount int) {
+	atomic.AddUint64(counter, uint64(amount))
+}
+
+func (s *ProcessingStats) Table(w io.Writer) {
+	titleVals := s.toVals()
+	data := make([][]string, len(titleVals))
+	for i, tv := range titleVals {
+		data[i] = []string{tv.name, strconv.Itoa(int(tv.val))}
+	}
+
+	table := tablewriter.NewWriter(w)
+
+	table.AppendBulk(data)
+	table.SetHeader([]string{"", s.Name})
+	table.SetBorder(false)
+	table.Render()
+
+}
+
+func ProcessingStatsTable(w io.Writer, stats ...*ProcessingStats) {
+	names := make([]string, len(stats)+1)
+
+	var data [][]string
+
+	for i := 0; i < len(stats); i++ {
+		stat := stats[i]
+		names[i+1] = stat.Name
+
+		titleVals := stat.toVals()
+
+		if i == 0 {
+			data = make([][]string, len(titleVals))
+		}
+
+		for j, tv := range titleVals {
+			if i == 0 {
+				data[j] = []string{tv.name, strconv.Itoa(int(tv.val))}
+			} else {
+				data[j] = append(data[j], strconv.Itoa(int(tv.val)))
+			}
+
+		}
+
+	}
+
+	table := tablewriter.NewWriter(w)
+
+	table.AppendBulk(data)
+	table.SetHeader(names)
+	table.SetBorder(false)
+	table.Render()
+
+}