diff options
author | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2018-03-21 17:21:46 +0100 |
---|---|---|
committer | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2018-04-02 08:06:21 +0200 |
commit | eb42774e587816b1fbcafbcea59ed65df703882a (patch) | |
tree | fdb62cf17355b47fa485941f3c3fffd604896daa /helpers | |
parent | f27977809ce5d5dce4db41db6323a4ad1b095985 (diff) |
Add support for a content dir set per language
A sample config:
```toml
defaultContentLanguage = "en"
defaultContentLanguageInSubdir = true
[Languages]
[Languages.en]
weight = 10
title = "In English"
languageName = "English"
contentDir = "content/english"
[Languages.nn]
weight = 20
title = "På Norsk"
languageName = "Norsk"
contentDir = "content/norwegian"
```
The value of `contentDir` can be any valid path, even absolute path references. The only restriction is that the content dirs cannot overlap.
The content files will be assigned a language by
1. The placement: `content/norwegian/post/my-post.md` will be read as Norwegian content.
2. The filename: `content/english/post/my-post.nn.md` will be read as Norwegian even if it lives in the English content folder.
The content directories will be merged into a big virtual filesystem with one simple rule: The most specific language file will win.
This means that if both `content/norwegian/post/my-post.md` and `content/english/post/my-post.nn.md` exists, they will be considered duplicates and the version inside `content/norwegian` will win.
Note that translations will be automatically assigned by Hugo by the content file's relative placement, so `content/norwegian/post/my-post.md` will be a translation of `content/english/post/my-post.md`.
If this does not work for you, you can connect the translations together by setting a `translationKey` in the content files' front matter.
Fixes #4523
Fixes #4552
Fixes #4553
Diffstat (limited to 'helpers')
-rw-r--r-- | helpers/language.go | 16 | ||||
-rw-r--r-- | helpers/language_test.go | 6 | ||||
-rw-r--r-- | helpers/path.go | 24 | ||||
-rw-r--r-- | helpers/path_test.go | 7 | ||||
-rw-r--r-- | helpers/pathspec.go | 192 | ||||
-rw-r--r-- | helpers/pathspec_test.go | 1 | ||||
-rw-r--r-- | helpers/testhelpers_test.go | 1 | ||||
-rw-r--r-- | helpers/url_test.go | 4 |
8 files changed, 219 insertions, 32 deletions
diff --git a/helpers/language.go b/helpers/language.go index 49a25ccf7..731e9b088 100644 --- a/helpers/language.go +++ b/helpers/language.go @@ -41,6 +41,14 @@ type Language struct { Title string Weight int + Disabled bool + + // If set per language, this tells Hugo that all content files without any + // language indicator (e.g. my-page.en.md) is in this language. + // This is usually a path relative to the working dir, but it can be an + // absolute directory referenece. It is what we get. + ContentDir string + Cfg config.Provider // These are params declared in the [params] section of the language merged with the @@ -66,7 +74,13 @@ func NewLanguage(lang string, cfg config.Provider) *Language { params[k] = v } ToLowerMap(params) - l := &Language{Lang: lang, Cfg: cfg, params: params, settings: make(map[string]interface{})} + + defaultContentDir := cfg.GetString("contentDir") + if defaultContentDir == "" { + panic("contentDir not set") + } + + l := &Language{Lang: lang, ContentDir: defaultContentDir, Cfg: cfg, params: params, settings: make(map[string]interface{})} return l } diff --git a/helpers/language_test.go b/helpers/language_test.go index 68ee3506d..4c4670321 100644 --- a/helpers/language_test.go +++ b/helpers/language_test.go @@ -22,11 +22,12 @@ import ( func TestGetGlobalOnlySetting(t *testing.T) { v := viper.New() + v.Set("defaultContentLanguageInSubdir", true) + v.Set("contentDir", "content") + v.Set("paginatePath", "page") lang := NewDefaultLanguage(v) lang.Set("defaultContentLanguageInSubdir", false) lang.Set("paginatePath", "side") - v.Set("defaultContentLanguageInSubdir", true) - v.Set("paginatePath", "page") require.True(t, lang.GetBool("defaultContentLanguageInSubdir")) require.Equal(t, "side", lang.GetString("paginatePath")) @@ -37,6 +38,7 @@ func TestLanguageParams(t *testing.T) { v := viper.New() v.Set("p1", "p1cfg") + v.Set("contentDir", "content") lang := NewDefaultLanguage(v) lang.SetParam("p1", "p1p") diff --git a/helpers/path.go b/helpers/path.go index 0a8544357..7ac9208bf 100644 --- a/helpers/path.go +++ b/helpers/path.go @@ -33,7 +33,7 @@ var ( ErrThemeUndefined = errors.New("no theme set") // ErrWalkRootTooShort is returned when the root specified for a file walk is shorter than 4 characters. - ErrWalkRootTooShort = errors.New("Path too short. Stop walking.") + ErrPathTooShort = errors.New("file path is too short") ) // filepathPathBridge is a bridge for common functionality in filepath vs path @@ -446,7 +446,7 @@ func SymbolicWalk(fs afero.Fs, root string, walker filepath.WalkFunc) error { // Sanity check if len(root) < 4 { - return ErrWalkRootTooShort + return ErrPathTooShort } // Handle the root first @@ -481,7 +481,7 @@ func SymbolicWalk(fs afero.Fs, root string, walker filepath.WalkFunc) error { } func getRealFileInfo(fs afero.Fs, path string) (os.FileInfo, string, error) { - fileInfo, err := LstatIfOs(fs, path) + fileInfo, err := LstatIfPossible(fs, path) realPath := path if err != nil { @@ -493,7 +493,7 @@ func getRealFileInfo(fs afero.Fs, path string) (os.FileInfo, string, error) { if err != nil { return nil, "", fmt.Errorf("Cannot read symbolic link '%s', error was: %s", path, err) } - fileInfo, err = LstatIfOs(fs, link) + fileInfo, err = LstatIfPossible(fs, link) if err != nil { return nil, "", fmt.Errorf("Cannot stat '%s', error was: %s", link, err) } @@ -514,16 +514,14 @@ func GetRealPath(fs afero.Fs, path string) (string, error) { return realPath, nil } -// Code copied from Afero's path.go -// if the filesystem is OsFs use Lstat, else use fs.Stat -func LstatIfOs(fs afero.Fs, path string) (info os.FileInfo, err error) { - _, ok := fs.(*afero.OsFs) - if ok { - info, err = os.Lstat(path) - } else { - info, err = fs.Stat(path) +// LstatIfPossible can be used to call Lstat if possible, else Stat. +func LstatIfPossible(fs afero.Fs, path string) (os.FileInfo, error) { + if lstater, ok := fs.(afero.Lstater); ok { + fi, _, err := lstater.LstatIfPossible(path) + return fi, err } - return + + return fs.Stat(path) } // SafeWriteToDisk is the same as WriteToDisk diff --git a/helpers/path_test.go b/helpers/path_test.go index d2c577dae..c2ac19675 100644 --- a/helpers/path_test.go +++ b/helpers/path_test.go @@ -57,8 +57,10 @@ func TestMakePath(t *testing.T) { for _, test := range tests { v := viper.New() - l := NewDefaultLanguage(v) + v.Set("contentDir", "content") v.Set("removePathAccents", test.removeAccents) + + l := NewDefaultLanguage(v) p, err := NewPathSpec(hugofs.NewMem(v), l) require.NoError(t, err) @@ -71,6 +73,8 @@ func TestMakePath(t *testing.T) { func TestMakePathSanitized(t *testing.T) { v := viper.New() + v.Set("contentDir", "content") + l := NewDefaultLanguage(v) p, _ := NewPathSpec(hugofs.NewMem(v), l) @@ -98,6 +102,7 @@ func TestMakePathSanitizedDisablePathToLower(t *testing.T) { v := viper.New() v.Set("disablePathToLower", true) + v.Set("contentDir", "content") l := NewDefaultLanguage(v) p, _ := NewPathSpec(hugofs.NewMem(v), l) diff --git a/helpers/pathspec.go b/helpers/pathspec.go index d35538b85..b18408590 100644 --- a/helpers/pathspec.go +++ b/helpers/pathspec.go @@ -17,6 +17,9 @@ import ( "fmt" "strings" + "github.com/spf13/afero" + + "github.com/gohugoio/hugo/common/types" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/hugofs" "github.com/spf13/cast" @@ -44,11 +47,13 @@ type PathSpec struct { theme string // Directories - contentDir string - themesDir string - layoutDir string - workingDir string - staticDirs []string + contentDir string + themesDir string + layoutDir string + workingDir string + staticDirs []string + absContentDirs []types.KeyValueStr + PublishDir string // The PathSpec looks up its config settings in both the current language @@ -65,6 +70,9 @@ type PathSpec struct { // The file systems to use Fs *hugofs.Fs + // The fine grained filesystems in play (resources, content etc.). + BaseFs *hugofs.BaseFs + // The config provider to use Cfg config.Provider } @@ -105,8 +113,65 @@ func NewPathSpec(fs *hugofs.Fs, cfg config.Provider) (*PathSpec, error) { languages = l } + defaultContentLanguage := cfg.GetString("defaultContentLanguage") + + // We will eventually pull out this badly placed path logic. + contentDir := cfg.GetString("contentDir") + workingDir := cfg.GetString("workingDir") + resourceDir := cfg.GetString("resourceDir") + publishDir := cfg.GetString("publishDir") + + if len(languages) == 0 { + // We have some old tests that does not test the entire chain, hence + // they have no languages. So create one so we get the proper filesystem. + languages = Languages{&Language{Lang: "en", ContentDir: contentDir}} + } + + absPuslishDir := AbsPathify(workingDir, publishDir) + if !strings.HasSuffix(absPuslishDir, FilePathSeparator) { + absPuslishDir += FilePathSeparator + } + // If root, remove the second '/' + if absPuslishDir == "//" { + absPuslishDir = FilePathSeparator + } + absResourcesDir := AbsPathify(workingDir, resourceDir) + if !strings.HasSuffix(absResourcesDir, FilePathSeparator) { + absResourcesDir += FilePathSeparator + } + if absResourcesDir == "//" { + absResourcesDir = FilePathSeparator + } + + contentFs, absContentDirs, err := createContentFs(fs.Source, workingDir, defaultContentLanguage, languages) + if err != nil { + return nil, err + } + + // Make sure we don't have any overlapping content dirs. That will never work. + for i, d1 := range absContentDirs { + for j, d2 := range absContentDirs { + if i == j { + continue + } + if strings.HasPrefix(d1.Value, d2.Value) || strings.HasPrefix(d2.Value, d1.Value) { + return nil, fmt.Errorf("found overlapping content dirs (%q and %q)", d1, d2) + } + } + } + + resourcesFs := afero.NewBasePathFs(fs.Source, absResourcesDir) + publishFs := afero.NewBasePathFs(fs.Destination, absPuslishDir) + + baseFs := &hugofs.BaseFs{ + ContentFs: contentFs, + ResourcesFs: resourcesFs, + PublishFs: publishFs, + } + ps := &PathSpec{ Fs: fs, + BaseFs: baseFs, Cfg: cfg, disablePathToLower: cfg.GetBool("disablePathToLower"), removePathAccents: cfg.GetBool("removePathAccents"), @@ -116,14 +181,15 @@ func NewPathSpec(fs *hugofs.Fs, cfg config.Provider) (*PathSpec, error) { Language: language, Languages: languages, defaultContentLanguageInSubdir: cfg.GetBool("defaultContentLanguageInSubdir"), - defaultContentLanguage: cfg.GetString("defaultContentLanguage"), + defaultContentLanguage: defaultContentLanguage, paginatePath: cfg.GetString("paginatePath"), BaseURL: baseURL, - contentDir: cfg.GetString("contentDir"), + contentDir: contentDir, themesDir: cfg.GetString("themesDir"), layoutDir: cfg.GetString("layoutDir"), - workingDir: cfg.GetString("workingDir"), + workingDir: workingDir, staticDirs: staticDirs, + absContentDirs: absContentDirs, theme: cfg.GetString("theme"), ProcessingStats: NewProcessingStats(lang), } @@ -135,13 +201,8 @@ func NewPathSpec(fs *hugofs.Fs, cfg config.Provider) (*PathSpec, error) { } } - publishDir := ps.AbsPathify(cfg.GetString("publishDir")) + FilePathSeparator - // If root, remove the second '/' - if publishDir == "//" { - publishDir = FilePathSeparator - } - - ps.PublishDir = publishDir + // TODO(bep) remove this, eventually + ps.PublishDir = absPuslishDir return ps, nil } @@ -165,6 +226,107 @@ func getStringOrStringSlice(cfg config.Provider, key string, id int) []string { return out } +func createContentFs(fs afero.Fs, + workingDir, + defaultContentLanguage string, + languages Languages) (afero.Fs, []types.KeyValueStr, error) { + + var contentLanguages Languages + var contentDirSeen = make(map[string]bool) + languageSet := make(map[string]bool) + + // The default content language needs to be first. + for _, language := range languages { + if language.Lang == defaultContentLanguage { + contentLanguages = append(contentLanguages, language) + contentDirSeen[language.ContentDir] = true + } + languageSet[language.Lang] = true + } + + for _, language := range languages { + if contentDirSeen[language.ContentDir] { + continue + } + if language.ContentDir == "" { + language.ContentDir = defaultContentLanguage + } + contentDirSeen[language.ContentDir] = true + contentLanguages = append(contentLanguages, language) + + } + + var absContentDirs []types.KeyValueStr + + fs, err := createContentOverlayFs(fs, workingDir, contentLanguages, languageSet, &absContentDirs) + return fs, absContentDirs, err + +} + +func createContentOverlayFs(source afero.Fs, + workingDir string, + languages Languages, + languageSet map[string]bool, + absContentDirs *[]types.KeyValueStr) (afero.Fs, error) { + if len(languages) == 0 { + return source, nil + } + + language := languages[0] + + contentDir := language.ContentDir + if contentDir == "" { + panic("missing contentDir") + } + + absContentDir := AbsPathify(workingDir, language.ContentDir) + if !strings.HasSuffix(absContentDir, FilePathSeparator) { + absContentDir += FilePathSeparator + } + + // If root, remove the second '/' + if absContentDir == "//" { + absContentDir = FilePathSeparator + } + + if len(absContentDir) < 6 { + return nil, fmt.Errorf("invalid content dir %q: %s", absContentDir, ErrPathTooShort) + } + + *absContentDirs = append(*absContentDirs, types.KeyValueStr{Key: language.Lang, Value: absContentDir}) + + overlay := hugofs.NewLanguageFs(language.Lang, languageSet, afero.NewBasePathFs(source, absContentDir)) + if len(languages) == 1 { + return overlay, nil + } + + base, err := createContentOverlayFs(source, workingDir, languages[1:], languageSet, absContentDirs) + if err != nil { + return nil, err + } + + return hugofs.NewLanguageCompositeFs(base, overlay), nil + +} + +// RelContentDir tries to create a path relative to the content root from +// the given filename. The return value is the path and language code. +func (p *PathSpec) RelContentDir(filename string) (string, string) { + for _, dir := range p.absContentDirs { + if strings.HasPrefix(filename, dir.Value) { + rel := strings.TrimPrefix(filename, dir.Value) + return strings.TrimPrefix(rel, FilePathSeparator), dir.Key + } + } + // Either not a content dir or already relative. + return filename, "" +} + +// ContentDirs returns all the content dirs (absolute paths). +func (p *PathSpec) ContentDirs() []types.KeyValueStr { + return p.absContentDirs +} + // PaginatePath returns the configured root path used for paginator pages. func (p *PathSpec) PaginatePath() string { return p.paginatePath diff --git a/helpers/pathspec_test.go b/helpers/pathspec_test.go index e10ccc639..dc2079e06 100644 --- a/helpers/pathspec_test.go +++ b/helpers/pathspec_test.go @@ -24,6 +24,7 @@ import ( func TestNewPathSpecFromConfig(t *testing.T) { v := viper.New() + v.Set("contentDir", "content") l := NewLanguage("no", v) v.Set("disablePathToLower", true) v.Set("removePathAccents", true) diff --git a/helpers/testhelpers_test.go b/helpers/testhelpers_test.go index 518a5bc23..215ae9188 100644 --- a/helpers/testhelpers_test.go +++ b/helpers/testhelpers_test.go @@ -25,6 +25,7 @@ func newTestDefaultPathSpec(configKeyValues ...interface{}) *PathSpec { func newTestCfg(fs *hugofs.Fs) *viper.Viper { v := viper.New() + v.Set("contentDir", "content") v.SetFs(fs.Source) diff --git a/helpers/url_test.go b/helpers/url_test.go index 9572547c7..0ca3c8df2 100644 --- a/helpers/url_test.go +++ b/helpers/url_test.go @@ -27,6 +27,7 @@ import ( func TestURLize(t *testing.T) { v := viper.New() + v.Set("contentDir", "content") l := NewDefaultLanguage(v) p, _ := NewPathSpec(hugofs.NewMem(v), l) @@ -88,6 +89,7 @@ func doTestAbsURL(t *testing.T, defaultInSubDir, addLanguage, multilingual bool, for _, test := range tests { v.Set("baseURL", test.baseURL) + v.Set("contentDir", "content") l := NewLanguage(lang, v) p, _ := NewPathSpec(hugofs.NewMem(v), l) @@ -166,6 +168,7 @@ func doTestRelURL(t *testing.T, defaultInSubDir, addLanguage, multilingual bool, for i, test := range tests { v.Set("baseURL", test.baseURL) v.Set("canonifyURLs", test.canonify) + v.Set("contentDir", "content") l := NewLanguage(lang, v) p, _ := NewPathSpec(hugofs.NewMem(v), l) @@ -254,6 +257,7 @@ func TestURLPrep(t *testing.T) { for i, d := range data { v := viper.New() v.Set("uglyURLs", d.ugly) + v.Set("contentDir", "content") l := NewDefaultLanguage(v) p, _ := NewPathSpec(hugofs.NewMem(v), l) |