diff options
author | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2018-03-21 17:21:46 +0100 |
---|---|---|
committer | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2018-04-02 08:06:21 +0200 |
commit | eb42774e587816b1fbcafbcea59ed65df703882a (patch) | |
tree | fdb62cf17355b47fa485941f3c3fffd604896daa /hugofs | |
parent | f27977809ce5d5dce4db41db6323a4ad1b095985 (diff) |
Add support for a content dir set per language
A sample config:
```toml
defaultContentLanguage = "en"
defaultContentLanguageInSubdir = true
[Languages]
[Languages.en]
weight = 10
title = "In English"
languageName = "English"
contentDir = "content/english"
[Languages.nn]
weight = 20
title = "På Norsk"
languageName = "Norsk"
contentDir = "content/norwegian"
```
The value of `contentDir` can be any valid path, even absolute path references. The only restriction is that the content dirs cannot overlap.
The content files will be assigned a language by
1. The placement: `content/norwegian/post/my-post.md` will be read as Norwegian content.
2. The filename: `content/english/post/my-post.nn.md` will be read as Norwegian even if it lives in the English content folder.
The content directories will be merged into a big virtual filesystem with one simple rule: The most specific language file will win.
This means that if both `content/norwegian/post/my-post.md` and `content/english/post/my-post.nn.md` exists, they will be considered duplicates and the version inside `content/norwegian` will win.
Note that translations will be automatically assigned by Hugo by the content file's relative placement, so `content/norwegian/post/my-post.md` will be a translation of `content/english/post/my-post.md`.
If this does not work for you, you can connect the translations together by setting a `translationKey` in the content files' front matter.
Fixes #4523
Fixes #4552
Fixes #4553
Diffstat (limited to 'hugofs')
-rw-r--r-- | hugofs/base_fs.go | 35 | ||||
-rw-r--r-- | hugofs/language_composite_fs.go | 51 | ||||
-rw-r--r-- | hugofs/language_composite_fs_test.go | 106 | ||||
-rw-r--r-- | hugofs/language_fs.go | 328 | ||||
-rw-r--r-- | hugofs/language_fs_test.go | 54 |
5 files changed, 574 insertions, 0 deletions
diff --git a/hugofs/base_fs.go b/hugofs/base_fs.go new file mode 100644 index 000000000..77af66dfe --- /dev/null +++ b/hugofs/base_fs.go @@ -0,0 +1,35 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "github.com/spf13/afero" +) + +// BaseFs contains the core base filesystems used by Hugo. The name "base" is used +// to underline that even if they can be composites, they all have a base path set to a specific +// resource folder, e.g "/my-project/content". So, no absolute filenames needed. +type BaseFs struct { + // The filesystem used to capture content. This can be a composite and + // language aware file system. + ContentFs afero.Fs + + // The filesystem used to store resources (processed images etc.). + // This usually maps to /my-project/resources. + ResourcesFs afero.Fs + + // The filesystem used to publish the rendered site. + // This usually maps to /my-project/public. + PublishFs afero.Fs +} diff --git a/hugofs/language_composite_fs.go b/hugofs/language_composite_fs.go new file mode 100644 index 000000000..2889f8a00 --- /dev/null +++ b/hugofs/language_composite_fs.go @@ -0,0 +1,51 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "github.com/spf13/afero" +) + +var ( + _ afero.Fs = (*languageCompositeFs)(nil) + _ afero.Lstater = (*languageCompositeFs)(nil) +) + +type languageCompositeFs struct { + *afero.CopyOnWriteFs +} + +// NewLanguageCompositeFs creates a composite and language aware filesystem. +// This is a hybrid filesystem. To get a specific file in Open, Stat etc., use the full filename +// to the target filesystem. This information is available in Readdir, Stat etc. via the +// special LanguageFileInfo FileInfo implementation. +func NewLanguageCompositeFs(base afero.Fs, overlay *LanguageFs) afero.Fs { + return afero.NewReadOnlyFs(&languageCompositeFs{afero.NewCopyOnWriteFs(base, overlay).(*afero.CopyOnWriteFs)}) +} + +// Open takes the full path to the file in the target filesystem. If it is a directory, it gets merged +// using the language as a weight. +func (fs *languageCompositeFs) Open(name string) (afero.File, error) { + f, err := fs.CopyOnWriteFs.Open(name) + if err != nil { + return nil, err + } + + fu, ok := f.(*afero.UnionFile) + if ok { + // This is a directory: Merge it. + fu.Merger = LanguageDirsMerger + } + return f, nil +} diff --git a/hugofs/language_composite_fs_test.go b/hugofs/language_composite_fs_test.go new file mode 100644 index 000000000..bb4ddf701 --- /dev/null +++ b/hugofs/language_composite_fs_test.go @@ -0,0 +1,106 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "path/filepath" + + "strings" + + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/require" +) + +func TestCompositeLanguagFsTest(t *testing.T) { + assert := require.New(t) + + languages := map[string]bool{ + "sv": true, + "en": true, + "nn": true, + } + msv := afero.NewMemMapFs() + baseSv := "/content/sv" + lfssv := NewLanguageFs("sv", languages, afero.NewBasePathFs(msv, baseSv)) + mnn := afero.NewMemMapFs() + baseNn := "/content/nn" + lfsnn := NewLanguageFs("nn", languages, afero.NewBasePathFs(mnn, baseNn)) + men := afero.NewMemMapFs() + baseEn := "/content/en" + lfsen := NewLanguageFs("en", languages, afero.NewBasePathFs(men, baseEn)) + + // The order will be sv, en, nn + composite := NewLanguageCompositeFs(lfsnn, lfsen) + composite = NewLanguageCompositeFs(composite, lfssv) + + afero.WriteFile(msv, filepath.Join(baseSv, "f1.txt"), []byte("some sv"), 0755) + afero.WriteFile(mnn, filepath.Join(baseNn, "f1.txt"), []byte("some nn"), 0755) + afero.WriteFile(men, filepath.Join(baseEn, "f1.txt"), []byte("some en"), 0755) + + // Swedish is the top layer. + assertLangFile(t, composite, "f1.txt", "sv") + + afero.WriteFile(msv, filepath.Join(baseSv, "f2.en.txt"), []byte("some sv"), 0755) + afero.WriteFile(mnn, filepath.Join(baseNn, "f2.en.txt"), []byte("some nn"), 0755) + afero.WriteFile(men, filepath.Join(baseEn, "f2.en.txt"), []byte("some en"), 0755) + + // English is in the middle, but the most specific language match wins. + //assertLangFile(t, composite, "f2.en.txt", "en") + + // Fetch some specific language versions + assertLangFile(t, composite, filepath.Join(baseNn, "f2.en.txt"), "nn") + assertLangFile(t, composite, filepath.Join(baseEn, "f2.en.txt"), "en") + assertLangFile(t, composite, filepath.Join(baseSv, "f2.en.txt"), "sv") + + // Read the root + f, err := composite.Open("/") + assert.NoError(err) + defer f.Close() + files, err := f.Readdir(-1) + assert.Equal(4, len(files)) + expected := map[string]bool{ + filepath.FromSlash("/content/en/f1.txt"): true, + filepath.FromSlash("/content/nn/f1.txt"): true, + filepath.FromSlash("/content/sv/f1.txt"): true, + filepath.FromSlash("/content/en/f2.en.txt"): true, + } + got := make(map[string]bool) + + for _, fi := range files { + fil, ok := fi.(*LanguageFileInfo) + assert.True(ok) + got[fil.Filename()] = true + } + assert.Equal(expected, got) +} + +func assertLangFile(t testing.TB, fs afero.Fs, filename, match string) { + f, err := fs.Open(filename) + if err != nil { + t.Fatal(err) + } + defer f.Close() + b, err := afero.ReadAll(f) + if err != nil { + t.Fatal(err) + } + + s := string(b) + if !strings.Contains(s, match) { + t.Fatalf("got %q expected it to contain %q", s, match) + + } +} diff --git a/hugofs/language_fs.go b/hugofs/language_fs.go new file mode 100644 index 000000000..95ec0831e --- /dev/null +++ b/hugofs/language_fs.go @@ -0,0 +1,328 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/spf13/afero" +) + +const hugoFsMarker = "__hugofs" + +var ( + _ LanguageAnnouncer = (*LanguageFileInfo)(nil) + _ FilePather = (*LanguageFileInfo)(nil) + _ afero.Lstater = (*LanguageFs)(nil) +) + +// LanguageAnnouncer is aware of its language. +type LanguageAnnouncer interface { + Lang() string + TranslationBaseName() string +} + +// FilePather is aware of its file's location. +type FilePather interface { + // Filename gets the full path and filename to the file. + Filename() string + + // Path gets the content relative path including file name and extension. + // The directory is relative to the content root where "content" is a broad term. + Path() string + + // RealName is FileInfo.Name in its original form. + RealName() string + + BaseDir() string +} + +var LanguageDirsMerger = func(lofi, bofi []os.FileInfo) ([]os.FileInfo, error) { + m := make(map[string]*LanguageFileInfo) + + for _, fi := range lofi { + fil, ok := fi.(*LanguageFileInfo) + if !ok { + return nil, fmt.Errorf("received %T, expected *LanguageFileInfo", fi) + } + m[fil.virtualName] = fil + } + + for _, fi := range bofi { + fil, ok := fi.(*LanguageFileInfo) + if !ok { + return nil, fmt.Errorf("received %T, expected *LanguageFileInfo", fi) + } + existing, found := m[fil.virtualName] + + if !found || existing.weight < fil.weight { + m[fil.virtualName] = fil + } + } + + merged := make([]os.FileInfo, len(m)) + i := 0 + for _, v := range m { + merged[i] = v + i++ + } + + return merged, nil +} + +type LanguageFileInfo struct { + os.FileInfo + lang string + baseDir string + realFilename string + relFilename string + name string + realName string + virtualName string + translationBaseName string + + // We add some weight to the files in their own language's content directory. + weight int +} + +func (fi *LanguageFileInfo) Filename() string { + return fi.realFilename +} + +func (fi *LanguageFileInfo) Path() string { + return fi.relFilename +} + +func (fi *LanguageFileInfo) RealName() string { + return fi.realName +} + +func (fi *LanguageFileInfo) BaseDir() string { + return fi.baseDir +} + +func (fi *LanguageFileInfo) Lang() string { + return fi.lang +} + +// TranslationBaseName returns the base filename without any extension or language +// identificator. +func (fi *LanguageFileInfo) TranslationBaseName() string { + return fi.translationBaseName +} + +// Name is the name of the file within this filesystem without any path info. +// It will be marked with language information so we can identify it as ours. +func (fi *LanguageFileInfo) Name() string { + return fi.name +} + +type languageFile struct { + afero.File + fs *LanguageFs +} + +// Readdir creates FileInfo entries by calling Lstat if possible. +func (l *languageFile) Readdir(c int) (ofi []os.FileInfo, err error) { + names, err := l.File.Readdirnames(c) + if err != nil { + return nil, err + } + + fis := make([]os.FileInfo, len(names)) + + for i, name := range names { + fi, _, err := l.fs.LstatIfPossible(filepath.Join(l.Name(), name)) + + if err != nil { + return nil, err + } + fis[i] = fi + } + + return fis, err +} + +type LanguageFs struct { + // This Fs is usually created with a BasePathFs + basePath string + lang string + nameMarker string + languages map[string]bool + afero.Fs +} + +func NewLanguageFs(lang string, languages map[string]bool, fs afero.Fs) *LanguageFs { + if lang == "" { + panic("no lang set for the language fs") + } + var basePath string + + if bfs, ok := fs.(*afero.BasePathFs); ok { + basePath, _ = bfs.RealPath("") + } + + marker := hugoFsMarker + "_" + lang + "_" + + return &LanguageFs{lang: lang, languages: languages, basePath: basePath, Fs: fs, nameMarker: marker} +} + +func (fs *LanguageFs) Lang() string { + return fs.lang +} + +func (fs *LanguageFs) Stat(name string) (os.FileInfo, error) { + name, err := fs.realName(name) + if err != nil { + return nil, err + } + + fi, err := fs.Fs.Stat(name) + if err != nil { + return nil, err + } + + return fs.newLanguageFileInfo(name, fi) +} + +func (fs *LanguageFs) Open(name string) (afero.File, error) { + name, err := fs.realName(name) + if err != nil { + return nil, err + } + f, err := fs.Fs.Open(name) + + if err != nil { + return nil, err + } + return &languageFile{File: f, fs: fs}, nil +} + +func (fs *LanguageFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + name, err := fs.realName(name) + if err != nil { + return nil, false, err + } + + var fi os.FileInfo + var b bool + + if lif, ok := fs.Fs.(afero.Lstater); ok { + fi, b, err = lif.LstatIfPossible(name) + } else { + fi, err = fs.Fs.Stat(name) + } + + if err != nil { + return nil, b, err + } + + lfi, err := fs.newLanguageFileInfo(name, fi) + + return lfi, b, err +} + +func (fs *LanguageFs) realPath(name string) (string, error) { + if baseFs, ok := fs.Fs.(*afero.BasePathFs); ok { + return baseFs.RealPath(name) + } + return name, nil +} + +func (fs *LanguageFs) realName(name string) (string, error) { + if strings.Contains(name, hugoFsMarker) { + if !strings.Contains(name, fs.nameMarker) { + return "", os.ErrNotExist + } + return strings.Replace(name, fs.nameMarker, "", 1), nil + } + + if fs.basePath == "" { + return name, nil + } + + return strings.TrimPrefix(name, fs.basePath), nil +} + +func (fs *LanguageFs) newLanguageFileInfo(filename string, fi os.FileInfo) (*LanguageFileInfo, error) { + filename = filepath.Clean(filename) + _, name := filepath.Split(filename) + + realName := name + virtualName := name + + realPath, err := fs.realPath(filename) + if err != nil { + return nil, err + } + + lang := fs.Lang() + + baseNameNoExt := "" + + if !fi.IsDir() { + + // Try to extract the language from the file name. + // Any valid language identificator in the name will win over the + // language set on the file system, e.g. "mypost.en.md". + baseName := filepath.Base(name) + ext := filepath.Ext(baseName) + baseNameNoExt = baseName + + if ext != "" { + baseNameNoExt = strings.TrimSuffix(baseNameNoExt, ext) + } + + fileLangExt := filepath.Ext(baseNameNoExt) + fileLang := strings.TrimPrefix(fileLangExt, ".") + + if fs.languages[fileLang] { + lang = fileLang + } + + baseNameNoExt = strings.TrimSuffix(baseNameNoExt, fileLangExt) + + // This connects the filename to the filesystem, not the language. + virtualName = baseNameNoExt + "." + lang + ext + + name = fs.nameMarker + name + } + + weight := 1 + // If this file's language belongs in this directory, add some weight to it + // to make it more important. + if lang == fs.Lang() { + weight = 2 + } + + if fi.IsDir() { + // For directories we always want to start from the union view. + realPath = strings.TrimPrefix(realPath, fs.basePath) + } + + return &LanguageFileInfo{ + lang: lang, + weight: weight, + realFilename: realPath, + realName: realName, + relFilename: strings.TrimPrefix(strings.TrimPrefix(realPath, fs.basePath), string(os.PathSeparator)), + name: name, + virtualName: virtualName, + translationBaseName: baseNameNoExt, + baseDir: fs.basePath, + FileInfo: fi}, nil +} diff --git a/hugofs/language_fs_test.go b/hugofs/language_fs_test.go new file mode 100644 index 000000000..ac17a1930 --- /dev/null +++ b/hugofs/language_fs_test.go @@ -0,0 +1,54 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "path/filepath" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/require" +) + +func TestLanguagFs(t *testing.T) { + languages := map[string]bool{ + "sv": true, + } + base := filepath.FromSlash("/my/base") + assert := require.New(t) + m := afero.NewMemMapFs() + bfs := afero.NewBasePathFs(m, base) + lfs := NewLanguageFs("sv", languages, bfs) + assert.NotNil(lfs) + assert.Equal("sv", lfs.Lang()) + err := afero.WriteFile(lfs, filepath.FromSlash("sect/page.md"), []byte("abc"), 0777) + assert.NoError(err) + fi, err := lfs.Stat(filepath.FromSlash("sect/page.md")) + assert.NoError(err) + assert.Equal("__hugofs_sv_page.md", fi.Name()) + + languager, ok := fi.(LanguageAnnouncer) + assert.True(ok) + + assert.Equal("sv", languager.Lang()) + + lfi, ok := fi.(*LanguageFileInfo) + assert.True(ok) + assert.Equal(filepath.FromSlash("/my/base/sect/page.md"), lfi.Filename()) + assert.Equal(filepath.FromSlash("sect/page.md"), lfi.Path()) + assert.Equal("page.sv.md", lfi.virtualName) + assert.Equal("__hugofs_sv_page.md", lfi.Name()) + assert.Equal("page.md", lfi.RealName()) + +} |