diff options
Diffstat (limited to 'hugofs/rootmapping_fs.go')
-rw-r--r-- | hugofs/rootmapping_fs.go | 566 |
1 files changed, 275 insertions, 291 deletions
diff --git a/hugofs/rootmapping_fs.go b/hugofs/rootmapping_fs.go index 2196be8e0..ea8b7e04d 100644 --- a/hugofs/rootmapping_fs.go +++ b/hugofs/rootmapping_fs.go @@ -27,15 +27,18 @@ import ( "github.com/spf13/afero" ) -var filepathSeparator = string(filepath.Separator) +var ( + filepathSeparator = string(filepath.Separator) +) // NewRootMappingFs creates a new RootMappingFs on top of the provided with -// of root mappings with some optional metadata about the root. +// root mappings with some optional metadata about the root. // Note that From represents a virtual root that maps to the actual filename in To. func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) { rootMapToReal := radix.New() + var virtualRoots []RootMapping - for i, rm := range rms { + for _, rm := range rms { (&rm).clean() fromBase := files.ResolveComponentFolder(rm.From) @@ -56,11 +59,13 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) { } // Extract "blog" from "content/blog" rm.path = strings.TrimPrefix(strings.TrimPrefix(rm.From, fromBase), filepathSeparator) - if rm.Meta != nil { - rm.Meta[metaKeyBaseDir] = rm.ToBasedir - rm.Meta[metaKeyMountRoot] = rm.path + if rm.Meta == nil { + rm.Meta = make(FileMeta) } + rm.Meta[metaKeyBaseDir] = rm.ToBasedir + rm.Meta[metaKeyMountRoot] = rm.path + meta := copyFileMeta(rm.Meta) if !fi.IsDir() { @@ -70,7 +75,7 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) { rm.fi = NewFileMetaInfo(fi, meta) - key := rm.rootKey() + key := filepathSeparator + rm.From var mappings []RootMapping v, found := rootMapToReal.Get(key) if found { @@ -80,30 +85,38 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) { mappings = append(mappings, rm) rootMapToReal.Insert(key, mappings) - rms[i] = rm + virtualRoots = append(virtualRoots, rm) } - rfs := &RootMappingFs{Fs: fs, - virtualRoots: rms, - rootMapToReal: rootMapToReal} + rootMapToReal.Insert(filepathSeparator, virtualRoots) + + rfs := &RootMappingFs{ + Fs: fs, + rootMapToReal: rootMapToReal, + } return rfs, nil } -// NewRootMappingFsFromFromTo is a convenicence variant of NewRootMappingFs taking -// From and To as string pairs. -func NewRootMappingFsFromFromTo(fs afero.Fs, fromTo ...string) (*RootMappingFs, error) { +func newRootMappingFsFromFromTo( + baseDir string, + fs afero.Fs, + fromTo ...string, +) (*RootMappingFs, error) { + rms := make([]RootMapping, len(fromTo)/2) for i, j := 0, 0; j < len(fromTo); i, j = i+1, j+2 { rms[i] = RootMapping{ - From: fromTo[j], - To: fromTo[j+1], + From: fromTo[j], + To: fromTo[j+1], + ToBasedir: baseDir, } } return NewRootMappingFs(fs, rms...) } +// RootMapping describes a virtual file or directory mount. type RootMapping struct { From string // The virtual mount. To string // The source directory or file. @@ -127,21 +140,16 @@ func (r RootMapping) filename(name string) string { return filepath.Join(r.To, strings.TrimPrefix(name, r.From)) } -func (r RootMapping) rootKey() string { - return r.From -} - // A RootMappingFs maps several roots into one. Note that the root of this filesystem // is directories only, and they will be returned in Readdir and Readdirnames // in the order given. type RootMappingFs struct { afero.Fs rootMapToReal *radix.Tree - virtualRoots []RootMapping - filter func(r RootMapping) bool } func (fs *RootMappingFs) Dirs(base string) ([]FileMetaInfo, error) { + base = filepathSeparator + fs.cleanName(base) roots := fs.getRootsWithPrefix(base) if roots == nil { @@ -176,138 +184,46 @@ func (fs *RootMappingFs) Dirs(base string) ([]FileMetaInfo, error) { return fss, nil } -// LstatIfPossible returns the os.FileInfo structure describing a given file. -func (fs *RootMappingFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { - fis, _, b, err := fs.doLstat(name, false) - if err != nil { - return nil, b, err - } - return fis[0], b, nil -} - -func (fs *RootMappingFs) virtualDirOpener(name string, isRoot bool) func() (afero.File, error) { - return func() (afero.File, error) { return &rootMappingFile{name: name, isRoot: isRoot, fs: fs}, nil } -} - -func (fs *RootMappingFs) doLstat(name string, allowMultiple bool) ([]FileMetaInfo, []FileMetaInfo, bool, error) { - if fs.isRoot(name) { - return []FileMetaInfo{newDirNameOnlyFileInfo(name, true, fs.virtualDirOpener(name, true))}, nil, false, nil - } - - roots := fs.getRoots(name) - rootsWithPrefix := fs.getRootsWithPrefix(name) - hasRootMappingsBelow := len(rootsWithPrefix) != 0 - - if len(roots) == 0 { - if hasRootMappingsBelow { - // No exact matches, but we have root mappings below name, - // let's make it look like a directory. - return []FileMetaInfo{newDirNameOnlyFileInfo(name, true, fs.virtualDirOpener(name, false))}, nil, false, nil - } - - return nil, nil, false, os.ErrNotExist - } - - // We may have a mapping for both static and static/subdir. - // These will not show in any Readdir so append them - // manually. - rootsInDir := fs.filterRootsBelow(rootsWithPrefix, name) - - var ( - fis []FileMetaInfo - dirs []FileMetaInfo - b bool - root RootMapping - err error - ) - - for _, root = range roots { - var fi os.FileInfo - fi, b, err = fs.statRoot(root, name) - if err != nil { - if os.IsNotExist(err) { - continue +// Filter creates a copy of this filesystem with only mappings matching a filter. +func (fs RootMappingFs) Filter(f func(m RootMapping) bool) *RootMappingFs { + rootMapToReal := radix.New() + fs.rootMapToReal.Walk(func(b string, v interface{}) bool { + rms := v.([]RootMapping) + var nrms []RootMapping + for _, rm := range rms { + if f(rm) { + nrms = append(nrms, rm) } - return nil, nil, false, err } - fim := fi.(FileMetaInfo) - - fis = append(fis, fim) - } - - for _, root = range rootsInDir { - - fi, _, err := fs.statRoot(root, "") - if err != nil { - if os.IsNotExist(err) { - continue - } - return nil, nil, false, err + if len(nrms) != 0 { + rootMapToReal.Insert(b, nrms) } - fim := fi.(FileMetaInfo) - dirs = append(dirs, fim) - } - - if len(fis) == 0 && len(dirs) == 0 { - return nil, nil, false, os.ErrNotExist - } - - if allowMultiple || len(fis) == 1 { - return fis, dirs, b, nil - } - - if len(fis) == 0 { - return nil, nil, false, os.ErrNotExist - } - - // Open it in this composite filesystem. - opener := func() (afero.File, error) { - return fs.Open(name) - } + return false + }) - return []FileMetaInfo{decorateFileInfo(fis[0], fs, opener, "", "", root.Meta)}, nil, b, nil + fs.rootMapToReal = rootMapToReal + return &fs } -// Open opens the namedrootMappingFile file for reading. -func (fs *RootMappingFs) Open(name string) (afero.File, error) { - if fs.isRoot(name) { - return &rootMappingFile{name: name, fs: fs, isRoot: true}, nil - } - - fis, dirs, _, err := fs.doLstat(name, true) +// LstatIfPossible returns the os.FileInfo structure describing a given file. +func (fs *RootMappingFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + fis, err := fs.doLstat(name) if err != nil { - return nil, err + return nil, false, err } + return fis[0], false, nil +} - if len(fis) == 1 { - fi := fis[0] - meta := fi.(FileMetaInfo).Meta() - f, err := meta.Open() - if err != nil { - return nil, err - } - - f = &rootMappingFile{File: f, fs: fs, name: name, meta: meta} - - if len(dirs) > 0 { - return &readDirDirsAppender{File: f, dirs: dirs}, nil - } - - return f, nil - } +// Open opens the named file for reading. +func (fs *RootMappingFs) Open(name string) (afero.File, error) { + fis, err := fs.doLstat(name) - f, err := fs.newUnionFile(fis...) if err != nil { return nil, err } - if len(dirs) > 0 { - return &readDirDirsAppender{File: f, dirs: dirs}, nil - } - - return f, nil - + return fs.newUnionFile(fis...) } // Stat returns the os.FileInfo structure describing a given file. If there is @@ -318,80 +234,51 @@ func (fs *RootMappingFs) Stat(name string) (os.FileInfo, error) { } -// Filter creates a copy of this filesystem with the applied filter. -func (fs RootMappingFs) Filter(f func(m RootMapping) bool) *RootMappingFs { - fs.filter = f - return &fs -} - -func (fs *RootMappingFs) isRoot(name string) bool { - return name == "" || name == filepathSeparator +func (fs *RootMappingFs) hasPrefix(prefix string) bool { + hasPrefix := false + fs.rootMapToReal.WalkPrefix(prefix, func(b string, v interface{}) bool { + hasPrefix = true + return true + }) + return hasPrefix } -func (fs *RootMappingFs) getRoots(name string) []RootMapping { - name = filepath.Clean(name) - _, v, found := fs.rootMapToReal.LongestPrefix(name) +func (fs *RootMappingFs) getRoot(key string) []RootMapping { + v, found := fs.rootMapToReal.Get(key) if !found { return nil } - rm := v.([]RootMapping) - - return fs.applyFilterToRoots(rm) + return v.([]RootMapping) } -func (fs *RootMappingFs) applyFilterToRoots(rm []RootMapping) []RootMapping { - if fs.filter == nil { - return rm +func (fs *RootMappingFs) getRoots(key string) (string, []RootMapping) { + s, v, found := fs.rootMapToReal.LongestPrefix(key) + if !found || (s == filepathSeparator && key != filepathSeparator) { + return "", nil } + return s, v.([]RootMapping) - var filtered []RootMapping - for _, m := range rm { - if fs.filter(m) { - filtered = append(filtered, m) - } - } +} + +func (fs *RootMappingFs) debug() { + fmt.Println("debug():") + fs.rootMapToReal.Walk(func(s string, v interface{}) bool { + fmt.Println("Key", s) + return false + }) - return filtered } func (fs *RootMappingFs) getRootsWithPrefix(prefix string) []RootMapping { - if fs.isRoot(prefix) { - return fs.virtualRoots - } - prefix = filepath.Clean(prefix) var roots []RootMapping - fs.rootMapToReal.WalkPrefix(prefix, func(b string, v interface{}) bool { roots = append(roots, v.([]RootMapping)...) return false }) - return fs.applyFilterToRoots(roots) -} - -// Filter out the mappings inside the name directory. -func (fs *RootMappingFs) filterRootsBelow(roots []RootMapping, name string) []RootMapping { - if len(roots) == 0 { - return nil - } - - sepCount := strings.Count(name, filepathSeparator) - var filtered []RootMapping - for _, x := range roots { - if name == x.From { - continue - } - - if strings.Count(x.From, filepathSeparator)-sepCount != 1 { - continue - } - - filtered = append(filtered, x) - - } - return filtered + return roots } func (fs *RootMappingFs) newUnionFile(fis ...FileMetaInfo) (afero.File, error) { @@ -400,6 +287,10 @@ func (fs *RootMappingFs) newUnionFile(fis ...FileMetaInfo) (afero.File, error) { if err != nil { return nil, err } + if len(fis) == 1 { + return f, nil + } + rf := &rootMappingFile{File: f, fs: fs, name: meta.Name(), meta: meta} if len(fis) == 1 { return rf, err @@ -439,148 +330,241 @@ func (fs *RootMappingFs) newUnionFile(fis ...FileMetaInfo) (afero.File, error) { } -func (fs *RootMappingFs) statRoot(root RootMapping, name string) (os.FileInfo, bool, error) { - filename := root.filename(name) +func (fs *RootMappingFs) cleanName(name string) string { + return strings.Trim(filepath.Clean(name), filepathSeparator) +} + +func (fs *RootMappingFs) collectDirEntries(prefix string) ([]os.FileInfo, error) { + prefix = filepathSeparator + fs.cleanName(prefix) + + var fis []os.FileInfo - var b bool - var fi os.FileInfo - var err error + seen := make(map[string]bool) // Prevent duplicate directories + level := strings.Count(prefix, filepathSeparator) - if ls, ok := fs.Fs.(afero.Lstater); ok { - fi, b, err = ls.LstatIfPossible(filename) + // First add any real files/directories. + rms := fs.getRoot(prefix) + for _, rm := range rms { + f, err := rm.fi.Meta().Open() if err != nil { - return nil, b, err + return nil, err } - - } else { - fi, err = fs.Fs.Stat(filename) + direntries, err := f.Readdir(-1) if err != nil { - return nil, b, err + f.Close() + return nil, err } - } - // Opens the real directory/file. - opener := func() (afero.File, error) { - return fs.Fs.Open(filename) - } + for _, fi := range direntries { + meta := fi.(FileMetaInfo).Meta() + mergeFileMeta(rm.Meta, meta) + if fi.IsDir() { + name := fi.Name() + if seen[name] { + continue + } + seen[name] = true + opener := func() (afero.File, error) { + return fs.Open(filepath.Join(rm.From, name)) + } + fi = newDirNameOnlyFileInfo(name, meta, false, opener) + } - if fi.IsDir() { - if name == "" { - name = root.From + fis = append(fis, fi) } - _, name = filepath.Split(name) - fi = newDirNameOnlyFileInfo(name, false, opener) - } - - return decorateFileInfo(fi, fs.Fs, opener, "", "", root.Meta), b, nil -} - -type rootMappingFile struct { - afero.File - fs *RootMappingFs - name string - meta FileMeta - isRoot bool -} - -type readDirDirsAppender struct { - afero.File - dirs []FileMetaInfo -} - -func (f *readDirDirsAppender) Readdir(count int) ([]os.FileInfo, error) { - fis, err := f.File.Readdir(count) - if err != nil { - return nil, err + f.Close() } - for _, dir := range f.dirs { - fis = append(fis, dir) - } - return fis, nil - -} - -func (f *readDirDirsAppender) Readdirnames(count int) ([]string, error) { - fis, err := f.Readdir(count) - if err != nil { - return nil, err - } - return fileInfosToNames(fis), nil -} + // Next add any file mounts inside the given directory. + prefixInside := prefix + filepathSeparator + fs.rootMapToReal.WalkPrefix(prefixInside, func(s string, v interface{}) bool { -func (f *rootMappingFile) Close() error { - if f.File == nil { - return nil - } - return f.File.Close() -} + if (strings.Count(s, filepathSeparator) - level) != 1 { + // This directory is not part of the current, but we + // need to include the first name part to make it + // navigable. + path := strings.TrimPrefix(s, prefixInside) + parts := strings.Split(path, filepathSeparator) + name := parts[0] -func (f *rootMappingFile) Name() string { - return f.name -} + if seen[name] { + return false + } + seen[name] = true + opener := func() (afero.File, error) { + return fs.Open(path) + } -func (f *rootMappingFile) Readdir(count int) ([]os.FileInfo, error) { - if f.File == nil { - filesn := make([]os.FileInfo, 0) - roots := f.fs.getRootsWithPrefix(f.name) - seen := make(map[string]bool) // Do not return duplicate directories + fi := newDirNameOnlyFileInfo(name, nil, false, opener) + fis = append(fis, fi) - j := 0 - for _, rm := range roots { - if count != -1 && j >= count { - break - } + return false + } + rms := v.([]RootMapping) + for _, rm := range rms { if !rm.fi.IsDir() { // A single file mount - filesn = append(filesn, rm.fi) + fis = append(fis, rm.fi) continue } - - from := rm.From - name := from - if !f.isRoot { - _, name = filepath.Split(from) - } - + name := filepath.Base(rm.From) if seen[name] { continue } seen[name] = true opener := func() (afero.File, error) { - return f.fs.Open(from) + return fs.Open(rm.From) } - j++ + fi := newDirNameOnlyFileInfo(name, rm.Meta, false, opener) - fi := newDirNameOnlyFileInfo(name, false, opener) + fis = append(fis, fi) - if rm.Meta != nil { - mergeFileMeta(rm.Meta, fi.Meta()) + } + + return false + }) + + return fis, nil +} + +func (fs *RootMappingFs) doLstat(name string) ([]FileMetaInfo, error) { + name = fs.cleanName(name) + key := filepathSeparator + name + + roots := fs.getRoot(key) + + if roots == nil { + if fs.hasPrefix(key) { + // We have directories mounted below this. + // Make it look like a directory. + return []FileMetaInfo{newDirNameOnlyFileInfo(name, nil, true, fs.virtualDirOpener(name))}, nil + } + + // Find any real files or directories with this key. + _, roots := fs.getRoots(key) + if roots == nil { + return nil, os.ErrNotExist + } + + var err error + var fis []FileMetaInfo + + for _, rm := range roots { + var fi FileMetaInfo + fi, _, err = fs.statRoot(rm, name) + if err == nil { + fis = append(fis, fi) } + } + + if fis != nil { + return fis, nil + } - filesn = append(filesn, fi) + if err == nil { + err = os.ErrNotExist } - return filesn, nil + + return nil, err } - if f.File == nil { - panic(fmt.Sprintf("no File for %q", f.name)) + fileCount := 0 + for _, root := range roots { + if !root.fi.IsDir() { + fileCount++ + } + if fileCount > 1 { + break + } } - fis, err := f.File.Readdir(count) + if fileCount == 0 { + // Dir only. + return []FileMetaInfo{newDirNameOnlyFileInfo(name, roots[0].Meta, true, fs.virtualDirOpener(name))}, nil + } + + if fileCount > 1 { + // Not supported by this filesystem. + return nil, errors.Errorf("found multiple files with name %q, use .Readdir or the source filesystem directly", name) + + } + + return []FileMetaInfo{roots[0].fi}, nil + +} + +func (fs *RootMappingFs) statRoot(root RootMapping, name string) (FileMetaInfo, bool, error) { + filename := root.filename(name) + + fi, b, err := lstatIfPossible(fs.Fs, filename) if err != nil { - return nil, err + return nil, b, err } - for i, fi := range fis { - fis[i] = decorateFileInfo(fi, f.fs, nil, "", "", f.meta) + var opener func() (afero.File, error) + if fi.IsDir() { + // Make sure metadata gets applied in Readdir. + opener = fs.realDirOpener(filename, root.Meta) + } else { + // Opens the real file directly. + opener = func() (afero.File, error) { + return fs.Fs.Open(filename) + } } - return fis, nil + return decorateFileInfo(fi, fs.Fs, opener, "", "", root.Meta), b, nil + +} + +func (fs *RootMappingFs) virtualDirOpener(name string) func() (afero.File, error) { + return func() (afero.File, error) { return &rootMappingFile{name: name, fs: fs}, nil } +} + +func (fs *RootMappingFs) realDirOpener(name string, meta FileMeta) func() (afero.File, error) { + return func() (afero.File, error) { + f, err := fs.Fs.Open(name) + if err != nil { + return nil, err + } + return &rootMappingFile{name: name, meta: meta, fs: fs, File: f}, nil + } +} + +type rootMappingFile struct { + afero.File + fs *RootMappingFs + name string + meta FileMeta +} + +func (f *rootMappingFile) Close() error { + if f.File == nil { + return nil + } + return f.File.Close() +} + +func (f *rootMappingFile) Name() string { + return f.name +} + +func (f *rootMappingFile) Readdir(count int) ([]os.FileInfo, error) { + if f.File != nil { + fis, err := f.File.Readdir(count) + if err != nil { + return nil, err + } + + for i, fi := range fis { + fis[i] = decorateFileInfo(fi, f.fs, nil, "", "", f.meta) + } + return fis, nil + } + return f.fs.collectDirEntries(f.name) } func (f *rootMappingFile) Readdirnames(count int) ([]string, error) { |