diff options
Diffstat (limited to 'lib/model/folder.go')
-rw-r--r-- | lib/model/folder.go | 84 |
1 files changed, 56 insertions, 28 deletions
diff --git a/lib/model/folder.go b/lib/model/folder.go index 18ef41a065..41b9fe9806 100644 --- a/lib/model/folder.go +++ b/lib/model/folder.go @@ -532,16 +532,20 @@ func (f *folder) scanSubdirs(subDirs []string) error { return nil } +const maxToRemove = 1000 + type scanBatch struct { - *db.FileInfoBatch - f *folder + f *folder + updateBatch *db.FileInfoBatch + toRemove []string } func (f *folder) newScanBatch() *scanBatch { b := &scanBatch{ - f: f, + f: f, + toRemove: make([]string, 0, maxToRemove), } - b.FileInfoBatch = db.NewFileInfoBatch(func(fs []protocol.FileInfo) error { + b.updateBatch = db.NewFileInfoBatch(func(fs []protocol.FileInfo) error { if err := b.f.getHealthErrorWithoutIgnores(); err != nil { l.Debugf("Stopping scan of folder %s due to: %s", b.f.Description(), err) return err @@ -552,9 +556,32 @@ func (f *folder) newScanBatch() *scanBatch { return b } -// Append adds the fileinfo to the batch for updating, and does a few checks. +func (b *scanBatch) Remove(item string) { + b.toRemove = append(b.toRemove, item) +} + +func (b *scanBatch) flushToRemove() { + if len(b.toRemove) > 0 { + b.f.fset.RemoveLocalItems(b.toRemove) + b.toRemove = b.toRemove[:0] + } +} + +func (b *scanBatch) Flush() error { + b.flushToRemove() + return b.updateBatch.Flush() +} + +func (b *scanBatch) FlushIfFull() error { + if len(b.toRemove) >= maxToRemove { + b.flushToRemove() + } + return b.updateBatch.FlushIfFull() +} + +// Update adds the fileinfo to the batch for updating, and does a few checks. // It returns false if the checks result in the file not going to be updated or removed. -func (b *scanBatch) Append(fi protocol.FileInfo, snap *db.Snapshot) bool { +func (b *scanBatch) Update(fi protocol.FileInfo, snap *db.Snapshot) bool { // Check for a "virtual" parent directory of encrypted files. We don't track // it, but check if anything still exists within and delete it otherwise. if b.f.Type == config.FolderTypeReceiveEncrypted && fi.IsDirectory() && protocol.IsEncryptedParent(fs.PathComponents(fi.Name)) { @@ -565,20 +592,21 @@ func (b *scanBatch) Append(fi protocol.FileInfo, snap *db.Snapshot) bool { } // Resolve receive-only items which are identical with the global state or // the global item is our own receive-only item. - // Except if they are in a receive-encrypted folder and are locally added. - // Those must never be sent in index updates and thus must retain the flag. switch gf, ok := snap.GetGlobal(fi.Name); { case !ok: case gf.IsReceiveOnlyChanged(): - if b.f.Type == config.FolderTypeReceiveOnly && fi.Deleted { - l.Debugf("%v scanning: Marking deleted item as not locally changed", b.f, fi) - fi.LocalFlags &^= protocol.FlagLocalReceiveOnly + if fi.IsDeleted() { + // Our item is deleted and the global item is our own receive only + // file. No point in keeping track of that. + b.Remove(fi.Name) + return true } case gf.IsEquivalentOptional(fi, b.f.modTimeWindow, false, false, protocol.FlagLocalReceiveOnly): - l.Debugf("%v scanning: Replacing scanned file info with global as it's equivalent", b.f, fi) + // What we have locally is equivalent to the global file. + l.Debugf("%v scanning: Merging identical locally changed item with global", b.f, fi) fi = gf } - b.FileInfoBatch.Append(fi) + b.updateBatch.Append(fi) return true } @@ -634,7 +662,7 @@ func (f *folder) scanSubdirsChangedAndNew(subDirs []string, batch *scanBatch) (i return changes, err } - if batch.Append(res.File, snap) { + if batch.Update(res.File, snap) { changes++ } @@ -642,7 +670,7 @@ func (f *folder) scanSubdirsChangedAndNew(subDirs []string, batch *scanBatch) (i case config.FolderTypeReceiveOnly, config.FolderTypeReceiveEncrypted: default: if nf, ok := f.findRename(snap, res.File, alreadyUsedOrExisting); ok { - if batch.Append(nf, snap) { + if batch.Update(nf, snap) { changes++ } } @@ -683,7 +711,7 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *scanBatch for _, file := range toIgnore { l.Debugln("marking file as ignored", file) nf := file.ConvertToIgnoredFileInfo() - if batch.Append(nf, snap) { + if batch.Update(nf, snap) { changes++ } if err := batch.FlushIfFull(); err != nil { @@ -713,7 +741,7 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *scanBatch l.Debugln("marking file as ignored", file) nf := file.ConvertToIgnoredFileInfo() - if batch.Append(nf, snap) { + if batch.Update(nf, snap) { changes++ } @@ -743,24 +771,24 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *scanBatch nf.Version = protocol.Vector{} } l.Debugln("marking file as deleted", nf) - if batch.Append(nf, snap) { + if batch.Update(nf, snap) { changes++ } case file.IsDeleted() && file.IsReceiveOnlyChanged(): switch f.Type { case config.FolderTypeReceiveOnly, config.FolderTypeReceiveEncrypted: - if gf, _ := snap.GetGlobal(file.Name); gf.IsDeleted() { + switch gf, ok := snap.GetGlobal(file.Name); { + case !ok: + case gf.IsReceiveOnlyChanged(): + l.Debugln("removing deleted, receive-only item that is globally receive-only from db", file) + batch.Remove(file.Name) + changes++ + case gf.IsDeleted(): // Our item is deleted and the global item is deleted too. We just // pretend it is a normal deleted file (nobody cares about that). - // Except if this is a receive-encrypted folder and it - // is a locally added file. Those must never be sent - // in index updates and thus must retain the flag. - if f.Type == config.FolderTypeReceiveEncrypted && gf.IsReceiveOnlyChanged() { - return true - } l.Debugf("%v scanning: Marking globally deleted item as not locally changed: %v", f, file.Name) file.LocalFlags &^= protocol.FlagLocalReceiveOnly - if batch.Append(file.ConvertDeletedToFileInfo(), snap) { + if batch.Update(file.ConvertDeletedToFileInfo(), snap) { changes++ } } @@ -769,7 +797,7 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *scanBatch // deleted and just the folder type/local flags changed. file.LocalFlags &^= protocol.FlagLocalReceiveOnly l.Debugln("removing receive-only flag on deleted item", file) - if batch.Append(file.ConvertDeletedToFileInfo(), snap) { + if batch.Update(file.ConvertDeletedToFileInfo(), snap) { changes++ } } @@ -788,7 +816,7 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *scanBatch for _, file := range toIgnore { l.Debugln("marking file as ignored", f) nf := file.ConvertToIgnoredFileInfo() - if batch.Append(nf, snap) { + if batch.Update(nf, snap) { changes++ } if iterError = batch.FlushIfFull(); iterError != nil { |