diff options
author | Jakob Borg <jakob@kastelo.net> | 2023-03-12 20:06:59 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-03-12 20:06:59 +0100 |
commit | 466b56ded1e43965e2de37997f6181f2a73b7bab (patch) | |
tree | c0ebe34311c4bea69f6d691805148a02047c69a7 | |
parent | 3ffe859fe817b94983dd5ab87227fe5f1322cdd3 (diff) |
lib/protocol: Cache expensive key operations (fixes #8599) (#8820)
This adds a cache to the expensive key generation operations. It's fixes
size LRU/MRU stuff to keep memory usage bounded under absurd conditions.
Also closes #8600.
-rw-r--r-- | cmd/syncthing/decrypt/decrypt.go | 10 | ||||
-rw-r--r-- | lib/connections/service.go | 13 | ||||
-rw-r--r-- | lib/model/model.go | 14 | ||||
-rw-r--r-- | lib/model/model_test.go | 53 | ||||
-rw-r--r-- | lib/model/requests_test.go | 72 | ||||
-rw-r--r-- | lib/model/testutils_test.go | 2 | ||||
-rw-r--r-- | lib/protocol/benchmark_test.go | 4 | ||||
-rw-r--r-- | lib/protocol/encryption.go | 115 | ||||
-rw-r--r-- | lib/protocol/encryption_test.go | 37 | ||||
-rw-r--r-- | lib/protocol/protocol.go | 10 | ||||
-rw-r--r-- | lib/protocol/protocol_test.go | 38 | ||||
-rw-r--r-- | lib/syncthing/syncthing.go | 5 |
12 files changed, 212 insertions, 161 deletions
diff --git a/cmd/syncthing/decrypt/decrypt.go b/cmd/syncthing/decrypt/decrypt.go index 5d4746ed9a..33a9bcd7ff 100644 --- a/cmd/syncthing/decrypt/decrypt.go +++ b/cmd/syncthing/decrypt/decrypt.go @@ -35,6 +35,7 @@ type CLI struct { TokenPath string `placeholder:"PATH" help:"Path to the token file within the folder (used to determine folder ID)"` folderKey *[32]byte + keyGen *protocol.KeyGenerator } type storedEncryptionToken struct { @@ -68,7 +69,8 @@ func (c *CLI) Run() error { } } - c.folderKey = protocol.KeyFromPassword(c.FolderID, c.Password) + c.keyGen = protocol.NewKeyGenerator() + c.folderKey = c.keyGen.KeyFromPassword(c.FolderID, c.Password) return c.walk() } @@ -151,7 +153,7 @@ func (c *CLI) process(srcFs fs.Filesystem, dstFs fs.Filesystem, path string) err // in native format, while protocol expects wire format (slashes). encFi.Name = osutil.NormalizedFilename(encFi.Name) - plainFi, err := protocol.DecryptFileInfo(*encFi, c.folderKey) + plainFi, err := protocol.DecryptFileInfo(c.keyGen, *encFi, c.folderKey) if err != nil { return fmt.Errorf("%s: decrypting metadata: %w", path, err) } @@ -162,7 +164,7 @@ func (c *CLI) process(srcFs fs.Filesystem, dstFs fs.Filesystem, path string) err var plainFd fs.File if dstFs != nil { - if err := dstFs.MkdirAll(filepath.Dir(plainFi.Name), 0700); err != nil { + if err := dstFs.MkdirAll(filepath.Dir(plainFi.Name), 0o700); err != nil { return fmt.Errorf("%s: %w", plainFi.Name, err) } @@ -209,7 +211,7 @@ func (c *CLI) decryptFile(encFi *protocol.FileInfo, plainFi *protocol.FileInfo, return fmt.Errorf("block count mismatch: encrypted %d != plaintext %d", len(encFi.Blocks), len(plainFi.Blocks)) } - fileKey := protocol.FileKey(plainFi.Name, c.folderKey) + fileKey := c.keyGen.FileKey(plainFi.Name, c.folderKey) for i, encBlock := range encFi.Blocks { // Read the encrypted block buf := make([]byte, encBlock.Size) diff --git a/lib/connections/service.go b/lib/connections/service.go index e504c8d202..44ff0f03f7 100644 --- a/lib/connections/service.go +++ b/lib/connections/service.go @@ -161,6 +161,7 @@ type service struct { natService *nat.Service evLogger events.Logger registry *registry.Registry + keyGen *protocol.KeyGenerator dialNow chan struct{} dialNowDevices map[protocol.DeviceID]struct{} @@ -171,7 +172,7 @@ type service struct { listenerTokens map[string]suture.ServiceToken } -func NewService(cfg config.Wrapper, myID protocol.DeviceID, mdl Model, tlsCfg *tls.Config, discoverer discover.Finder, bepProtocolName string, tlsDefaultCommonName string, evLogger events.Logger, registry *registry.Registry) Service { +func NewService(cfg config.Wrapper, myID protocol.DeviceID, mdl Model, tlsCfg *tls.Config, discoverer discover.Finder, bepProtocolName string, tlsDefaultCommonName string, evLogger events.Logger, registry *registry.Registry, keyGen *protocol.KeyGenerator) Service { spec := svcutil.SpecWithInfoLogger(l) service := &service{ Supervisor: suture.New("connections.Service", spec), @@ -190,6 +191,7 @@ func NewService(cfg config.Wrapper, myID protocol.DeviceID, mdl Model, tlsCfg *t natService: nat.NewService(myID, cfg), evLogger: evLogger, registry: registry, + keyGen: keyGen, dialNowDevicesMut: sync.NewMutex(), dialNow: make(chan struct{}, 1), @@ -411,7 +413,7 @@ func (s *service) handleHellos(ctx context.Context) error { // connections are limited. rd, wr := s.limiter.getLimiters(remoteID, c, c.IsLocal()) - protoConn := protocol.NewConnection(remoteID, rd, wr, c, s.model, c, deviceCfg.Compression, s.cfg.FolderPasswords(remoteID)) + protoConn := protocol.NewConnection(remoteID, rd, wr, c, s.model, c, deviceCfg.Compression, s.cfg.FolderPasswords(remoteID), s.keyGen) go func() { <-protoConn.Closed() s.dialNowDevicesMut.Lock() @@ -426,6 +428,7 @@ func (s *service) handleHellos(ctx context.Context) error { continue } } + func (s *service) connect(ctx context.Context) error { // Map of when to earliest dial each given device + address again nextDialAt := make(nextDialRegistry) @@ -1020,8 +1023,10 @@ func urlsToStrings(urls []*url.URL) []string { return strings } -var warningLimiters = make(map[protocol.DeviceID]*rate.Limiter) -var warningLimitersMut = sync.NewMutex() +var ( + warningLimiters = make(map[protocol.DeviceID]*rate.Limiter) + warningLimitersMut = sync.NewMutex() +) func warningFor(dev protocol.DeviceID, msg string) { warningLimitersMut.Lock() diff --git a/lib/model/model.go b/lib/model/model.go index c728212245..2263c5c831 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -142,6 +142,7 @@ type model struct { folderIOLimiter *util.Semaphore fatalChan chan error started chan struct{} + keyGen *protocol.KeyGenerator // fields protected by fmut fmut sync.RWMutex @@ -174,9 +175,7 @@ var _ config.Verifier = &model{} type folderFactory func(*model, *db.FileSet, *ignore.Matcher, config.FolderConfiguration, versioner.Versioner, events.Logger, *util.Semaphore) service -var ( - folderFactories = make(map[config.FolderType]folderFactory) -) +var folderFactories = make(map[config.FolderType]folderFactory) var ( errDeviceUnknown = errors.New("unknown device") @@ -205,7 +204,7 @@ var ( // NewModel creates and starts a new model. The model starts in read-only mode, // where it sends index information to connected peers and responds to requests // for file data without altering the local folder in any way. -func NewModel(cfg config.Wrapper, id protocol.DeviceID, clientName, clientVersion string, ldb *db.Lowlevel, protectedFiles []string, evLogger events.Logger) Model { +func NewModel(cfg config.Wrapper, id protocol.DeviceID, clientName, clientVersion string, ldb *db.Lowlevel, protectedFiles []string, evLogger events.Logger, keyGen *protocol.KeyGenerator) Model { spec := svcutil.SpecWithDebugLogger(l) m := &model{ Supervisor: suture.New("model", spec), @@ -227,6 +226,7 @@ func NewModel(cfg config.Wrapper, id protocol.DeviceID, clientName, clientVersio folderIOLimiter: util.NewSemaphore(cfg.Options().MaxFolderConcurrency()), fatalChan: make(chan error), started: make(chan struct{}), + keyGen: keyGen, // fields protected by fmut fmut: sync.NewRWMutex(), @@ -1462,7 +1462,7 @@ func (m *model) ccCheckEncryption(fcfg config.FolderConfiguration, folderDevice } if isEncryptedRemote { - passwordToken := protocol.PasswordToken(fcfg.ID, folderDevice.EncryptionPassword) + passwordToken := protocol.PasswordToken(m.keyGen, fcfg.ID, folderDevice.EncryptionPassword) match := false if hasTokenLocal { match = bytes.Equal(passwordToken, ccDeviceInfos.local.EncryptionPasswordToken) @@ -2483,7 +2483,7 @@ func (m *model) generateClusterConfig(device protocol.DeviceID) (protocol.Cluste if deviceCfg.DeviceID == m.id && hasEncryptionToken { protocolDevice.EncryptionPasswordToken = encryptionToken } else if folderDevice.EncryptionPassword != "" { - protocolDevice.EncryptionPasswordToken = protocol.PasswordToken(folderCfg.ID, folderDevice.EncryptionPassword) + protocolDevice.EncryptionPasswordToken = protocol.PasswordToken(m.keyGen, folderCfg.ID, folderDevice.EncryptionPassword) if folderDevice.DeviceID == device { passwords[folderCfg.ID] = folderDevice.EncryptionPassword } @@ -3264,7 +3264,7 @@ func readEncryptionToken(cfg config.FolderConfiguration) ([]byte, error) { func writeEncryptionToken(token []byte, cfg config.FolderConfiguration) error { tokenName := encryptionTokenPath(cfg) - fd, err := cfg.Filesystem(nil).OpenFile(tokenName, fs.OptReadWrite|fs.OptCreate, 0666) + fd, err := cfg.Filesystem(nil).OpenFile(tokenName, fs.OptReadWrite|fs.OptCreate, 0o666) if err != nil { return err } diff --git a/lib/model/model_test.go b/lib/model/model_test.go index ec27cd6193..66aafbcfcc 100644 --- a/lib/model/model_test.go +++ b/lib/model/model_test.go @@ -271,7 +271,7 @@ func BenchmarkRequestOut(b *testing.B) { fc := newFakeConnection(device1, m) for _, f := range files { - fc.addFile(f.Name, 0644, protocol.FileInfoTypeFile, []byte("some data to return")) + fc.addFile(f.Name, 0o644, protocol.FileInfoTypeFile, []byte("some data to return")) } m.AddConnection(fc, protocol.Hello{}) must(b, m.Index(device1, "default", files)) @@ -296,7 +296,7 @@ func BenchmarkRequestInSingleFile(b *testing.B) { rand.Read(buf) mustRemove(b, defaultFs.RemoveAll("request")) defer func() { mustRemove(b, defaultFs.RemoveAll("request")) }() - must(b, defaultFs.MkdirAll("request/for/a/file/in/a/couple/of/dirs", 0755)) + must(b, defaultFs.MkdirAll("request/for/a/file/in/a/couple/of/dirs", 0o755)) writeFile(b, defaultFs, "request/for/a/file/in/a/couple/of/dirs/128k", buf) b.ResetTimer() @@ -1148,8 +1148,8 @@ func TestAutoAcceptNameConflict(t *testing.T) { id := srand.String(8) label := srand.String(8) - testOs.MkdirAll(id, 0777) - testOs.MkdirAll(label, 0777) + testOs.MkdirAll(id, 0o777) + testOs.MkdirAll(label, 0o777) defer os.RemoveAll(id) defer os.RemoveAll(label) m, cancel := newState(t, defaultAutoAcceptCfg) @@ -1198,7 +1198,7 @@ func TestAutoAcceptFallsBackToID(t *testing.T) { id := srand.String(8) label := srand.String(8) t.Log(id, label) - testOs.MkdirAll(label, 0777) + testOs.MkdirAll(label, 0o777) defer os.RemoveAll(label) defer os.RemoveAll(id) defer cleanupModel(m) @@ -1330,7 +1330,8 @@ func TestAutoAcceptEnc(t *testing.T) { Folders: []protocol.Folder{{ ID: id, Label: id, - }}} + }}, + } } // Earlier tests might cause the connection to get closed, thus ClusterConfig @@ -1486,7 +1487,7 @@ func changeIgnores(t *testing.T, m *testModel, expected []string) { func TestIgnores(t *testing.T) { // Assure a clean start state mustRemove(t, defaultFs.RemoveAll(config.DefaultMarkerName)) - mustRemove(t, defaultFs.MkdirAll(config.DefaultMarkerName, 0644)) + mustRemove(t, defaultFs.MkdirAll(config.DefaultMarkerName, 0o644)) writeFile(t, defaultFs, ".stignore", []byte(".*\nquux\n")) m := setupModel(t, defaultCfgWrapper) @@ -1548,7 +1549,7 @@ func TestIgnores(t *testing.T) { func TestEmptyIgnores(t *testing.T) { // Assure a clean start state mustRemove(t, defaultFs.RemoveAll(config.DefaultMarkerName)) - must(t, defaultFs.MkdirAll(config.DefaultMarkerName, 0644)) + must(t, defaultFs.MkdirAll(config.DefaultMarkerName, 0o644)) m := setupModel(t, defaultCfgWrapper) defer cleanupModel(m) @@ -1634,7 +1635,7 @@ func TestROScanRecovery(t *testing.T) { waitForState(t, sub, "default", "folder path missing") - testOs.Mkdir(fcfg.Path, 0700) + testOs.Mkdir(fcfg.Path, 0o700) waitForState(t, sub, "default", config.ErrMarkerMissing.Error()) @@ -1687,7 +1688,7 @@ func TestRWScanRecovery(t *testing.T) { waitForState(t, sub, "default", "folder path missing") - testOs.Mkdir(fcfg.Path, 0700) + testOs.Mkdir(fcfg.Path, 0o700) waitForState(t, sub, "default", config.ErrMarkerMissing.Error()) @@ -2147,10 +2148,10 @@ func TestIssue2782(t *testing.T) { if err := os.RemoveAll(testDir); err != nil { t.Skip(err) } - if err := os.MkdirAll(testDir+"/syncdir", 0755); err != nil { + if err := os.MkdirAll(testDir+"/syncdir", 0o755); err != nil { t.Skip(err) } - if err := os.WriteFile(testDir+"/syncdir/file", []byte("hello, world\n"), 0644); err != nil { + if err := os.WriteFile(testDir+"/syncdir/file", []byte("hello, world\n"), 0o644); err != nil { t.Skip(err) } if err := os.Symlink("syncdir", testDir+"/synclink"); err != nil { @@ -2480,7 +2481,7 @@ func TestIssue2571(t *testing.T) { defer os.RemoveAll(testFs.URI()) for _, dir := range []string{"toLink", "linkTarget"} { - must(t, testFs.MkdirAll(dir, 0775)) + must(t, testFs.MkdirAll(dir, 0o775)) fd, err := testFs.Create(filepath.Join(dir, "a")) must(t, err) fd.Close() @@ -2518,8 +2519,8 @@ func TestIssue4573(t *testing.T) { testFs := fcfg.Filesystem(nil) defer os.RemoveAll(testFs.URI()) - must(t, testFs.MkdirAll("inaccessible", 0755)) - defer testFs.Chmod("inaccessible", 0777) + must(t, testFs.MkdirAll("inaccessible", 0o755)) + defer testFs.Chmod("inaccessible", 0o777) file := filepath.Join("inaccessible", "a") fd, err := testFs.Create(file) @@ -2529,7 +2530,7 @@ func TestIssue4573(t *testing.T) { m := setupModel(t, w) defer cleanupModel(m) - must(t, testFs.Chmod("inaccessible", 0000)) + must(t, testFs.Chmod("inaccessible", 0o000)) m.ScanFolder("default") @@ -2561,7 +2562,7 @@ func TestInternalScan(t *testing.T) { for _, dir := range baseDirs { sub := filepath.Join(dir, "subDir") for _, dir := range []string{dir, sub} { - if err := testFs.MkdirAll(dir, 0775); err != nil { + if err := testFs.MkdirAll(dir, 0o775); err != nil { t.Fatalf("%v: %v", dir, err) } } @@ -2633,7 +2634,7 @@ func TestCustomMarkerName(t *testing.T) { waitForState(t, sub, "default", "folder path missing") - testOs.Mkdir(fcfg.Path, 0700) + testOs.Mkdir(fcfg.Path, 0o700) fd := testOs.Create(filepath.Join(fcfg.Path, "myfile")) fd.Close() @@ -2646,7 +2647,7 @@ func TestRemoveDirWithContent(t *testing.T) { tfs := fcfg.Filesystem(nil) defer cleanupModelAndRemoveDir(m, tfs.URI()) - tfs.MkdirAll("dirwith", 0755) + tfs.MkdirAll("dirwith", 0o755) content := filepath.Join("dirwith", "content") fd, err := tfs.Create(content) must(t, err) @@ -2712,7 +2713,7 @@ func TestIssue4475(t *testing.T) { // This should result in the directory being recreated and added to the // db locally. - must(t, testFs.MkdirAll("delDir", 0755)) + must(t, testFs.MkdirAll("delDir", 0o755)) m.ScanFolder("default") @@ -2721,7 +2722,7 @@ func TestIssue4475(t *testing.T) { } fileName := filepath.Join("delDir", "file") - conn.addFile(fileName, 0644, protocol.FileInfoTypeFile, nil) + conn.addFile(fileName, 0o644, protocol.FileInfoTypeFile, nil) conn.sendIndexUpdate() // Is there something we could trigger on instead of just waiting? @@ -2805,7 +2806,7 @@ func TestVersionRestore(t *testing.T) { file = filepath.FromSlash(file) } dir := filepath.Dir(file) - must(t, filesystem.MkdirAll(dir, 0755)) + must(t, filesystem.MkdirAll(dir, 0o755)) if fd, err := filesystem.Create(file); err != nil { t.Fatal(err) } else if _, err := fd.Write([]byte(file)); err != nil { @@ -3185,7 +3186,7 @@ func TestConnCloseOnRestart(t *testing.T) { br := &testutils.BlockingRW{} nw := &testutils.NoopRW{} - m.AddConnection(protocol.NewConnection(device1, br, nw, testutils.NoopCloser{}, m, new(protocolmocks.ConnectionInfo), protocol.CompressionNever, nil), protocol.Hello{}) + m.AddConnection(protocol.NewConnection(device1, br, nw, testutils.NoopCloser{}, m, new(protocolmocks.ConnectionInfo), protocol.CompressionNever, nil, m.keyGen), protocol.Hello{}) m.pmut.RLock() if len(m.closed) != 1 { t.Fatalf("Expected just one conn (len(m.conn) == %v)", len(m.conn)) @@ -3654,7 +3655,7 @@ func TestBlockListMap(t *testing.T) { // Change type must(t, ffs.Remove("four")) - must(t, ffs.Mkdir("four", 0644)) + must(t, ffs.Mkdir("four", 0o644)) m.ScanFolders() @@ -3933,7 +3934,7 @@ func TestIssue6961(t *testing.T) { // Remote, invalid (receive-only) and existing file must(t, m.Index(device2, fcfg.ID, []protocol.FileInfo{{Name: name, RawInvalid: true, Sequence: 1}})) // Create a local file - if fd, err := tfs.OpenFile(name, fs.OptCreate, 0666); err != nil { + if fd, err := tfs.OpenFile(name, fs.OptCreate, 0o666); err != nil { t.Fatal(err) } else { fd.Close() @@ -4038,7 +4039,7 @@ func TestCcCheckEncryption(t *testing.T) { defer cleanupModel(m) pw := "foo" - token := protocol.PasswordToken(fcfg.ID, pw) + token := protocol.PasswordToken(m.keyGen, fcfg.ID, pw) m.folderEncryptionPasswordTokens[fcfg.ID] = token testCases := []struct { diff --git a/lib/model/requests_test.go b/lib/model/requests_test.go index a3a3dfc826..5f940cbe8f 100644 --- a/lib/model/requests_test.go +++ b/lib/model/requests_test.go @@ -55,7 +55,7 @@ func TestRequestSimple(t *testing.T) { // Send an update for the test file, wait for it to sync and be reported back. contents := []byte("test file contents\n") - fc.addFile("testfile", 0644, protocol.FileInfoTypeFile, contents) + fc.addFile("testfile", 0o644, protocol.FileInfoTypeFile, contents) fc.sendIndexUpdate() select { case <-done: @@ -101,7 +101,7 @@ func TestSymlinkTraversalRead(t *testing.T) { // Send an update for the symlink, wait for it to sync and be reported back. contents := []byte("..") - fc.addFile("symlink", 0644, protocol.FileInfoTypeSymlink, contents) + fc.addFile("symlink", 0o644, protocol.FileInfoTypeSymlink, contents) fc.sendIndexUpdate() <-done @@ -151,7 +151,7 @@ func TestSymlinkTraversalWrite(t *testing.T) { // Send an update for the symlink, wait for it to sync and be reported back. contents := []byte("..") - fc.addFile("symlink", 0644, protocol.FileInfoTypeSymlink, contents) + fc.addFile("symlink", 0o644, protocol.FileInfoTypeSymlink, contents) fc.sendIndexUpdate() <-done @@ -159,9 +159,9 @@ func TestSymlinkTraversalWrite(t *testing.T) { // blocks for any of them to come back, or index entries. Hopefully none // of that should happen. contents = []byte("testdata testdata\n") - fc.addFile("symlink/testfile", 0644, protocol.FileInfoTypeFile, contents) - fc.addFile("symlink/testdir", 0644, protocol.FileInfoTypeDirectory, contents) - fc.addFile("symlink/testsyml", 0644, protocol.FileInfoTypeSymlink, contents) + fc.addFile("symlink/testfile", 0o644, protocol.FileInfoTypeFile, contents) + fc.addFile("symlink/testdir", 0o644, protocol.FileInfoTypeDirectory, contents) + fc.addFile("symlink/testsyml", 0o644, protocol.FileInfoTypeSymlink, contents) fc.sendIndexUpdate() select { @@ -203,7 +203,7 @@ func TestRequestCreateTmpSymlink(t *testing.T) { }) // Send an update for the test file, wait for it to sync and be reported back. - fc.addFile(name, 0644, protocol.FileInfoTypeSymlink, []byte("..")) + fc.addFile(name, 0o644, protocol.FileInfoTypeSymlink, []byte("..")) fc.sendIndexUpdate() select { @@ -257,7 +257,7 @@ func TestRequestVersioningSymlinkAttack(t *testing.T) { } // Send an update for the test file, wait for it to sync and be reported back. - fc.addFile("foo", 0644, protocol.FileInfoTypeSymlink, []byte(tmpdir)) + fc.addFile("foo", 0o644, protocol.FileInfoTypeSymlink, []byte(tmpdir)) fc.sendIndexUpdate() waitForIdx() @@ -267,8 +267,8 @@ func TestRequestVersioningSymlinkAttack(t *testing.T) { waitForIdx() // Recreate foo and a file in it with some data - fc.updateFile("foo", 0755, protocol.FileInfoTypeDirectory, nil) - fc.addFile("foo/test", 0644, protocol.FileInfoTypeFile, []byte("testtesttest")) + fc.updateFile("foo", 0o755, protocol.FileInfoTypeDirectory, nil) + fc.addFile("foo/test", 0o644, protocol.FileInfoTypeFile, []byte("testtesttest")) fc.sendIndexUpdate() waitForIdx() @@ -286,7 +286,6 @@ func TestRequestVersioningSymlinkAttack(t *testing.T) { func TestPullInvalidIgnoredSO(t *testing.T) { t.Skip("flaky") pullInvalidIgnored(t, config.FolderTypeSendOnly) - } func TestPullInvalidIgnoredSR(t *testing.T) { @@ -322,11 +321,11 @@ func pullInvalidIgnored(t *testing.T, ft config.FolderType) { ign := "ignoredNonExisting" ignExisting := "ignoredExisting" - fc.addFile(invIgn, 0644, protocol.FileInfoTypeFile, contents) - fc.addFile(invDel, 0644, protocol.FileInfoTypeFile, contents) + fc.addFile(invIgn, 0o644, protocol.FileInfoTypeFile, contents) + fc.addFile(invDel, 0o644, protocol.FileInfoTypeFile, contents) fc.deleteFile(invDel) - fc.addFile(ign, 0644, protocol.FileInfoTypeFile, contents) - fc.addFile(ignExisting, 0644, protocol.FileInfoTypeFile, contents) + fc.addFile(ign, 0o644, protocol.FileInfoTypeFile, contents) + fc.addFile(ignExisting, 0o644, protocol.FileInfoTypeFile, contents) writeFile(t, fss, ignExisting, otherContents) done := make(chan struct{}) @@ -549,8 +548,8 @@ func TestParentDeletion(t *testing.T) { child := filepath.Join(parent, "bar") received := make(chan []protocol.FileInfo) - fc.addFile(parent, 0777, protocol.FileInfoTypeDirectory, nil) - fc.addFile(child, 0777, protocol.FileInfoTypeDirectory, nil) + fc.addFile(parent, 0o777, protocol.FileInfoTypeDirectory, nil) + fc.addFile(child, 0o777, protocol.FileInfoTypeDirectory, nil) fc.setIndexFn(func(_ context.Context, folder string, fs []protocol.FileInfo) error { received <- fs return nil @@ -588,7 +587,7 @@ func TestParentDeletion(t *testing.T) { } // Recreate the child dir on the remote - fc.updateFile(child, 0777, protocol.FileInfoTypeDirectory, nil) + fc.updateFile(child, 0o777, protocol.FileInfoTypeDirectory, nil) fc.sendIndexUpdate() // Wait for the child dir to be recreated and sent to the remote @@ -634,7 +633,7 @@ func TestRequestSymlinkWindows(t *testing.T) { return nil }) - fc.addFile("link", 0644, protocol.FileInfoTypeSymlink, nil) + fc.addFile("link", 0o644, protocol.FileInfoTypeSymlink, nil) fc.sendIndexUpdate() select { @@ -714,7 +713,7 @@ func TestRequestRemoteRenameChanged(t *testing.T) { b: []byte("bData"), } for _, n := range [2]string{a, b} { - fc.addFile(n, 0644, protocol.FileInfoTypeFile, data[n]) + fc.addFile(n, 0o644, protocol.FileInfoTypeFile, data[n]) } fc.sendIndexUpdate() select { @@ -772,7 +771,7 @@ func TestRequestRemoteRenameChanged(t *testing.T) { return nil }) - fd, err := tfs.OpenFile(b, fs.OptReadWrite, 0644) + fd, err := tfs.OpenFile(b, fs.OptReadWrite, 0o644) if err != nil { t.Fatal(err) } @@ -784,7 +783,7 @@ func TestRequestRemoteRenameChanged(t *testing.T) { // rename fc.deleteFile(a) - fc.updateFile(b, 0644, protocol.FileInfoTypeFile, data[a]) + fc.updateFile(b, 0o644, protocol.FileInfoTypeFile, data[a]) // Make sure the remote file for b is newer and thus stays global -> local conflict fc.mut.Lock() for i := range fc.files { @@ -843,7 +842,7 @@ func TestRequestRemoteRenameConflict(t *testing.T) { b: []byte("bData"), } for _, n := range [2]string{a, b} { - fc.addFile(n, 0644, protocol.FileInfoTypeFile, data[n]) + fc.addFile(n, 0o644, protocol.FileInfoTypeFile, data[n]) } fc.sendIndexUpdate() select { @@ -859,7 +858,7 @@ func TestRequestRemoteRenameConflict(t *testing.T) { must(t, equalContents(filepath.Join(tmpDir, n), data[n])) } - fd, err := tfs.OpenFile(b, fs.OptReadWrite, 0644) + fd, err := tfs.OpenFile(b, fs.OptReadWrite, 0o644) if err != nil { t.Fatal(err) } @@ -883,7 +882,7 @@ func TestRequestRemoteRenameConflict(t *testing.T) { // rename fc.deleteFile(a) - fc.updateFile(b, 0644, protocol.FileInfoTypeFile, data[a]) + fc.updateFile(b, 0o644, protocol.FileInfoTypeFile, data[a]) fc.sendIndexUpdate() select { case <-recv: @@ -933,7 +932,7 @@ func TestRequestDeleteChanged(t *testing.T) { // setup a := "a" data := []byte("aData") - fc.addFile(a, 0644, protocol.FileInfoTypeFile, data) + fc.addFile(a, 0o644, protocol.FileInfoTypeFile, data) fc.sendIndexUpdate() select { case <-done: @@ -952,7 +951,7 @@ func TestRequestDeleteChanged(t *testing.T) { return nil }) - fd, err := tfs.OpenFile(a, fs.OptReadWrite, 0644) + fd, err := tfs.OpenFile(a, fs.OptReadWrite, 0o644) if err != nil { t.Fatal(err) } @@ -999,7 +998,7 @@ func TestNeedFolderFiles(t *testing.T) { data := []byte("foo") num := 20 for i := 0; i < num; i++ { - fc.addFile(strconv.Itoa(i), 0644, protocol.FileInfoTypeFile, data) + fc.addFile(strconv.Itoa(i), 0o644, protocol.FileInfoTypeFile, data) } fc.sendIndexUpdate() @@ -1146,7 +1145,7 @@ func TestRequestLastFileProgress(t *testing.T) { }) contents := []byte("test file contents\n") - fc.addFile("testfile", 0644, protocol.FileInfoTypeFile, contents) + fc.addFile("testfile", 0o644, protocol.FileInfoTypeFile, contents) fc.sendIndexUpdate() select { @@ -1280,7 +1279,7 @@ func TestRequestIndexSenderClusterConfigBeforeStart(t *testing.T) { dir2 := "bar" // Initialise db with an entry and then stop everything again - must(t, tfs.Mkdir(dir1, 0777)) + must(t, tfs.Mkdir(dir1, 0o777)) m := newModel(t, w, myID, "syncthing", "dev", nil) defer cleanupModelAndRemoveDir(m, tfs.URI()) m.ServeBackground() @@ -1290,7 +1289,7 @@ func TestRequestIndexSenderClusterConfigBeforeStart(t *testing.T) { // Add connection (sends incoming cluster config) before starting the new model m = &testModel{ - model: NewModel(m.cfg, m.id, m.clientName, m.clientVersion, m.db, m.protectedFiles, m.evLogger).(*model), + model: NewModel(m.cfg, m.id, m.clientName, m.clientVersion, m.db, m.protectedFiles, m.evLogger, protocol.NewKeyGenerator()).(*model), evCancel: m.evCancel, stopped: make(chan struct{}), } @@ -1326,7 +1325,7 @@ func TestRequestIndexSenderClusterConfigBeforeStart(t *testing.T) { } // Check that an index is sent for the newly added item - must(t, tfs.Mkdir(dir2, 0777)) + must(t, tfs.Mkdir(dir2, 0o777)) m.ScanFolders() select { case <-timeout: @@ -1346,8 +1345,9 @@ func TestRequestReceiveEncrypted(t *testing.T) { fcfg.Type = config.FolderTypeReceiveEncrypted setFolder(t, w, fcfg) - encToken := protocol.PasswordToken(fcfg.ID, "pw") - must(t, tfs.Mkdir(config.DefaultMarkerName, 0777)) + keyGen := protocol.NewKeyGenerator() + encToken := protocol.PasswordToken(keyGen, fcfg.ID, "pw") + must(t, tfs.Mkdir(config.DefaultMarkerName, 0o777)) must(t, writeEncryptionToken(encToken, fcfg)) m := setupModel(t, w) @@ -1407,7 +1407,7 @@ func TestRequestReceiveEncrypted(t *testing.T) { name := "foo" data := make([]byte, 2000) rand.Read(data) - fc.addFile(name, 0664, protocol.FileInfoTypeFile, data) + fc.addFile(name, 0o664, protocol.FileInfoTypeFile, data) fc.sendIndexUpdate() select { @@ -1467,7 +1467,7 @@ func TestRequestGlobalInvalidToValid(t *testing.T) { // Setup device with valid file, do not send index yet contents := []byte("test file contents\n") - fc.addFile(name, 0644, protocol.FileInfoTypeFile, contents) + fc.addFile(name, 0o644, protocol.FileInfoTypeFile, contents) // Third device ignoring the same file fc.mut.Lock() diff --git a/lib/model/testutils_test.go b/lib/model/testutils_test.go index aceb44062d..7a63aef5b8 100644 --- a/lib/model/testutils_test.go +++ b/lib/model/testutils_test.go @@ -154,7 +154,7 @@ func newModel(t testing.TB, cfg config.Wrapper, id protocol.DeviceID, clientName if err != nil { t.Fatal(err) } - m := NewModel(cfg, id, clientName, clientVersion, ldb, protectedFiles, evLogger).(*model) + m := NewModel(cfg, id, clientName, clientVersion, ldb, protectedFiles, evLogger, protocol.NewKeyGenerator()).(*model) ctx, cancel := context.WithCancel(context.Background()) go evLogger.Serve(ctx) return &testModel{ diff --git a/lib/protocol/benchmark_test.go b/lib/protocol/benchmark_test.go index 2515404503..c740e0725e 100644 --- a/lib/protocol/benchmark_test.go +++ b/lib/protocol/benchmark_test.go @@ -60,9 +60,9 @@ func benchmarkRequestsTLS(b *testing.B, conn0, conn1 net.Conn) { func benchmarkRequestsConnPair(b *testing.B, conn0, conn1 net.Conn) { // Start up Connections on them - c0 := NewConnection(LocalDeviceID, conn0, conn0, testutils.NoopCloser{}, new(fakeModel), new(mockedConnectionInfo), CompressionMetadata, nil) + c0 := NewConnection(LocalDeviceID, conn0, conn0, testutils.NoopCloser{}, new(fakeModel), new(mockedConnectionInfo), CompressionMetadata, nil, testKeyGen) c0.Start() - c1 := NewConnection(LocalDeviceID, conn1, conn1, testutils.NoopCloser{}, new(fakeModel), new(mockedConnectionInfo), CompressionMetadata, nil) + c1 := NewConnection(LocalDeviceID, conn1, conn1, testutils.NoopCloser{}, new(fakeModel), new(mockedConnectionInfo), CompressionMetadata, nil, testKeyGen) c1.Start() // Satisfy the assertions in the protocol by sending an initial cluster config diff --git a/lib/protocol/encryption.go b/lib/protocol/encryption.go index c5c66d4866..ea2a4ad01c 100644 --- a/lib/protocol/encryption.go +++ b/lib/protocol/encryption.go @@ -17,6 +17,7 @@ import ( "sync" "github.com/gogo/protobuf/proto" + lru "github.com/hashicorp/golang-lru/v2" "github.com/miscreant/miscreant.go" "github.com/syncthing/syncthing/lib/rand" "github.com/syncthing/syncthing/lib/sha256" @@ -34,6 +35,8 @@ const ( maxPathComponent = 200 // characters encryptedDirExtension = ".syncthing-enc" // for top level dirs miscreantAlgo = "AES-SIV" + folderKeyCacheEntries = 1000 + fileKeyCacheEntries = 5000 ) // The encryptedModel sits between the encrypted device and the model. It @@ -42,12 +45,21 @@ const ( type encryptedModel struct { model Model folderKeys *folderKeyRegistry + keyGen *KeyGenerator +} + +func newEncryptedModel(model Model, folderKeys *folderKeyRegistry, keyGen *KeyGenerator) encryptedModel { + return encryptedModel{ + model: model, + folderKeys: folderKeys, + keyGen: keyGen, + } } func (e encryptedModel) Index(deviceID DeviceID, folder string, files []FileInfo) error { if folderKey, ok := e.folderKeys.get(folder); ok { // incoming index data to be decrypted - if err := decryptFileInfos(files, folderKey); err != nil { + if err := decryptFileInfos(e.keyGen, files, folderKey); err != nil { return err } } @@ -57,7 +69,7 @@ func (e encryptedModel) Index(deviceID DeviceID, folder string, files []FileInfo func (e encryptedModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) error { if folderKey, ok := e.folderKeys.get(folder); ok { // incoming index data to be decrypted - if err := decryptFileInfos(files, folderKey); err != nil { + if err := decryptFileInfos(e.keyGen, files, folderKey); err != nil { return err } } @@ -86,7 +98,7 @@ func (e encryptedModel) Request(deviceID DeviceID, folder, name string, blockNo, // Decrypt the block hash. - fileKey := FileKey(realName, folderKey) + fileKey := e.keyGen.FileKey(realName, folderKey) var additional [8]byte binary.BigEndian.PutUint64(additional[:], uint64(realOffset)) realHash, err := decryptDeterministic(hash, fileKey, additional[:]) @@ -145,6 +157,16 @@ type encryptedConnection struct { ConnectionInfo conn *rawConnection folderKeys *folderKeyRegistry + keyGen *KeyGenerator +} + +func newEncryptedConnection(ci ConnectionInfo, conn *rawConnection, folderKeys *folderKeyRegistry, keyGen *KeyGenerator) encryptedConnection { + return encryptedConnection{ + ConnectionInfo: ci, + conn: conn, + folderKeys: folderKeys, + keyGen: keyGen, + } |