Merge pull request #1535 from ThomasWaldmann/merge-1.0-maint

Merge 1.0 maint

4 files changed, 98 insertions, 77 deletions

diff --git a/src/borg/remote.py b/src/borg/remote.py
index cd382ecf2..604506cd0 100644
--- a/src/borg/remote.py
+++ b/src/borg/remote.py
@@ -42,6 +42,14 @@ class InvalidRPCMethod(Error):
     """RPC method {} is not valid"""
 
 
+class UnexpectedRPCDataFormatFromClient(Error):
+    """Borg {}: Got unexpected RPC data format from client."""
+
+
+class UnexpectedRPCDataFormatFromServer(Error):
+    """Got unexpected RPC data format from server."""
+
+
 class RepositoryServer:  # pragma: no cover
     rpc_methods = (
         '__len__',
@@ -84,13 +92,18 @@ class RepositoryServer:  # pragma: no cover
             if r:
                 data = os.read(stdin_fd, BUFSIZE)
                 if not data:
-                    self.repository.close()
+                    if self.repository is not None:
+                        self.repository.close()
+                    else:
+                        os.write(stderr_fd, "Borg {}: Got connection close before repository was opened.\n"
+                                 .format(__version__).encode())
                     return
                 unpacker.feed(data)
                 for unpacked in unpacker:
                     if not (isinstance(unpacked, tuple) and len(unpacked) == 4):
-                        self.repository.close()
-                        raise Exception("Unexpected RPC data format.")
+                        if self.repository is not None:
+                            self.repository.close()
+                        raise UnexpectedRPCDataFormatFromClient(__version__)
                     type, msgid, method, args = unpacked
                     method = method.decode('ascii')
                     try:
@@ -354,7 +367,7 @@ This problem will go away as soon as the server has been upgraded to 1.0.7+.
                     self.unpacker.feed(data)
                     for unpacked in self.unpacker:
                         if not (isinstance(unpacked, tuple) and len(unpacked) == 4):
-                            raise Exception("Unexpected RPC data format.")
+                            raise UnexpectedRPCDataFormatFromServer()
                         type, msgid, error, res = unpacked
                         if msgid in self.ignore_responses:
                             self.ignore_responses.remove(msgid)
diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py
index fd7eb5fc8..dfa540e01 100644
--- a/src/borg/testsuite/archiver.py
+++ b/src/borg/testsuite/archiver.py
@@ -285,8 +285,14 @@ class ArchiverTestCaseBase(BaseTestCase):
         # Symlink
         if are_symlinks_supported():
             os.symlink('somewhere', os.path.join(self.input_path, 'link1'))
-        if xattr.is_enabled(self.input_path):
-            xattr.setxattr(os.path.join(self.input_path, 'file1'), 'user.foo', b'bar')
+        self.create_regular_file('fusexattr', size=1)
+        if not xattr.XATTR_FAKEROOT and xattr.is_enabled(self.input_path):
+            # ironically, due to the way how fakeroot works, comparing fuse file xattrs to orig file xattrs
+            # will FAIL if fakeroot supports xattrs, thus we only set the xattr if XATTR_FAKEROOT is False.
+            # This is because fakeroot with xattr-support does not propagate xattrs of the underlying file
+            # into "fakeroot space". Because the xattrs exposed by borgfs are these of an underlying file
+            # (from fakeroots point of view) they are invisible to the test process inside the fakeroot.
+            xattr.setxattr(os.path.join(self.input_path, 'fusexattr'), 'user.foo', b'bar')
             # XXX this always fails for me
             # ubuntu 14.04, on a TMP dir filesystem with user_xattr, using fakeroot
             # same for newer ubuntu and centos.
@@ -362,7 +368,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
             self.assert_in(name, list_output)
         self.assert_dirs_equal('input', 'output/input')
         info_output = self.cmd('info', self.repository_location + '::test')
-        item_count = 3 if has_lchflags else 4  # one file is UF_NODUMP
+        item_count = 4 if has_lchflags else 5  # one file is UF_NODUMP
         self.assert_in('Number of files: %d' % item_count, info_output)
         shutil.rmtree(self.cache_path)
         with environment_variable(BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK='yes'):
@@ -1399,7 +1405,9 @@ class ArchiverTestCase(ArchiverTestCaseBase):
             with open(in_fn, 'rb') as in_f, open(out_fn, 'rb') as out_f:
                 assert in_f.read() == out_f.read()
             # list/read xattrs
-            if xattr.is_enabled(self.input_path):
+            in_fn = 'input/fusexattr'
+            out_fn = os.path.join(mountpoint, 'input', 'fusexattr')
+            if not xattr.XATTR_FAKEROOT and xattr.is_enabled(self.input_path):
                 assert xattr.listxattr(out_fn) == ['user.foo', ]
                 assert xattr.getxattr(out_fn, 'user.foo') == b'bar'
             else:
diff --git a/src/borg/testsuite/hashindex.py b/src/borg/testsuite/hashindex.py
index a7d757145..5ddb85171 100644
--- a/src/borg/testsuite/hashindex.py
+++ b/src/borg/testsuite/hashindex.py
@@ -24,19 +24,19 @@ class HashIndexTestCase(BaseTestCase):
         self.assert_equal(len(idx), 0)
         # Test set
         for x in range(100):
-            idx[bytes('%-32d' % x, 'ascii')] = make_value(x)
+            idx[H(x)] = make_value(x)
         self.assert_equal(len(idx), 100)
         for x in range(100):
-            self.assert_equal(idx[bytes('%-32d' % x, 'ascii')], make_value(x))
+            self.assert_equal(idx[H(x)], make_value(x))
         # Test update
         for x in range(100):
-            idx[bytes('%-32d' % x, 'ascii')] = make_value(x * 2)
+            idx[H(x)] = make_value(x * 2)
         self.assert_equal(len(idx), 100)
         for x in range(100):
-            self.assert_equal(idx[bytes('%-32d' % x, 'ascii')], make_value(x * 2))
+            self.assert_equal(idx[H(x)], make_value(x * 2))
         # Test delete
         for x in range(50):
-            del idx[bytes('%-32d' % x, 'ascii')]
+            del idx[H(x)]
         self.assert_equal(len(idx), 50)
         idx_name = tempfile.NamedTemporaryFile()
         idx.write(idx_name.name)
@@ -48,7 +48,7 @@ class HashIndexTestCase(BaseTestCase):
         idx = cls.read(idx_name.name)
         self.assert_equal(len(idx), 50)
         for x in range(50, 100):
-            self.assert_equal(idx[bytes('%-32d' % x, 'ascii')], make_value(x * 2))
+            self.assert_equal(idx[H(x)], make_value(x * 2))
         idx.clear()
         self.assert_equal(len(idx), 0)
         idx.write(idx_name.name)
@@ -57,11 +57,11 @@ class HashIndexTestCase(BaseTestCase):
 
     def test_nsindex(self):
         self._generic_test(NSIndex, lambda x: (x, x),
-                           '80fba5b40f8cf12f1486f1ba33c9d852fb2b41a5b5961d3b9d1228cf2aa9c4c9')
+                           'b96ec1ddabb4278cc92261ee171f7efc979dc19397cc5e89b778f05fa25bf93f')
 
     def test_chunkindex(self):
         self._generic_test(ChunkIndex, lambda x: (x, x, x),
-                           '1d71865e72e3c3af18d3c7216b6fa7b014695eaa3ed7f14cf9cd02fba75d1c95')
+                           '9d437a1e145beccc790c69e66ba94fc17bd982d83a401c9c6e524609405529d8')
 
     def test_resize(self):
         n = 2000  # Must be >= MIN_BUCKETS
@@ -71,11 +71,11 @@ class HashIndexTestCase(BaseTestCase):
         initial_size = os.path.getsize(idx_name.name)
         self.assert_equal(len(idx), 0)
         for x in range(n):
-            idx[bytes('%-32d' % x, 'ascii')] = x, x
+            idx[H(x)] = x, x
         idx.write(idx_name.name)
         self.assert_true(initial_size < os.path.getsize(idx_name.name))
         for x in range(n):
-            del idx[bytes('%-32d' % x, 'ascii')]
+            del idx[H(x)]
         self.assert_equal(len(idx), 0)
         idx.write(idx_name.name)
         self.assert_equal(initial_size, os.path.getsize(idx_name.name))
@@ -83,7 +83,7 @@ class HashIndexTestCase(BaseTestCase):
     def test_iteritems(self):
         idx = NSIndex()
         for x in range(100):
-            idx[bytes('%-0.32d' % x, 'ascii')] = x, x
+            idx[H(x)] = x, x
         all = list(idx.iteritems())
         self.assert_equal(len(all), 100)
         second_half = list(idx.iteritems(marker=all[49][0]))
diff --git a/src/borg/testsuite/repository.py b/src/borg/testsuite/repository.py
index 67c250e8d..620bcaf3e 100644
--- a/src/borg/testsuite/repository.py
+++ b/src/borg/testsuite/repository.py
@@ -45,21 +45,21 @@ class RepositoryTestCaseBase(BaseTestCase):
         self.repository = self.open(exclusive=exclusive)
 
     def add_keys(self):
-        self.repository.put(b'00000000000000000000000000000000', b'foo')
-        self.repository.put(b'00000000000000000000000000000001', b'bar')
-        self.repository.put(b'00000000000000000000000000000003', b'bar')
+        self.repository.put(H(0), b'foo')
+        self.repository.put(H(1), b'bar')
+        self.repository.put(H(3), b'bar')
         self.repository.commit()
-        self.repository.put(b'00000000000000000000000000000001', b'bar2')
-        self.repository.put(b'00000000000000000000000000000002', b'boo')
-        self.repository.delete(b'00000000000000000000000000000003')
+        self.repository.put(H(1), b'bar2')
+        self.repository.put(H(2), b'boo')
+        self.repository.delete(H(3))
 
 
 class RepositoryTestCase(RepositoryTestCaseBase):
 
     def test1(self):
         for x in range(100):
-            self.repository.put(('%-32d' % x).encode('ascii'), b'SOMEDATA')
-        key50 = ('%-32d' % 50).encode('ascii')
+            self.repository.put(H(x), b'SOMEDATA')
+        key50 = H(50)
         self.assert_equal(self.repository.get(key50), b'SOMEDATA')
         self.repository.delete(key50)
         self.assert_raises(Repository.ObjectNotFound, lambda: self.repository.get(key50))
@@ -70,69 +70,69 @@ class RepositoryTestCase(RepositoryTestCaseBase):
             for x in range(100):
                 if x == 50:
                     continue
-                self.assert_equal(repository2.get(('%-32d' % x).encode('ascii')), b'SOMEDATA')
+                self.assert_equal(repository2.get(H(x)), b'SOMEDATA')
 
     def test2(self):
         """Test multiple sequential transactions
         """
-        self.repository.put(b'00000000000000000000000000000000', b'foo')
-        self.repository.put(b'00000000000000000000000000000001', b'foo')
+        self.repository.put(H(0), b'foo')
+        self.repository.put(H(1), b'foo')
         self.repository.commit()
-        self.repository.delete(b'00000000000000000000000000000000')
-        self.repository.put(b'00000000000000000000000000000001', b'bar')
+        self.repository.delete(H(0))
+        self.repository.put(H(1), b'bar')
         self.repository.commit()
-        self.assert_equal(self.repository.get(b'00000000000000000000000000000001'), b'bar')
+        self.assert_equal(self.repository.get(H(1)), b'bar')
 
     def test_consistency(self):
         """Test cache consistency
         """
-        self.repository.put(b'00000000000000000000000000000000', b'foo')
-        self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo')
-        self.repository.put(b'00000000000000000000000000000000', b'foo2')
-        self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo2')
-        self.repository.put(b'00000000000000000000000000000000', b'bar')
-        self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'bar')
-        self.repository.delete(b'00000000000000000000000000000000')
-        self.assert_raises(Repository.ObjectNotFound, lambda: self.repository.get(b'00000000000000000000000000000000'))
+        self.repository.put(H(0), b'foo')
+        self.assert_equal(self.repository.get(H(0)), b'foo')
+        self.repository.put(H(0), b'foo2')
+        self.assert_equal(self.repository.get(H(0)), b'foo2')
+        self.repository.put(H(0), b'bar')
+        self.assert_equal(self.repository.get(H(0)), b'bar')
+        self.repository.delete(H(0))
+        self.assert_raises(Repository.ObjectNotFound, lambda: self.repository.get(H(0)))
 
     def test_consistency2(self):
         """Test cache consistency2
         """
-        self.repository.put(b'00000000000000000000000000000000', b'foo')
-        self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo')
+        self.repository.put(H(0), b'foo')
+        self.assert_equal(self.repository.get(H(0)), b'foo')
         self.repository.commit()
-        self.repository.put(b'00000000000000000000000000000000', b'foo2')
-        self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo2')
+        self.repository.put(H(0), b'foo2')
+        self.assert_equal(self.repository.get(H(0)), b'foo2')
         self.repository.rollback()
-        self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo')
+        self.assert_equal(self.repository.get(H(0)), b'foo')
 
     def test_overwrite_in_same_transaction(self):
         """Test cache consistency2
         """
-        self.repository.put(b'00000000000000000000000000000000', b'foo')
-        self.repository.put(b'00000000000000000000000000000000', b'foo2')
+        self.repository.put(H(0), b'foo')
+        self.repository.put(H(0), b'foo2')
         self.repository.commit()
-        self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo2')
+        self.assert_equal(self.repository.get(H(0)), b'foo2')
 
     def test_single_kind_transactions(self):
         # put
-        self.repository.put(b'00000000000000000000000000000000', b'foo')
+        self.repository.put(H(0), b'foo')
         self.repository.commit()
         self.repository.close()
         # replace
         self.repository = self.open()
         with self.repository:
-            self.repository.put(b'00000000000000000000000000000000', b'bar')
+            self.repository.put(H(0), b'bar')
             self.repository.commit()
         # delete
         self.repository = self.open()
         with self.repository:
-            self.repository.delete(b'00000000000000000000000000000000')
+            self.repository.delete(H(0))
             self.repository.commit()
 
     def test_list(self):
         for x in range(100):
-            self.repository.put(('%-32d' % x).encode('ascii'), b'SOMEDATA')
+            self.repository.put(H(x), b'SOMEDATA')
         all = self.repository.list()
         self.assert_equal(len(all), 100)
         first_half = self.repository.list(limit=50)
@@ -145,10 +145,10 @@ class RepositoryTestCase(RepositoryTestCaseBase):
 
     def test_max_data_size(self):
         max_data = b'x' * MAX_DATA_SIZE
-        self.repository.put(b'00000000000000000000000000000000', max_data)
-        self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), max_data)
+        self.repository.put(H(0), max_data)
+        self.assert_equal(self.repository.get(H(0)), max_data)
         self.assert_raises(IntegrityError,
-                           lambda: self.repository.put(b'00000000000000000000000000000001', max_data + b'x'))
+                           lambda: self.repository.put(H(1), max_data + b'x'))
 
 
 class LocalRepositoryTestCase(RepositoryTestCaseBase):
@@ -163,22 +163,22 @@ class LocalRepositoryTestCase(RepositoryTestCaseBase):
         assert self.repository.compact[0] == 41 + 9
 
     def test_sparse1(self):
-        self.repository.put(b'00000000000000000000000000000000', b'foo')
-        self.repository.put(b'00000000000000000000000000000001', b'123456789')
+        self.repository.put(H(0), b'foo')
+        self.repository.put(H(1), b'123456789')
         self.repository.commit()
-        self.repository.put(b'00000000000000000000000000000001', b'bar')
+        self.repository.put(H(1), b'bar')
         self._assert_sparse()
 
     def test_sparse2(self):
-        self.repository.put(b'00000000000000000000000000000000', b'foo')
-        self.repository.put(b'00000000000000000000000000000001', b'123456789')
+        self.repository.put(H(0), b'foo')
+        self.repository.put(H(1), b'123456789')
         self.repository.commit()
-        self.repository.delete(b'00000000000000000000000000000001')
+        self.repository.delete(H(1))
         self._assert_sparse()
 
     def test_sparse_delete(self):
-        self.repository.put(b'00000000000000000000000000000000', b'1245')
-        self.repository.delete(b'00000000000000000000000000000000')
+        self.repository.put(H(0), b'1245')
+        self.repository.delete(H(0))
         self.repository.io._write_fd.sync()
 
         # The on-line tracking works on a per-object basis...
@@ -267,7 +267,7 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
             self.assert_equal(len(self.repository), 3)
 
     def test_ignores_commit_tag_in_data(self):
-        self.repository.put(b'0' * 32, LoggedIO.COMMIT)
+        self.repository.put(H(0), LoggedIO.COMMIT)
         self.reopen()
         with self.repository:
             io = self.repository.io
@@ -359,19 +359,19 @@ class RepositoryAppendOnlyTestCase(RepositoryTestCaseBase):
     def test_append_only(self):
         def segments_in_repository():
             return len(list(self.repository.io.segment_iterator()))
-        self.repository.put(b'00000000000000000000000000000000', b'foo')
+        self.repository.put(H(0), b'foo')
         self.repository.commit()
 
         self.repository.append_only = False
         assert segments_in_repository() == 2
-        self.repository.put(b'00000000000000000000000000000000', b'foo')
+        self.repository.put(H(0), b'foo')
         self.repository.commit()
         # normal: compact squashes the data together, only one segment
         assert segments_in_repository() == 4
 
         self.repository.append_only = True
         assert segments_in_repository() == 4
-        self.repository.put(b'00000000000000000000000000000000', b'foo')
+        self.repository.put(H(0), b'foo')
         self.repository.commit()
         # append only: does not compact, only new segments written
         assert segments_in_repository() == 6
@@ -385,7 +385,7 @@ class RepositoryFreeSpaceTestCase(RepositoryTestCaseBase):
         self.reopen()
 
         with self.repository:
-            self.repository.put(b'00000000000000000000000000000000', b'foobar')
+            self.repository.put(H(0), b'foobar')
             with pytest.raises(Repository.InsufficientFreeSpaceError):
                 self.repository.commit()
 
@@ -393,13 +393,13 @@ class RepositoryFreeSpaceTestCase(RepositoryTestCaseBase):
 class RepositoryAuxiliaryCorruptionTestCase(RepositoryTestCaseBase):
     def setUp(self):
         super().setUp()
-        self.repository.put(b'00000000000000000000000000000000', b'foo')
+        self.repository.put(H(0), b'foo')
         self.repository.commit()
         self.repository.close()
 
     def do_commit(self):
         with self.repository:
-            self.repository.put(b'00000000000000000000000000000000', b'fox')
+            self.repository.put(H(0), b'fox')
             self.repository.commit()
 
     def test_corrupted_hints(self):
@@ -453,12 +453,12 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase):
 
     def get_objects(self, *ids):
         for id_ in ids:
-            self.repository.get(('%032d' % id_).encode('ascii'))
+            self.repository.get(H(id_))
 
     def add_objects(self, segments):
         for ids in segments:
             for id_ in ids:
-                self.repository.put(('%032d' % id_).encode('ascii'), b'data')
+                self.repository.put(H(id_), b'data')
             self.repository.commit()
 
     def get_head(self):
@@ -469,7 +469,7 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase):
 
     def corrupt_object(self, id_):
         idx = self.open_index()
-        segment, offset = idx[('%032d' % id_).encode('ascii')]
+        segment, offset = idx[H(id_)]
         with open(os.path.join(self.tmppath, 'repository', 'data', '0', str(segment)), 'r+b') as fd:
             fd.seek(offset)
             fd.write(b'BOOM')
@@ -560,8 +560,8 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase):
         self.assert_equal(set([1, 2, 3, 4, 5, 6]), self.list_objects())
 
     def test_crash_before_compact(self):
-        self.repository.put(bytes(32), b'data')
-        self.repository.put(bytes(32), b'data2')
+        self.repository.put(H(0), b'data')
+        self.repository.put(H(0), b'data2')
         # Simulate a crash before compact
         with patch.object(Repository, 'compact_segments') as compact:
             self.repository.commit()
@@ -569,7 +569,7 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase):
         self.reopen()
         with self.repository:
             self.check(repair=True)
-            self.assert_equal(self.repository.get(bytes(32)), b'data2')
+            self.assert_equal(self.repository.get(H(0)), b'data2')
 
 
 class RemoteRepositoryTestCase(RepositoryTestCase):