diff options
author | Thomas Waldmann <tw@waldmann-edv.de> | 2016-07-11 01:23:27 +0200 |
---|---|---|
committer | Thomas Waldmann <tw@waldmann-edv.de> | 2016-07-11 01:23:27 +0200 |
commit | 3baa8a3728e3a94c8531edc83ae2a7e1d16e64bc (patch) | |
tree | 01f6de438c3d663bbedf3b5a982e70615faf9555 | |
parent | f0930b89cee2e389eb33a28184e543511ccbe78e (diff) | |
parent | b8198c4ff1f1c61422ffcf32e48e15d600e5ea8a (diff) |
Merge branch '1.0-maint'
# Conflicts:
# docs/changes.rst
# docs/usage/mount.rst.inc
# src/borg/archive.py
# src/borg/archiver.py
# src/borg/fuse.py
# src/borg/testsuite/archiver.py
-rw-r--r-- | docs/api.rst | 38 | ||||
-rw-r--r-- | docs/changes.rst | 113 | ||||
-rw-r--r-- | docs/conf.py | 2 | ||||
-rw-r--r-- | docs/resources.rst | 18 | ||||
-rw-r--r-- | docs/usage/check.rst.inc | 9 | ||||
-rw-r--r-- | docs/usage/help.rst.inc | 78 | ||||
-rw-r--r-- | docs/usage/mount.rst.inc | 7 | ||||
-rw-r--r-- | setup.py | 2 | ||||
-rw-r--r-- | src/borg/_hashindex.c | 8 | ||||
-rw-r--r-- | src/borg/archive.py | 61 | ||||
-rw-r--r-- | src/borg/archiver.py | 22 | ||||
-rw-r--r-- | src/borg/fuse.py | 70 | ||||
-rw-r--r-- | src/borg/hashindex.pyx | 5 | ||||
-rw-r--r-- | src/borg/helpers.py | 5 | ||||
-rw-r--r-- | src/borg/locking.py | 16 | ||||
-rw-r--r-- | src/borg/shellpattern.py | 2 | ||||
-rw-r--r-- | src/borg/testsuite/__init__.py | 18 | ||||
-rw-r--r-- | src/borg/testsuite/archiver.py | 182 |
18 files changed, 434 insertions, 222 deletions
diff --git a/docs/api.rst b/docs/api.rst index 718cb58e0..31a22fbb3 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -6,63 +6,55 @@ API Documentation :members: :undoc-members: -.. automodule:: borg.upgrader - :members: - :undoc-members: - .. automodule:: borg.archive :members: :undoc-members: -.. automodule:: borg.fuse - :members: - :undoc-members: - -.. automodule:: borg.platform +.. automodule:: borg.repository :members: :undoc-members: -.. automodule:: borg.locking +.. automodule:: borg.remote :members: :undoc-members: -.. automodule:: borg.shellpattern +.. automodule:: borg.cache :members: :undoc-members: -.. automodule:: borg.repository +.. automodule:: borg.key :members: :undoc-members: -.. automodule:: borg.lrucache +.. automodule:: borg.logger :members: :undoc-members: -.. automodule:: borg.remote +.. automodule:: borg.helpers :members: :undoc-members: -.. automodule:: borg.xattr +.. automodule:: borg.locking :members: :undoc-members: -.. automodule:: borg.helpers +.. automodule:: borg.shellpattern :members: :undoc-members: -.. automodule:: borg.cache +.. automodule:: borg.lrucache :members: :undoc-members: -.. automodule:: borg.key +.. automodule:: borg.fuse :members: :undoc-members: -.. automodule:: borg.logger +.. automodule:: borg.xattr :members: :undoc-members: -.. automodule:: borg.platform_darwin +.. automodule:: borg.platform :members: :undoc-members: @@ -75,7 +67,7 @@ API Documentation :undoc-members: .. automodule:: borg.compress - :members: + :members: get_compressor, Compressor, CompressorBase :undoc-members: .. automodule:: borg.chunker @@ -85,7 +77,3 @@ API Documentation .. automodule:: borg.crypto :members: :undoc-members: - -.. automodule:: borg.platform_freebsd - :members: - :undoc-members: diff --git a/docs/changes.rst b/docs/changes.rst index 5267551c7..15479a5ab 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -123,6 +123,51 @@ Other changes: - ChunkBuffer: add test for leaving partial chunk in buffer, fixes #945 +Version 1.0.6rc1 (2016-07-10) +----------------------------- + +New features: + +- borg check --repair: heal damaged files if missing chunks re-appear (e.g. if + the previously missing chunk was added again in a later backup archive), + #148. (*) Also improved logging. + +Bug fixes: + +- sync_dir: silence fsync() failing with EINVAL, #1287 + Some network filesystems (like smbfs) don't support this and we use this in + repository code. +- borg mount (FUSE): + + - fix directories being shadowed when contained paths were also specified, + #1295 + - raise I/O Error (EIO) on damaged files (unless -o allow_damaged_files is + used), #1302. (*) +- borg extract: warn if a damaged file is extracted, #1299. (*) +- Added some missing return code checks (ChunkIndex._add, hashindex_resize). +- borg check: fix/optimize initial hash table size, avoids resize of the table. + +Other changes: + +- tests: + + - add more FUSE tests, #1284 + - deduplicate fuse (u)mount code + - fix borg binary test issues, #862 +- docs: + + - changelog: added release dates to older borg releases + - fix some sphinx (docs generator) warnings, #881 + +Notes: + +(*) Some features depend on information (chunks_healthy list) added to item +metadata when a file with missing chunks was "repaired" using all-zero +replacement chunks. The chunks_healthy list is generated since borg 1.0.4, +thus borg can't recognize such "repaired" (but content-damaged) files if the +repair was done with an older borg version. + + Version 1.0.5 (2016-07-07) -------------------------- @@ -247,8 +292,8 @@ Other changes: - add .eggs to .gitignore -Version 1.0.3 -------------- +Version 1.0.3 (2016-05-20) +-------------------------- Bug fixes: @@ -277,8 +322,8 @@ Other changes: - borg create help: document format tags, #894 -Version 1.0.2 -------------- +Version 1.0.2 (2016-04-16) +-------------------------- Bug fixes: @@ -313,8 +358,8 @@ Other changes: - fix confusing usage of "repo" as archive name (use "arch") -Version 1.0.1 -------------- +Version 1.0.1 (2016-04-08) +-------------------------- New features: @@ -365,8 +410,8 @@ Other changes: - Document logo font. Recreate logo png. Remove GIMP logo file. -Version 1.0.0 -------------- +Version 1.0.0 (2016-03-05) +-------------------------- The major release number change (0.x -> 1.x) indicates bigger incompatible changes, please read the compatibility notes, adapt / test your scripts and @@ -449,8 +494,8 @@ Other changes: - FAQ: how to limit bandwidth -Version 1.0.0rc2 ----------------- +Version 1.0.0rc2 (2016-02-28) +----------------------------- New features: @@ -491,8 +536,8 @@ Other changes: - "connection closed by remote": add FAQ entry and point to issue #636 -Version 1.0.0rc1 ----------------- +Version 1.0.0rc1 (2016-02-07) +----------------------------- New features: @@ -541,8 +586,8 @@ Other changes: - misc. updates and fixes -Version 0.30.0 --------------- +Version 0.30.0 (2016-01-23) +--------------------------- Compatibility notes: @@ -619,8 +664,8 @@ Other changes: - add gcc gcc-c++ to redhat/fedora/corora install docs, fixes #583 -Version 0.29.0 --------------- +Version 0.29.0 (2015-12-13) +--------------------------- Compatibility notes: @@ -695,8 +740,8 @@ Other changes: - fix wrong installation instructions for archlinux -Version 0.28.2 --------------- +Version 0.28.2 (2015-11-15) +--------------------------- New features: @@ -719,8 +764,8 @@ Other changes: - minor install docs improvements -Version 0.28.1 --------------- +Version 0.28.1 (2015-11-08) +--------------------------- Bug fixes: @@ -734,8 +779,8 @@ Other changes: - fix build on readthedocs -Version 0.28.0 --------------- +Version 0.28.0 (2015-11-08) +--------------------------- Compatibility notes: @@ -832,8 +877,8 @@ Other changes: - minor development docs update -Version 0.27.0 --------------- +Version 0.27.0 (2015-10-07) +--------------------------- New features: @@ -867,8 +912,8 @@ Other changes: - hint to single-file pyinstaller binaries from README -Version 0.26.1 --------------- +Version 0.26.1 (2015-09-28) +--------------------------- This is a minor update, just docs and new pyinstaller binaries. @@ -880,8 +925,8 @@ This is a minor update, just docs and new pyinstaller binaries. Note: if you did a python-based installation, there is no need to upgrade. -Version 0.26.0 --------------- +Version 0.26.0 (2015-09-19) +--------------------------- New features: @@ -941,8 +986,8 @@ Other changes: - Darwin (OS X Yosemite) -Version 0.25.0 --------------- +Version 0.25.0 (2015-08-29) +--------------------------- Compatibility notes: @@ -1008,8 +1053,8 @@ Other changes: - split install docs into system-specific preparations and generic instructions -Version 0.24.0 --------------- +Version 0.24.0 (2015-08-09) +--------------------------- Incompatible changes (compared to 0.23): @@ -1112,8 +1157,8 @@ Other changes: - some easy micro optimizations -Version 0.23.0 --------------- +Version 0.23.0 (2015-06-11) +--------------------------- Incompatible changes (compared to attic, fork related): diff --git a/docs/conf.py b/docs/conf.py index a90c8c936..516702751 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -55,6 +55,8 @@ version = sw_version.split('-')[0] # The full version, including alpha/beta/rc tags. release = version +suppress_warnings = ['image.nonlocal_uri'] + # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. #language = None diff --git a/docs/resources.rst b/docs/resources.rst index 4ae184946..2ef0f021f 100644 --- a/docs/resources.rst +++ b/docs/resources.rst @@ -17,21 +17,15 @@ Some of them refer to attic, but you can do the same stuff (and more) with borgb - `TW's slides for borgbackup talks / lightning talks <https://slides.com/thomaswaldmann>`_ (just grab the latest ones) -- "Attic / Borg Backup" talk from GPN 2015 (video, german audio, english slides): - `media.ccc.de <https://media.ccc.de/browse/conferences/gpn/gpn15/gpn15-6942-attic_borg_backup.html#video>`_ - or - `youtube <https://www.youtube.com/watch?v=Nb5nXEKSN-k>`_ +- `Attic / Borg Backup talk from GPN 2015 (media.ccc.de) <https://media.ccc.de/browse/conferences/gpn/gpn15/gpn15-6942-attic_borg_backup.html#video>`_ +- `Attic / Borg Backup talk from GPN 2015 (youtube) <https://www.youtube.com/watch?v=Nb5nXEKSN-k>`_ -- "Attic" talk from Easterhegg 2015 (video, german audio, english slides): - `media.ccc.de <https://media.ccc.de/v/eh15_-_49_-__-_saal_-_201504042130_-_attic_-_the_holy_grail_of_backups_-_thomas#video>`_ - or - `youtube <https://www.youtube.com/watch?v=96VEAAFDtJw>`_ +- `Attic talk from Easterhegg 2015 (media.ccc.de) <https://media.ccc.de/v/eh15_-_49_-__-_saal_-_201504042130_-_attic_-_the_holy_grail_of_backups_-_thomas#video>`_ +- `Attic talk from Easterhegg 2015 (youtube) <https://www.youtube.com/watch?v=96VEAAFDtJw>`_ -- "Attic Backup: Mount your encrypted backups over ssh", 2014 (video, english): - `youtube <https://www.youtube.com/watch?v=BVXDFv9YMp8>`_ +- `Attic Backup: Mount your encrypted backups over ssh (youtube) <https://www.youtube.com/watch?v=BVXDFv9YMp8>`_ -- "Evolution of Borg", Oct 2015 (gource visualization of attic and borg development): - `youtube <https://www.youtube.com/watch?v=K4k_4wDkG6Q>`_ +- `Evolution of Borg (youtube) <https://www.youtube.com/watch?v=K4k_4wDkG6Q>`_ Software -------- diff --git a/docs/usage/check.rst.inc b/docs/usage/check.rst.inc index 187172ffd..17d6cf492 100644 --- a/docs/usage/check.rst.inc +++ b/docs/usage/check.rst.inc @@ -59,9 +59,12 @@ Second, the consistency and correctness of the archive metadata is verified: - Check if archive metadata chunk is present. if not, remove archive from manifest. - For all files (items) in the archive, for all chunks referenced by these - files, check if chunk is present (if not and we are in repair mode, replace - it with a same-size chunk of zeros). This requires reading of archive and - file metadata, but not data. + files, check if chunk is present. + If a chunk is not present and we are in repair mode, replace it with a same-size + replacement chunk of zeros. + If a previously lost chunk reappears (e.g. via a later backup) and we are in + repair mode, the all-zero replacement chunk will be replaced by the correct chunk. + This requires reading of archive and file metadata, but not data. - If we are in repair mode and we checked all the archives: delete orphaned chunks from the repo. - if you use a remote repo server via ssh:, the archive check is executed on diff --git a/docs/usage/help.rst.inc b/docs/usage/help.rst.inc index 30d8c4591..f6618b677 100644 --- a/docs/usage/help.rst.inc +++ b/docs/usage/help.rst.inc @@ -1,48 +1,9 @@ .. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! -.. _borg_placeholders: - -borg help placeholders -~~~~~~~~~~~~~~~~~~~~~~ -:: - - -Repository (or Archive) URLs and --prefix values support these placeholders: - -{hostname} - - The (short) hostname of the machine. - -{fqdn} - - The full name of the machine. - -{now} - - The current local date and time. - -{utcnow} - - The current UTC date and time. - -{user} - - The user name (or UID, if no name is available) of the user running borg. - -{pid} - - The current process ID. - -Examples:: - - borg create /path/to/repo::{hostname}-{user}-{utcnow} ... - borg create /path/to/repo::{hostname}-{now:%Y-%m-%d_%H:%M:%S} ... - borg prune --prefix '{hostname}-' ... .. _borg_patterns: borg help patterns ~~~~~~~~~~~~~~~~~~ -:: Exclusion patterns support four separate styles, fnmatch, shell, regular @@ -131,3 +92,42 @@ Examples:: sh:/home/*/.thumbnails EOF $ borg create --exclude-from exclude.txt backup / + +.. _borg_placeholders: + +borg help placeholders +~~~~~~~~~~~~~~~~~~~~~~ + + +Repository (or Archive) URLs and --prefix values support these placeholders: + +{hostname} + + The (short) hostname of the machine. + +{fqdn} + + The full name of the machine. + +{now} + + The current local date and time. + +{utcnow} + + The current UTC date and time. + +{user} + + The user name (or UID, if no name is available) of the user running borg. + +{pid} + + The current process ID. + +Examples:: + + borg create /path/to/repo::{hostname}-{user}-{utcnow} ... + borg create /path/to/repo::{hostname}-{now:%Y-%m-%d_%H:%M:%S} ... + borg prune --prefix '{hostname}-' ... + diff --git a/docs/usage/mount.rst.inc b/docs/usage/mount.rst.inc index 83310d8b5..cac84a437 100644 --- a/docs/usage/mount.rst.inc +++ b/docs/usage/mount.rst.inc @@ -38,6 +38,13 @@ used in fstab entries: To allow a regular user to use fstab entries, add the ``user`` option: ``/path/to/repo /mnt/point fuse.borgfs defaults,noauto,user 0 0`` +For mount options, see the fuse(8) manual page. Additional mount options +supported by borg: + +- allow_damaged_files: by default damaged files (where missing chunks were + replaced with runs of zeros by borg check --repair) are not readable and + return EIO (I/O error). Set this option to read such files. + The BORG_MOUNT_DATA_CACHE_ENTRIES environment variable is meant for advanced users to tweak the performance. It sets the number of cached data chunks; additional memory usage can be up to ~8 MiB times this number. The default is the number @@ -187,7 +187,7 @@ class build_usage(Command): params = {"topic": topic, "underline": '~' * len('borg help ' + topic)} doc.write(".. _borg_{topic}:\n\n".format(**params)) - doc.write("borg help {topic}\n{underline}\n::\n\n".format(**params)) + doc.write("borg help {topic}\n{underline}\n\n".format(**params)) doc.write(Archiver.helptext[topic]) else: params = {"command": command, diff --git a/src/borg/_hashindex.c b/src/borg/_hashindex.c index 4599fe0fa..bfa3ef09b 100644 --- a/src/borg/_hashindex.c +++ b/src/borg/_hashindex.c @@ -100,6 +100,8 @@ static int hashindex_delete(HashIndex *index, const void *key); static void *hashindex_next_key(HashIndex *index, const void *key); /* Private API */ +static void hashindex_free(HashIndex *index); + static int hashindex_index(HashIndex *index, const void *key) { @@ -148,7 +150,11 @@ hashindex_resize(HashIndex *index, int capacity) return 0; } while((key = hashindex_next_key(index, key))) { - hashindex_set(new, key, key + key_size); + if(!hashindex_set(new, key, key + key_size)) { + /* This can only happen if there's a bug in the code calculating capacity */ + hashindex_free(new); + return 0; + } } free(index->buckets); index->buckets = new->buckets; diff --git a/src/borg/archive.py b/src/borg/archive.py index 90df7f162..b4572772d 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -419,6 +419,7 @@ Number of files: {0.stats.nfiles}'''.format( :param hardlink_masters: maps paths to (chunks, link_target) for extracting subtrees with hardlinks correctly :param original_path: 'path' key as stored in archive """ + has_damaged_chunks = 'chunks_healthy' in item if dry_run or stdout: if 'chunks' in item: for _, data in self.pipeline.fetch_many([c.id for c in item.chunks], is_preloaded=True): @@ -426,6 +427,9 @@ Number of files: {0.stats.nfiles}'''.format( sys.stdout.buffer.write(data) if stdout: sys.stdout.buffer.flush() + if has_damaged_chunks: + logger.warning('File %s has damaged (all-zero) chunks. Try running borg check --repair.' % + remove_surrogates(item[b'path'])) return original_path = original_path or item.path @@ -481,6 +485,9 @@ Number of files: {0.stats.nfiles}'''.format( fd.truncate(pos) fd.flush() self.restore_attrs(path, item, fd=fd.fileno()) + if has_damaged_chunks: + logger.warning('File %s has damaged (all-zero) chunks. Try running borg check --repair.' % + remove_surrogates(item.path)) if hardlink_masters: # Update master entry with extracted file path, so that following hardlinks don't extract twice. hardlink_masters[item.get('source') or original_path] = (None, path) @@ -924,7 +931,7 @@ class ArchiveChecker: """ # Explicitly set the initial hash table capacity to avoid performance issues # due to hash table "resonance" - capacity = int(len(self.repository) * 1.2) + capacity = int(len(self.repository) * 1.35 + 1) # > len * 1.0 / HASH_MAX_LOAD (see _hashindex.c) self.chunks = ChunkIndex(capacity) marker = None while True: @@ -1033,31 +1040,53 @@ class ArchiveChecker: self.repository.put(id_, cdata) def verify_file_chunks(item): - """Verifies that all file chunks are present + """Verifies that all file chunks are present. - Missing file chunks will be replaced with new chunks of the same - length containing all zeros. + Missing file chunks will be replaced with new chunks of the same length containing all zeros. + If a previously missing file chunk re-appears, the replacement chunk is replaced by the correct one. """ offset = 0 chunk_list = [] chunks_replaced = False - for chunk_id, size, csize in item.chunks: + has_chunks_healthy = 'chunks_healthy' in item + chunks_current = item.chunks + chunks_healthy = item.chunks_healthy if has_chunks_healthy else chunks_current + assert len(chunks_current) == len(chunks_healthy) + for chunk_current, chunk_healthy in zip(chunks_current, chunks_healthy): + chunk_id, size, csize = chunk_healthy if chunk_id not in self.chunks: - # If a file chunk is missing, create an all empty replacement chunk - logger.error('{}: Missing file chunk detected (Byte {}-{})'.format(item.path, offset, offset + size)) - self.error_found = chunks_replaced = True - data = bytes(size) - chunk_id = self.key.id_hash(data) - cdata = self.key.encrypt(Chunk(data)) - csize = len(cdata) - add_reference(chunk_id, size, csize, cdata) + # a chunk of the healthy list is missing + if chunk_current == chunk_healthy: + logger.error('{}: New missing file chunk detected (Byte {}-{}). ' + 'Replacing with all-zero chunk.'.format(item.path, offset, offset + size)) + self.error_found = chunks_replaced = True + data = bytes(size) + chunk_id = self.key.id_hash(data) + cdata = self.key.encrypt(Chunk(data)) + csize = len(cdata) + add_reference(chunk_id, size, csize, cdata) + else: + logger.info('{}: Previously missing file chunk is still missing (Byte {}-{}). It has a ' + 'all-zero replacement chunk already.'.format(item.path, offset, offset + size)) + chunk_id, size, csize = chunk_current + add_reference(chunk_id, size, csize) else: - add_reference(chunk_id, size, csize) - chunk_list.append((chunk_id, size, csize)) + if chunk_current == chunk_healthy: + # normal case, all fine. + add_reference(chunk_id, size, csize) + else: + logger.info('{}: Healed previously missing file chunk! ' + '(Byte {}-{}).'.format(item.path, offset, offset + size)) + add_reference(chunk_id, size, csize) + mark_as_possibly_superseded(chunk_current[0]) # maybe orphaned the all-zero replacement chunk + chunk_list.append([chunk_id, size, csize]) # list-typed element as chunks_healthy is list-of-lists offset += size - if chunks_replaced and 'chunks_healthy' not in item: + if chunks_replaced and not has_chunks_healthy: # if this is first repair, remember the correct chunk IDs, so we can maybe heal the file later item.chunks_healthy = item.chunks + if has_chunks_healthy and chunk_list == chunks_healthy: + logger.info('{}: Completely healed previously damaged file!'.format(item.path)) + del item.chunks_healthy item.chunks = chunk_list def robust_iterator(archive): diff --git a/src/borg/archiver.py b/src/borg/archiver.py index d010022a1..b2e98fb6e 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -1134,8 +1134,7 @@ class Archiver: re:^/home/[^/]\.tmp/ sh:/home/*/.thumbnails EOF - $ borg create --exclude-from exclude.txt backup / - ''') + $ borg create --exclude-from exclude.txt backup /\n\n''') helptext['placeholders'] = textwrap.dedent(''' Repository (or Archive) URLs and --prefix values support these placeholders: @@ -1167,8 +1166,7 @@ class Archiver: borg create /path/to/repo::{hostname}-{user}-{utcnow} ... borg create /path/to/repo::{hostname}-{now:%Y-%m-%d_%H:%M:%S} ... - borg prune --prefix '{hostname}-' ... - ''') + borg prune --prefix '{hostname}-' ...\n\n''') def do_help(self, parser, commands, args): if not args.topic: @@ -1331,9 +1329,12 @@ class Archiver: - Check if archive metadata chunk is present. if not, remove archive from manifest. - For all files (items) in the archive, for all chunks referenced by these - files, check if chunk is present (if not and we are in repair mode, replace - it with a same-size chunk of zeros). This requires reading of archive and - file metadata, but not data. + files, check if chunk is present. + If a chunk is not present and we are in repair mode, replace it with a same-size + replacement chunk of zeros. + If a previously lost chunk reappears (e.g. via a later backup) and we are in + repair mode, the all-zero replacement chunk will be replaced by the correct chunk. + This requires reading of archive and file metadata, but not data. - If we are in repair mode and we checked all the archives: delete orphaned chunks from the repo. - if you use a remote repo server via ssh:, the archive check is executed on @@ -1739,6 +1740,13 @@ class Archiver: To allow a regular user to use fstab entries, add the ``user`` option: ``/path/to/repo /mnt/point fuse.borgfs defaults,noauto,user 0 0`` + For mount options, see the fuse(8) manual page. Additional mount options + supported by borg: + + - allow_damaged_files: by default damaged files (where missing chunks were + replaced with runs of zeros by borg check --repair) are not readable and + return EIO (I/O error). Set this option to read such files. + The BORG_MOUNT_DATA_CACHE_ENTRIES environment variable is meant for advanced users to tweak the performance. It sets the number of cached data chunks; additional memory usage can be up to ~8 MiB times this number. The default is the number diff --git a/src/borg/fuse.py b/src/borg/fuse.py index eeb4c7f80..93530f892 100644 --- a/src/borg/fuse.py +++ b/src/borg/fuse.py @@ -14,7 +14,7 @@ from .logger import create_logger logger = create_logger() from .archive import Archive -from .helpers import daemonize +from .helpers import daemonize, safe_encode from .item import Item from .lrucache import LRUCache @@ -50,6 +50,9 @@ class ItemCache: class FuseOperations(llfuse.Operations): """Export archive as a fuse filesystem """ + + allow_damaged_files = False + def __init__(self, key, repository, manifest, archive, cached_repo): super().__init__() self._inode_count = 0 @@ -79,6 +82,32 @@ class FuseOperations(llfuse.Operations): self.contents[1][os.fsencode(archive_name)] = archive_inode self.pending_archives[archive_inode] = Archive(repository, key, manifest, archive_name) + def mount(self, mountpoint, mount_options, foreground=False): + """Mount filesystem on *mountpoint* with *mount_options*.""" + options = ['fsname=borgfs', 'ro'] + if mount_options: + options.extend(mount_options.split(',')) + try: + options.remove('allow_damaged_files') + self.allow_damaged_files = True + except ValueError: + pass + llfuse.init(self, mountpoint, options) + if not foreground: + daemonize() + + # If the file system crashes, we do not want to umount because in that + # case the mountpoint suddenly appears to become empty. This can have + # nasty consequences, imagine the user has e.g. an active rsync mirror + # job - seeing the mountpoint empty, rsync would delete everything in the + # mirror. + umount = False + try: + signal = fuse_main() + umount = (signal is None) # no crash and no signal -> umount request + finally: + llfuse.close(umount) + def process_archive(self, archive, prefix=[]): """Build fuse inode hierarchy from archive metadata """ @@ -88,6 +117,16 @@ class FuseOperations(llfuse.Operations): unpacker.feed(data) for item in unpacker: item = Item(internal_dict=item) + try: + # This can happen if an archive was created with a command line like + # $ borg create ... dir1/file dir1 + # In this case the code below will have created a default_dir inode for dir1 already. + inode = self._find_inode(safe_encode(item.path), prefix) + except KeyError: + pass + else: + self.items[inode] = item + continue segments = prefix + os.fsencode(os.path.normpath(item.path)).split(b'/') del item.path num_segments = len(segments) @@ -214,6 +253,15 @@ class FuseOperations(llfuse.Operations): return self.getattr(inode) def open(self, inode, flags, ctx=None): + if not self.allow_damaged_files: + item = self.get_item(inode) + if 'chunks_healthy' in item: + # Processed archive items don't carry the path anymore; for converting the inode + # to the path we'd either have to store the inverse of the current structure, + # or search the entire archive. So we just don't print it. It's easy to correlate anyway. + logger.warning('File has damaged (all-zero) chunks. Try running borg check --repair. ' + 'Mount with allow_damaged_files to read damaged files.') + raise llfuse.FUSEError(errno.EIO) return inode def opendir(self, inode, ctx=None): @@ -254,23 +302,3 @@ class FuseOperations(llfuse.Operations): def readlink(self, inode, ctx=None): item = self.get_item(inode) return os.fsencode(item.source) - - def mount(self, mountpoint, extra_options, foreground=False): - options = ['fsname=borgfs', 'ro'] - if extra_options: - options.extend(extra_options.split(',')) - llfuse.init(self, mountpoint, options) - if not foreground: - daemonize() - - # If the file system crashes, we do not want to umount because in that - # case the mountpoint suddenly appears to become empty. This can have - # nasty consequences, imagine the user has e.g. an active rsync mirror - # job - seeing the mountpoint empty, rsync would delete everything in the - # mirror. - umount = False - try: - signal = fuse_main() - umount = (signal is None) # no crash and no signal -> umount request - finally: - llfuse.close(umount) diff --git a/src/borg/hashindex.pyx b/src/borg/hashindex.pyx index 724f2ee84..389cf2563 100644 --- a/src/borg/hashindex.pyx +++ b/src/borg/hashindex.pyx @@ -18,8 +18,6 @@ cdef extern from "_hashindex.c": HashIndex *hashindex_read(char *path) HashIndex *hashindex_init(int capacity, int key_size, int value_size) void hashindex_free(HashIndex *index) - void hashindex_merge(HashIndex *index, HashIndex *other) - void hashindex_add(HashIndex *index, void *key, void *value) int hashindex_get_size(HashIndex *index) int hashindex_write(HashIndex *index, char *path) void *hashindex_get(HashIndex *index, void *key) @@ -323,7 +321,8 @@ cdef class ChunkIndex(IndexBase): values[1] = data[1] values[2] = data[2] else: - hashindex_set(self.index, key, data) + if not hashindex_set(self.index, key, data): + raise Exception('hashindex_set failed') def merge(self, ChunkIndex other): cdef void *key = NULL diff --git a/src/borg/helpers.py b/src/borg/helpers.py index cf4d14e05..401f1ea7c 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py |