From 5788219ff43de193f89d4fa48afd5c8638d4efb9 Mon Sep 17 00:00:00 2001 From: enkore Date: Wed, 17 May 2017 00:09:41 +0200 Subject: borg export-tar (#2519) --- docs/man/borg-export-tar.1 | 142 +++++++++++++++++++++ docs/usage.rst | 18 +++ docs/usage/export-tar.rst.inc | 73 +++++++++++ src/borg/archiver.py | 281 +++++++++++++++++++++++++++++++++++++++++ src/borg/helpers.py | 13 +- src/borg/testsuite/__init__.py | 17 ++- src/borg/testsuite/archiver.py | 37 ++++++ 7 files changed, 574 insertions(+), 7 deletions(-) create mode 100644 docs/man/borg-export-tar.1 create mode 100644 docs/usage/export-tar.rst.inc diff --git a/docs/man/borg-export-tar.1 b/docs/man/borg-export-tar.1 new file mode 100644 index 000000000..ecbefc142 --- /dev/null +++ b/docs/man/borg-export-tar.1 @@ -0,0 +1,142 @@ +.\" Man page generated from reStructuredText. +. +.TH BORG-EXPORT-TAR 1 "2017-05-16" "" "borg backup tool" +.SH NAME +borg-export-tar \- Export archive contents as a tarball +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.sp +borg export\-tar ARCHIVE FILE PATH +.SH DESCRIPTION +.sp +This command creates a tarball from an archive. +.sp +When giving \(aq\-\(aq as the output FILE, Borg will write a tar stream to standard output. +.sp +By default (\-\-tar\-filter=auto) Borg will detect whether the FILE should be compressed +based on its file extension and pipe the tarball through an appropriate filter +before writing it to FILE: +.INDENT 0.0 +.IP \(bu 2 +\&.tar.gz: gzip +.IP \(bu 2 +\&.tar.bz2: bzip2 +.IP \(bu 2 +\&.tar.xz: xz +.UNINDENT +.sp +Alternatively a \-\-tar\-filter program may be explicitly specified. It should +read the uncompressed tar stream from stdin and write a compressed/filtered +tar stream to stdout. +.sp +The generated tarball uses the GNU tar format. +.sp +export\-tar is a lossy conversion: +BSD flags, ACLs, extended attributes (xattrs), atime and ctime are not exported. +Timestamp resolution is limited to whole seconds, not the nanosecond resolution +otherwise supported by Borg. +.sp +A \-\-sparse option (as found in borg extract) is not supported. +.sp +By default the entire archive is extracted but a subset of files and directories +can be selected by passing a list of \fBPATHs\fP as arguments. +The file selection can further be restricted by using the \fB\-\-exclude\fP option. +.sp +See the output of the "borg help patterns" command for more help on exclude patterns. +.sp +\fB\-\-progress\fP can be slower than no progress display, since it makes one additional +pass over the archive metadata. +.SH OPTIONS +.sp +See \fIborg\-common(1)\fP for common options of Borg commands. +.SS arguments +.INDENT 0.0 +.TP +.B ARCHIVE +archive to export +.TP +.B FILE +output tar file. "\-" to write to stdout instead. +.TP +.B PATH +paths to extract; patterns are supported +.UNINDENT +.SS optional arguments +.INDENT 0.0 +.TP +.B \-\-tar\-filter +filter program to pipe data through +.TP +.B \-\-list +output verbose list of items (files, dirs, ...) +.TP +.BI \-e \ PATTERN\fP,\fB \ \-\-exclude \ PATTERN +exclude paths matching PATTERN +.TP +.BI \-\-exclude\-from \ EXCLUDEFILE +read exclude patterns from EXCLUDEFILE, one per line +.TP +.BI \-\-pattern \ PATTERN +include/exclude paths matching PATTERN +.TP +.BI \-\-patterns\-from \ PATTERNFILE +read include/exclude patterns from PATTERNFILE, one per line +.TP +.BI \-\-strip\-components \ NUMBER +Remove the specified number of leading path elements. Pathnames with fewer elements will be silently skipped. +.UNINDENT +.SH EXAMPLES +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +# export as uncompressed tar +$ borg export\-tar /path/to/repo::Monday Monday.tar + +# exclude some types, compress using gzip +$ borg export\-tar /path/to/repo::Monday Monday.tar.gz \-\-exclude \(aq*.so\(aq + +# use higher compression level with gzip +$ borg export\-tar testrepo::linux \-\-tar\-filter="gzip \-9" Monday.tar.gz + +# export a gzipped tar, but instead of storing it on disk, +# upload it to a remote site using curl. +$ borg export\-tar ... \-\-tar\-filter="gzip" \- | curl \-\-data\-binary @\- https://somewhere/to/POST +.ft P +.fi +.UNINDENT +.UNINDENT +.SH SEE ALSO +.sp +\fIborg\-common(1)\fP +.SH AUTHOR +The Borg Collective +.\" Generated by docutils manpage writer. +. diff --git a/docs/usage.rst b/docs/usage.rst index d2b0904a6..33c145350 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -492,6 +492,24 @@ Examples Comment: This is a better comment ... +.. include:: usage/export-tar.rst.inc + +Examples +~~~~~~~~ +:: + + # export as uncompressed tar + $ borg export-tar /path/to/repo::Monday Monday.tar + + # exclude some types, compress using gzip + $ borg export-tar /path/to/repo::Monday Monday.tar.gz --exclude '*.so' + + # use higher compression level with gzip + $ borg export-tar testrepo::linux --tar-filter="gzip -9" Monday.tar.gz + + # export a gzipped tar, but instead of storing it on disk, + # upload it to a remote site using curl. + $ borg export-tar ... --tar-filter="gzip" - | curl --data-binary @- https://somewhere/to/POST .. include:: usage/with-lock.rst.inc diff --git a/docs/usage/export-tar.rst.inc b/docs/usage/export-tar.rst.inc new file mode 100644 index 000000000..af5fb5458 --- /dev/null +++ b/docs/usage/export-tar.rst.inc @@ -0,0 +1,73 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + +.. _borg_export-tar: + +borg export-tar +--------------- +:: + + borg export-tar ARCHIVE FILE PATH + +positional arguments + ARCHIVE + archive to export + FILE + output tar file. "-" to write to stdout instead. + PATH + paths to extract; patterns are supported + +optional arguments + ``--tar-filter`` + | filter program to pipe data through + ``--list`` + | output verbose list of items (files, dirs, ...) + ``-e PATTERN``, ``--exclude PATTERN`` + | exclude paths matching PATTERN + ``--exclude-from EXCLUDEFILE`` + | read exclude patterns from EXCLUDEFILE, one per line + ``--pattern PATTERN`` + | include/exclude paths matching PATTERN + ``--patterns-from PATTERNFILE`` + | read include/exclude patterns from PATTERNFILE, one per line + ``--strip-components NUMBER`` + | Remove the specified number of leading path elements. Pathnames with fewer elements will be silently skipped. + +`Common options`_ + | + +Description +~~~~~~~~~~~ + +This command creates a tarball from an archive. + +When giving '-' as the output FILE, Borg will write a tar stream to standard output. + +By default (--tar-filter=auto) Borg will detect whether the FILE should be compressed +based on its file extension and pipe the tarball through an appropriate filter +before writing it to FILE: + +- .tar.gz: gzip +- .tar.bz2: bzip2 +- .tar.xz: xz + +Alternatively a --tar-filter program may be explicitly specified. It should +read the uncompressed tar stream from stdin and write a compressed/filtered +tar stream to stdout. + +The generated tarball uses the GNU tar format. + +export-tar is a lossy conversion: +BSD flags, ACLs, extended attributes (xattrs), atime and ctime are not exported. +Timestamp resolution is limited to whole seconds, not the nanosecond resolution +otherwise supported by Borg. + +A --sparse option (as found in borg extract) is not supported. + +By default the entire archive is extracted but a subset of files and directories +can be selected by passing a list of ``PATHs`` as arguments. +The file selection can further be restricted by using the ``--exclude`` option. + +See the output of the "borg help patterns" command for more help on exclude patterns. + +``--progress`` can be slower than no progress display, since it makes one additional +pass over the archive metadata. \ No newline at end of file diff --git a/src/borg/archiver.py b/src/borg/archiver.py index ecde3f7d6..80a0dc4ba 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -15,6 +15,7 @@ import signal import stat import subprocess import sys +import tarfile import textwrap import time import traceback @@ -61,6 +62,7 @@ from .helpers import ErrorIgnoringTextIOWrapper from .helpers import ProgressIndicatorPercent from .helpers import basic_json_data, json_print from .helpers import replace_placeholders +from .helpers import ChunkIteratorFileWrapper from .patterns import ArgparsePatternAction, ArgparseExcludeFileAction, ArgparsePatternFileAction, parse_exclude_pattern from .patterns import PatternMatcher from .item import Item @@ -694,6 +696,219 @@ class Archiver: pi.finish() return self.exit_code + @with_repository() + @with_archive + def do_export_tar(self, args, repository, manifest, key, archive): + """Export archive contents as a tarball""" + self.output_list = args.output_list + + # A quick note about the general design of tar_filter and tarfile; + # The tarfile module of Python can provide some compression mechanisms + # by itself, using the builtin gzip, bz2 and lzma modules (and "tarmodes" + # such as "w:xz"). + # + # Doing so would have three major drawbacks: + # For one the compressor runs on the same thread as the program using the + # tarfile, stealing valuable CPU time from Borg and thus reducing throughput. + # Then this limits the available options - what about lz4? Brotli? zstd? + # The third issue is that systems can ship more optimized versions than those + # built into Python, e.g. pigz or pxz, which can use more than one thread for + # compression. + # + # Therefore we externalize compression by using a filter program, which has + # none of these drawbacks. The only issue of using an external filter is + # that it has to be installed -- hardly a problem, considering that + # the decompressor must be installed as well to make use of the exported tarball! + + filter = None + if args.tar_filter == 'auto': + # Note that filter remains None if tarfile is '-'. + if args.tarfile.endswith('.tar.gz'): + filter = 'gzip' + elif args.tarfile.endswith('.tar.bz2'): + filter = 'bzip2' + elif args.tarfile.endswith('.tar.xz'): + filter = 'xz' + logger.debug('Automatically determined tar filter: %s', filter) + else: + filter = args.tar_filter + + if args.tarfile == '-': + tarstream, tarstream_close = sys.stdout.buffer, False + else: + tarstream, tarstream_close = open(args.tarfile, 'wb'), True + + if filter: + # When we put a filter between us and the final destination, + # the selected output (tarstream until now) becomes the output of the filter (=filterout). + # The decision whether to close that or not remains the same. + filterout = tarstream + filterout_close = tarstream_close + # There is no deadlock potential here (the subprocess docs warn about this), because + # communication with the process is a one-way road, i.e. the process can never block + # for us to do something while we block on the process for something different. + filtercmd = shlex.split(filter) + logger.debug('--tar-filter command line: %s', filtercmd) + filterproc = subprocess.Popen(filtercmd, stdin=subprocess.PIPE, stdout=filterout) + # Always close the pipe, otherwise the filter process would not notice when we are done. + tarstream = filterproc.stdin + tarstream_close = True + + # The | (pipe) symbol instructs tarfile to use a streaming mode of operation + # where it never seeks on the passed fileobj. + tar = tarfile.open(fileobj=tarstream, mode='w|') + + self._export_tar(args, archive, tar) + + # This does not close the fileobj (tarstream) we passed to it -- a side effect of the | mode. + tar.close() + + if tarstream_close: + tarstream.close() + + if filter: + logger.debug('Done creating tar, waiting for filter to die...') + rc = filterproc.wait() + if rc: + logger.error('--tar-filter exited with code %d, output file is likely unusable!', rc) + self.exit_code = set_ec(EXIT_ERROR) + else: + logger.debug('filter exited with code %d', rc) + + if filterout_close: + filterout.close() + + return self.exit_code + + def _export_tar(self, args, archive, tar): + matcher = self.build_matcher(args.patterns, args.paths) + + progress = args.progress + output_list = args.output_list + strip_components = args.strip_components + partial_extract = not matcher.empty() or strip_components + hardlink_masters = {} if partial_extract else None + + def peek_and_store_hardlink_masters(item, matched): + if (partial_extract and not matched and hardlinkable(item.mode) and + item.get('hardlink_master', True) and 'source' not in item): + hardlink_masters[item.get('path')] = (item.get('chunks'), None) + + filter = self.build_filter(matcher, peek_and_store_hardlink_masters, strip_components) + + if progress: + pi = ProgressIndicatorPercent(msg='%5.1f%% Processing: %s', step=0.1, msgid='extract') + pi.output('Calculating size') + extracted_size = sum(item.get_size(hardlink_masters) for item in archive.iter_items(filter)) + pi.total = extracted_size + else: + pi = None + + def item_content_stream(item): + """ + Return a file-like object that reads from the chunks of *item*. + """ + chunk_iterator = archive.pipeline.fetch_many([chunk_id for chunk_id, _, _ in item.chunks]) + if pi: + info = [remove_surrogates(item.path)] + return ChunkIteratorFileWrapper(chunk_iterator, + lambda read_bytes: pi.show(increase=len(read_bytes), info=info)) + else: + return ChunkIteratorFileWrapper(chunk_iterator) + + def item_to_tarinfo(item, original_path): + """ + Transform a Borg *item* into a tarfile.TarInfo object. + + Return a tuple (tarinfo, stream), where stream may be a file-like object that represents + the file contents, if any, and is None otherwise. When *tarinfo* is None, the *item* + cannot be represented as a TarInfo object and should be skipped. + """ + + # If we would use the PAX (POSIX) format (which we currently don't), + # we can support most things that aren't possible with classic tar + # formats, including GNU tar, such as: + # atime, ctime, possibly Linux capabilities (security.* xattrs) + # and various additions supported by GNU tar in POSIX mode. + + stream = None + tarinfo = tarfile.TarInfo() + tarinfo.name = item.path + tarinfo.mtime = item.mtime / 1e9 + tarinfo.mode = stat.S_IMODE(item.mode) + tarinfo.uid = item.uid + tarinfo.gid = item.gid + tarinfo.uname = item.user or '' + tarinfo.gname = item.group or '' + # The linkname in tar has the same dual use the 'source' attribute of Borg items, + # i.e. for symlinks it means the destination, while for hardlinks it refers to the + # file. + # Since hardlinks in tar have a different type code (LNKTYPE) the format might + # support hardlinking arbitrary objects (including symlinks and directories), but + # whether implementations actually support that is a whole different question... + tarinfo.linkname = "" + + modebits = stat.S_IFMT(item.mode) + if modebits == stat.S_IFREG: + tarinfo.type = tarfile.REGTYPE + if 'source' in item: + source = os.sep.join(item.source.split(os.sep)[strip_components:]) + if hardlink_masters is None: + linkname = source + else: + chunks, linkname = hardlink_masters.get(item.source, (None, source)) + if linkname: + # Master was already added to the archive, add a hardlink reference to it. + tarinfo.type = tarfile.LNKTYPE + tarinfo.linkname = linkname + elif chunks is not None: + # The item which has the chunks was not put into the tar, therefore + # we do that now and update hardlink_masters to reflect that. + item.chunks = chunks + tarinfo.size = item.get_size() + stream = item_content_stream(item) + hardlink_masters[item.get('source') or original_path] = (None, item.path) + else: + tarinfo.size = item.get_size() + stream = item_content_stream(item) + elif modebits == stat.S_IFDIR: + tarinfo.type = tarfile.DIRTYPE + elif modebits == stat.S_IFLNK: + tarinfo.type = tarfile.SYMTYPE + tarinfo.linkname = item.source + elif modebits == stat.S_IFBLK: + tarinfo.type = tarfile.BLKTYPE + tarinfo.devmajor = os.major(item.rdev) + tarinfo.devminor = os.minor(item.rdev) + elif modebits == stat.S_IFCHR: + tarinfo.type = tarfile.CHRTYPE + tarinfo.devmajor = os.major(item.rdev) + tarinfo.devminor = os.minor(item.rdev) + elif modebits == stat.S_IFIFO: + tarinfo.type = tarfile.FIFOTYPE + else: + self.print_warning('%s: unsupported file type %o for tar export', remove_surrogates(item.path), modebits) + set_ec(EXIT_WARNING) + return None, stream + return tarinfo, stream + + for item in archive.iter_items(filter, preload=True): + orig_path = item.path + if strip_components: + item.path = os.sep.join(orig_path.split(os.sep)[strip_components:]) + tarinfo, stream = item_to_tarinfo(item, orig_path) + if tarinfo: + if output_list: + logging.getLogger('borg.output.list').info(remove_surrogates(orig_path)) + tar.addfile(tarinfo, stream) + + if pi: + pi.finish() + + for pattern in matcher.get_unmatched_include_patterns(): + self.print_warning("Include pattern '%s' never matched.", pattern) + return self.exit_code + @with_repository() @with_archive def do_diff(self, args, repository, manifest, key, archive): @@ -2605,6 +2820,72 @@ class Archiver: subparser.add_argument('paths', metavar='PATH', nargs='*', type=str, help='paths to extract; patterns are supported') + export_tar_epilog = process_epilog(""" + This command creates a tarball from an archive. + + When giving '-' as the output FILE, Borg will write a tar stream to standard output. + + By default (--tar-filter=auto) Borg will detect whether the FILE should be compressed + based on its file extension and pipe the tarball through an appropriate filter + before writing it to FILE: + + - .tar.gz: gzip + - .tar.bz2: bzip2 + - .tar.xz: xz + + Alternatively a --tar-filter program may be explicitly specified. It should + read the uncompressed tar stream from stdin and write a compressed/filtered + tar stream to stdout. + + The generated tarball uses the GNU tar format. + + export-tar is a lossy conversion: + BSD flags, ACLs, extended attributes (xattrs), atime and ctime are not exported. + Timestamp resolution is limited to whole seconds, not the nanosecond resolution + otherwise supported by Borg. + + A --sparse option (as found in borg extract) is not supported. + + By default the entire archive is extracted but a subset of files and directories + can be selected by passing a list of ``PATHs`` as arguments. + The file selection can further be restricted by using the ``--exclude`` option. + + See the output of the "borg help patterns" command for more help on exclude patterns. + + ``--progress`` can be slower than no progress display, since it makes one additional + pass over the archive metadata. + """) + subparser = subparsers.add_parser('export-tar', parents=[common_parser], add_help=False, + description=self.do_export_tar.__doc__, + epilog=export_tar_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help='create tarball from archive') + subparser.set_defaults(func=self.do_export_tar) + subparser.add_argument('--tar-filter', dest='tar_filter', default='auto', + help='filter program to pipe data through') + subparser.add_argument('--list', dest='output_list', + action='store_true', default=False, + help='output verbose list of items (files, dirs, ...)') + subparser.add_argument('-e', '--exclude', dest='patterns', + type=parse_exclude_pattern, action='append', + metavar="PATTERN", help='exclude paths matching PATTERN') + subparser.add_argument('--exclude-from', action=ArgparseExcludeFileAction, + metavar='EXCLUDEFILE', help='read exclude patterns from EXCLUDEFILE, one per line') + subparser.add_argument('--pattern', action=ArgparsePatternAction, + metavar="PATTERN", help='include/exclude paths matching PATTERN') + subparser.add_argument('--patterns-from', action=ArgparsePatternFileAction, + metavar='PATTERNFILE', help='read include/exclude patterns from PATTERNFILE, one per line') + subparser.add_argument('--strip-components', dest='strip_components', + type=int, default=0, metavar='NUMBER', + help='Remove the specified number of leading path elements. Pathnames with fewer elements will be silently skipped.') + subparser.add_argument('location', metavar='ARCHIVE', + type=location_validator(archive=True), + help='archive to export') + subparser.add_argument('tarfile', metavar='FILE', + help='output tar file. "-" to write to stdout instead.') + subparser.add_argument('paths', metavar='PATH', nargs='*', type=str, + help='paths to extract; patterns are supported') + diff_epilog = process_epilog(""" This command finds differences (file contents, user/group/mode) between archives. diff --git a/src/borg/helpers.py b/src/borg/helpers.py index c30271435..b67250ebc 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -1622,11 +1622,20 @@ class ItemFormatter(BaseFormatter): class ChunkIteratorFileWrapper: """File-like wrapper for chunk iterators""" - def __init__(self, chunk_iterator): + def __init__(self, chunk_iterator, read_callback=None): + """ + *chunk_iterator* should be an iterator yielding bytes. These will be buffered + internally as necessary to satisfy .read() calls. + + *read_callback* will be called with one argument, some byte string that has + just been read and will be subsequently returned to a caller of .read(). + It can be used to update a progress display. + """ self.chunk_iterator = chunk_iterator self.chunk_offset = 0 self.chunk = b'' self.exhausted = False + self.read_callback = read_callback def _refill(self): remaining = len(self.chunk) - self.chunk_offset @@ -1655,6 +1664,8 @@ class ChunkIteratorFileWrapper: read_data = self._read(nbytes) nbytes -= len(read_data) parts.append(read_data) + if self.read_callback: + self.read_callback(read_data) return b''.join(parts) diff --git a/src/borg/testsuite/__init__.py b/src/borg/testsuite/__init__.py index fea632cb2..38f5d4ab1 100644 --- a/src/borg/testsuite/__init__.py +++ b/src/borg/testsuite/__init__.py @@ -150,7 +150,7 @@ class BaseTestCase(unittest.TestCase): diff = filecmp.dircmp(dir1, dir2) self._assert_dirs_equal_cmp(diff, **kwargs) - def _assert_dirs_equal_cmp(self, diff, ignore_bsdflags=False, ignore_xattrs=False): + def _assert_dirs_equal_cmp(self, diff, ignore_bsdflags=False, ignore_xattrs=False, ignore_ns=False): self.assert_equal(diff.left_only, []) self.assert_equal(diff.right_only, []) self.assert_equal(diff.diff_files, []) @@ -162,25 +162,30 @@ class BaseTestCase(unittest.TestCase): s2 = os.lstat(path2) # Assume path2 is on FUSE if st_dev is different fuse = s1.st_dev != s2.st_dev - attrs = ['st_mode', 'st_uid', 'st_gid', 'st_rdev'] + attrs = ['st_uid', 'st_gid', 'st_rdev'] if not fuse or not os.path.isdir(path1): # dir nlink is always 1 on our fuse filesystem attrs.append('st_nlink') d1 = [filename] + [getattr(s1, a) for a in attrs] d2 = [filename] + [getattr(s2, a) for a in attrs] + d1.insert(1, oct(s1.st_mode)) + d2.insert(1, oct(s2.st_mode)) if not ignore_bsdflags: d1.append(get_flags(path1, s1)) d2.append(get_flags(path2, s2)) # ignore st_rdev if file is not a block/char device, fixes #203 - if not stat.S_ISCHR(d1[1]) and not stat.S_ISBLK(d1[1]): + if not stat.S_ISCHR(s1.st_mode) and not stat.S_ISBLK(s1.st_mode): d1[4] = None - if not stat.S_ISCHR(d2[1]) and not stat.S_ISBLK(d2[1]): + if not stat.S_ISCHR(s2.st_mode) and not stat.S_ISBLK(s2.st_mode): d2[4] = None # If utime isn't fully supported, borg can't set mtime. # Therefore, we shouldn't test it in that case. if is_utime_fully_supported(): # Older versions of llfuse do not support ns precision properly - if fuse and not have_fuse_mtime_ns: + if ignore_ns: + d1.append(int(s1.st_mtime_ns / 1e9)) + d2.append(int(s2.st_mtime_ns / 1e9)) + elif fuse and not have_fuse_mtime_ns: d1.append(round(s1.st_mtime_ns, -4)) d2.append(round(s2.st_mtime_ns, -4)) else: @@ -191,7 +196,7 @@ class BaseTestCase(unittest.TestCase): d2.append(no_selinux(get_all(path2, follow_symlinks=False))) self.assert_equal(d1, d2) for sub_diff in diff.subdirs.values(): - self._assert_dirs_equal_cmp(sub_diff, ignore_bsdflags=ignore_bsdflags, ignore_xattrs=ignore_xattrs) + self._assert_dirs_equal_cmp(sub_diff, ignore_bsdflags=ignore_bsdflags, ignore_xattrs=ignore_xattrs, ignore_ns=ignore_ns) @contextmanager def fuse_mount(self, location, mountpoint, *options): diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index f0e860304..dd3285fd3 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -96,6 +96,14 @@ def exec_cmd(*args, archiver=None, fork=False, exe=None, **kw): sys.stdin, sys.stdout, sys.stderr = stdin, stdout, stderr +def have_gnutar(): + if not shutil.which('tar'): + return False + popen = subprocess.Popen(['tar', '--version'], stdout=subprocess.PIPE) + stdout, stderr = popen.communicate() + return b'GNU tar' in stdout + + # check if the binary "borg.exe" is available (for local testing a symlink to virtualenv/bin/borg should do) try: exec_cmd('help', exe='borg.exe', fork=True) @@ -2354,6 +2362,35 @@ id: 2 / e29442 3506da 4e1ea7 / 25f62a 5a3d41 - 02 assert '_meta' in result assert '_items' in result + requires_gnutar = pytest.mark.skipif(not have_gnutar(), reason='GNU tar must be installed for this test.') + requires_gzip = pytest.mark.skipif(not shutil.which('gzip'), reason='gzip must be installed for this test.') + + @requires_gnutar + def test_export_tar(self): + self.create_test_files() + os.unlink('input/flagfile') + self.cmd('init', '--encryption=repokey', self.repository_location) + self.cmd('create', self.repository_location + '::test', 'input') + self.cmd('export-tar', self.repository_location + '::test', 'simple.tar') + with changedir('output'): + # This probably assumes GNU tar. Note -p switch to extract permissions regardless of umask. + subprocess.check_output(['tar', 'xpf', '../simple.tar']) + self.assert_dirs_equal('input', 'output/input', ignore_bsdflags=True, ignore_xattrs=True, ignore_ns=True) + + @requires_gnutar + @requires_gzip + def test_export_tar_gz(self): + if not shutil.which('gzip'): + pytest.skip('gzip is not installed') + self.create_test_files() + os.unlink('input/flagfile') + self.cmd('init', '--encryption=repokey', self.repository_location) + self.cmd('create', self.repository_location + '::test', 'input') + self.cmd('export-tar', self.repository_location + '::test', 'simple.tar.gz') + with changedir('output'): + subprocess.check_output(['tar', 'xpf', '../simple.tar.gz']) + self.assert_dirs_equal('input', 'output/input', ignore_bsdflags=True, ignore_xattrs=True, ignore_ns=True) + @unittest.skipUnless('binary' in BORG_EXES, 'no borg.exe available') class ArchiverTestCaseBinary(ArchiverTestCase): -- cgit v1.2.3