setup.py: move build_man / build_usage code to setup_docs.py

brings setup.py down from 800 to 300 lines.

1 files changed, 532 insertions, 0 deletions

diff --git a/setup_docs.py b/setup_docs.py
new file mode 100644
index 000000000..e2bc1f97a
--- /dev/null
+++ b/setup_docs.py
@@ -0,0 +1,532 @@
+# -*- encoding: utf-8 *-*
+# Support code for building docs (build_usage, build_man)
+
+import os
+import io
+import re
+import sys
+import textwrap
+from collections import OrderedDict
+from datetime import datetime
+
+from setuptools import Command
+
+
+def format_metavar(option):
+    if option.nargs in ('*', '...'):
+        return '[%s...]' % option.metavar
+    elif option.nargs == '?':
+        return '[%s]' % option.metavar
+    elif option.nargs is None:
+        return option.metavar
+    else:
+        raise ValueError('Can\'t format metavar %s, unknown nargs %s!' % (option.metavar, option.nargs))
+
+
+class build_usage(Command):
+    description = "generate usage for each command"
+
+    user_options = [
+        ('output=', 'O', 'output directory'),
+    ]
+
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        print('generating usage docs')
+        import borg
+        borg.doc_mode = 'build_man'
+        if not os.path.exists('docs/usage'):
+            os.mkdir('docs/usage')
+        # allows us to build docs without the C modules fully loaded during help generation
+        from borg.archiver import Archiver
+        parser = Archiver(prog='borg').build_parser()
+        # borgfs has a separate man page to satisfy debian's "every program from a package
+        # must have a man page" requirement, but it doesn't need a separate HTML docs page
+        #borgfs_parser = Archiver(prog='borgfs').build_parser()
+
+        self.generate_level("", parser, Archiver)
+
+    def generate_level(self, prefix, parser, Archiver, extra_choices=None):
+        is_subcommand = False
+        choices = {}
+        for action in parser._actions:
+            if action.choices is not None and 'SubParsersAction' in str(action.__class__):
+                is_subcommand = True
+                for cmd, parser in action.choices.items():
+                    choices[prefix + cmd] = parser
+        if extra_choices is not None:
+            choices.update(extra_choices)
+        if prefix and not choices:
+            return
+        print('found commands: %s' % list(choices.keys()))
+
+        for command, parser in sorted(choices.items()):
+            if command.startswith('debug'):
+                print('skipping', command)
+                continue
+            print('generating help for %s' % command)
+
+            if self.generate_level(command + " ", parser, Archiver):
+                continue
+
+            with open('docs/usage/%s.rst.inc' % command.replace(" ", "_"), 'w') as doc:
+                doc.write(".. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!\n\n")
+                if command == 'help':
+                    for topic in Archiver.helptext:
+                        params = {"topic": topic,
+                                  "underline": '~' * len('borg help ' + topic)}
+                        doc.write(".. _borg_{topic}:\n\n".format(**params))
+                        doc.write("borg help {topic}\n{underline}\n\n".format(**params))
+                        doc.write(Archiver.helptext[topic])
+                else:
+                    params = {"command": command,
+                              "command_": command.replace(' ', '_'),
+                              "underline": '-' * len('borg ' + command)}
+                    doc.write(".. _borg_{command_}:\n\n".format(**params))
+                    doc.write("borg {command}\n{underline}\n.. code-block:: none\n\n    borg [common options] {command}".format(**params))
+                    self.write_usage(parser, doc)
+                    epilog = parser.epilog
+                    parser.epilog = None
+                    self.write_options(parser, doc)
+                    doc.write("\n\nDescription\n~~~~~~~~~~~\n")
+                    doc.write(epilog)
+
+        if 'create' in choices:
+            common_options = [group for group in choices['create']._action_groups if group.title == 'Common options'][0]
+            with open('docs/usage/common-options.rst.inc', 'w') as doc:
+                self.write_options_group(common_options, doc, False, base_indent=0)
+
+        return is_subcommand
+
+    def write_usage(self, parser, fp):
+        if any(len(o.option_strings) for o in parser._actions):
+            fp.write(' [options]')
+        for option in parser._actions:
+            if option.option_strings:
+                continue
+            fp.write(' ' + format_metavar(option))
+        fp.write('\n\n')
+
+    def write_options(self, parser, fp):
+        def is_positional_group(group):
+            return any(not o.option_strings for o in group._group_actions)
+
+        # HTML output:
+        # A table using some column-spans
+
+        def html_write(s):
+            for line in s.splitlines():
+                fp.write('    ' + line + '\n')
+
+        rows = []
+        for group in parser._action_groups:
+            if group.title == 'Common options':
+                # (no of columns used, columns, ...)
+                rows.append((1, '.. class:: borg-common-opt-ref\n\n:ref:`common_options`'))
+            else:
+                if not group._group_actions:
+                    continue
+                group_header = '**%s**' % group.title
+                if group.description:
+                    group_header += ' — ' + group.description
+                rows.append((1, group_header))
+                if is_positional_group(group):
+                    for option in group._group_actions:
+                        rows.append((3, '', '``%s``' % option.metavar, option.help or ''))
+                else:
+                    for option in group._group_actions:
+                        if option.metavar:
+                            option_fmt = '``%s ' + option.metavar + '``'
+                        else:
+                            option_fmt = '``%s``'
+                        option_str = ', '.join(option_fmt % s for s in option.option_strings)
+                        option_desc = textwrap.dedent((option.help or '') % option.__dict__)
+                        rows.append((3, '', option_str, option_desc))
+
+        fp.write('.. only:: html\n\n')
+        table = io.StringIO()
+        table.write('.. class:: borg-options-table\n\n')
+        self.rows_to_table(rows, table.write)
+        fp.write(textwrap.indent(table.getvalue(), ' ' * 4))
+
+        # LaTeX output:
+        # Regular rST option lists (irregular column widths)
+        latex_options = io.StringIO()
+        for group in parser._action_groups:
+            if group.title == 'Common options':
+                latex_options.write('\n\n:ref:`common_options`\n')
+                latex_options.write('    |')
+            else:
+                self.write_options_group(group, latex_options)
+        fp.write('\n.. only:: latex\n\n')
+        fp.write(textwrap.indent(latex_options.getvalue(), ' ' * 4))
+
+    def rows_to_table(self, rows, write):
+        def write_row_separator():
+            write('+')
+            for column_width in column_widths:
+                write('-' * (column_width + 1))
+                write('+')
+            write('\n')
+
+        # Find column count and width
+        column_count = max(columns for columns, *_ in rows)
+        column_widths = [0] * column_count
+        for columns, *cells in rows:
+            for i in range(columns):
+                # "+ 1" because we want a space between the cell contents and the delimiting "|" in the output
+                column_widths[i] = max(column_widths[i], len(cells[i]) + 1)
+
+        for columns, *original_cells in rows:
+            write_row_separator()
+            # If a cell contains newlines, then the row must be split up in individual rows
+            # where each cell contains no newline.
+            rowspanning_cells = []
+            original_cells = list(original_cells)
+            while any('\n' in cell for cell in original_cells):
+                cell_bloc = []
+                for i, cell in enumerate(original_cells):
+                    pre, _, original_cells[i] = cell.partition('\n')
+                    cell_bloc.append(pre)
+                rowspanning_cells.append(cell_bloc)
+            rowspanning_cells.append(original_cells)
+            for cells in rowspanning_cells:
+                for i, column_width in enumerate(column_widths):
+                    if i < columns:
+                        write('| ')
+                        write(cells[i].ljust(column_width))
+                    else:
+                        write('  ')
+                        write(''.ljust(column_width))
+                write('|\n')
+
+        write_row_separator()
+        # This bit of JavaScript kills the <colgroup> that is invariably inserted by docutils,
+        # but does absolutely no good here. It sets bogus column widths which cannot be overridden
+        # with CSS alone.
+        # Since this is HTML-only output, it would be possible to just generate a <table> directly,
+        # but then we'd lose rST formatting.
+        write(textwrap.dedent("""
+        .. raw:: html
+
+            <script type='text/javascript'>
+            $(document).ready(function () {
+                $('.borg-options-table colgroup').remove();
+            })
+            </script>
+        """))
+
+    def write_options_group(self, group, fp, with_title=True, base_indent=4):
+        def is_positional_group(group):
+            return any(not o.option_strings for o in group._group_actions)
+
+        indent = ' ' * base_indent
+
+        if is_positional_group(group):
+            for option in group._group_actions:
+                fp.write(option.metavar + '\n')
+                fp.write(textwrap.indent(option.help or '', ' ' * base_indent) + '\n')
+            return
+
+        if not group._group_actions:
+            return
+
+        if with_title:
+            fp.write('\n\n')
+            fp.write(group.title + '\n')
+
+        opts = OrderedDict()
+
+        for option in group._group_actions:
+            if option.metavar:
+                option_fmt = '%s ' + option.metavar
+            else:
+                option_fmt = '%s'
+            option_str = ', '.join(option_fmt % s for s in option.option_strings)
+            option_desc = textwrap.dedent((option.help or '') % option.__dict__)
+            opts[option_str] = textwrap.indent(option_desc, ' ' * 4)
+
+        padding = len(max(opts)) + 1
+
+        for option, desc in opts.items():
+            fp.write(indent + option.ljust(padding) + desc + '\n')
+
+
+class build_man(Command):
+    description = 'build man pages'
+
+    user_options = []
+
+    see_also = {
+        'create': ('delete', 'prune', 'check', 'patterns', 'placeholders', 'compression'),
+        'recreate': ('patterns', 'placeholders', 'compression'),
+        'list': ('info', 'diff', 'prune', 'patterns'),
+        'info': ('list', 'diff'),
+        'init': ('create', 'delete', 'check', 'list', 'key-import', 'key-export', 'key-change-passphrase'),
+        'key-import': ('key-export', ),
+        'key-export': ('key-import', ),
+        'mount': ('umount', 'extract'),  # Would be cooler if these two were on the same page
+        'umount': ('mount', ),
+        'extract': ('mount', ),
+    }
+
+    rst_prelude = textwrap.dedent("""
+    .. role:: ref(title)
+
+    .. |project_name| replace:: Borg
+
+    """)
+
+    usage_group = {
+        'break-lock': 'lock',
+        'with-lock': 'lock',
+
+        'change-passphrase': 'key',
+        'key_change-passphrase': 'key',
+        'key_export': 'key',
+        'key_import': 'key',
+        'key_migrate-to-repokey': 'key',
+
+        'export-tar': 'tar',
+
+        'benchmark_crud': 'benchmark',
+
+        'umount': 'mount',
+    }
+
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        print('building man pages (in docs/man)', file=sys.stderr)
+        import borg
+        borg.doc_mode = 'build_man'
+        os.makedirs('docs/man', exist_ok=True)
+        # allows us to build docs without the C modules fully loaded during help generation
+        from borg.archiver import Archiver
+        parser = Archiver(prog='borg').build_parser()
+        borgfs_parser = Archiver(prog='borgfs').build_parser()
+
+        self.generate_level('', parser, Archiver, {'borgfs': borgfs_parser})
+        self.build_topic_pages(Archiver)
+        self.build_intro_page()
+
+    def generate_level(self, prefix, parser, Archiver, extra_choices=None):
+        is_subcommand = False
+        choices = {}
+        for action in parser._actions:
+            if action.choices is not None and 'SubParsersAction' in str(action.__class__):
+                is_subcommand = True
+                for cmd, parser in action.choices.items():
+                    choices[prefix + cmd] = parser
+        if extra_choices is not None:
+            choices.update(extra_choices)
+        if prefix and not choices:
+            return
+
+        for command, parser in sorted(choices.items()):
+            if command.startswith('debug') or command == 'help':
+                continue
+
+            if command == "borgfs":
+                man_title = command
+            else:
+                man_title = 'borg-' + command.replace(' ', '-')
+            print('building man page', man_title + '(1)', file=sys.stderr)
+
+            is_intermediary = self.generate_level(command + ' ', parser, Archiver)
+
+            doc, write = self.new_doc()
+            self.write_man_header(write, man_title, parser.description)
+
+            self.write_heading(write, 'SYNOPSIS')
+            if is_intermediary:
+                subparsers = [action for action in parser._actions if 'SubParsersAction' in str(action.__class__)][0]
+                for subcommand in subparsers.choices:
+                    write('| borg', '[common options]', command, subcommand, '...')
+                    self.see_also.setdefault(command, []).append('%s-%s' % (command, subcommand))
+            else:
+                if command == "borgfs":
+                    write(command, end='')
+                else:
+                    write('borg', '[common options]', command, end='')
+                self.write_usage(write, parser)
+            write('\n')
+
+            description, _, notes = parser.epilog.partition('\n.. man NOTES')
+
+            if description:
+                self.write_heading(write, 'DESCRIPTION')
+                write(description)
+
+            if not is_intermediary:
+                self.write_heading(write, 'OPTIONS')
+                write('See `borg-common(1)` for common options of Borg commands.')
+                write()
+                self.write_options(write, parser)
+
+                self.write_examples(write, command)
+
+            if notes:
+                self.write_heading(write, 'NOTES')
+                write(notes)
+
+            self.write_see_also(write, man_title)
+
+            self.gen_man_page(man_title, doc.getvalue())
+
+        # Generate the borg-common(1) man page with the common options.
+        if 'create' in choices:
+            doc, write = self.new_doc()
+            man_title = 'borg-common'
+            self.write_man_header(write, man_title, 'Common options of Borg commands')
+
+            common_options = [group for group in choices['create']._action_groups if group.title == 'Common options'][0]
+
+            self.write_heading(write, 'SYNOPSIS')
+            self.write_options_group(write, common_options)
+            self.write_see_also(write, man_title)
+            self.gen_man_page(man_title, doc.getvalue())
+
+        return is_subcommand
+
+    def build_topic_pages(self, Archiver):
+        for topic, text in Archiver.helptext.items():
+            doc, write = self.new_doc()
+            man_title = 'borg-' + topic
+            print('building man page', man_title + '(1)', file=sys.stderr)
+
+            self.write_man_header(write, man_title, 'Details regarding ' + topic)
+            self.write_heading(write, 'DESCRIPTION')
+            write(text)
+            self.gen_man_page(man_title, doc.getvalue())
+
+    def build_intro_page(self):
+        print('building man page borg(1)', file=sys.stderr)
+        with open('docs/man_intro.rst') as fd:
+            man_intro = fd.read()
+        self.gen_man_page('borg', self.rst_prelude + man_intro)
+
+    def new_doc(self):
+        doc = io.StringIO(self.rst_prelude)
+        doc.read()
+        write = self.printer(doc)
+        return doc, write
+
+    def printer(self, fd):
+        def write(*args, **kwargs):
+            print(*args, file=fd, **kwargs)
+        return write
+
+    def write_heading(self, write, header, char='-', double_sided=False):
+        write()
+        if double_sided:
+            write(char * len(header))
+        write(header)
+        write(char * len(header))
+        write()
+
+    def write_man_header(self, write, title, description):
+        self.write_heading(write, title, '=', double_sided=True)
+        self.write_heading(write, description, double_sided=True)
+        # man page metadata
+        write(':Author: The Borg Collective')
+        write(':Date:', datetime.utcnow().date().isoformat())
+        write(':Manual section: 1')
+        write(':Manual group: borg backup tool')
+        write()
+
+    def write_examples(self, write, command):
+        command = command.replace(' ', '_')
+        with open('docs/usage/%s.rst' % self.usage_group.get(command, command)) as fd:
+            usage = fd.read()
+            usage_include = '.. include:: %s.rst.inc' % command
+            begin = usage.find(usage_include)
+            end = usage.find('.. include', begin + 1)
+            # If a command has a dedicated anchor, it will occur before the command's include.
+            if 0 < usage.find('.. _', begin + 1) < end:
+                end = usage.find('.. _', begin + 1)
+            examples = usage[begin:end]
+            examples = examples.replace(usage_include, '')
+            examples = examples.replace('Examples\n~~~~~~~~', '')
+            examples = examples.replace('Miscellaneous Help\n------------------', '')
+            examples = examples.replace('``docs/misc/prune-example.txt``:', '``docs/misc/prune-example.txt``.')
+            examples = examples.replace('.. highlight:: none\n', '')  # we don't support highlight
+            examples = re.sub('^(~+)$', lambda matches: '+' * len(matches.group(0)), examples, flags=re.MULTILINE)
+            examples = examples.strip()
+        if examples:
+            self.write_heading(write, 'EXAMPLES', '-')
+            write(examples)
+
+    def write_see_also(self, write, man_title):
+        see_also = self.see_also.get(man_title.replace('borg-', ''), ())
+        see_also = ['`borg-%s(1)`' % s for s in see_also]
+        see_also.insert(0, '`borg-common(1)`')
+        self.write_heading(write, 'SEE ALSO')
+        write(', '.join(see_also))
+
+    def gen_man_page(self, name, rst):
+        from docutils.writers import manpage
+        from docutils.core import publish_string
+        from docutils.nodes import inline
+        from docutils.parsers.rst import roles
+
+        def issue(name, rawtext, text, lineno, inliner, options={}, content=[]):
+            return [inline(rawtext, '#' + text)], []
+
+        roles.register_local_role('issue', issue)
+        # We give the source_path so that docutils can find relative includes
+        # as-if the document where located in the docs/ directory.
+        man_page = publish_string(source=rst, source_path='docs/%s.rst' % name, writer=manpage.Writer())
+        with open('docs/man/%s.1' % name, 'wb') as fd:
+            fd.write(man_page)
+
+    def write_usage(self, write, parser):
+        if any(len(o.option_strings) for o in parser._actions):
+            write(' [options] ', end='')
+        for option in parser._actions:
+            if option.option_strings:
+                continue
+            write(format_metavar(option), end=' ')
+
+    def write_options(self, write, parser):
+        for group in parser._action_groups:
+            if group.title == 'Common options' or not group._group_actions:
+                continue
+            title = 'arguments' if group.title == 'positional arguments' else group.title
+            self.write_heading(write, title, '+')
+            self.write_options_group(write, group)
+
+    def write_options_group(self, write, group):
+        def is_positional_group(group):
+            return any(not o.option_strings for o in group._group_actions)
+
+        if is_positional_group(group):
+            for option in group._group_actions:
+                write(option.metavar)
+                write(textwrap.indent(option.help or '', ' ' * 4))
+            return
+
+        opts = OrderedDict()
+
+        for option in group._group_actions:
+            if option.metavar:
+                option_fmt = '%s ' + option.metavar
+            else:
+                option_fmt = '%s'
+            option_str = ', '.join(option_fmt % s for s in option.option_strings)
+            option_desc = textwrap.dedent((option.help or '') % option.__dict__)
+            opts[option_str] = textwrap.indent(option_desc, ' ' * 4)
+
+        padding = len(max(opts)) + 1
+
+        for option, desc in opts.items():
+            write(option.ljust(padding), desc)