diff options
author | Thomas Waldmann <tw@waldmann-edv.de> | 2018-06-12 21:06:01 +0200 |
---|---|---|
committer | Thomas Waldmann <tw@waldmann-edv.de> | 2018-06-12 21:06:01 +0200 |
commit | 06f23784588629e172afe9213252aad2de16b5c3 (patch) | |
tree | 4acb2079fcb1492bb9e6a3aa837ed2c1010f3615 /setup_docs.py | |
parent | 39a7078741c6796f6b33ad27f6df9061dc6ccf37 (diff) |
setup.py: move build_man / build_usage code to setup_docs.py
brings setup.py down from 800 to 300 lines.
Diffstat (limited to 'setup_docs.py')
-rw-r--r-- | setup_docs.py | 532 |
1 files changed, 532 insertions, 0 deletions
diff --git a/setup_docs.py b/setup_docs.py new file mode 100644 index 000000000..e2bc1f97a --- /dev/null +++ b/setup_docs.py @@ -0,0 +1,532 @@ +# -*- encoding: utf-8 *-* +# Support code for building docs (build_usage, build_man) + +import os +import io +import re +import sys +import textwrap +from collections import OrderedDict +from datetime import datetime + +from setuptools import Command + + +def format_metavar(option): + if option.nargs in ('*', '...'): + return '[%s...]' % option.metavar + elif option.nargs == '?': + return '[%s]' % option.metavar + elif option.nargs is None: + return option.metavar + else: + raise ValueError('Can\'t format metavar %s, unknown nargs %s!' % (option.metavar, option.nargs)) + + +class build_usage(Command): + description = "generate usage for each command" + + user_options = [ + ('output=', 'O', 'output directory'), + ] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + print('generating usage docs') + import borg + borg.doc_mode = 'build_man' + if not os.path.exists('docs/usage'): + os.mkdir('docs/usage') + # allows us to build docs without the C modules fully loaded during help generation + from borg.archiver import Archiver + parser = Archiver(prog='borg').build_parser() + # borgfs has a separate man page to satisfy debian's "every program from a package + # must have a man page" requirement, but it doesn't need a separate HTML docs page + #borgfs_parser = Archiver(prog='borgfs').build_parser() + + self.generate_level("", parser, Archiver) + + def generate_level(self, prefix, parser, Archiver, extra_choices=None): + is_subcommand = False + choices = {} + for action in parser._actions: + if action.choices is not None and 'SubParsersAction' in str(action.__class__): + is_subcommand = True + for cmd, parser in action.choices.items(): + choices[prefix + cmd] = parser + if extra_choices is not None: + choices.update(extra_choices) + if prefix and not choices: + return + print('found commands: %s' % list(choices.keys())) + + for command, parser in sorted(choices.items()): + if command.startswith('debug'): + print('skipping', command) + continue + print('generating help for %s' % command) + + if self.generate_level(command + " ", parser, Archiver): + continue + + with open('docs/usage/%s.rst.inc' % command.replace(" ", "_"), 'w') as doc: + doc.write(".. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!\n\n") + if command == 'help': + for topic in Archiver.helptext: + params = {"topic": topic, + "underline": '~' * len('borg help ' + topic)} + doc.write(".. _borg_{topic}:\n\n".format(**params)) + doc.write("borg help {topic}\n{underline}\n\n".format(**params)) + doc.write(Archiver.helptext[topic]) + else: + params = {"command": command, + "command_": command.replace(' ', '_'), + "underline": '-' * len('borg ' + command)} + doc.write(".. _borg_{command_}:\n\n".format(**params)) + doc.write("borg {command}\n{underline}\n.. code-block:: none\n\n borg [common options] {command}".format(**params)) + self.write_usage(parser, doc) + epilog = parser.epilog + parser.epilog = None + self.write_options(parser, doc) + doc.write("\n\nDescription\n~~~~~~~~~~~\n") + doc.write(epilog) + + if 'create' in choices: + common_options = [group for group in choices['create']._action_groups if group.title == 'Common options'][0] + with open('docs/usage/common-options.rst.inc', 'w') as doc: + self.write_options_group(common_options, doc, False, base_indent=0) + + return is_subcommand + + def write_usage(self, parser, fp): + if any(len(o.option_strings) for o in parser._actions): + fp.write(' [options]') + for option in parser._actions: + if option.option_strings: + continue + fp.write(' ' + format_metavar(option)) + fp.write('\n\n') + + def write_options(self, parser, fp): + def is_positional_group(group): + return any(not o.option_strings for o in group._group_actions) + + # HTML output: + # A table using some column-spans + + def html_write(s): + for line in s.splitlines(): + fp.write(' ' + line + '\n') + + rows = [] + for group in parser._action_groups: + if group.title == 'Common options': + # (no of columns used, columns, ...) + rows.append((1, '.. class:: borg-common-opt-ref\n\n:ref:`common_options`')) + else: + if not group._group_actions: + continue + group_header = '**%s**' % group.title + if group.description: + group_header += ' — ' + group.description + rows.append((1, group_header)) + if is_positional_group(group): + for option in group._group_actions: + rows.append((3, '', '``%s``' % option.metavar, option.help or '')) + else: + for option in group._group_actions: + if option.metavar: + option_fmt = '``%s ' + option.metavar + '``' + else: + option_fmt = '``%s``' + option_str = ', '.join(option_fmt % s for s in option.option_strings) + option_desc = textwrap.dedent((option.help or '') % option.__dict__) + rows.append((3, '', option_str, option_desc)) + + fp.write('.. only:: html\n\n') + table = io.StringIO() + table.write('.. class:: borg-options-table\n\n') + self.rows_to_table(rows, table.write) + fp.write(textwrap.indent(table.getvalue(), ' ' * 4)) + + # LaTeX output: + # Regular rST option lists (irregular column widths) + latex_options = io.StringIO() + for group in parser._action_groups: + if group.title == 'Common options': + latex_options.write('\n\n:ref:`common_options`\n') + latex_options.write(' |') + else: + self.write_options_group(group, latex_options) + fp.write('\n.. only:: latex\n\n') + fp.write(textwrap.indent(latex_options.getvalue(), ' ' * 4)) + + def rows_to_table(self, rows, write): + def write_row_separator(): + write('+') + for column_width in column_widths: + write('-' * (column_width + 1)) + write('+') + write('\n') + + # Find column count and width + column_count = max(columns for columns, *_ in rows) + column_widths = [0] * column_count + for columns, *cells in rows: + for i in range(columns): + # "+ 1" because we want a space between the cell contents and the delimiting "|" in the output + column_widths[i] = max(column_widths[i], len(cells[i]) + 1) + + for columns, *original_cells in rows: + write_row_separator() + # If a cell contains newlines, then the row must be split up in individual rows + # where each cell contains no newline. + rowspanning_cells = [] + original_cells = list(original_cells) + while any('\n' in cell for cell in original_cells): + cell_bloc = [] + for i, cell in enumerate(original_cells): + pre, _, original_cells[i] = cell.partition('\n') + cell_bloc.append(pre) + rowspanning_cells.append(cell_bloc) + rowspanning_cells.append(original_cells) + for cells in rowspanning_cells: + for i, column_width in enumerate(column_widths): + if i < columns: + write('| ') + write(cells[i].ljust(column_width)) + else: + write(' ') + write(''.ljust(column_width)) + write('|\n') + + write_row_separator() + # This bit of JavaScript kills the <colgroup> that is invariably inserted by docutils, + # but does absolutely no good here. It sets bogus column widths which cannot be overridden + # with CSS alone. + # Since this is HTML-only output, it would be possible to just generate a <table> directly, + # but then we'd lose rST formatting. + write(textwrap.dedent(""" + .. raw:: html + + <script type='text/javascript'> + $(document).ready(function () { + $('.borg-options-table colgroup').remove(); + }) + </script> + """)) + + def write_options_group(self, group, fp, with_title=True, base_indent=4): + def is_positional_group(group): + return any(not o.option_strings for o in group._group_actions) + + indent = ' ' * base_indent + + if is_positional_group(group): + for option in group._group_actions: + fp.write(option.metavar + '\n') + fp.write(textwrap.indent(option.help or '', ' ' * base_indent) + '\n') + return + + if not group._group_actions: + return + + if with_title: + fp.write('\n\n') + fp.write(group.title + '\n') + + opts = OrderedDict() + + for option in group._group_actions: + if option.metavar: + option_fmt = '%s ' + option.metavar + else: + option_fmt = '%s' + option_str = ', '.join(option_fmt % s for s in option.option_strings) + option_desc = textwrap.dedent((option.help or '') % option.__dict__) + opts[option_str] = textwrap.indent(option_desc, ' ' * 4) + + padding = len(max(opts)) + 1 + + for option, desc in opts.items(): + fp.write(indent + option.ljust(padding) + desc + '\n') + + +class build_man(Command): + description = 'build man pages' + + user_options = [] + + see_also = { + 'create': ('delete', 'prune', 'check', 'patterns', 'placeholders', 'compression'), + 'recreate': ('patterns', 'placeholders', 'compression'), + 'list': ('info', 'diff', 'prune', 'patterns'), + 'info': ('list', 'diff'), + 'init': ('create', 'delete', 'check', 'list', 'key-import', 'key-export', 'key-change-passphrase'), + 'key-import': ('key-export', ), + 'key-export': ('key-import', ), + 'mount': ('umount', 'extract'), # Would be cooler if these two were on the same page + 'umount': ('mount', ), + 'extract': ('mount', ), + } + + rst_prelude = textwrap.dedent(""" + .. role:: ref(title) + + .. |project_name| replace:: Borg + + """) + + usage_group = { + 'break-lock': 'lock', + 'with-lock': 'lock', + + 'change-passphrase': 'key', + 'key_change-passphrase': 'key', + 'key_export': 'key', + 'key_import': 'key', + 'key_migrate-to-repokey': 'key', + + 'export-tar': 'tar', + + 'benchmark_crud': 'benchmark', + + 'umount': 'mount', + } + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + print('building man pages (in docs/man)', file=sys.stderr) + import borg + borg.doc_mode = 'build_man' + os.makedirs('docs/man', exist_ok=True) + # allows us to build docs without the C modules fully loaded during help generation + from borg.archiver import Archiver + parser = Archiver(prog='borg').build_parser() + borgfs_parser = Archiver(prog='borgfs').build_parser() + + self.generate_level('', parser, Archiver, {'borgfs': borgfs_parser}) + self.build_topic_pages(Archiver) + self.build_intro_page() + + def generate_level(self, prefix, parser, Archiver, extra_choices=None): + is_subcommand = False + choices = {} + for action in parser._actions: + if action.choices is not None and 'SubParsersAction' in str(action.__class__): + is_subcommand = True + for cmd, parser in action.choices.items(): + choices[prefix + cmd] = parser + if extra_choices is not None: + choices.update(extra_choices) + if prefix and not choices: + return + + for command, parser in sorted(choices.items()): + if command.startswith('debug') or command == 'help': + continue + + if command == "borgfs": + man_title = command + else: + man_title = 'borg-' + command.replace(' ', '-') + print('building man page', man_title + '(1)', file=sys.stderr) + + is_intermediary = self.generate_level(command + ' ', parser, Archiver) + + doc, write = self.new_doc() + self.write_man_header(write, man_title, parser.description) + + self.write_heading(write, 'SYNOPSIS') + if is_intermediary: + subparsers = [action for action in parser._actions if 'SubParsersAction' in str(action.__class__)][0] + for subcommand in subparsers.choices: + write('| borg', '[common options]', command, subcommand, '...') + self.see_also.setdefault(command, []).append('%s-%s' % (command, subcommand)) + else: + if command == "borgfs": + write(command, end='') + else: + write('borg', '[common options]', command, end='') + self.write_usage(write, parser) + write('\n') + + description, _, notes = parser.epilog.partition('\n.. man NOTES') + + if description: + self.write_heading(write, 'DESCRIPTION') + write(description) + + if not is_intermediary: + self.write_heading(write, 'OPTIONS') + write('See `borg-common(1)` for common options of Borg commands.') + write() + self.write_options(write, parser) + + self.write_examples(write, command) + + if notes: + self.write_heading(write, 'NOTES') + write(notes) + + self.write_see_also(write, man_title) + + self.gen_man_page(man_title, doc.getvalue()) + + # Generate the borg-common(1) man page with the common options. + if 'create' in choices: + doc, write = self.new_doc() + man_title = 'borg-common' + self.write_man_header(write, man_title, 'Common options of Borg commands') + + common_options = [group for group in choices['create']._action_groups if group.title == 'Common options'][0] + + self.write_heading(write, 'SYNOPSIS') + self.write_options_group(write, common_options) + self.write_see_also(write, man_title) + self.gen_man_page(man_title, doc.getvalue()) + + return is_subcommand + + def build_topic_pages(self, Archiver): + for topic, text in Archiver.helptext.items(): + doc, write = self.new_doc() + man_title = 'borg-' + topic + print('building man page', man_title + '(1)', file=sys.stderr) + + self.write_man_header(write, man_title, 'Details regarding ' + topic) + self.write_heading(write, 'DESCRIPTION') + write(text) + self.gen_man_page(man_title, doc.getvalue()) + + def build_intro_page(self): + print('building man page borg(1)', file=sys.stderr) + with open('docs/man_intro.rst') as fd: + man_intro = fd.read() + self.gen_man_page('borg', self.rst_prelude + man_intro) + + def new_doc(self): + doc = io.StringIO(self.rst_prelude) + doc.read() + write = self.printer(doc) + return doc, write + + def printer(self, fd): + def write(*args, **kwargs): + print(*args, file=fd, **kwargs) + return write + + def write_heading(self, write, header, char='-', double_sided=False): + write() + if double_sided: + write(char * len(header)) + write(header) + write(char * len(header)) + write() + + def write_man_header(self, write, title, description): + self.write_heading(write, title, '=', double_sided=True) + self.write_heading(write, description, double_sided=True) + # man page metadata + write(':Author: The Borg Collective') + write(':Date:', datetime.utcnow().date().isoformat()) + write(':Manual section: 1') + write(':Manual group: borg backup tool') + write() + + def write_examples(self, write, command): + command = command.replace(' ', '_') + with open('docs/usage/%s.rst' % self.usage_group.get(command, command)) as fd: + usage = fd.read() + usage_include = '.. include:: %s.rst.inc' % command + begin = usage.find(usage_include) + end = usage.find('.. include', begin + 1) + # If a command has a dedicated anchor, it will occur before the command's include. + if 0 < usage.find('.. _', begin + 1) < end: + end = usage.find('.. _', begin + 1) + examples = usage[begin:end] + examples = examples.replace(usage_include, '') + examples = examples.replace('Examples\n~~~~~~~~', '') + examples = examples.replace('Miscellaneous Help\n------------------', '') + examples = examples.replace('``docs/misc/prune-example.txt``:', '``docs/misc/prune-example.txt``.') + examples = examples.replace('.. highlight:: none\n', '') # we don't support highlight + examples = re.sub('^(~+)$', lambda matches: '+' * len(matches.group(0)), examples, flags=re.MULTILINE) + examples = examples.strip() + if examples: + self.write_heading(write, 'EXAMPLES', '-') + write(examples) + + def write_see_also(self, write, man_title): + see_also = self.see_also.get(man_title.replace('borg-', ''), ()) + see_also = ['`borg-%s(1)`' % s for s in see_also] + see_also.insert(0, '`borg-common(1)`') + self.write_heading(write, 'SEE ALSO') + write(', '.join(see_also)) + + def gen_man_page(self, name, rst): + from docutils.writers import manpage + from docutils.core import publish_string + from docutils.nodes import inline + from docutils.parsers.rst import roles + + def issue(name, rawtext, text, lineno, inliner, options={}, content=[]): + return [inline(rawtext, '#' + text)], [] + + roles.register_local_role('issue', issue) + # We give the source_path so that docutils can find relative includes + # as-if the document where located in the docs/ directory. + man_page = publish_string(source=rst, source_path='docs/%s.rst' % name, writer=manpage.Writer()) + with open('docs/man/%s.1' % name, 'wb') as fd: + fd.write(man_page) + + def write_usage(self, write, parser): + if any(len(o.option_strings) for o in parser._actions): + write(' [options] ', end='') + for option in parser._actions: + if option.option_strings: + continue + write(format_metavar(option), end=' ') + + def write_options(self, write, parser): + for group in parser._action_groups: + if group.title == 'Common options' or not group._group_actions: + continue + title = 'arguments' if group.title == 'positional arguments' else group.title + self.write_heading(write, title, '+') + self.write_options_group(write, group) + + def write_options_group(self, write, group): + def is_positional_group(group): + return any(not o.option_strings for o in group._group_actions) + + if is_positional_group(group): + for option in group._group_actions: + write(option.metavar) + write(textwrap.indent(option.help or '', ' ' * 4)) + return + + opts = OrderedDict() + + for option in group._group_actions: + if option.metavar: + option_fmt = '%s ' + option.metavar + else: + option_fmt = '%s' + option_str = ', '.join(option_fmt % s for s in option.option_strings) + option_desc = textwrap.dedent((option.help or '') % option.__dict__) + opts[option_str] = textwrap.indent(option_desc, ' ' * 4) + + padding = len(max(opts)) + 1 + + for option, desc in opts.items(): + write(option.ljust(padding), desc) |