From 452ebe908c35f7550373c314fc84ed0cf591121b Mon Sep 17 00:00:00 2001 From: Drew DeVault Date: Sat, 6 Jul 2019 13:43:15 -0400 Subject: Implement annotations --- gitsrht/annotations.py | 263 ++++++++++++++++++++++++++++++++++++++++++++ gitsrht/blueprints/api.py | 30 +++++ gitsrht/blueprints/repo.py | 14 ++- gitsrht/templates/blob.html | 2 +- scss/main.scss | 39 +++++++ 5 files changed, 344 insertions(+), 4 deletions(-) create mode 100644 gitsrht/annotations.py diff --git a/gitsrht/annotations.py b/gitsrht/annotations.py new file mode 100644 index 0000000..b9508fd --- /dev/null +++ b/gitsrht/annotations.py @@ -0,0 +1,263 @@ +from pygments.formatter import Formatter +from pygments.token import Token, STANDARD_TYPES +from pygments.util import string_types, iteritems +from srht.markdown import markdown +from urllib.parse import urlparse + +_escape_html_table = { + ord('&'): u'&', + ord('<'): u'<', + ord('>'): u'>', + ord('"'): u'"', + ord("'"): u''', +} + +def escape_html(text, table=_escape_html_table): + return text.translate(table) + +def _get_ttype_class(ttype): + fname = STANDARD_TYPES.get(ttype) + if fname: + return fname + aname = '' + while fname is None: + aname = '-' + ttype[-1] + aname + ttype = ttype.parent + fname = STANDARD_TYPES.get(ttype) + return fname + aname + +# Fork of the pygments HtmlFormatter (BSD licensed) +# The main difference is that it relies on AnnotatedFormatter to escape the +# HTML tags in the source. Other features we don't use are removed to keep it +# slim. +class _BaseFormatter(Formatter): + def __init__(self): + super().__init__() + self._create_stylesheet() + + def get_style_defs(self, arg=None): + """ + Return CSS style definitions for the classes produced by the current + highlighting style. ``arg`` can be a string or list of selectors to + insert before the token type classes. + """ + if arg is None: + arg = ".highlight" + if isinstance(arg, string_types): + args = [arg] + else: + args = list(arg) + + def prefix(cls): + if cls: + cls = '.' + cls + tmp = [] + for arg in args: + tmp.append((arg and arg + ' ' or '') + cls) + return ', '.join(tmp) + + styles = [(level, ttype, cls, style) + for cls, (style, ttype, level) in iteritems(self.class2style) + if cls and style] + styles.sort() + lines = ['%s { %s } /* %s */' % (prefix(cls), style, repr(ttype)[6:]) + for (level, ttype, cls, style) in styles] + return '\n'.join(lines) + + def _get_css_class(self, ttype): + """Return the css class of this token type prefixed with + the classprefix option.""" + ttypeclass = _get_ttype_class(ttype) + if ttypeclass: + return ttypeclass + return '' + + def _get_css_classes(self, ttype): + """Return the css classes of this token type prefixed with + the classprefix option.""" + cls = self._get_css_class(ttype) + while ttype not in STANDARD_TYPES: + ttype = ttype.parent + cls = self._get_css_class(ttype) + ' ' + cls + return cls + + def _create_stylesheet(self): + t2c = self.ttype2class = {Token: ''} + c2s = self.class2style = {} + for ttype, ndef in self.style: + name = self._get_css_class(ttype) + style = '' + if ndef['color']: + style += 'color: #%s; ' % ndef['color'] + if ndef['bold']: + style += 'font-weight: bold; ' + if ndef['italic']: + style += 'font-style: italic; ' + if ndef['underline']: + style += 'text-decoration: underline; ' + if ndef['bgcolor']: + style += 'background-color: #%s; ' % ndef['bgcolor'] + if ndef['border']: + style += 'border: 1px solid #%s; ' % ndef['border'] + if style: + t2c[ttype] = name + # save len(ttype) to enable ordering the styles by + # hierarchy (necessary for CSS cascading rules!) + c2s[name] = (style[:-2], ttype, len(ttype)) + + def _format_lines(self, tokensource): + lsep = "\n" + # for lookup only + getcls = self.ttype2class.get + c2s = self.class2style + + lspan = '' + line = [] + for ttype, value in tokensource: + cls = self._get_css_classes(ttype) + cspan = cls and '' % cls or '' + + parts = value.split('\n') + + # for all but the last line + for part in parts[:-1]: + if line: + if lspan != cspan: + line.extend(((lspan and ''), cspan, part, + (cspan and ''), lsep)) + else: # both are the same + line.extend((part, (lspan and ''), lsep)) + yield 1, ''.join(line) + line = [] + elif part: + yield 1, ''.join((cspan, part, (cspan and ''), lsep)) + else: + yield 1, lsep + # for the last line + if line and parts[-1]: + if lspan != cspan: + line.extend(((lspan and ''), cspan, parts[-1])) + lspan = cspan + else: + line.append(parts[-1]) + elif parts[-1]: + line = [cspan, parts[-1]] + lspan = cspan + # else we neither have to open a new span nor set lspan + + if line: + line.extend(((lspan and ''), lsep)) + yield 1, ''.join(line) + + def _wrap_div(self, inner): + yield 0, f"
" + for tup in inner: + yield tup + yield 0, '
\n' + + def _wrap_pre(self, inner): + yield 0, '
'
+        for tup in inner:
+            yield tup
+        yield 0, '
' + + def wrap(self, source, outfile): + """ + Wrap the ``source``, which is a generator yielding + individual lines, in custom generators. See docstring + for `format`. Can be overridden. + """ + return self._wrap_div(self._wrap_pre(source)) + + def format_unencoded(self, tokensource, outfile): + source = self._format_lines(tokensource) + source = self.wrap(source, outfile) + for t, piece in source: + outfile.write(piece) + +def validate_annotation(valid, anno): + valid.expect("type" in anno, "'type' is required") + if not valid.ok: + return + valid.expect(anno["type"] in ["link", "markdown"], + f"'{anno['type']} is not a valid annotation type'") + if anno["type"] == "link": + for field in ["lineno", "colno", "len"]: + valid.expect(field in anno, "f'{field}' is required") + valid.expect(field not in anno or isinstance(anno[field], int), + "f'{field}' must be an integer") + valid.expect("to" in anno, "'to' is required") + valid.expect("title" not in anno or isinstance(anno["title"], str), + "'title' must be a string") + elif anno["type"] == "markdown": + for field in ["lineno"]: + valid.expect(field in anno, "f'{field}' is required") + valid.expect(field not in anno or isinstance(anno[field], int), + "f'{field}' must be an integer") + for field in ["title", "content"]: + valid.expect(field in anno, "f'{field}' is required") + valid.expect(field not in anno or isinstance(anno[field], str), + "f'{field}' must be a string") + +class AnnotatedFormatter(_BaseFormatter): + def __init__(self, annos, link_prefix): + super().__init__() + self.annos = dict() + self.link_prefix = link_prefix + for anno in (annos or list()): + lineno = int(anno["lineno"]) + self.annos.setdefault(lineno, list()) + self.annos[lineno].append(anno) + self.annos[lineno] = sorted(self.annos[lineno], + key=lambda anno: anno.get("from", -1)) + + def _annotate_token(self, token, colno, annos): + # TODO: Extend this to support >1 anno per token + for anno in annos: + if anno["type"] == "link": + start = anno["colno"] - 1 + end = anno["colno"] + anno["len"] - 1 + target = anno["to"] + title = anno.get("title", "") + url = urlparse(target) + if url.scheme == "": + target = self.link_prefix + "/" + target + if start <= colno < end: + return (f"{escape_html(token)}""") + elif anno["type"] == "markdown": + if "\n" not in token: + continue + title = anno["title"] + content = anno["content"] + content = markdown(content, baselevel=6, + link_prefix=self.link_prefix) + annotation = f"
{title}{content}
\n" + token = escape_html(token).replace("\n", annotation, 1) + return token + # Other types? + return escape_html(token) + + def _wrap_source(self, source): + lineno = 0 + colno = 0 + for ttype, token in source: + parts = token.splitlines(True) + _lineno = lineno + for part in parts: + annos = self.annos.get(_lineno + 1, []) + if any(annos): + yield ttype, self._annotate_token(part, colno, annos) + else: + yield ttype, escape_html(part) + _lineno += 1 + if "\n" in token: + lineno += sum(1 if c == "\n" else 0 for c in token) + colno = len(token[token.rindex("\n")+1:]) + else: + colno += len(token) + + def _format_lines(self, source): + yield from super()._format_lines(self._wrap_source(source)) diff --git a/gitsrht/blueprints/api.py b/gitsrht/blueprints/api.py index 58bcb8b..fa223ab 100644 --- a/gitsrht/blueprints/api.py +++ b/gitsrht/blueprints/api.py @@ -1,13 +1,17 @@ import base64 +import json import pygit2 from flask import Blueprint, current_app, request, send_file, abort +from gitsrht.annotations import validate_annotation from gitsrht.blueprints.repo import lookup_ref, get_log, collect_refs from gitsrht.git import Repository as GitRepository, commit_time, annotate_tree from gitsrht.webhooks import RepoWebhook from io import BytesIO from scmsrht.blueprints.api import get_user, get_repo +from scmsrht.redis import redis from srht.api import paginated_response from srht.oauth import current_token, oauth +from srht.validation import Validation data = Blueprint("api.data", __name__) @@ -133,6 +137,32 @@ def repo_tree_GET(username, reponame, ref, path): abort(404) return tree_to_dict(tree) +@data.route("/api/repos//annotate", methods=["PUT"]) +@data.route("/api//repos//annotate", methods=["PUT"]) +@oauth("data:read") +def repo_annotate_PUT(username, reponame): + user = get_user(username) + repo = get_repo(user, reponame) + + valid = Validation(request) + + for oid, annotations in valid.source.items(): + valid.expect(isinstance(oid, str), "blob keys must be strings") + valid.expect(isinstance(annotations, list), + "blob values must be lists of annotations") + if not valid.ok: + return valid.response + for anno in annotations: + validate_annotation(valid, anno) + if not valid.ok: + return valid.response + # TODO: more validation on annotation structure + redis.set(f"git.sr.ht:git:annotations:{oid}", json.dumps(annotations)) + # Invalidate rendered markup cache + redis.delete(f"git.sr.ht:git:highlight:{oid}") + + return { }, 200 + @data.route("/api/repos//blob/", defaults={"username": None, "path": ""}) @data.route("/api/repos//blob//", diff --git a/gitsrht/blueprints/repo.py b/gitsrht/blueprints/repo.py index fe9e068..a12bf8e 100644 --- a/gitsrht/blueprints/repo.py +++ b/gitsrht/blueprints/repo.py @@ -1,13 +1,15 @@ import binascii +import json import os import pygit2 import pygments -import sys import subprocess +import sys from datetime import timedelta from jinja2 import Markup from flask import Blueprint, render_template, abort, send_file, request from flask import Response, url_for +from gitsrht.annotations import AnnotatedFormatter from gitsrht.editorconfig import EditorConfig from gitsrht.git import Repository as GitRepository, commit_time, annotate_tree from gitsrht.git import diffstat @@ -45,8 +47,14 @@ def get_readme(repo, tip, link_prefix=None): return get_formatted_readme("git.sr.ht:git", file_finder, content_getter, link_prefix=link_prefix) -def _highlight_file(name, data, blob_id): - return get_highlighted_file("git.sr.ht:git", name, blob_id, data) +def _highlight_file(repo, ref, name, data, blob_id): + annotations = redis.get(f"git.sr.ht:git:annotations:{blob_id}") + if annotations: + annotations = json.loads(annotations.decode()) + link_prefix = url_for( + 'repo.tree', owner=repo.owner, repo=repo.name, ref=ref) + return get_highlighted_file("git.sr.ht:git", name, blob_id, data, + formatter=AnnotatedFormatter(annotations, link_prefix)) def render_empty_repo(owner, repo): origin = cfg("git.sr.ht", "origin") diff --git a/gitsrht/templates/blob.html b/gitsrht/templates/blob.html index fe978e5..f04c288 100644 --- a/gitsrht/templates/blob.html +++ b/gitsrht/templates/blob.html @@ -74,7 +74,7 @@ pre, body { id="L{{loop.index}}" >{{loop.index}}{% if not loop.last %} {% endif %}{% endfor %} - {{ highlight_file(entry.name, data, blob.id.hex) }} + {{ highlight_file(repo, ref, entry.name, data, blob.id.hex) }} {% else %}
diff --git a/scss/main.scss b/scss/main.scss index 968bceb..cc086a8 100644 --- a/scss/main.scss +++ b/scss/main.scss @@ -176,3 +176,42 @@ pre { img { max-width: 100%; } + +// Annotations +.highlight { + a.annotation { + color: inherit; + background: lighten($primary, 45); + border-bottom: 1px dotted $gray-800; + + &:hover { + text-decoration: none; + border-bottom: 1px solid $gray-800; + } + } + + details { + display: inline; + margin-left: 3rem; + color: $gray-600; + + &[open] { + display: block; + color: inherit; + background: $gray-100; + width: 30rem; + margin: 0; + white-space: normal; + font: inherit; + position: absolute; + + summary { + background: $gray-300; + } + + ul:last-child { + margin-bottom: 0; + } + } + } +} -- cgit v1.2.3