From 958fb1391461214f120624176e8817e932106096 Mon Sep 17 00:00:00 2001
From: Ludovic Chabant <ludovic@chabant.com>
Date: Fri, 8 Mar 2019 06:05:32 +0000
Subject: Refactor post update hook to use scmsrht

This also:
- Computes the "build note" only once (instead of once per manifet)
- Cleans up unused imports
---
 gitsrht-update-hook |  34 +-------
 gitsrht/worker.py   | 236 ++++++++++++++++++++--------------------------------
 2 files changed, 95 insertions(+), 175 deletions(-)

diff --git a/gitsrht-update-hook b/gitsrht-update-hook
index fed9971..7712377 100755
--- a/gitsrht-update-hook
+++ b/gitsrht-update-hook
@@ -1,19 +1,15 @@
 #!/usr/bin/env python3
-from srht.config import cfg, cfgi
+from srht.config import cfg
 from srht.database import DbSession
 db = DbSession(cfg("git.sr.ht", "connection-string"))
-from gitsrht.types import User, Repository, RepoVisibility
+from gitsrht.types import Repository, RepoVisibility
 db.init()
 from configparser import ConfigParser
 from datetime import datetime
-from pygit2 import Repository as GitRepository, Commit, Tag
-import shlex
-import subprocess
+from gitsrht.worker import do_post_update
 import sys
-import re
 
 op = sys.argv[0]
-builds_sr_ht = cfg("builds.sr.ht", "origin", default=None)
 origin = cfg("git.sr.ht", "origin")
 
 if op == "hooks/post-update":
@@ -42,26 +38,4 @@ if op == "hooks/post-update":
     repo.updated = datetime.utcnow()
     db.session.commit()
 
-    git_repo = GitRepository(repo.path)
-    oids = set()
-    for ref in refs:
-        try:
-            if re.match(r"^[0-9a-z]{40}$", ref): # commit
-                commit = git_repo.get(ref)
-            elif ref.startswith("refs/"): # ref
-                target_id = git_repo.lookup_reference(ref).target
-                commit = git_repo.get(target_id)
-                if isinstance(commit, Tag):
-                    commit = git_repo.get(commit.target)
-            else:
-                continue
-            if not isinstance(commit, Commit):
-                continue
-            if commit.id in oids:
-                continue
-            oids.add(commit.id)
-        except:
-            continue
-        if builds_sr_ht:
-            from gitsrht.worker import do_post_update
-            do_post_update(repo, git_repo, commit)
+    do_post_update(repo, refs)
diff --git a/gitsrht/worker.py b/gitsrht/worker.py
index ebfd120..d18ac6e 100644
--- a/gitsrht/worker.py
+++ b/gitsrht/worker.py
@@ -1,38 +1,27 @@
+import html
+import os
+import os.path
+import re
+import requests
+import yaml
+from buildsrht.manifest import Manifest
+from pygit2 import Repository as GitRepository, Commit, Tag
+from scmsrht.worker import BuildSubmitterBase
 from srht.config import cfg
-from srht.database import DbSession, db
+from srht.database import db
+from srht.oauth import OAuthScope
+from urllib.parse import urlparse
+
 if not hasattr(db, "session"):
-    db = DbSession(cfg("git.sr.ht", "connection-string"))
     import gitsrht.types
+    from srht.database import DbSession
+    db = DbSession(cfg("git.sr.ht", "connection-string"))
     db.init()
 
-from celery import Celery
-from srht.oauth import OAuthScope
-from buildsrht.manifest import Manifest
-import requests
-import html
-import yaml
-import os
-import re
-
-worker = Celery('git', broker=cfg("git.sr.ht", "redis"))
 builds_sr_ht = cfg("builds.sr.ht", "origin")
 builds_client_id = cfg("builds.sr.ht", "oauth-client-id")
 git_sr_ht = cfg("git.sr.ht", "origin")
 
-@worker.task
-def _do_webhook(url, payload, headers=None, **kwargs):
-    if "timeout" not in kwargs:
-        kwargs["timeout"] = 15
-    return requests.post(url, json=payload, headers=headers, **kwargs)
-    # TODO: Store the response somewhere I guess
-
-def do_webhook(url, payload, headers=None):
-    try:
-        return _do_webhook(url, payload, headers, timeout=3)
-    except requests.exceptions.Timeout:
-        _do_webhook.delay(url, payload, headers)
-        return None
-
 def first_line(text):
     try:
         i = text.index("\n")
@@ -41,122 +30,79 @@ def first_line(text):
     else:
         return text[:i + 1]
 
-def _auto_setup_sub_source(repo, m):
-    sources = m.get('sources', [])
-    re_repo_url = re.compile(r'^{}/(?P<usr>[^/]+)/{}/?$'.format(
-        re.escape(git_sr_ht), re.escape(repo.name)))
-    for i, s in enumerate(sources):
-        m = re_repo_url.search(s)
-        if m:
-            owner = repo.owner.canonical_name
-            if owner != m.group('usr'):
-                new_source = '{}/{}/{}'.format(
-                    git_sr_ht, repo.owner.canonical_name, repo.name)
-                sources[i] = new_source
-                print("auto-setup: changing source {} -> {}".format(
-                    s, new_source))
-                break
-
-def _auto_setup_auto_source(repo, m):
-    sources = m.setdefault('sources', [])
-    this_url = '{}/{}/{}'.format(
-        git_sr_ht, repo.owner.canonical_name, repo.name)
-    if this_url not in sources:
-        sources.append(this_url)
-        print("auto-setup: adding source {}".format(this_url))
-
-_auto_setup_funcs = {
-    'sub_source': _auto_setup_sub_source,
-    #'auto_source': _auto_setup_auto_source
-}
-
-def _auto_setup_manifest(repo, m):
-    auto_setup = m.setdefault('autosetup', 'sub_source')
-    if not auto_setup:
-        return
-
-    if isinstance(auto_setup, str):
-        auto_setup = [s.strip() for s in auto_setup.split(',')]
-
-    for s in auto_setup:
-        func = _auto_setup_funcs.get(s)
-        if func:
-            func(repo, m)
-        else:
-            print("Warning: unknown build manifest auto-setup function: {}".format(s))
-
-def submit_builds(repo, git_repo, commit):
-    manifest_blobs = dict()
-    if ".build.yml" in commit.tree:
-        build_yml = commit.tree[".build.yml"]
-        if build_yml.type == 'blob':
-            manifest_blobs[".build.yml"] = build_yml
-    elif ".builds"  in commit.tree:
-        build_dir = commit.tree[".builds"]
-        if build_dir.type == 'tree':
-            manifest_blobs.update(
-                {
-                    blob.name: blob
-                    for blob in git_repo.get(build_dir.id)
-                    if blob.type == 'blob' and (
-                        blob.name.endswith('.yml')
-                        or blob.name.endswith('.yaml')
-                    )
-                }
-            )
-
-    manifests = {}
-    for name, blob in manifest_blobs.items():
-        m = git_repo.get(blob.id).data.decode()
-        m = yaml.safe_load(m)
-        _auto_setup_manifest(repo, m)
-        manifests[name] = m
-
-    if not any(manifests):
-        return
-
-    for name, m in iter(manifests.items()):
-        m = Manifest(m)
-        if m.sources:
-            m.sources = [source if os.path.basename(source) != repo.name
-                    else source + "#" + str(commit.id) for source in m.sources]
-        manifests[name] = m
-    token = repo.owner.oauth_token
-    scopes = repo.owner.oauth_token_scopes
-    scopes = [OAuthScope(s) for s in scopes.split(",")]
-    if not any(s for s in scopes
-            if s.client_id == builds_client_id and s.access == 'write'):
-        print("Warning: log out and back in on the website to enable builds integration")
-        return
-    for name, manifest in iter(manifests.items()):
-        resp = do_webhook(builds_sr_ht + "/api/jobs", {
-            "manifest": yaml.dump(manifest.to_dict(), default_flow_style=False),
-            # TODO: orgs
-            "tags": [repo.name] + [name] if name else [],
-            "note": "{}\n\n[{}]({}) &mdash; [{}](mailto:{})".format(
-                # TODO: cgit replacement
-                html.escape(first_line(commit.message)),
-                str(commit.id)[:7],
-                "{}/{}/{}/commit/{}".format(
-                    git_sr_ht,
-                    "~" + repo.owner.username,
-                    repo.name,
-                    str(commit.id)),
-                commit.author.name,
-                commit.author.email,
-            )
-        }, { "Authorization": "token " + token })
-        if not resp or resp.status_code != 200:
-            print("Failed to submit build job" + (" " + name) if name else "")
-            return
-        build_id = resp.json().get("id")
-        if name != ".build.yml":
-            print("Build started: https://builds.sr.ht/~{}/job/{} [{}]".format(
-                repo.owner.username, build_id, name))
-        else:
-            print("Build started: https://builds.sr.ht/~{}/job/{}".format(
-                repo.owner.username, build_id))
-
-def do_post_update(repo, git_repo, commit):
-    if builds_sr_ht:
-        submit_builds(repo, git_repo, commit)
+class GitBuildSubmitter(BuildSubmitterBase):
+    def __init__(self, repo, git_repo):
+        super().__init__(git_sr_ht, 'git', repo)
+        self.git_repo = git_repo
+
+    def find_manifests(self, commit):
+        manifest_blobs = dict()
+        if ".build.yml" in commit.tree:
+            build_yml = commit.tree[".build.yml"]
+            if build_yml.type == 'blob':
+                manifest_blobs[".build.yml"] = build_yml
+        elif ".builds"  in commit.tree:
+            build_dir = commit.tree[".builds"]
+            if build_dir.type == 'tree':
+                manifest_blobs.update(
+                    {
+                        blob.name: blob
+                        for blob in self.git_repo.get(build_dir.id)
+                        if blob.type == 'blob' and (
+                            blob.name.endswith('.yml')
+                            or blob.name.endswith('.yaml')
+                        )
+                    }
+                )
+
+        manifests = {}
+        for name, blob in manifest_blobs.items():
+            m = self.git_repo.get(blob.id).data.decode()
+            manifests[name] = m
+        return manifests
+
+    def get_commit_id(self, commit):
+        return str(commit.id)
+
+    def get_commit_note(self, commit):
+        return "{}\n\n[{}]({}) &mdash; [{}](mailto:{})".format(
+            # TODO: cgit replacement
+            html.escape(first_line(commit.message)),
+            str(commit.id)[:7],
+            "{}/{}/{}/commit/{}".format(
+                git_sr_ht,
+                "~" + self.repo.owner.username,
+                self.repo.name,
+                str(commit.id)),
+            commit.author.name,
+            commit.author.email,
+        )
+
+def do_post_update(repo, refs):
+    if not builds_sr_ht:
+        return False
+
+    git_repo = GitRepository(repo.path)
+    oids = set()
+    for ref in refs:
+        try:
+            if re.match(r"^[0-9a-z]{40}$", ref): # commit
+                commit = git_repo.get(ref)
+            elif ref.startswith("refs/"): # ref
+                target_id = git_repo.lookup_reference(ref).target
+                commit = git_repo.get(target_id)
+                if isinstance(commit, Tag):
+                    commit = git_repo.get(commit.target)
+            else:
+                continue
+            if not isinstance(commit, Commit):
+                continue
+            if commit.id in oids:
+                continue
+            oids.add(commit.id)
+        except:
+            continue
+
+        if builds_sr_ht:
+            s = GitBuildSubmitter(repo, git_repo)
+            s.submit(commit)
-- 
cgit v1.2.3