~sircmpwn/dispatch.sr.ht

35faf0f4a3d948ae8b2780337b6a49fadc8f85e2 — Drew DeVault 5 months ago 15c40ad
Generalize build submission
A dispatchsrht/builds.py => dispatchsrht/builds.py +86 -0
@@ 0,0 1,86 @@
import base64
import json
import re
import requests
import yaml
from cryptography.fernet import Fernet
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
from flask import url_for
from srht.config import cfg
from typing import Any, Callable, Dict, Iterable, Tuple

_root = cfg("dispatch.sr.ht", "origin")
_builds_sr_ht = cfg("builds.sr.ht", "origin", default=None)
_secret_key = cfg("sr.ht", "secret-key")
_kdf = PBKDF2HMAC(
        algorithm=hashes.SHA256(),
        length=32,
        salt=_secret_key.encode(),
        iterations=100000,
        backend=default_backend())
_key = base64.urlsafe_b64encode(_kdf.derive(_secret_key.encode()))
_fernet = Fernet(_key)

if _builds_sr_ht:
    from buildsrht.manifest import Manifest

def first_line(text):
    """Returns the first line of a string. Useful for commit messages."""
    if not "\n" in text:
        return text
    return text[:text.index("\n") + 1]

def encrypt_notify_url(route, payload):
    encpayload = _fernet.encrypt(
            json.dumps(payload).encode()).decode()
    return _root + url_for(route, payload=encpayload)

def decrypt_notify_payload(payload):
    return json.loads(_fernet.decrypt(payload.encode()).decode())

def submit_build(
        build_tag: str,
        manifests: Iterable[Tuple[str, Manifest]],
        oauth_token: str,
        note: str=None,
        secrets: bool=False,
        preparing: Callable[[str], Any]=None,
        submitted: Callable[[str, str], str]=None) -> str:
    """
    Submits a build, or builds, to builds.sr.ht. Returns a user-friendly
    summary of the builds submitted.

    @build_tag:      Build tag for this set of manifests, usually a repo name
    @manifests:      List of build manifests to submit and their names
    @oauth_token:    The user's builds.sr.ht-authorized OAuth token
    @note:           Note to add to build submission, e.g. commit message
    @secrets:        Whether to enable secrets for this build
    @preparing:      A callable called when each manifest is being prepared
                     Called with the manifest name.
    @submitted:      A callable called when each manifest has been submitted.
                     Called with the manifest name and the job URL. Should
                     return a brief statement for the summary string.
    """
    build_urls = []
    for name, manifest in manifests:
        if preparing:
            preparing(name)
        build_tag = re.sub(r"[^a-z0-9_.-]", "", build_tag.lower())
        if name:
            name = re.sub(r"[^a-z0-9_.-]", "", name.lower())
        resp = requests.post(_builds_sr_ht + "/api/jobs", json={
            "manifest": yaml.dump(manifest.to_dict(), default_flow_style=False),
            "tags": [build_tag] + ([name] if name else []),
            "note": note,
            "secrets": secrets,
        }, headers={
            "Authorization": "token " + oauth_token,
        })
        if resp.status_code != 200:
            return resp.text
        build_id = resp.json()["id"]
        if submitted:
            build_urls.append(submitted(name, build_id))
    return "Started builds:\n\n" + "\n".join(build_urls)

M dispatchsrht/tasks/github/auth.py => dispatchsrht/tasks/github/auth.py +87 -102
@@ 2,13 2,8 @@ import base64
import html
import json
import sqlalchemy as sa
import re
import requests
import yaml
from cryptography.fernet import Fernet
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
from flask import redirect, request, url_for
from flask_login import current_user
from functools import wraps


@@ 18,16 13,17 @@ from srht.config import cfg
from srht.database import Base, db
from srht.flask import loginrequired, csrf_bypass
from dispatchsrht.app import app

def _first_line(text):
    if not "\n" in text:
        return text
    return text[:text.index("\n") + 1]
from dispatchsrht.builds import first_line, encrypt_notify_url
from dispatchsrht.builds import decrypt_notify_payload, submit_build

_github_client_id = cfg("dispatch.sr.ht::github",
        "oauth-client-id", default=None)
_github_client_secret = cfg("dispatch.sr.ht::github",
        "oauth-client-secret", default=None)
_builds_sr_ht = cfg("builds.sr.ht", "origin", default=None)

if _builds_sr_ht:
    from buildsrht.manifest import Manifest, Trigger

class GitHubAuthorization(Base):
    __tablename__ = "github_authorization"


@@ 94,27 90,49 @@ def github_callback():
    db.session.commit()
    return redirect(state)

_root = cfg("dispatch.sr.ht", "origin")
_builds_sr_ht = cfg("builds.sr.ht", "origin", default=None)
_secret_key = cfg("sr.ht", "secret-key")
_kdf = PBKDF2HMAC(
        algorithm=hashes.SHA256(),
        length=32,
        salt=_secret_key.encode(),
        iterations=100000,
        backend=default_backend())
_key = base64.urlsafe_b64encode(_kdf.derive(_secret_key.encode()))
_fernet = Fernet(_key)

def submit_build(hook, repo, commit, base=None,
context = lambda name: "builds.sr.ht" + (f": {name}" if name else "")

def source_url(repo, base, git_commit, source):
    if not source.endswith("/" + base.name):
        return source
    if base.name != repo.name:
        return base.name + "::" + repo.clone_url + "#" + git_commit.sha
    if repo.private:
        return repo.ssh_url + "#" + git_commit.sha
    return repo.clone_url + "#" + git_commit.sha

def update_preparing(base_commit):
    def go(name):
        try:
            base_commit.create_status("pending", _builds_sr_ht,
                    "preparing builds.sr.ht job", context=context(name))
        except GithubException:
            pass
    return go

def update_submitted(base_commit, username):
    def go(name, build_id):
        build_url = "{}/~{}/job/{}".format(
                _builds_sr_ht, username, build_id)
        try:
            base_commit.create_status("pending", build_url,
                    "builds.sr.ht job is running", context=context(name))
            return build_url
        except GithubException:
            return ""
    return go

def submit_github_build(hook, repo, commit, base=None,
        secrets=False, env=dict(), extras=dict()):
    if base == None:
        base = repo

    auth = GitHubAuthorization.query.filter(
        GitHubAuthorization.user_id == hook.user_id).first()
    if not auth:
        return "You have not authorized us to access your GitHub account", 401
    github = Github(auth.oauth_token)

    try:
        repo = github.get_repo(repo["full_name"])
        base = github.get_repo(base["full_name"])


@@ 125,104 143,71 @@ def submit_build(hook, repo, commit, base=None,
            "Did you revoke our access?"), 401
    base_commit = base.get_commit(sha)
    git_commit = commit.commit

    try:
        manifest = repo.get_contents(".build.yml", ref=git_commit.sha)
        files = [repo.get_contents(".build.yml", ref=git_commit.sha)]
    except GithubException:
        manifest = None
    if manifest is not None:
        manifests = [manifest]
    else:
        try:
            manifests = repo.get_dir_contents(
            files = repo.get_dir_contents(
                    ".builds", ref=git_commit.sha) or []
            manifests = [repo.get_contents(m.path, ref=git_commit.sha)
                    for m in manifests]
            files = [repo.get_contents(
                f.path, ref=git_commit.sha) for f in files]
        except GithubException:
            manifests = []
    if not manifests:
            files = []
    if not files:
        return "There are no build manifest in this repository"
    def source_url(source):
        if not source.endswith("/" + base.name):
            return source
        if base.name != repo.name:
            return base.name + "::" + repo.clone_url + "#" + git_commit.sha
        if repo.private:
            return repo.ssh_url + "#" + git_commit.sha
        return repo.clone_url + "#" + git_commit.sha
    build_urls = []
    for manifest in manifests:
        name = manifest.name
        manifest = base64.b64decode(manifest.content)
        from buildsrht.manifest import Manifest, Trigger

    manifests = list()
    for f in files:
        name = f.name
        manifest = base64.b64decode(f.content)
        try:
            manifest = Manifest(yaml.safe_load(manifest))
        except Exception as ex:
            return f"There are errors in {name}:\n{str(ex)}", 400

        if manifest.sources:
            manifest.sources = [source_url(s) for s in manifest.sources]
        context = "builds.sr.ht" + (f": {name}" if name else "")
        try:
            status = base_commit.create_status("pending", _builds_sr_ht,
                    "preparing builds.sr.ht job", context=context)
        except GithubException:
            return "Unable to add status to commit", 400
        complete_url = completion_url(base.full_name, auth.user.username,
                auth.oauth_token, commit.sha, context, extras)
        manifest.triggers.append(Trigger({
            "action": "webhook",
            "condition": "always",
            "url": complete_url,
        }))
            manifest.sources = [source_url(repo, base, git_commit, s)
                    for s in manifest.sources]
        if not manifest.environment:
            manifest.environment = env
        else:
            manifest.environment.update(env)
        repo_name = re.sub(r"[^a-z0-9_.-]", "", repo.name.lower())
        if name:
            name = re.sub(r"[^a-z0-9_.-]", "", name.lower())
        resp = requests.post(_builds_sr_ht + "/api/jobs", json={
            "manifest": yaml.dump(manifest.to_dict(), default_flow_style=False),
            "tags": [repo_name] + ([name] if name else []),
            "note": "{}\n\n[{}]({}) — [{}](mailto:{})".format(
                html.escape(_first_line(git_commit.message)),
                str(git_commit.sha)[:7], commit.html_url,
                git_commit.author.name,
                git_commit.author.email,
            ),
            "secrets": secrets,
        }, headers={
            "Authorization": "token " + hook.user.oauth_token,
        })
        if resp.status_code != 200:
            return resp.text
        build_id = resp.json()["id"]
        build_url = "{}/~{}/job/{}".format(
                _builds_sr_ht, auth.user.username, build_id)
        status = base_commit.create_status("pending", build_url,
                "builds.sr.ht job is running",
                context=context)
        build_urls.append(build_url)
    return "Started builds:\n\n" + "\n".join(build_urls)

def completion_url(full_name, username, oauth_token, sha, context, extras):
    complete_request = {
        "full_name": full_name,
        "oauth_token": oauth_token,
        "username": username,
        "sha": sha,
        "context": context,
    }
    complete_request.update(extras)
    complete_payload = _fernet.encrypt(
            json.dumps(complete_request).encode()).decode()
    complete_url = _root + url_for("github_complete_build",
            payload=complete_payload)
    return complete_url

        notify_payload = {
            "full_name": base.full_name,
            "oauth_token": auth.oauth_token,
            "username": auth.user.username,
            "sha": commit.sha,
            "context": context(name),
        }
        if extras:
            notify_payload.update(extras)
        notify_url = encrypt_notify_url("github_complete_build", notify_payload)

        manifest.triggers.append(Trigger({
            "action": "webhook",
            "condition": "always",
            "url": notify_url,
        }))

        manifests.append((name, manifest))

    note = "{}\n\n[{}]({}) — [{}](mailto:{})".format(
            html.escape(first_line(git_commit.message)),
            str(git_commit.sha)[:7], commit.html_url,
            git_commit.author.name,
            git_commit.author.email)

    return submit_build(repo.name, manifests, hook.user.oauth_token,
            note=note, secrets=secrets,
            preparing=update_preparing(base_commit),
            submitted=update_submitted(base_commit, auth.user.username))

@csrf_bypass
@app.route("/github/complete_build/<payload>", methods=["POST"])
def github_complete_build(payload):
    payload = json.loads(_fernet.decrypt(payload.encode()).decode())
    payload = decrypt_notify_payload(payload)
    github = Github(payload["oauth_token"])
    repo = github.get_repo(payload["full_name"])
    commit = repo.get_commit(payload["sha"])

M dispatchsrht/tasks/github/github_commit_to_build.py => dispatchsrht/tasks/github/github_commit_to_build.py +2 -2
@@ 12,7 12,7 @@ from srht.validation import Validation
from dispatchsrht.tasks import TaskDef
from dispatchsrht.tasks.github.auth import GitHubAuthorization
from dispatchsrht.tasks.github.auth import githubloginrequired
from dispatchsrht.tasks.github.auth import submit_build
from dispatchsrht.tasks.github.auth import submit_github_build
from dispatchsrht.types import Task

_root = cfg("dispatch.sr.ht", "origin")


@@ 83,7 83,7 @@ class GitHubCommitToBuild(TaskDef):
        ref = valid.require("ref")
        if not valid.ok:
            return "Got request, but it has no commits"
        return submit_build(hook, repo, commit, env={
        return submit_github_build(hook, repo, commit, env={
            "GITHUB_DELIVERY": request.headers.get("X-GitHub-Delivery"),
            "GITHUB_EVENT": request.headers.get("X-GitHub-Event"),
            "GITHUB_REF": ref,

M dispatchsrht/tasks/github/github_pr_to_build.py => dispatchsrht/tasks/github/github_pr_to_build.py +2 -2
@@ 13,7 13,7 @@ from srht.validation import Validation
from dispatchsrht.tasks import TaskDef
from dispatchsrht.tasks.github.auth import GitHubAuthorization
from dispatchsrht.tasks.github.auth import githubloginrequired
from dispatchsrht.tasks.github.auth import submit_build
from dispatchsrht.tasks.github.auth import submit_github_build
from dispatchsrht.types import Task

_root = cfg("dispatch.sr.ht", "origin")


@@ 117,7 117,7 @@ class GitHubPRToBuild(TaskDef):
        secrets = hook.secrets
        if not base_repo["private"]:
            secrets = False
        return submit_build(hook, head_repo, head, base_repo,
        return submit_github_build(hook, head_repo, head, base_repo,
                secrets=secrets, extras={
                    "automerge": hook.automerge, 
                    "pr": pr["number"]