~sircmpwn/dispatch.sr.ht

3a56828e48216d66de8fa71fc4db5270e009eb75 — Drew DeVault 5 months ago 51150a1 0.12.0
Support running builds.sr.ht CI for GitLab commits
A dispatchsrht/alembic/versions/101d96a6baaf_add_gitlab_tables.py => dispatchsrht/alembic/versions/101d96a6baaf_add_gitlab_tables.py +61 -0
@@ 0,0 1,61 @@
"""Add gitlab tables

Revision ID: 101d96a6baaf
Revises: 986fd25d5184
Create Date: 2019-10-23 12:40:05.563827

"""

# revision identifiers, used by Alembic.
revision = '101d96a6baaf'
down_revision = '986fd25d5184'

from alembic import op
import sqlalchemy as sa
import sqlalchemy_utils as sau


def upgrade():
    op.create_table('gitlab_authorization',
        sa.Column('id', sa.Integer, primary_key=True),
        sa.Column('created', sa.DateTime, nullable=False),
        sa.Column('updated', sa.DateTime, nullable=False),
        sa.Column('user_id', sa.Integer, sa.ForeignKey("user.id")),
        sa.Column('upstream', sa.Unicode, nullable=False),
        sa.Column('oauth_token', sa.Unicode(512), nullable=False))

    op.create_table('gitlab_commit_to_build',
        sa.Column('id', sau.UUIDType, primary_key=True),
        sa.Column('created', sa.DateTime, nullable=False),
        sa.Column('updated', sa.DateTime, nullable=False),
        sa.Column('user_id', sa.Integer,
            sa.ForeignKey("user.id", ondelete="CASCADE")),
        sa.Column('task_id', sa.Integer,
            sa.ForeignKey("task.id", ondelete="CASCADE")),
        sa.Column('repo_name', sa.Unicode, nullable=False),
        sa.Column('repo_id', sa.Integer, nullable=False),
        sa.Column('web_url', sa.Unicode, nullable=False),
        sa.Column('gitlab_webhook_id', sa.Integer, nullable=False),
        sa.Column('secrets', sa.Boolean, nullable=False, server_default='t'),
        sa.Column('upstream', sa.Unicode, nullable=False))

    op.create_table('gitlab_mr_to_build',
        sa.Column('id', sau.UUIDType, primary_key=True),
        sa.Column('created', sa.DateTime, nullable=False),
        sa.Column('updated', sa.DateTime, nullable=False),
        sa.Column('user_id', sa.Integer,
                sa.ForeignKey("user.id", ondelete="CASCADE")),
        sa.Column('task_id', sa.Integer,
                sa.ForeignKey("task.id", ondelete="CASCADE")),
        sa.Column('repo_name', sa.Unicode(1024), nullable=False),
        sa.Column('repo_id', sa.Integer, nullable=False),
        sa.Column('web_url', sa.Unicode, nullable=False),
        sa.Column('gitlab_webhook_id', sa.Integer, nullable=False),
        sa.Column('upstream', sa.Unicode, nullable=False),
        sa.Column('private', sa.Boolean, nullable=False, server_default='f'),
        sa.Column('secrets', sa.Boolean, nullable=False, server_default='f'))

def downgrade():
    op.drop_table('gitlab_authorization')
    op.drop_table('gitlab_commit_to_build')
    op.drop_table('gitlab_mr_to_build')

M dispatchsrht/builds.py => dispatchsrht/builds.py +6 -2
@@ 44,6 44,7 @@ def submit_build(
        build_tag: str,
        manifests: Iterable[Tuple[str, Manifest]],
        oauth_token: str,
        username: str,
        note: str=None,
        secrets: bool=False,
        preparing: Callable[[str], Any]=None,


@@ 81,6 82,9 @@ def submit_build(
        if resp.status_code != 200:
            return resp.text
        build_id = resp.json()["id"]
        build_url = "{}/~{}/job/{}".format(
                _builds_sr_ht, username, build_id)
        build_urls.append((name, build_url))
        if submitted:
            build_urls.append(submitted(name, build_id))
    return "Started builds:\n\n" + "\n".join(build_urls)
            submitted(name, build_id)
    return build_urls

M dispatchsrht/tasks/__init__.py => dispatchsrht/tasks/__init__.py +1 -0
@@ 1,2 1,3 @@
from dispatchsrht.tasks.taskdef import TaskDef, taskdefs
import dispatchsrht.tasks.github
import dispatchsrht.tasks.gitlab

M dispatchsrht/tasks/github/common.py => dispatchsrht/tasks/github/common.py +13 -10
@@ 1,20 1,20 @@
import base64
import html
import json
import sqlalchemy as sa
import requests
import sqlalchemy as sa
import yaml
from dispatchsrht.app import app
from dispatchsrht.builds import decrypt_notify_payload, submit_build
from dispatchsrht.builds import first_line, encrypt_notify_url
from flask import redirect, request, url_for
from flask_login import current_user
from functools import wraps
from github import Github, GithubException
from urllib.parse import urlencode
from srht.config import cfg
from srht.database import Base, db
from srht.flask import loginrequired, csrf_bypass
from dispatchsrht.app import app
from dispatchsrht.builds import first_line, encrypt_notify_url
from dispatchsrht.builds import decrypt_notify_payload, submit_build
from urllib.parse import urlencode

_github_client_id = cfg("dispatch.sr.ht::github",
        "oauth-client-id", default=None)


@@ 112,14 112,13 @@ def update_preparing(base_commit):

def update_submitted(base_commit, username):
    def go(name, build_id):
        build_url = "{}/~{}/job/{}".format(
                _builds_sr_ht, username, build_id)
        try:
            build_url = "{}/~{}/job/{}".format(
                    _builds_sr_ht, username, build_id)
            base_commit.create_status("pending", build_url,
                    "builds.sr.ht job is running", context=context(name))
            return build_url
        except GithubException:
            return ""
            pass
    return go

def submit_github_build(hook, repo, commit, base=None,


@@ 199,10 198,14 @@ def submit_github_build(hook, repo, commit, base=None,
            git_commit.author.name,
            git_commit.author.email)

    return submit_build(repo.name, manifests, hook.user.oauth_token,
    urls = submit_build(repo.name, manifests,
            hook.user.oauth_token, hook.user.username,
            note=note, secrets=secrets,
            preparing=update_preparing(base_commit),
            submitted=update_submitted(base_commit, auth.user.username))
    if isinstance(urls, str):
        return urls
    return "Submitted:\n\n" + "\n".join([f"{n}: {u}" for n, u in urls])

@csrf_bypass
@app.route("/github/complete_build/<payload>", methods=["POST"])

M dispatchsrht/tasks/github/github_commit_to_build.py => dispatchsrht/tasks/github/github_commit_to_build.py +8 -8
@@ 1,19 1,19 @@
import sqlalchemy as sa
import sqlalchemy_utils as sau
from github import Github
from dispatchsrht.tasks import TaskDef
from dispatchsrht.tasks.github.common import GitHubAuthorization
from dispatchsrht.tasks.github.common import githubloginrequired
from dispatchsrht.tasks.github.common import submit_github_build
from dispatchsrht.types import Task
from flask import Blueprint, redirect, request, render_template, url_for, abort
from flask_login import current_user
from github import Github
from jinja2 import Markup
from uuid import UUID, uuid4
from srht.database import Base, db
from srht.config import cfg
from srht.database import Base, db
from srht.flask import icon, csrf_bypass
from srht.validation import Validation
from dispatchsrht.tasks import TaskDef
from dispatchsrht.tasks.github.common import GitHubAuthorization
from dispatchsrht.tasks.github.common import githubloginrequired
from dispatchsrht.tasks.github.common import submit_github_build
from dispatchsrht.types import Task
from uuid import UUID, uuid4

_root = cfg("dispatch.sr.ht", "origin")
_builds_sr_ht = cfg("builds.sr.ht", "origin", default=None)

A dispatchsrht/tasks/gitlab/__init__.py => dispatchsrht/tasks/gitlab/__init__.py +6 -0
@@ 0,0 1,6 @@
import dispatchsrht.tasks.gitlab.gitlab_commit_to_build

# Merge requests are disabled due to the following problem:
# https://gitlab.com/gitlab-org/gitlab/issues/16491
#
#import dispatchsrht.tasks.gitlab.gitlab_mr_to_build

A dispatchsrht/tasks/gitlab/common.py => dispatchsrht/tasks/gitlab/common.py +250 -0
@@ 0,0 1,250 @@
import html
import json
import requests
import sqlalchemy as sa
import yaml
from dispatchsrht.app import app
from dispatchsrht.builds import encrypt_notify_url, first_line, submit_build
from dispatchsrht.builds import decrypt_notify_payload
from flask import abort, redirect, render_template, request, url_for
from flask_login import current_user
from functools import wraps
from srht.config import cfg, cfgb
from srht.database import Base, db
from srht.flask import csrf_bypass, loginrequired
from urllib.parse import urlencode

_root = cfg("dispatch.sr.ht", "origin")
_builds_sr_ht = cfg("builds.sr.ht", "origin", default=None)
_gitlab_enabled = cfgb("dispatch.sr.ht::gitlab", "enabled")

if _builds_sr_ht:
    from buildsrht.manifest import Manifest, Trigger

if _gitlab_enabled:
    from gitlab import Gitlab
    from gitlab.exceptions import GitlabError

class GitLabAuthorization(Base):
    __tablename__ = "gitlab_authorization"
    id = sa.Column(sa.Integer, primary_key=True)
    created = sa.Column(sa.DateTime, nullable=False)
    updated = sa.Column(sa.DateTime, nullable=False)
    user_id = sa.Column(sa.Integer, sa.ForeignKey("user.id"))
    user = sa.orm.relationship("User")
    upstream = sa.Column(sa.Unicode, nullable=False)
    oauth_token = sa.Column(sa.Unicode(512), nullable=False)

def gitlab_redirect(upstream, return_to):
    gl_authorize_url = f"https://{upstream}/oauth/authorize"
    gl_client = cfg("dispatch.sr.ht::gitlab", upstream, default=None)
    if not gl_client:
        return redirect(url_for("gitlab_no_instance"))
    [instance_name, client_id, secret] = gl_client.split(":")
    parameters = {
        "client_id": client_id,
        "scope": "api",
        "state": return_to,
        "response_type": "code",
        "redirect_uri": _root + url_for("gitlab_callback", upstream=upstream),
    }
    return redirect("{}?{}".format(gl_authorize_url, urlencode(parameters)))

def gitlabloginrequired(f):
    @wraps(f)
    def wrapper(upstream, *args, **kwargs):
        auth = GitLabAuthorization.query.filter(
                GitLabAuthorization.user_id == current_user.id,
                GitLabAuthorization.upstream == upstream,
            ).first()
        if not auth:
            return gitlab_redirect(upstream, request.path)
        try:
            gitlab = Gitlab(f"https://{upstream}", oauth_token=auth.oauth_token)
            return f(gitlab, upstream, *args, **kwargs)
        except GitlabError:
            db.session.delete(auth)
            db.session.commit()
            return gitlab_redirect(upstream, request.path)
    return loginrequired(wrapper)

@app.route("/gitlab/no-instance")
def gitlab_no_instance():
    return render_template("gitlab/no-instance.html")

@app.route("/gitlab/callback/<upstream>")
@loginrequired
def gitlab_callback(upstream):
    code = request.args.get("code")
    state = request.args.get("state")
    gl_client = cfg("dispatch.sr.ht::gitlab", upstream, default=None)
    if not gl_client:
        abort(400)
    [instance_name, client_id, secret] = gl_client.split(":")

    resp = requests.post(
        f"https://{upstream}/oauth/token", headers={
            "Accept": "application/json"
        }, data={
            "grant_type": "authorization_code",
            "client_id": client_id,
            "client_secret": secret,
            "code": code,
            "redirect_uri": _root + url_for("gitlab_callback", upstream=upstream),
        })
    if resp.status_code != 200:
        # TODO: Proper error page
        print(resp.text)
        return "An error occured"
    json = resp.json()
    access_token = json.get("access_token")
    auth = GitLabAuthorization()
    auth.user_id = current_user.id
    auth.oauth_token = access_token
    auth.upstream = upstream
    db.session.add(auth)
    db.session.commit()
    return redirect(state)

def source_url(project, commit, url, source):
    if not url.endswith("/" + project.attributes["name"]):
        return url
    if source.attributes['name'] != project.attributes['name']:
        return (project.name + "::"
                + source.attributes['http_url_to_repo']
                + "#" + commit.sha)
    if project.attributes['visibility'] == 'private':
        return project.attributes['ssh_url_to_repo'] + "#" + commit.get_id()
    return project.attributes['http_url_to_repo'] + "#" + commit.get_id()

context = lambda name: "builds.sr.ht" + (f": {name}" if name else "")

def update_preparing(commit):
    def go(name):
        try:
            status = commit.statuses.create({
                "state": "pending",
                "context": context(name),
                "target_url": _builds_sr_ht,
            })
        except GitlabError:
            pass
    return go

def update_submitted(commit, username):
    def go(name, build_id):
        build_url = "{}/~{}/job/{}".format(
                _builds_sr_ht, username, build_id)
        try:
            status = commit.statuses.create({
                "state": "running",
                "context": context(name),
                "target_url": build_url,
            })
            return build_url
        except GitlabError:
            return ""
    return go

def submit_gitlab_build(auth, hook, project, commit,
        source=None, env=dict(), is_mr=False):
    if source is None:
        source = project
    try:
        files = [source.files.get(file_path=".build.yml",
            ref=commit.get_id())]
    except GitlabError:
        try:
            tree = source.repository_tree(
                    path=".builds", ref=commit.get_id())
            files = [source.files.get(file_path=e['path'],
                        ref=commit.get_id())
                    for e in tree if e['path'].endswith(".yml")]
        except GitlabError:
            return "There are no build manifests in this repository."

    env.update({
        "GITLAB_REPOSITORY": hook.repo_name,
        "GITLAB_EVENT": request.headers.get('X-Gitlab-Event'),
    })

    manifests = list()
    for f in files:
        name = f.attributes["file_name"]
        manifest = f.decode()
        try:
            manifest = Manifest(yaml.safe_load(manifest))
        except Exception as ex:
            return f"There are errors in {name}:\n{str(ex)}", 400

        if manifest.sources:
            manifest.sources = [source_url(project, commit, url, source)
                    for url in manifest.sources]
        if not manifest.environment:
            manifest.environment = env
        else:
            manifest.environment.update(env)

        notify_payload = {
            "context": context(name),
            "oauth_token": auth.oauth_token,
            "project_id": project.get_id(),
            "sha": commit.get_id(),
            "upstream": hook.upstream,
            "username": auth.user.username,
        }

        notify_url = encrypt_notify_url(
                "gitlab_complete_build", notify_payload)

        manifest.triggers.append(Trigger({
            "action": "webhook",
            "condition": "always",
            "url": notify_url,
        }))

        manifests.append((name, manifest))

    note = "{}\n\n[{}]({}) &mdash; [{}](mailto:{})".format(
            html.escape(first_line(commit.attributes["message"])),
            str(commit.get_id())[:7], "{}/commit/{}".format(
                project.attributes["web_url"], commit.get_id()),
            commit.attributes["committer_name"],
            commit.attributes["committer_email"])

    return submit_build(project.attributes['name'], manifests,
            hook.user.oauth_token, hook.user.username,
            note=note, secrets=hook.secrets,
            preparing=update_preparing(commit),
            submitted=update_submitted(commit, auth.user.username))

@csrf_bypass
@app.route("/gitlab/complete_build/<payload>", methods=["POST"])
def gitlab_complete_build(payload):
    result = json.loads(request.data.decode('utf-8'))
    build_id = result["id"]
    status = result["status"]

    payload = decrypt_notify_payload(payload)
    context = payload["context"]
    oauth_token = payload["oauth_token"]
    project_id = payload["project_id"]
    sha = payload["sha"]
    upstream = payload["upstream"]
    username = payload["username"]

    gitlab = Gitlab(f"https://{upstream}", oauth_token=oauth_token)
    project = gitlab.projects.get(project_id)
    commit = project.commits.get(sha)

    build_url = "{}/~{}/job/{}".format(
            _builds_sr_ht, username, build_id)

    status = commit.statuses.create({
        "state": "success" if status == "success" else "failed",
        "context": context,
        "target_url": build_url,
        "description": "completed successfully" if status == "success" else "failed",
    })

    return f"Sent build status to {upstream}"

A dispatchsrht/tasks/gitlab/gitlab_commit_to_build.py => dispatchsrht/tasks/gitlab/gitlab_commit_to_build.py +168 -0
@@ 0,0 1,168 @@
import sqlalchemy as sa
import sqlalchemy_utils as sau
from dispatchsrht.tasks.gitlab.common import GitLabAuthorization
from dispatchsrht.tasks.gitlab.common import gitlabloginrequired
from dispatchsrht.tasks.gitlab.common import submit_gitlab_build
from dispatchsrht.tasks import TaskDef
from dispatchsrht.types import Task
from flask import Blueprint, redirect, render_template, request, url_for
from flask_login import current_user
from jinja2 import Markup
from srht.config import cfg, cfgb, cfgkeys
from srht.database import Base, db
from srht.flask import icon, csrf_bypass, loginrequired
from srht.validation import Validation
from uuid import UUID, uuid4

_root = cfg("dispatch.sr.ht", "origin")
_builds_sr_ht = cfg("builds.sr.ht", "origin", default=None)
_gitlab_enabled = cfgb("dispatch.sr.ht::gitlab", "enabled")

if _gitlab_enabled:
    from gitlab import Gitlab

class GitLabCommitToBuild(TaskDef):
    name = "gitlab_commit_to_build"
    enabled = bool(_gitlab_enabled and _builds_sr_ht)

    def description():
        return (icon("gitlab") + Markup(" GitLab commits ") +
            icon("caret-right") + Markup(" builds.sr.ht jobs"))

    class _GitLabCommitToBuildRecord(Base):
        __tablename__ = "gitlab_commit_to_build"
        id = sa.Column(sau.UUIDType, primary_key=True)
        created = sa.Column(sa.DateTime, nullable=False)
        updated = sa.Column(sa.DateTime, nullable=False)
        user_id = sa.Column(sa.Integer,
                sa.ForeignKey("user.id", ondelete="CASCADE"))
        user = sa.orm.relationship("User")
        task_id = sa.Column(sa.Integer,
                sa.ForeignKey("task.id", ondelete="CASCADE"))
        task = sa.orm.relationship("Task")
        repo_name = sa.Column(sa.Unicode, nullable=False)
        repo_id = sa.Column(sa.Integer, nullable=False)
        web_url = sa.Column(sa.Unicode, nullable=False)
        gitlab_webhook_id = sa.Column(sa.Integer, nullable=False)
        secrets = sa.Column(sa.Boolean, nullable=False, server_default='t')
        upstream = sa.Column(sa.Unicode, nullable=False)

    blueprint = Blueprint("gitlab_commit_to_build",
            __name__, template_folder="gitlab_commit_to_build")

    def edit_GET(task):
        record = GitLabCommitToBuild._GitLabCommitToBuildRecord.query.filter(
            GitLabCommitToBuild._GitLabCommitToBuildRecord.task_id == task.id
        ).one_or_none()
        if not record:
            abort(404)
        return render_template("gitlab/edit.html", task=task, record=record)

    def edit_POST(task):
        record = GitLabCommitToBuild._GitLabCommitToBuildRecord.query.filter(
            GitLabCommitToBuild._GitLabCommitToBuildRecord.task_id == task.id
        ).one_or_none()
        valid = Validation(request)
        secrets = valid.optional("secrets", cls=bool, default=False)
        record.secrets = bool(secrets)
        db.session.commit()
        return redirect(url_for("html.edit_task", task_id=task.id))

    @blueprint.route("/configure")
    @loginrequired
    def configure():
        canonical_upstream = cfg("dispatch.sr.ht::gitlab", "canonical-upstream")
        upstreams = [k for k in cfgkeys("dispatch.sr.ht::gitlab") if k not in [
            "enabled", "canonical-upstream", canonical_upstream,
        ]]
        return render_template("gitlab/select-instance.html",
                canonical_upstream=canonical_upstream, upstreams=upstreams,
                instance_name=lambda inst: cfg("dispatch.sr.ht::gitlab",
                    inst).split(":")[0])

    @blueprint.route("/configure/<upstream>")
    @gitlabloginrequired
    def configure_repo_GET(gitlab, upstream):
        repos = gitlab.projects.list(owned=True)
        repos = sorted(repos, key=lambda r: r.attributes["name_with_namespace"])
        existing = GitLabCommitToBuild._GitLabCommitToBuildRecord.query.filter(
                GitLabCommitToBuild._GitLabCommitToBuildRecord.user_id == current_user.id,
                GitLabCommitToBuild._GitLabCommitToBuildRecord.upstream == upstream).all()
        existing = [e.repo for e in existing]
        return render_template("gitlab/select-repo.html",
                repos=repos, existing=existing)

    @blueprint.route("/configure/<upstream>", methods=["POST"])
    @gitlabloginrequired
    def configure_repo_POST(gitlab, upstream):
        valid = Validation(request)
        repo_id = valid.require("repo_id")
        if not valid.ok:
            return "quit yo hackin bullshit"
        project = gitlab.projects.get(int(repo_id))
        if not project:
            return "quit yo hackin bullshit"

        task = Task()
        task.name = "{}::gitlab_commit_to_build".format(
                project.attributes["name_with_namespace"])
        task.user_id = current_user.id
        task._taskdef = "gitlab_commit_to_build"
        db.session.add(task)
        db.session.flush()

        record = GitLabCommitToBuild._GitLabCommitToBuildRecord()
        record.id = uuid4()
        record.user_id = current_user.id
        record.task_id = task.id
        record.gitlab_webhook_id = -1
        record.repo_name = project.attributes['name_with_namespace']
        record.repo_id = project.id
        record.web_url = project.attributes['web_url']
        record.upstream = upstream
        db.session.add(record)
        db.session.flush()

        hook = project.hooks.create({
            "url": _root + url_for("gitlab_commit_to_build._webhook",
                record_id=record.id),
            "push_events": 1,
        })
        record.gitlab_webhook_id = hook.id
        db.session.commit()

        return redirect(url_for("html.edit_task", task_id=task.id))

    @csrf_bypass
    @blueprint.route("/webhook/<record_id>", methods=["POST"])
    def _webhook(record_id):
        record_id = UUID(record_id)
        hook = GitLabCommitToBuild._GitLabCommitToBuildRecord.query.filter(
                GitLabCommitToBuild._GitLabCommitToBuildRecord.id == record_id
            ).one_or_none()
        if not hook:
            return "Unknown hook " + str(record_id), 404
        auth = GitLabAuthorization.query.filter(
                GitLabAuthorization.user_id == hook.user_id,
                GitLabAuthorization.upstream == hook.upstream,
            ).first()
        if not auth:
            return "Invalid authorization for this hook"
        gitlab = Gitlab(f"https://{hook.upstream}",
                oauth_token=auth.oauth_token)

        valid = Validation(request)
        commit = valid.require("after")
        ref = valid.require("ref")
        if not valid.ok:
            return "Unexpected hook payload"

        project = gitlab.projects.get(hook.repo_id)
        commit = project.commits.get(commit)

        urls = submit_gitlab_build(auth, hook, project, commit, env={
            "GITLAB_REF": ref,
        })
        if isinstance(urls, str):
            return urls
        return "Submitted:\n\n" + "\n".join([f"{n}: {u}" for n, u in urls])

A dispatchsrht/tasks/gitlab/gitlab_mr_to_build.py => dispatchsrht/tasks/gitlab/gitlab_mr_to_build.py +181 -0
@@ 0,0 1,181 @@
import sqlalchemy as sa
import sqlalchemy_utils as sau
from dispatchsrht.tasks.gitlab.common import GitLabAuthorization
from dispatchsrht.tasks.gitlab.common import gitlabloginrequired
from dispatchsrht.tasks.gitlab.common import submit_gitlab_build
from dispatchsrht.tasks import TaskDef
from dispatchsrht.types import Task
from flask import Blueprint, redirect, render_template, request, url_for
from flask_login import current_user
from jinja2 import Markup
from srht.config import cfg, cfgb, cfgkeys
from srht.database import Base, db
from srht.flask import icon, csrf_bypass, loginrequired
from srht.validation import Validation
from uuid import UUID, uuid4

_root = cfg("dispatch.sr.ht", "origin")
_builds_sr_ht = cfg("builds.sr.ht", "origin", default=None)
_gitlab_enabled = cfgb("dispatch.sr.ht::gitlab", "enabled")

if _gitlab_enabled:
    from gitlab import Gitlab

class GitLabMRToBuild(TaskDef):
    name = "gitlab_mr_to_build"
    enabled = bool(_gitlab_enabled and _builds_sr_ht)

    def description():
        return (icon("gitlab") + Markup(" GitLab merge requests ") +
            icon("caret-right") + Markup(" builds.sr.ht jobs"))

    class _GitLabMRToBuildRecord(Base):
        __tablename__ = "gitlab_mr_to_build"
        id = sa.Column(sau.UUIDType, primary_key=True)
        created = sa.Column(sa.DateTime, nullable=False)
        updated = sa.Column(sa.DateTime, nullable=False)
        user_id = sa.Column(sa.Integer,
                sa.ForeignKey("user.id", ondelete="CASCADE"))
        user = sa.orm.relationship("User")
        task_id = sa.Column(sa.Integer,
                sa.ForeignKey("task.id", ondelete="CASCADE"))
        task = sa.orm.relationship("Task")
        repo_name = sa.Column(sa.Unicode, nullable=False)
        repo_id = sa.Column(sa.Integer, nullable=False)
        web_url = sa.Column(sa.Unicode, nullable=False)
        gitlab_webhook_id = sa.Column(sa.Integer, nullable=False)
        upstream = sa.Column(sa.Unicode, nullable=False)
        private = sa.Column(sa.Boolean, nullable=False, server_default='f')
        secrets = sa.Column(sa.Boolean, nullable=False, server_default='f')

    blueprint = Blueprint("gitlab_mr_to_build",
            __name__, template_folder="gitlab_mr_to_build")

    def edit_GET(task):
        record = GitLabMRToBuild._GitLabMRToBuildRecord.query.filter(
            GitLabMRToBuild._GitLabMRToBuildRecord.task_id == task.id
        ).one_or_none()
        if not record:
            abort(404)
        return render_template("gitlab/edit.html", task=task, record=record)

    def edit_POST(task):
        record = GitLabMRToBuild._GitLabMRToBuildRecord.query.filter(
            GitLabMRToBuild._GitLabMRToBuildRecord.task_id == task.id
        ).one_or_none()
        valid = Validation(request)
        # TODO: Check if the repo is public/private and enable secrets if so
        secrets = valid.optional("secrets", cls=bool, default=False)
        record.secrets = bool(secrets)
        db.session.commit()
        return redirect(url_for("html.edit_task", task_id=task.id))

    @blueprint.route("/configure")
    @loginrequired
    def configure():
        canonical_upstream = cfg("dispatch.sr.ht::gitlab", "canonical-upstream")
        upstreams = [k for k in cfgkeys("dispatch.sr.ht::gitlab") if k not in [
            "enabled", "canonical-upstream", canonical_upstream,
        ]]
        return render_template("gitlab/select-instance.html",
                canonical_upstream=canonical_upstream, upstreams=upstreams,
                instance_name=lambda inst: cfg("dispatch.sr.ht::gitlab",
                    inst).split(":")[0])

    @blueprint.route("/configure/<upstream>")
    @gitlabloginrequired
    def configure_repo_GET(gitlab, upstream):
        repos = gitlab.projects.list(owned=True)
        repos = sorted(repos, key=lambda r: r.attributes["name_with_namespace"])
        existing = GitLabMRToBuild._GitLabMRToBuildRecord.query.filter(
                GitLabMRToBuild._GitLabMRToBuildRecord.user_id == current_user.id,
                GitLabMRToBuild._GitLabMRToBuildRecord.upstream == upstream).all()
        existing = [e.repo for e in existing]
        return render_template("gitlab/select-repo.html",
                repos=repos, existing=existing)

    @blueprint.route("/configure/<upstream>", methods=["POST"])
    @gitlabloginrequired
    def configure_repo_POST(gitlab, upstream):
        valid = Validation(request)
        repo_id = valid.require("repo_id")
        if not valid.ok:
            return "quit yo hackin bullshit"
        project = gitlab.projects.get(int(repo_id))
        if not project:
            return "quit yo hackin bullshit"

        task = Task()
        task.name = "{}::gitlab_mr_to_build".format(
                project.attributes["name_with_namespace"])
        task.user_id = current_user.id
        task._taskdef = "gitlab_mr_to_build"
        db.session.add(task)
        db.session.flush()

        record = GitLabMRToBuild._GitLabMRToBuildRecord()
        record.id = uuid4()
        record.user_id = current_user.id
        record.task_id = task.id
        record.gitlab_webhook_id = -1
        record.repo_name = project.attributes['name_with_namespace']
        record.repo_id = project.id
        record.web_url = project.attributes['web_url']
        record.upstream = upstream
        db.session.add(record)
        db.session.flush()

        hook = project.hooks.create({
            "url": _root + url_for("gitlab_mr_to_build._webhook",
                record_id=record.id),
            "merge_requests_events": 1,
        })
        record.gitlab_webhook_id = hook.id
        db.session.commit()

        return redirect(url_for("html.edit_task", task_id=task.id))

    @csrf_bypass
    @blueprint.route("/webhook/<record_id>", methods=["POST"])
    def _webhook(record_id):
        record_id = UUID(record_id)
        hook = GitLabMRToBuild._GitLabMRToBuildRecord.query.filter(
                GitLabMRToBuild._GitLabMRToBuildRecord.id == record_id
            ).one_or_none()
        if not hook:
            return "Unknown hook " + str(record_id), 404
        auth = GitLabAuthorization.query.filter(
                GitLabAuthorization.user_id == hook.user_id,
                GitLabAuthorization.upstream == hook.upstream,
            ).first()
        if not auth:
            return "Invalid authorization for this hook"
        gitlab = Gitlab(f"https://{hook.upstream}",
                oauth_token=auth.oauth_token)

        valid = Validation(request)
        object_attrs = valid.require("object_attributes")
        if not valid.ok:
            return "Unexpected hook payload"
        source = object_attrs["source"]
        last_commit = object_attrs["last_commit"]

        project = gitlab.projects.get(hook.repo_id)
        source = gitlab.projects.get(source["id"])
        commit = project.commits.get(last_commit["id"])
        merge_req = project.mergerequests.get(object_attrs["iid"])

        urls = submit_gitlab_build(auth, hook, project, commit, source, {
            "GITLAB_MR_NUMBER": object_attrs["iid"],
            "GITLAB_MR_TITLE": object_attrs["title"],
            "GITLAB_BASE_REPO": project.attributes["name_with_namespace"],
            "GITLAB_HEAD_REPO": source.attributes["name_with_namespace"],
        })
        if isinstance(urls, str):
            return urls

        summary = "\n\nbuilds.sr.ht jobs:\n\n" + (
                "\n".join([f"[{n}]({url}): :clock1: running" for n, u in urls]))
        merge_req.description += summary
        merge_req.save()
        return 

A dispatchsrht/tasks/gitsrht/__init__.py => dispatchsrht/tasks/gitsrht/__init__.py +1 -0
@@ 0,0 1,1 @@
import dispatchsrht.tasks.gitsrht.gitsrht_tag_to_email

A dispatchsrht/tasks/gitsrht/gitsrht_tag_to_email.py => dispatchsrht/tasks/gitsrht/gitsrht_tag_to_email.py +5 -0
@@ 0,0 1,5 @@
from dispatchsrht.tasks import TaskDef
from dispatchsrht.types import Task

_root = cfg("dispatch.sr.ht", "origin")
_builds_sr_ht = cfg("builds.sr.ht", "origin", default=None)

M dispatchsrht/templates/edit.html => dispatchsrht/templates/edit.html +10 -12
@@ 3,9 3,9 @@
<title>Configure {{ task.name }} - dispatch.sr.ht</title>
{% endblock %}
{% block body %}
<div class="container-fluid">
  <div class="row">
    <div class="col-md-12 header-tabbed">
<div class="header-tabbed">
  <div class="container">
    <ul class="nav nav-tabs">
      <h2>
        {{task.name}}
      </h2>


@@ 14,15 14,13 @@
        class="nav-link {% if view == title %}active{% endif %}"
        href="{{ path }}">{{ title }}</a>
      {% endmacro %}
      <ul class="nav nav-tabs">
        <li class="nav-item">
          {{link(url_for("html.edit_task", task_id=task.id), "summary")}}
        </li>
        <li class="nav-item">
          {{link(url_for("html.delete_task", task_id=task.id), "delete")}}
        </li>
      </ul>
    </div>
      <li class="nav-item">
        {{link(url_for("html.edit_task", task_id=task.id), "summary")}}
      </li>
      <li class="nav-item">
        {{link(url_for("html.delete_task", task_id=task.id), "delete")}}
      </li>
    </ul>
  </div>
</div>
<div class="container">

M dispatchsrht/templates/github/select-repo.html => dispatchsrht/templates/github/select-repo.html +0 -1
@@ 3,7 3,6 @@
<div class="container">
  <div class="row">
    <div class="col-md-4">
      <h2>dispatch</h2>
      <p>
        This task will run a builds.sr.ht job for each commit pushed to a
        GitHub repository. The repository should provide the build manifest in

A dispatchsrht/templates/gitlab/edit.html => dispatchsrht/templates/gitlab/edit.html +79 -0
@@ 0,0 1,79 @@
{# vim: set ft=htmldjango : #}
<form method="POST">
  {{csrf_token()}}
  <p>
    Submits build manifests from
    <a
      href="{{record.web_url}}"
      target="_blank"
      rel="nofollow noopener"
    >{{icon("gitlab")}} {{record.repo_name}}</a>
    for every
    {% if task._taskdef == "gitlab_commit_to_build" %}
      commit.
    {% else %}
      merge request.
    {% endif %}
    If your repo has a <code>.build.yml</code> file, it will be used as the
    manifest. If your repo has a <code>.builds</code> directory with several
    manifests inside, they will all be submitted together.
  </p>
  <h3>Options</h3>
  {% if task._taskdef == "gitlab_commit_to_build" %}
  <div class="form-group">
    <div class="form-check">
      <input
        name="secrets"
        id="secrets"
        class="form-check-input"
        type="checkbox"
        {{"checked" if record.secrets else ""}}
      />
      <label for="secrets" class="form-check-label">
        Include secrets in builds
      </label>
    </div>
  </div>
  {% else %}
  <div class="form-group">
    {% if record.private %}
    <div class="alert alert-danger">
      <strong>Warning</strong>: Enable secrets for this hook with care. Anyone
      who can submit a pull request will be able to extract secrets from the
      build environment if you enable secrets for this repository.
    </div>
    {% endif %}
    <div class="form-check">
      {% if not record.private %}
      <input class="form-check-input" type="checkbox" disabled />
      <label class="form-check-label">
        <s>Include secrets in builds</s>
      </label>
      <small class="form-text text-muted">
        Secrets are disabled for merge requests on public repos.
      </small>
      {% else %}
      <input
        name="secrets"
        id="secrets"
        class="form-check-input"
        type="checkbox"
        {{"checked" if record.secrets else ""}}
      />
      <label for="secrets" class="form-check-label">
        Include secrets in builds
      </label>
      {% endif %}
    </div>
  </div>
  {% endif %}
  <button type="submit" class="btn btn-primary">
    Save changes
    {{icon("caret-right")}}
  </button>
</form>
{% if saved %}
<div class="alert alert-success">
  Changes saved.
</div>
{% endif %}

A dispatchsrht/templates/gitlab/select-instance.html => dispatchsrht/templates/gitlab/select-instance.html +66 -0
@@ 0,0 1,66 @@
{% extends "layout.html" %}
{% block body %}
<div class="container">
  <div class="row">
    <div class="col-md-8">
      <h3>Choose a GitLab instance</h3>
      <p>Which GitLab instance would you like to use?</p>
      <div class="event-list configure">
        <div class="event">
          <h4>
            {{icon('gitlab')}} {{instance_name(canonical_upstream)}}
            <a
              href="{{url_for('.configure_repo_GET', upstream=canonical_upstream)}}"
              class="btn btn-primary btn-lg pull-right"
            >Continue with {{canonical_upstream}} {{icon('caret-right')}}</a>
          </h4>
        </div>
      </div>
      <details>
        <summary>Choose another instance...</summary>
        <div class="event-list configure">
          {% for upstream in upstreams %}
          <div class="event">
            <h4>
              {{icon('gitlab')}} {{upstream}}
              <a
                href="{{url_for('.configure_repo_GET', upstream=upstream)}}"
                class="btn btn-primary btn-lg pull-right"
              >Continue with {{upstream}} {{icon('caret-right')}}</a>
            </h4>
          </div>
          {% endfor %}
          <p>
            Is your instance missing? Gitlab instances have to be manually
            approved by the {{cfg('sr.ht', 'site-name')}} administrators.
            Send an email to
            <a
              href="mailto:{{cfg('sr.ht', 'owner-email')}}"
            >{{cfg('sr.ht', 'owner-name')}} &lt;{{cfg('sr.ht', 'owner-email')}}&gt;</a>
            to request yours.
          </p>
        </div>
      </details>
      <div class="alert alert-danger" style="margin-top: 1rem">
        <strong>Notice:</strong> By proceeding, you will be granting
        {{cfg("sr.ht", "site-name")}} full access to your GitLab account.
        This is a limitation of the GitLab API; we cannot request narrower
        permissions.
      </div>
    </div>
    <div class="col-md-4">
      <p>
        This task will run a builds.sr.ht job for each commit pushed to a
        GitLab repository. The location of build manifests in your GitLab
        repository is compatible with git.sr.ht.
      </p>
      <a
        href="https://man.sr.ht/builds.sr.ht/#gitsrht"
        target="_blank"
        class="btn btn-link"
      >Read the documentation {{icon('caret-right')}}</a>
    </div>
  </div>
</div>
{% endblock %}


A dispatchsrht/templates/gitlab/select-repo.html => dispatchsrht/templates/gitlab/select-repo.html +51 -0
@@ 0,0 1,51 @@
{% extends "layout.html" %}
{% block body %}
<div class="container">
  <div class="row">
    <div class="col-md-8">
      <h3>Choose a GitLab project</h3>
      <div class="event-list configure">
      {% for repo in repos %}
        <form class="event" method="POST">
          {{csrf_token()}}
          <input type="hidden" name="repo_id" value="{{ repo.id }}" />
          <h4>
            {{icon("gitlab")}}
            {% if repo.full_name not in existing %}
            <button
              type="submit"
              class="pull-right btn btn-primary btn-lg"
            >Add task {{icon("caret-right")}}</button>
            {% else %}
            <button
              class="pull-right btn btn-default btn-lg"
              disabled
            >Already configured</button>
            {% endif %}
            {{ repo.attributes['name_with_namespace'] }}
            <a
              href="{{ repo.attributes['web_url'] }}"
              target="_blank"
              rel="noopener"
            >{{icon("external-link-alt")}}</a>
          </h4>
        </form>
      {% endfor %}
      </div>
    </div>
    <div class="col-md-4">
      <p>
        This task will run a builds.sr.ht job for each commit pushed to a
        GitLab repository. The location of build manifests in your GitLab
        repository is compatible with git.sr.ht.
      </p>
      <a
        href="https://man.sr.ht/builds.sr.ht/#gitsrht"
        target="_blank"
        class="btn btn-link"
      >Read the documentation {{icon('caret-right')}}</a>
    </div>
  </div>
</div>
{% endblock %}


M dispatchsrht/templates/task-settings.html => dispatchsrht/templates/task-settings.html +1 -1
@@ 1,6 1,6 @@
{% extends "edit.html" %}
{% block content %}
<div class="col-md-8">
<div class="col-md-12">
  {{taskdef.edit_GET(task)|safe}}
</div>
{% endblock %}