~sircmpwn/git.sr.ht

a3fe57b6844709eac64fdca60370e5c227979fd3 — Drew DeVault 2 months ago b2bae9b
Add plumbing API and pygit2 backend implementation

This new API exposes the raw odb and refdb data and the provided pygit2
backend implementation allows users to create pygit2 repositories which
are backed by the git.sr.ht API over the network, rather than by local
storage.
5 files changed, 266 insertions(+), 28 deletions(-)

M gitsrht/app.py
A gitsrht/blueprints/api/__init__.py
A gitsrht/blueprints/api/plumbing.py
R gitsrht/blueprints/{api.py => api/porcelain.py}
A gitsrht/pygit2_backend.py
M gitsrht/app.py => gitsrht/app.py +3 -2
@@ 22,13 22,14 @@ class GitApp(ScmSrhtFlask):
                repository_class=Repository, user_class=User,
                repo_api=GitRepoApi(), oauth_service=oauth_service)

        from gitsrht.blueprints.api import data
        from gitsrht.blueprints.api import plumbing, porcelain
        from gitsrht.blueprints.artifacts import artifacts
        from gitsrht.blueprints.email import mail
        from gitsrht.blueprints.repo import repo
        from gitsrht.blueprints.stats import stats

        self.register_blueprint(data)
        self.register_blueprint(plumbing)
        self.register_blueprint(porcelain)
        self.register_blueprint(mail)
        self.register_blueprint(repo)
        self.register_blueprint(stats)

A gitsrht/blueprints/api/__init__.py => gitsrht/blueprints/api/__init__.py +2 -0
@@ 0,0 1,2 @@
from gitsrht.blueprints.api.plumbing import plumbing
from gitsrht.blueprints.api.porcelain import porcelain

A gitsrht/blueprints/api/plumbing.py => gitsrht/blueprints/api/plumbing.py +70 -0
@@ 0,0 1,70 @@
import base64
import binascii
import pygit2
from flask import Blueprint, Response, abort, request
from gitsrht.git import Repository as GitRepository
from scmsrht.blueprints.api import get_user, get_repo
from srht.oauth import oauth

plumbing = Blueprint("api.plumbing", __name__)

def libgit2_object_type_to_str(otype):
    return {
        pygit2.GIT_OBJ_COMMIT: "commit",
        pygit2.GIT_OBJ_TREE: "tree",
        pygit2.GIT_OBJ_BLOB: "blob",
        pygit2.GIT_OBJ_TAG: "tag",
    }[otype]

@plumbing.route("/api/repos/<reponame>/odb/<oid>", defaults={"username": None})
@plumbing.route("/api/<username>/repos/<reponame>/odb/<oid>")
@oauth("data:read")
def repo_get_object(username, reponame, oid):
    user = get_user(username)
    repo = get_repo(user, reponame)
    with GitRepository(repo.path) as git_repo:
        try:
            otype, odata = git_repo.odb.read(oid)
        except KeyError:
            return "object not found", 404
        return Response(odata, headers={
            "X-Git-Object-Type": libgit2_object_type_to_str(otype),
        }, content_type="application/octet-stream")

@plumbing.route("/api/repos/<reponame>/lookup/<oid_prefix>",
        defaults={"username": None})
@plumbing.route("/api/<username>/repos/<reponame>/lookup/<oid_prefix>")
@oauth("data:read")
def repo_lookup_prefix(username, reponame, oid_prefix):
    user = get_user(username)
    repo = get_repo(user, reponame)
    with GitRepository(repo.path) as git_repo:
        # XXX: This will look up anything, not just a partially qualified Oid
        try:
            o = git_repo.revparse_single(oid_prefix)
        except KeyError:
            return "object not found", 404
        except ValueError:
            return "ambiguous oid", 409
        return o.oid.hex

@plumbing.route("/api/repos/<reponame>/refdb/<path:refname>",
        defaults={"username": None})
@plumbing.route("/api/<username>/repos/<reponame>/refdb/<path:refname>")
@oauth("data:read")
def repo_get_ref(username, reponame, refname):
    user = get_user(username)
    repo = get_repo(user, reponame)
    with GitRepository(repo.path) as git_repo:
        try:
            ref = git_repo.lookup_reference(refname)
        except pygit2.InvalidSpecError:
            return "invalid reference", 400
        except KeyError:
            return "unknown reference", 404
        if isinstance(ref.target, pygit2.Oid):
            # direct reference
            return f"{ref.target.hex} {ref.peel().oid.hex}"
        else:
            # symbolic reference
            return str(ref.target)

R gitsrht/blueprints/api.py => gitsrht/blueprints/api/porcelain.py +26 -26
@@ 19,7 19,7 @@ from srht.oauth import current_token, oauth
from srht.redis import redis
from srht.validation import Validation

data = Blueprint("api.data", __name__)
porcelain = Blueprint("api.porcelain", __name__)

# See also gitsrht-update-hook/types.go
def commit_to_dict(c):


@@ 66,8 66,8 @@ def ref_to_dict(artifacts, ref):
        "artifacts": [a.to_dict() for a in artifacts.get(target, [])],
    }

@data.route("/api/repos/<reponame>/refs", defaults={"username": None})
@data.route("/api/<username>/repos/<reponame>/refs")
@porcelain.route("/api/repos/<reponame>/refs", defaults={"username": None})
@porcelain.route("/api/<username>/repos/<reponame>/refs")
@oauth("data:read")
def repo_refs_GET(username, reponame):
    user = get_user(username)


@@ 94,8 94,8 @@ def repo_refs_GET(username, reponame):
            "results_per_page": len(refs),
        }

@data.route("/api/repos/<reponame>/artifacts/<path:refname>", defaults={"username": None}, methods=["POST"])
@data.route("/api/<username>/repos/<reponame>/artifacts/<path:refname>", methods=["POST"])
@porcelain.route("/api/repos/<reponame>/artifacts/<path:refname>", defaults={"username": None}, methods=["POST"])
@porcelain.route("/api/<username>/repos/<reponame>/artifacts/<path:refname>", methods=["POST"])
@oauth("data:write")
def repo_refs_by_name_POST(username, reponame, refname):
    user = get_user(username)


@@ 124,17 124,17 @@ def repo_refs_by_name_POST(username, reponame, refname):
        return artifact.to_dict()

# dear god, this routing
@data.route("/api/repos/<reponame>/log",
@porcelain.route("/api/repos/<reponame>/log",
        defaults={"username": None, "ref": None, "path": ""})
@data.route("/api/repos/<reponame>/log/<path:ref>",
@porcelain.route("/api/repos/<reponame>/log/<path:ref>",
        defaults={"username": None, "path": ""})
@data.route("/api/repos/<reponame>/log/<ref>/<path:path>",
@porcelain.route("/api/repos/<reponame>/log/<ref>/<path:path>",
        defaults={"username": None})
@data.route("/api/<username>/repos/<reponame>/log",
@porcelain.route("/api/<username>/repos/<reponame>/log",
        defaults={"ref": None, "path": ""})
@data.route("/api/<username>/repos/<reponame>/log/<path:ref>",
@porcelain.route("/api/<username>/repos/<reponame>/log/<path:ref>",
        defaults={"path": ""})
@data.route("/api/<username>/repos/<reponame>/log/<ref>/<path:path>")
@porcelain.route("/api/<username>/repos/<reponame>/log/<ref>/<path:path>")
@oauth("data:read")
def repo_commits_GET(username, reponame, ref, path):
    user = get_user(username)


@@ 161,17 161,17 @@ def repo_commits_GET(username, reponame, ref, path):
            "results_per_page": commits_per_page
        }

@data.route("/api/repos/<reponame>/tree",
@porcelain.route("/api/repos/<reponame>/tree",
        defaults={"username": None, "ref": None, "path": ""})
@data.route("/api/repos/<reponame>/tree/<path:ref>",
@porcelain.route("/api/repos/<reponame>/tree/<path:ref>",
        defaults={"username": None, "path": ""})
@data.route("/api/repos/<reponame>/tree/<ref>/<path:path>",
@porcelain.route("/api/repos/<reponame>/tree/<ref>/<path:path>",
        defaults={"username": None})
@data.route("/api/<username>/repos/<reponame>/tree",
@porcelain.route("/api/<username>/repos/<reponame>/tree",
        defaults={"ref": None, "path": ""})
@data.route("/api/<username>/repos/<reponame>/tree/<path:ref>",
@porcelain.route("/api/<username>/repos/<reponame>/tree/<path:ref>",
        defaults={"path": ""})
@data.route("/api/<username>/repos/<reponame>/tree/<ref>/<path:path>")
@porcelain.route("/api/<username>/repos/<reponame>/tree/<ref>/<path:path>")
@oauth("data:read")
def repo_tree_GET(username, reponame, ref, path):
    user = get_user(username)


@@ 201,13 201,13 @@ def repo_tree_GET(username, reponame, ref, path):
        return tree_to_dict(tree)

# TODO: remove fallback routes
@data.route("/api/repos/<reponame>/annotate", methods=["PUT"],
@porcelain.route("/api/repos/<reponame>/annotate", methods=["PUT"],
        defaults={"username": None, "commit": "master"})
@data.route("/api/<username>/repos/<reponame>/annotate", methods=["PUT"],
@porcelain.route("/api/<username>/repos/<reponame>/annotate", methods=["PUT"],
        defaults={"commit": "master"})
@data.route("/api/repos/<reponame>/<commit>/annotate", methods=["PUT"],
@porcelain.route("/api/repos/<reponame>/<commit>/annotate", methods=["PUT"],
        defaults={"username": None})
@data.route("/api/<username>/repos/<reponame>/<commit>/annotate", methods=["PUT"])
@porcelain.route("/api/<username>/repos/<reponame>/<commit>/annotate", methods=["PUT"])
@oauth("repo:write")
def repo_annotate_PUT(username, reponame, commit):
    user = get_user(username)


@@ 245,13 245,13 @@ def repo_annotate_PUT(username, reponame, commit):

    return { "updated": nblobs }, 200

@data.route("/api/repos/<reponame>/blob/<path:ref>",
@porcelain.route("/api/repos/<reponame>/blob/<path:ref>",
        defaults={"username": None, "path": ""})
@data.route("/api/repos/<reponame>/blob/<ref>/<path:path>",
@porcelain.route("/api/repos/<reponame>/blob/<ref>/<path:path>",
        defaults={"username": None})
@data.route("/api/<username>/blob/<reponame>/blob/<path:ref>",
@porcelain.route("/api/<username>/blob/<reponame>/blob/<path:ref>",
        defaults={"path": ""})
@data.route("/api/<username>/repos/<reponame>/blob/<ref>/<path:path>")
@porcelain.route("/api/<username>/repos/<reponame>/blob/<ref>/<path:path>")
@oauth("data:read")
def repo_blob_GET(username, reponame, ref, path):
    user = get_user(username)


@@ 317,5 317,5 @@ def _webhook_create(sub, valid, username, reponame):
    sub.sync = valid.optional("sync", cls=bool, default=False)
    return sub

RepoWebhook.api_routes(data, "/api/<username>/repos/<reponame>",
RepoWebhook.api_routes(porcelain, "/api/<username>/repos/<reponame>",
        filters=_webhook_filters, create=_webhook_create)

A gitsrht/pygit2_backend.py => gitsrht/pygit2_backend.py +165 -0
@@ 0,0 1,165 @@
import pygit2
import requests
from srht.config import get_origin

def str_to_libgit2_object_type(otype):
    return {
        "commit": pygit2.GIT_OBJ_COMMIT,
        "tree": pygit2.GIT_OBJ_TREE,
        "blob": pygit2.GIT_OBJ_BLOB,
        "tag": pygit2.GIT_OBJ_TAG,
    }[otype]

_gitsrht = get_origin("git.sr.ht")

class OdbBackend(pygit2.OdbBackend):
    def __init__(self, authorization, owner_name, repo_name,
            upstream=_gitsrht, session=None):
        super().__init__()
        self.base_url = f"{upstream}/api/{owner_name}/repos/{repo_name}"
        self.authorization = authorization
        if session == None:
            self.session = requests.Session()
        else:
            self.session = session

    def _get(self, path, *args, **kwargs):
        headers = kwargs.pop("headers", dict())
        return self.session.get(f"{self.base_url}{path}",
                headers={**self.authorization, **headers})

    def _head(self, path, *args, **kwargs):
        headers = kwargs.pop("headers", dict())
        return self.session.head(f"{self.base_url}{path}",
                headers={**self.authorization, **headers})

    def exists(self, oid):
        r = self._head(f"/lookup/{str(oid)}")
        return r.status_code != 404

    def exists_prefix(self, oid_prefix):
        r = self._get(f"/lookup/{str(oid_prefix)}")
        if r.status_code == 404:
            raise KeyError(r.text)
        elif r.status_code == 409:
            raise ValueError(r.text)
        return r.text

    def read(self, oid):
        r = self._get(f"/odb/{str(oid)}")
        if r.status_code == 404:
            raise KeyError(r.text)
        elif r.status_code == 409:
            raise ValueError(r.text)
        otype = r.headers["X-Git-Object-Type"]
        otype = str_to_libgit2_object_type(otype)
        return otype, r.content

    def read_header(self, oid):
        r = self._head(f"/odb/{str(oid)}")
        if r.status_code == 404:
            raise KeyError(r.text)
        elif r.status_code == 409:
            raise ValueError(r.text)
        otype = r.headers["X-Git-Object-Type"]
        otype = str_to_libgit2_object_type(otype)
        length = int(r.headers["Content-Length"])
        return otype, length

    def read_prefix(self, oid_prefix):
        oid = self.exists_prefix(oid_prefix)
        return (oid, *self.read(oid))

    def __iter__(self):
        raise NotImplementedError()

    def refresh(self):
        pass # no-op

class RefdbBackend(pygit2.RefdbBackend):
    def __init__(self, authorization, owner_name, repo_name,
            upstream=_gitsrht, session=None):
        super().__init__()
        self.base_url = f"{upstream}/api/{owner_name}/repos/{repo_name}"
        self.authorization = authorization
        if session == None:
            self.session = requests.Session()
        else:
            self.session = session

    def _get(self, path, *args, **kwargs):
        headers = kwargs.pop("headers", dict())
        return requests.get(f"{self.base_url}{path}",
                headers={**self.authorization, **headers})

    def _head(self, path, *args, **kwargs):
        headers = kwargs.pop("headers", dict())
        return requests.head(f"{self.base_url}{path}",
                headers={**self.authorization, **headers})

    def exists(self, ref):
        r = self._head(f"/refdb/{ref}")
        if r.status_code == 404:
            return False
        elif r.status_code == 200:
            return True
        else:
            raise Exception(r.text)

    def lookup(self, ref):
        r = self._get(f"/refdb/{ref}")
        if r.status_code == 404:
            raise KeyError(r.text)
        elif r.status_code != 200:
            raise Exception(r.text)
        if " " in r.text:
            target, peel = r.text.split(" ", 1)
            return pygit2.Reference(ref, target, peel)
        else:
            return pygit2.Reference(ref, r.text)

    def write(self, ref, force, who, message, old, old_target):
        raise NotImplementedError()

    def rename(self, old_name, new_name, force, who, message):
        raise NotImplementedError()

    def delete(self, ref_name, old_id, old_target):
        raise NotImplementedError()

    def has_log(self, ref_name):
        raise NotImplementedError()

    def ensure_log(self, ref_name):
        raise NotImplementedError()

    def __iter__(self):
        raise NotImplementedError()

    def __next__(self):
        raise NotImplementedError()

class GitSrhtRepository(pygit2.Repository):
    """
    A pygit2.Repository which is backed by the git.sr.ht API rather than by
    local storage.
    """
    def __init__(self, authorization, owner_name, repo_name, upstream=_gitsrht):
        """
        authorization: a dictionary of headers providing API authorization
        (e.g. from srht.api.get_authorization)

        owner_name: the canonical name of the repository owner
        """
        super().__init__()
        self.session = requests.Session()
        odb = pygit2.Odb()
        odb_backend = OdbBackend(authorization, owner_name, repo_name,
                upstream=upstream, session=self.session)
        odb.add_backend(odb_backend, 1)
        refdb = pygit2.Refdb.new(self)
        refdb_backend = RefdbBackend(authorization, owner_name, repo_name,
                upstream=upstream, session=self.session)
        refdb.set_backend(refdb_backend)
        self.set_odb(odb)
        self.set_refdb(refdb)