proof of concept: serve local git repos via http

test-git-serve

@@ -0,0 +1,29 @@
+#!/bin/sh
+
+set -eux
+
+# setup a fake little git hosting world
+rm -rf tmp
+git clone --bare . tmp/projects/cockpit-project/bots
+touch tmp/projects/cockpit-project/bots/git-daemon-export-ok                  # read access...
+git --git-dir tmp/projects/cockpit-project/bots config http.receivepack true  # ... and write
+
+# serve it
+./git-serve -p 4444 tmp/projects&
+sleep 0.1  # avoid races
+
+# client can check out a copy, make changes, push
+(
+   git clone http://localhost:4444/cockpit-project/bots tmp/clone
+   cd tmp/clone
+   git commit --allow-empty -m 'clone, commit, push'
+   git push
+)
+rm -rf tmp/clone
+
+
+# and we can see their changes
+git --git-dir tmp/projects/cockpit-project/bots show
+
+# kill server
+pkill -P $$

test/simhub.py

@@ -0,0 +1,184 @@
+#!/usr/bin/env python3
+
+import argparse
+import asyncio
+import contextlib
+import email
+import hashlib
+import logging
+import os
+import socket
+import subprocess
+import tempfile
+from collections.abc import Sequence
+from datetime import datetime
+from pathlib import Path
+from typing import Self
+
+from aiohttp import web
+from aiohttp.helpers import ETag
+from aiohttp.typedefs import Handler
+from yarl import URL
+
+from lib.aio.jsonutil import JsonObject, get_str
+
+# routing tables for 'api', 'clone' and 'raw' endpoints
+api = web.RouteTableDef()
+clone = web.RouteTableDef()
+raw = web.RouteTableDef()
+
+
+@web.middleware
+async def conditions(request: web.Request, handler: Handler) -> web.StreamResponse:
+    response = await handler(request)
+    if response.etag and request.if_none_match and response.etag == request.if_none_match:
+        return web.Response(status=304)
+    if (response.last_modified and request.if_modified_since and request.if_modified_since <= response.last_modified):
+        return web.Response(status=304)
+    return response
+
+
+class Repository:
+    def __init__(self, gitdir: Path, issues: Sequence[JsonObject] = ()) -> None:
+        self.gitdir = gitdir
+        self.issues = list(issues)
+
+    async def git(self, *cmd: str, **kwargs) -> str:
+        proc = await asyncio.create_subprocess_exec('git', *cmd, stdout=subprocess.PIPE, cwd=self.gitdir)
+        return (await proc.communicate())[0].decode()
+
+    def issue(self, nr: int | str) -> JsonObject:
+        return self.issues[int(nr)]
+
+
+class SimHub(contextlib.AsyncExitStack):
+    def __init__(self, path: Path | None = None, addr: str = 'localhost', port: int = 0) -> None:
+        super().__init__()
+
+        self.path = path or Path(self.enter_context(tempfile.TemporaryDirectory()))
+        self.repos = dict[str, Repository]()
+
+        self.app = web.Application(middlewares=[conditions])
+        self.app['simhub'] = self
+        self.app.add_routes(clone)
+        self.app.add_routes(api)
+        self.app.add_routes(raw)
+
+        self.listener = socket.socket()
+        self.listener.bind((addr, port))
+        addr, port = self.listener.getsockname()
+
+        self.api = URL.build(scheme='http', host=addr, port=port, path='/')
+
+    async def __aenter__(self) -> Self:
+        self.listener.listen()
+
+        self.runner = web.AppRunner(self.app)
+        await self.runner.setup()
+        self.push_async_callback(self.runner.cleanup)
+
+        self.site = web.SockSite(self.runner, self.listener)
+        await self.site.start()
+
+        return self
+
+    def get_repo(self, owner: str, name: str) -> Repository:
+        return self.repos[f'{owner}/{name}']
+
+    async def clone(self, name: str, clone_from: str, issues: Sequence[JsonObject] = ()) -> None:
+        repo = Repository(self.path / name)
+        repo.gitdir.mkdir(parents=True)
+        await repo.git('clone', '--bare', '--mirror', clone_from, str(repo.gitdir))
+        (repo.gitdir / 'git-daemon-export-ok').touch()
+        self.repos[name] = repo
+
+
+# Request helpers which reach inside of SimHub
+def simhub(request: web.Request) -> SimHub:
+    simhub = request.app['simhub']
+    assert isinstance(simhub, SimHub)
+    return simhub
+
+
+def repository(request: web.Request) -> Repository:
+    return simhub(request).get_repo(request.match_info['owner'], request.match_info['repo'])
+
+
+def json_response(obj: JsonObject) -> web.Response:
+    response = web.json_response(obj)
+    response.etag = ETag(hashlib.sha256(str(obj).encode()).hexdigest())
+    if last_modified := get_str(obj, '.last-modified', None):
+        response.last_modified = datetime.fromisoformat(last_modified)
+    return response
+
+
+# API endpoints
+@api.get(r'/repos/{owner}/{repo}')
+async def get_repo(request: web.Request) -> web.Response:
+    output = await repository(request).git('symbolic-ref', '--short', 'HEAD')
+    return json_response({'default_branch': output.strip()})
+
+
+@api.get(r'/repos/{owner}/{repo}/git/refs/heads/{branch}')
+async def get_branch(request: web.Request) -> web.Response:
+    output = await repository(request).git('rev-parse', f'refs/heads/{request.match_info["branch"]}')
+    return json_response({'object': {'sha': output.strip()}})
+
+
+@api.get(r'/repos/{owner}/{repo}/issues/{nr:\d+}')
+async def get_issue(request: web.Request) -> web.Response:
+    return json_response(repository(request).issues[int(request.match_info['nr'])])
+
+
+@api.get(r'/repos/{owner}/{repo}/pulls/{nr:\d+}')
+async def get_pull(request: web.Request) -> web.Response:
+    return json_response(repository(request).issues[int(request.match_info['nr'])])
+
+
+# clone endpoint
+@clone.post('/{owner}/{repo}/git-upload-pack')
+@clone.get('/{owner}/{repo}/info/refs')
+async def git_http_backend(request: web.Request) -> web.Response:
+    proc = await asyncio.create_subprocess_exec(
+        'git', 'http-backend', stdin=subprocess.PIPE, stdout=subprocess.PIPE, env={
+            **os.environ,
+            'REQUEST_METHOD': request.method,
+            'PATH_INFO': request.path,
+            'QUERY_STRING': request.query_string,
+            'GIT_PROTOCOL': request.headers.get('git-protocol', ''),
+            'CONTENT_TYPE': request.headers.get('content-type', ''),
+            'GIT_PROJECT_ROOT': str(simhub(request).path)
+        }
+    )
+    data = await request.read() if request.method == 'POST' else b''
+    stdout, _ = await proc.communicate(data)
+    headers, _, body = stdout.partition(b'\r\n\r\n')
+    message = email.parser.BytesParser().parsebytes(headers, headersonly=True)
+    status, _, reason = message.get('Status', '200 OK').partition(' ')
+    return web.Response(status=int(status), reason=reason, headers=dict(message), body=body)
+
+    return await repository(request).http_backend(request)
+
+
+# raw endpoint
+@raw.get('/{owner}/{repo}/{ref:[^{}/:]+}/{path:[^{}:]+}')
+async def get_raw(request: web.Request) -> web.Response:
+    objname = request.match_info['ref'] + ':' + request.match_info['path']
+    return web.Response(text=await repository(request).git('cat-file', '-p', objname))
+
+
+async def main() -> None:
+    logging.basicConfig(level=logging.DEBUG)
+
+    parser = argparse.ArgumentParser(description="Serve a single git repository via HTTP")
+    parser.add_argument('--addr', '-a', default='127.0.0.1', help="Address to bind to")
+    parser.add_argument('--port', '-p', type=int, default=0, help="Port number to bind to")
+    args = parser.parse_args()
+
+    async with SimHub(addr=args.addr, port=args.port) as simhub:
+        print(simhub.api)
+        await asyncio.sleep(86400)
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

test/test_job_runner.py

@@ -0,0 +1,34 @@
+import asyncio
+import os
+from pathlib import Path
+from test.simhub import SimHub
+from typing import AsyncIterator
+
+import pytest
+
+
+@pytest.fixture
+async def simhub(tmpdir_factory: pytest.TempdirFactory) -> AsyncIterator[SimHub]:
+    async with SimHub(Path(tmpdir_factory.mktemp('simhub'))) as hub:
+        await hub.clone('cockpit-project/bots', os.getcwd())
+        yield hub
+
+
+async def test_simhub(simhub: SimHub, tmp_path: Path) -> None:
+    job_runner_toml = tmp_path / 'job-runner.toml'
+
+    job_runner_toml.write_text(f'''
+    [container]
+    run-args = [
+        "--network=host"
+    ]
+
+    [forge.github]
+    api-url="{simhub.api}"
+    clone-url="{simhub.api}"
+    content-url="{simhub.api}"
+    token=""
+    ''')
+    proc = await asyncio.create_subprocess_exec('./job-runner', f'-F{job_runner_toml}', '--debug', 'run', 'cockpit-project/bots')
+    await proc.wait()
+    print('bzzt')