From 1c5bea40432df62df657fcc6750bbd7da5e78631 Mon Sep 17 00:00:00 2001 From: Sam Willcocks Date: Mon, 14 Apr 2025 16:59:19 +0100 Subject: [PATCH 01/13] gitlab: add gitlab integration --- buildbot_nix/buildbot_nix/__init__.py | 4 + buildbot_nix/buildbot_nix/gitlab_project.py | 319 ++++++++++++++++++++ buildbot_nix/buildbot_nix/models.py | 28 ++ nixosModules/master.nix | 87 +++++- 4 files changed, 435 insertions(+), 3 deletions(-) create mode 100644 buildbot_nix/buildbot_nix/gitlab_project.py diff --git a/buildbot_nix/buildbot_nix/__init__.py b/buildbot_nix/buildbot_nix/__init__.py index 036e9b24b..be81dc286 100644 --- a/buildbot_nix/buildbot_nix/__init__.py +++ b/buildbot_nix/buildbot_nix/__init__.py @@ -18,6 +18,7 @@ from .errors import BuildbotNixError from .gitea_projects import GiteaBackend from .github_projects import GithubBackend +from .gitlab_project import GitlabBackend from .local_worker import NixLocalWorker from .models import AuthBackendConfig, BuildbotNixConfig from .nix_build import BuildConfig @@ -63,6 +64,9 @@ def _initialize_backends(self) -> dict[str, GitBackend]: if self.config.gitea is not None: backends["gitea"] = GiteaBackend(self.config.gitea, self.config.url) + if self.config.gitlab is not None: + backends["gitlab"] = GitlabBackend(self.config.gitlab, self.config.url) + if self.config.pull_based is not None: backends["pull_based"] = PullBasedBacked(self.config.pull_based) diff --git a/buildbot_nix/buildbot_nix/gitlab_project.py b/buildbot_nix/buildbot_nix/gitlab_project.py new file mode 100644 index 000000000..41c915588 --- /dev/null +++ b/buildbot_nix/buildbot_nix/gitlab_project.py @@ -0,0 +1,319 @@ +import os +import signal +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + +from buildbot.changes.base import ChangeSource +from buildbot.config.builder import BuilderConfig +from buildbot.plugins import util +from buildbot.reporters.base import ReporterBase +from buildbot.reporters.gitlab import GitLabStatusPush +from buildbot.www.auth import AuthBase +from buildbot.www.avatar import AvatarBase +from pydantic import BaseModel +from twisted.logger import Logger +from twisted.python import log + +from buildbot_nix.common import ( + ThreadDeferredBuildStep, + atomic_write_file, + filter_repos_by_topic, + http_request, + model_dump_project_cache, + model_validate_project_cache, + paginated_github_request, + slugify_project_name, +) +from buildbot_nix.models import GitlabConfig, Interpolate +from buildbot_nix.nix_status_generator import BuildNixEvalStatusGenerator +from buildbot_nix.projects import GitBackend, GitProject + +tlog = Logger() + + +class NamespaceData(BaseModel): + path: str + kind: str + + +class RepoData(BaseModel): + id: int + name_with_namespace: str + path: str + path_with_namespace: str + ssh_url_to_repo: str + web_url: str + namespace: NamespaceData + default_branch: str + topics: list[str] + + +class GitlabProject(GitProject): + config: GitlabConfig + data: RepoData + + def __init__(self, config: GitlabConfig, data: RepoData) -> None: + self.config = config + self.data = data + + def get_project_url(self) -> str: + url = urlparse(self.config.instance_url) + return f"{url.scheme}://git:%(secret:{self.config.token_file})s@{url.hostname}/{self.data.path_with_namespace}" + + def create_change_source(self) -> ChangeSource | None: + return None + + @property + def pretty_type(self) -> str: + return "Gitlab" + + @property + def type(self) -> str: + return "gitlab" + + @property + def nix_ref_type(self) -> str: + return "gitlab" + + @property + def repo(self) -> str: + return self.data.path + + @property + def owner(self) -> str: + return self.data.namespace.path + + @property + def name(self) -> str: + return self.data.name_with_namespace + + @property + def url(self) -> str: + return self.data.web_url + + @property + def project_id(self) -> str: + return slugify_project_name(self.data.path_with_namespace) + + @property + def default_branch(self) -> str: + return self.data.default_branch + + @property + def topics(self) -> list[str]: + return self.data.topics + + @property + def belongs_to_org(self) -> bool: + return self.data.namespace.kind == "group" + + @property + def private_key_path(self) -> Path | None: + return None + + @property + def known_hosts_path(self) -> Path | None: + return None + + +class GitlabBackend(GitBackend): + config: GitlabConfig + instance_url: str + + def __init__(self, config: GitlabConfig, instance_url: str) -> None: + self.config = config + self.instance_url = instance_url + + def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig: + factory = util.BuildFactory() + factory.addStep( + ReloadGitlabProjects(self.config, self.config.project_cache_file), + ) + factory.addStep( + CreateGitlabProjectHooks( + self.config, + self.instance_url, + ) + ) + return util.BuilderConfig( + name=self.reload_builder_name, + workernames=worker_names, + factory=factory, + ) + + def create_reporter(self) -> ReporterBase: + return GitLabStatusPush( + token=Interpolate(self.config.token), + context=Interpolate("buildbot/%(prop:status_name)s"), + baseURL=self.config.instance_url, + generators=[ + BuildNixEvalStatusGenerator(), + ], + ) + + def create_change_hook(self) -> dict[str, Any]: + return dict(secret=self.config.webhook_secret) + + def load_projects(self) -> list["GitProject"]: + if not self.config.project_cache_file.exists(): + return [] + + repos: list[RepoData] = filter_repos_by_topic( + self.config.topic, + sorted( + model_validate_project_cache(RepoData, self.config.project_cache_file), + key=lambda repo: repo.path_with_namespace, + ), + lambda repo: repo.topics, + ) + tlog.info(f"Loading {len(repos)} cached repos.") + + return [ + GitlabProject(self.config, RepoData.model_validate(repo)) for repo in repos + ] + + def are_projects_cached(self) -> bool: + return self.config.project_cache_file.exists() + + def create_auth(self) -> AuthBase: + raise NotImplementedError + + def create_avatar_method(self) -> AvatarBase | None: + return None + + @property + def reload_builder_name(self) -> str: + return "reload-gitlab-projects" + + @property + def type(self) -> str: + return "gitlab" + + @property + def pretty_type(self) -> str: + return "Gitlab" + + @property + def change_hook_name(self) -> str: + return "gitlab" + + +class ReloadGitlabProjects(ThreadDeferredBuildStep): + name = "reload_gitlab_projects" + + config: GitlabConfig + project_cache_file: Path + + def __init__( + self, + config: GitlabConfig, + project_cache_file: Path, + **kwargs: Any, + ) -> None: + self.config = config + self.project_cache_file = project_cache_file + super().__init__(**kwargs) + + def run_deferred(self) -> None: + repos: list[RepoData] = filter_repos_by_topic( + self.config.topic, + refresh_projects(self.config, self.project_cache_file), + lambda repo: repo.topics, + ) + atomic_write_file(self.project_cache_file, model_dump_project_cache(repos)) + + def run_post(self) -> Any: + return util.SUCCESS + + +class CreateGitlabProjectHooks(ThreadDeferredBuildStep): + name = "create_gitlab_project_hooks" + + config: GitlabConfig + instance_url: str + + def __init__(self, config: GitlabConfig, instance_url: str, **kwargs: Any) -> None: + self.config = config + self.instance_url = instance_url + super().__init__(**kwargs) + + def run_deferred(self) -> None: + repos = model_validate_project_cache(RepoData, self.config.project_cache_file) + for repo in repos: + create_project_hook( + token=self.config.token, + webhook_secret=self.config.webhook_secret, + project_id=repo.id, + gitlab_url=self.config.instance_url, + instance_url=self.instance_url, + ) + + def run_post(self) -> Any: + os.kill(os.getpid(), signal.SIGHUP) + return util.SUCCESS + + +def refresh_projects(config: GitlabConfig, cache_file: Path) -> list[RepoData]: + # access level 40 == Maintainer. See https://docs.gitlab.com/api/members/#roles + return [ + RepoData.model_validate(repo) + for repo in paginated_github_request( + f"{config.instance_url}/api/v4/projects?min_access_level=40&pagination=keyset&per_page=100&order_by=id&sort=asc", + config.token, + ) + ] + + +def create_project_hook( + token: str, + webhook_secret: str, + project_id: int, + gitlab_url: str, + instance_url: str, +) -> None: + hook_url = instance_url + "change_hook/gitlab" + for hook in paginated_github_request( + f"{gitlab_url}/api/v4/projects/{project_id}/hooks", + token, + ): + if hook["url"] == hook_url: + log.msg(f"hook for gitlab project {project_id} already exists") + return + log.msg(f"creating hook for gitlab project {project_id}") + http_request( + f"{gitlab_url}/api/v4/projects/{project_id}/hooks", + method="POST", + headers={ + "Authorization": f"Bearer {token}", + "Accept": "application/json", + "Content-Type": "application/json", + }, + data=dict( + name="buildbot hook", + url=hook_url, + enable_ssl_verification=True, + token=webhook_secret, + # We don't need to be informed of most events + confidential_issues_events=False, + confidential_note_events=False, + deployment_events=False, + feature_flag_events=False, + issues_events=False, + job_events=False, + merge_requests_events=False, + note_events=False, + pipeline_events=False, + releases_events=False, + wiki_page_events=False, + resource_access_token_events=False, + ), + ) + + +if __name__ == "__main__": + c = GitlabConfig( + topic=None, + ) + + print(refresh_projects(c, Path("deleteme-gitlab-cache"))) diff --git a/buildbot_nix/buildbot_nix/models.py b/buildbot_nix/buildbot_nix/models.py index ab16139a8..139d7450b 100644 --- a/buildbot_nix/buildbot_nix/models.py +++ b/buildbot_nix/buildbot_nix/models.py @@ -161,6 +161,33 @@ def oauth_secret(self) -> str: ) +class GitlabConfig(BaseModel): + instance_url: str = Field(default="https://gitlab.com") + topic: str | None + + token_file: Path = Field(default=Path("gitlab-token")) + webhook_secret_file: Path = Field(default=Path("gitlab-webhook-secret")) + + oauth_id: str | None + oauth_secret_file: Path | None + + project_cache_file: Path = Field(default=Path("gitlab-project-cache.json")) + + @property + def token(self) -> str: + return read_secret_file(self.token_file) + + @property + def webhook_secret(self) -> str: + return read_secret_file(self.webhook_secret_file) + + @property + def oauth_secret(self) -> str: + if self.oauth_secret_file is None: + raise InternalError + return read_secret_file(self.oauth_secret_file) + + class PostBuildStep(BaseModel): name: str environment: Mapping[str, str | Interpolate] @@ -281,6 +308,7 @@ class BuildbotNixConfig(BaseModel): eval_worker_count: int | None = None gitea: GiteaConfig | None = None github: GitHubConfig | None = None + gitlab: GitlabConfig | None pull_based: PullBasedConfig | None outputs_path: Path | None = None post_build_steps: list[PostBuildStep] = [] diff --git a/nixosModules/master.nix b/nixosModules/master.nix index c47393c0e..d0777310f 100644 --- a/nixosModules/master.nix +++ b/nixosModules/master.nix @@ -242,6 +242,7 @@ in type = lib.types.enum [ "gitea" "github" + "gitlab" ]; }; @@ -300,6 +301,42 @@ in }; }; + gitlab = { + enable = lib.mkEnableOption "Enable Gitlab integration"; + tokenFile = lib.mkOption { + type = lib.types.path; + description = "Gitlab token file"; + }; + webhookSecretFile = lib.mkOption { + type = lib.types.path; + description = "Gitlab webhook secret file"; + }; + instanceUrl = lib.mkOption { + type = lib.types.str; + description = "Gitlab instance url"; + example = "https://gitlab.example.com"; + default = "https://gitlab.com"; + }; + topic = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = "build-with-buildbot"; + description = '' + Projects that have this topic will be built by buildbot. + If null, all projects that the buildbot Gitea user has access to, are built. + ''; + }; + oauthId = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = "Gitlab oauth id"; + }; + oauthSecretFile = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + description = "Gitlab oauth secret file"; + }; + }; + gitea = { enable = lib.mkEnableOption "Enable Gitea integration" // { default = cfg.authBackend == "gitea"; @@ -710,6 +747,11 @@ in cfg.authBackend == "gitea" -> (cfg.gitea.oauthId != null && cfg.gitea.oauthSecretFile != null); message = ''config.services.buildbot-nix.master.authBackend is set to "gitea", then config.services.buildbot-nix.master.gitea.oauthId and config.services.buildbot-nix.master.gitea.oauthSecretFile have to be set.''; } + { + assertion = + cfg.authBackend == "gitlab" -> (cfg.gitlab.oauthId != null && cfg.gitlab.oauthSecretFile != null); + message = ''config.services.buildbot-nix.master.authBackend is set to "gitlab", then config.services.buildbot-nix.master.gitlab.oauthId and config.services.buildbot-nix.master.gitlab.oauthSecretFile have to be set.''; + } { assertion = cfg.authBackend == "github" -> cfg.github.enable; message = '' @@ -722,6 +764,12 @@ in If `cfg.authBackend` is set to `"gitea"` the GitHub backend must be enabled with `cfg.gitea.enable`; ''; } + { + assertion = cfg.authBackend == "gitlab" -> cfg.gitlab.enable; + message = '' + If `cfg.authBackend is set to `"gitlab"` the Gitlab backend must be enabled with `cfg.gitlab.enable`; + ''; + } ]; services.buildbot-master = { @@ -752,6 +800,19 @@ in (pkgs.formats.json { }).generate "buildbot-nix-config.json" { db_url = cfg.dbUrl; auth_backend = cfg.authBackend; + gitlab = + if !cfg.gitlab.enable then + null + else + { + instance_url = cfg.gitlab.instanceUrl; + topic = cfg.gitea.topic; + oauth_id = cfg.gitlab.oauthId; + oauth_secret_file = "gitlab-oauth-secret"; + token_file = "gitlab-token"; + webhook_secret_file = "gitlab-webhook-secret"; + project_cache_file = "gitlab-project-cache.json"; + }; gitea = if !cfg.gitea.enable then null @@ -867,10 +928,15 @@ in ]) ++ lib.optional (cfg.authBackend == "gitea") "gitea-oauth-secret:${cfg.gitea.oauthSecretFile}" ++ lib.optional (cfg.authBackend == "github") "github-oauth-secret:${cfg.github.oauthSecretFile}" + ++ lib.optional (cfg.authBackend == "gitlab") "gitlab-oauth-secret:${cfg.gitlab.oauthSecretFile}" ++ lib.optionals cfg.gitea.enable [ "gitea-token:${cfg.gitea.tokenFile}" "gitea-webhook-secret:${cfg.gitea.webhookSecretFile}" ] + ++ lib.optionals cfg.gitlab.enable [ + "gitlab-token:${cfg.gitlab.tokenFile}" + "gitlab-webhook-secret:${cfg.gitlab.webhookSecretFile}" + ] ++ lib.mapAttrsToList ( repoName: path: "effects-secret__${cleanUpRepoName repoName}:${path}" ) cfg.effects.perRepoSecretFiles @@ -967,15 +1033,26 @@ in ]; } (lib.mkIf (cfg.authBackend == "httpbasicauth") { set-basic-auth = true; }) + (lib.mkIf + (lib.elem cfg.accessMode.fullyPrivate.backend [ + "github" + "gitea" + "gitlab" + ]) + { + email-domain = "*"; + } + ) (lib.mkIf (lib.elem cfg.accessMode.fullyPrivate.backend [ "github" "gitea" ]) { + # https://github.com/oauth2-proxy/oauth2-proxy/issues/1724 + scope = "read:user user:email repo"; github-user = lib.concatStringsSep "," (cfg.accessMode.fullyPrivate.users ++ cfg.admins); github-team = cfg.accessMode.fullyPrivate.teams; - email-domain = "*"; } ) (lib.mkIf (cfg.accessMode.fullyPrivate.backend == "github") { provider = "github"; }) @@ -986,6 +1063,12 @@ in redeem-url = "${cfg.gitea.instanceUrl}/login/oauth/access_token"; validate-url = "${cfg.gitea.instanceUrl}/api/v1/user/emails"; }) + (lib.mkIf (cfg.accessMode.fullyPrivate.backend == "gitlab") { + provider = "gitlab"; + provider-display-name = "Gitlab"; + gitlab-groups = cfg.accessMode.fullyPrivate.teams; + oidc-issuer-url = cfg.gitlab.instanceUrl; + }) ]; }; @@ -998,8 +1081,6 @@ in client_secret = "$(cat ${cfg.accessMode.fullyPrivate.clientSecretFile})" cookie_secret = "$(cat ${cfg.accessMode.fullyPrivate.cookieSecretFile})" basic_auth_password = "$(cat ${cfg.httpBasicAuthPasswordFile})" - # https://github.com/oauth2-proxy/oauth2-proxy/issues/1724 - scope = "read:user user:email repo" EOF ''; }; From db64e5eeb3731989572d0058177b305b6e678208 Mon Sep 17 00:00:00 2001 From: Sam Willcocks Date: Tue, 15 Apr 2025 13:11:38 +0100 Subject: [PATCH 02/13] update link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a7c1f010e..a065d0a17 100644 --- a/README.md +++ b/README.md @@ -472,7 +472,7 @@ The following instances run on GitHub: [Configuration](https://github.com/nix-community/infra/tree/master/modules/nixos) | [Instance](https://buildbot.nix-community.org/) - [**Mic92's dotfiles**](https://github.com/Mic92/dotfiles): - [Configuration](https://github.com/Mic92/dotfiles/blob/main/nixos/eve/modules/buildbot.nix) + [Configuration](https://github.com/Mic92/dotfiles/blob/main/machines/eve/modules/buildbot.nix) | [Instance](https://buildbot.thalheim.io/) - [**Technical University Munich**](https://dse.in.tum.de/): [Configuration](https://github.com/TUM-DSE/doctor-cluster-config/tree/master/modules/buildbot) From 8472df6038440758cb7fdfc7e943511071402b13 Mon Sep 17 00:00:00 2001 From: Sam Willcocks Date: Wed, 16 Apr 2025 14:36:08 +0100 Subject: [PATCH 03/13] use a proper token when creating statuses --- buildbot_nix/buildbot_nix/gitlab_project.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/buildbot_nix/buildbot_nix/gitlab_project.py b/buildbot_nix/buildbot_nix/gitlab_project.py index 41c915588..a33d1b976 100644 --- a/buildbot_nix/buildbot_nix/gitlab_project.py +++ b/buildbot_nix/buildbot_nix/gitlab_project.py @@ -144,7 +144,7 @@ def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig: def create_reporter(self) -> ReporterBase: return GitLabStatusPush( - token=Interpolate(self.config.token), + token=self.config.token, context=Interpolate("buildbot/%(prop:status_name)s"), baseURL=self.config.instance_url, generators=[ @@ -309,11 +309,3 @@ def create_project_hook( resource_access_token_events=False, ), ) - - -if __name__ == "__main__": - c = GitlabConfig( - topic=None, - ) - - print(refresh_projects(c, Path("deleteme-gitlab-cache"))) From 8a27ef7afc7e0fadaaa76921976dfafab9ebd4f6 Mon Sep 17 00:00:00 2001 From: Sam Willcocks Date: Wed, 16 Apr 2025 15:19:34 +0100 Subject: [PATCH 04/13] maybe fix commit triggers --- buildbot_nix/buildbot_nix/gitlab_project.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/buildbot_nix/buildbot_nix/gitlab_project.py b/buildbot_nix/buildbot_nix/gitlab_project.py index a33d1b976..b502e68e3 100644 --- a/buildbot_nix/buildbot_nix/gitlab_project.py +++ b/buildbot_nix/buildbot_nix/gitlab_project.py @@ -39,6 +39,7 @@ class NamespaceData(BaseModel): class RepoData(BaseModel): id: int + name: str name_with_namespace: str path: str path_with_namespace: str @@ -86,7 +87,12 @@ def owner(self) -> str: @property def name(self) -> str: - return self.data.name_with_namespace + # This needs to match what buildbot uses in change hooks to map an incoming change + # to a project: https://github.com/buildbot/buildbot/blob/master/master/buildbot/www/hooks/gitlab.py#L45 + # I suspect this will result in clashes if you have identically-named projects in + # different namespaces, as is totally valid in gitlab. + # Using self.data.name_with_namespace would be more robust. + return self.data.name @property def url(self) -> str: From c47fdfadb2d204212eff1f30f17d8151762f0d6e Mon Sep 17 00:00:00 2001 From: Sam Willcocks Date: Wed, 23 Apr 2025 13:47:14 +0100 Subject: [PATCH 05/13] gitlab: add avatar method --- buildbot_nix/buildbot_nix/gitlab_project.py | 101 +++++++++++++++++++- 1 file changed, 99 insertions(+), 2 deletions(-) diff --git a/buildbot_nix/buildbot_nix/gitlab_project.py b/buildbot_nix/buildbot_nix/gitlab_project.py index b502e68e3..4ef4415f5 100644 --- a/buildbot_nix/buildbot_nix/gitlab_project.py +++ b/buildbot_nix/buildbot_nix/gitlab_project.py @@ -1,17 +1,21 @@ import os import signal +from collections.abc import Generator from pathlib import Path from typing import Any -from urllib.parse import urlparse +from urllib.parse import urlencode, urlparse from buildbot.changes.base import ChangeSource from buildbot.config.builder import BuilderConfig from buildbot.plugins import util from buildbot.reporters.base import ReporterBase from buildbot.reporters.gitlab import GitLabStatusPush +from buildbot.util import httpclientservice +from buildbot.www import resource from buildbot.www.auth import AuthBase from buildbot.www.avatar import AvatarBase from pydantic import BaseModel +from twisted.internet import defer from twisted.logger import Logger from twisted.python import log @@ -186,7 +190,7 @@ def create_auth(self) -> AuthBase: raise NotImplementedError def create_avatar_method(self) -> AvatarBase | None: - return None + return AvatarGitlab(config=self.config) @property def reload_builder_name(self) -> str: @@ -260,6 +264,99 @@ def run_post(self) -> Any: return util.SUCCESS +class AvatarGitlab(AvatarBase): + name = "gitlab" + + config: GitlabConfig + + def __init__( + self, + config: GitlabConfig, + debug: bool = False, + verify: bool = True, + ) -> None: + self.config = config + self.debug = debug + self.verify = verify + + self.master = None + self.client: httpclientservice.HTTPSession | None = None + + def _get_http_client( + self, + ) -> httpclientservice.HTTPSession: + if self.client is not None: + return self.client + + headers = { + "User-Agent": "Buildbot", + "Authorization": f"Bearer {self.config.token}", + } + + self.client = httpclientservice.HTTPSession( + self.master.httpservice, # type: ignore[attr-defined] + self.config.instance_url, + headers=headers, + debug=self.debug, + verify=self.verify, + ) + + return self.client + + @defer.inlineCallbacks + def getUserAvatar( # noqa: N802 + self, + email: str, + username: str | None, + size: str | int, + defaultAvatarUrl: str, # noqa: N803 + ) -> Generator[defer.Deferred, str | None, None]: + if isinstance(size, int): + size = str(size) + avatar_url = None + if username is not None: + avatar_url = yield self._get_avatar_by_username(username) + if avatar_url is None: + avatar_url = yield self._get_avatar_by_email(email, size) + if not avatar_url: + avatar_url = defaultAvatarUrl + raise resource.Redirect(avatar_url) + + @defer.inlineCallbacks + def _get_avatar_by_username( + self, username: str + ) -> Generator[defer.Deferred, Any, str | None]: + qs = urlencode(dict(username=username)) + http = self._get_http_client() + users = yield http.get(f"/api/v4/users?{qs}") + users = yield users.json() + if len(users) == 1: + return users[0]["avatar_url"] + if len(users) > 1: + # TODO: log warning + ... + return None + + @defer.inlineCallbacks + def _get_avatar_by_email( + self, email: str, size: str | None + ) -> Generator[defer.Deferred, Any, str | None]: + http = self._get_http_client() + q = dict(email=email) + if size is not None: + q["size"] = size + qs = urlencode(q) + res = yield http.get(f"/api/v4/avatar?{qs}") + data = yield res.json() + # N.B: Gitlab's public avatar endpoint returns a gravatar url if there isn't an + # account with a matching public email - so it should always return *something*. + # See: https://docs.gitlab.com/api/avatar/#get-details-on-an-account-avatar + if "avatar_url" in data: + return data["avatar_url"] + else: + return None + + def refresh_projects(config: GitlabConfig, cache_file: Path) -> list[RepoData]: # access level 40 == Maintainer. See https://docs.gitlab.com/api/members/#roles return [ From cedb3208c0aa728b65fa6f34045685da0b311e64 Mon Sep 17 00:00:00 2001 From: Sam Willcocks Date: Wed, 23 Apr 2025 15:40:03 +0100 Subject: [PATCH 06/13] gitlab: use correct url to fix build triggers --- buildbot_nix/buildbot_nix/gitlab_project.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/buildbot_nix/buildbot_nix/gitlab_project.py b/buildbot_nix/buildbot_nix/gitlab_project.py index 4ef4415f5..d289fdc33 100644 --- a/buildbot_nix/buildbot_nix/gitlab_project.py +++ b/buildbot_nix/buildbot_nix/gitlab_project.py @@ -100,7 +100,10 @@ def name(self) -> str: @property def url(self) -> str: - return self.data.web_url + # Not `web_url`: the buildbot gitlab hook dialect uses repository.url which seems + # to be the ssh url in practice. + # See: https://github.com/buildbot/buildbot/blob/master/master/buildbot/www/hooks/gitlab.py#L271 + return self.data.ssh_url_to_repo @property def project_id(self) -> str: From 2e1c276af31d8ca175e604c4a288620f252e54b5 Mon Sep 17 00:00:00 2001 From: Sam Willcocks Date: Wed, 23 Apr 2025 15:40:28 +0100 Subject: [PATCH 07/13] enable mypy on all unixes --- formatter/flake-module.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/formatter/flake-module.nix b/formatter/flake-module.nix index 6d798a437..fe83b9ef3 100644 --- a/formatter/flake-module.nix +++ b/formatter/flake-module.nix @@ -18,7 +18,7 @@ ]; programs.mypy = { - enable = pkgs.stdenv.buildPlatform.isLinux; + enable = pkgs.stdenv.buildPlatform.isUnix; package = pkgs.buildbot.python.pkgs.mypy; directories."." = { modules = [ @@ -39,7 +39,7 @@ # the mypy module adds `./buildbot_nix/**/*.py` which does not appear to work # furthermore, saying `directories.""` will lead to `/buildbot_nix/**/*.py` which # is obviously incorrect... - settings.formatter."mypy-" = lib.mkIf pkgs.stdenv.buildPlatform.isLinux { + settings.formatter."mypy-" = lib.mkIf pkgs.stdenv.buildPlatform.isUnix { includes = [ "buildbot_nix/**/*.py" "buildbot_effects/**/*.py" From 0e4ede3cead3a2a119e518141251c18a9ccac99d Mon Sep 17 00:00:00 2001 From: Sam Willcocks Date: Wed, 23 Apr 2025 16:03:27 +0100 Subject: [PATCH 08/13] gitlab: fix commit status description --- buildbot_nix/buildbot_nix/gitlab_project.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildbot_nix/buildbot_nix/gitlab_project.py b/buildbot_nix/buildbot_nix/gitlab_project.py index d289fdc33..d67085558 100644 --- a/buildbot_nix/buildbot_nix/gitlab_project.py +++ b/buildbot_nix/buildbot_nix/gitlab_project.py @@ -8,6 +8,7 @@ from buildbot.changes.base import ChangeSource from buildbot.config.builder import BuilderConfig from buildbot.plugins import util +from buildbot.process.properties import Interpolate from buildbot.reporters.base import ReporterBase from buildbot.reporters.gitlab import GitLabStatusPush from buildbot.util import httpclientservice @@ -29,7 +30,7 @@ paginated_github_request, slugify_project_name, ) -from buildbot_nix.models import GitlabConfig, Interpolate +from buildbot_nix.models import GitlabConfig from buildbot_nix.nix_status_generator import BuildNixEvalStatusGenerator from buildbot_nix.projects import GitBackend, GitProject From d911dba57fe730652369f2924fd661c8c074b2a6 Mon Sep 17 00:00:00 2001 From: Sam Willcocks Date: Mon, 19 May 2025 14:15:42 +0100 Subject: [PATCH 09/13] fix pyproject license metadata --- buildbot_effects/pyproject.toml | 2 +- buildbot_nix/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buildbot_effects/pyproject.toml b/buildbot_effects/pyproject.toml index 9c810b610..7ff6ba6f1 100644 --- a/buildbot_effects/pyproject.toml +++ b/buildbot_effects/pyproject.toml @@ -9,7 +9,7 @@ authors = [ ] description = "CI effects for buildbot-nix" requires-python = ">=3.12" -license = {text = "MIT"} +license = {file = "../LICENSE.md"} classifiers = [ "Programming Language :: Python :: 3", "Development Status :: 5 - Production/Stable", diff --git a/buildbot_nix/pyproject.toml b/buildbot_nix/pyproject.toml index 1e29060a7..fcd038028 100644 --- a/buildbot_nix/pyproject.toml +++ b/buildbot_nix/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "buildbot-nix" -license = {text = "MIT"} +license = {file = "../LICENSE.md"} authors = [ { name = "Jörg Thalheim", email = "joerg@thalheim.io" }, ] From 0b392937d3e5ed788e7a1bbfb99e0160d72c469b Mon Sep 17 00:00:00 2001 From: Sam Willcocks Date: Tue, 30 Sep 2025 20:00:09 +0100 Subject: [PATCH 10/13] update for latest, fix lints --- buildbot_nix/buildbot_nix/gitlab_project.py | 88 ++++++++++++--------- buildbot_nix/buildbot_nix/models.py | 6 ++ 2 files changed, 56 insertions(+), 38 deletions(-) diff --git a/buildbot_nix/buildbot_nix/gitlab_project.py b/buildbot_nix/buildbot_nix/gitlab_project.py index d67085558..eeda16fd1 100644 --- a/buildbot_nix/buildbot_nix/gitlab_project.py +++ b/buildbot_nix/buildbot_nix/gitlab_project.py @@ -21,9 +21,10 @@ from twisted.python import log from buildbot_nix.common import ( + RepoAccessors, ThreadDeferredBuildStep, atomic_write_file, - filter_repos_by_topic, + filter_repos, http_request, model_dump_project_cache, model_validate_project_cache, @@ -167,19 +168,23 @@ def create_reporter(self) -> ReporterBase: ) def create_change_hook(self) -> dict[str, Any]: - return dict(secret=self.config.webhook_secret) + return {"secret": self.config.webhook_secret} def load_projects(self) -> list["GitProject"]: if not self.config.project_cache_file.exists(): return [] - repos: list[RepoData] = filter_repos_by_topic( - self.config.topic, + repos: list[RepoData] = filter_repos( + self.config.filters, sorted( model_validate_project_cache(RepoData, self.config.project_cache_file), key=lambda repo: repo.path_with_namespace, ), - lambda repo: repo.topics, + RepoAccessors( + repo_name=lambda repo: repo.name, + user=lambda repo: repo.namespace.path, + topics=lambda repo: repo.topics, + ), ) tlog.info(f"Loading {len(repos)} cached repos.") @@ -230,10 +235,14 @@ def __init__( super().__init__(**kwargs) def run_deferred(self) -> None: - repos: list[RepoData] = filter_repos_by_topic( - self.config.topic, - refresh_projects(self.config, self.project_cache_file), - lambda repo: repo.topics, + repos: list[RepoData] = filter_repos( + self.config.filters, + refresh_projects(self.config), + RepoAccessors( + repo_name=lambda repo: repo.name, + user=lambda repo: repo.namespace.path, + topics=lambda repo: repo.topics, + ), ) atomic_write_file(self.project_cache_file, model_dump_project_cache(repos)) @@ -276,8 +285,8 @@ class AvatarGitlab(AvatarBase): def __init__( self, config: GitlabConfig, - debug: bool = False, - verify: bool = True, + debug: bool = False, # noqa: FBT002 + verify: bool = True, # noqa: FBT002 ) -> None: self.config = config self.debug = debug @@ -289,6 +298,7 @@ def __init__( def _get_http_client( self, ) -> httpclientservice.HTTPSession: + assert self.master is not None # noqa: S101 if self.client is not None: return self.client @@ -298,7 +308,7 @@ def _get_http_client( } self.client = httpclientservice.HTTPSession( - self.master.httpservice, # type: ignore[attr-defined] + self.master.httpservice, self.config.instance_url, headers=headers, debug=self.debug, @@ -310,18 +320,20 @@ def _get_http_client( @defer.inlineCallbacks def getUserAvatar( # noqa: N802 self, - email: str, - username: str | None, + email: bytes, + username: bytes | None, size: str | int, defaultAvatarUrl: str, # noqa: N803 ) -> Generator[defer.Deferred, str | None, None]: if isinstance(size, int): size = str(size) + username_str = username.decode("utf-8") if username else None + email_str = email.decode("utf-8") avatar_url = None - if username is not None: - avatar_url = yield self._get_avatar_by_username(username) + if username_str is not None: + avatar_url = yield self._get_avatar_by_username(username_str) if avatar_url is None: - avatar_url = yield self._get_avatar_by_email(email, size) + avatar_url = yield self._get_avatar_by_email(email_str, size) if not avatar_url: avatar_url = defaultAvatarUrl raise resource.Redirect(avatar_url) @@ -330,7 +342,7 @@ def getUserAvatar( # noqa: N802 def _get_avatar_by_username( self, username: str ) -> Generator[defer.Deferred, Any, str | None]: - qs = urlencode(dict(username=username)) + qs = urlencode({"username": username}) http = self._get_http_client() users = yield http.get(f"/api/v4/users?{qs}") users = yield users.json() @@ -346,7 +358,7 @@ def _get_avatar_by_email( self, email: str, size: str | None ) -> Generator[defer.Deferred, Any, str | None]: http = self._get_http_client() - q = dict(email=email) + q = {"email": email} if size is not None: q["size"] = size qs = urlencode(q) @@ -361,7 +373,7 @@ def _get_avatar_by_email( return None -def refresh_projects(config: GitlabConfig, cache_file: Path) -> list[RepoData]: +def refresh_projects(config: GitlabConfig) -> list[RepoData]: # access level 40 == Maintainer. See https://docs.gitlab.com/api/members/#roles return [ RepoData.model_validate(repo) @@ -396,23 +408,23 @@ def create_project_hook( "Accept": "application/json", "Content-Type": "application/json", }, - data=dict( - name="buildbot hook", - url=hook_url, - enable_ssl_verification=True, - token=webhook_secret, + data={ + "name": "buildbot hook", + "url": hook_url, + "enable_ssl_verification": True, + "token": webhook_secret, # We don't need to be informed of most events - confidential_issues_events=False, - confidential_note_events=False, - deployment_events=False, - feature_flag_events=False, - issues_events=False, - job_events=False, - merge_requests_events=False, - note_events=False, - pipeline_events=False, - releases_events=False, - wiki_page_events=False, - resource_access_token_events=False, - ), + "confidential_issues_events": False, + "confidential_note_events": False, + "deployment_events": False, + "feature_flag_events": False, + "issues_events": False, + "job_events": False, + "merge_requests_events": False, + "note_events": False, + "pipeline_events": False, + "releases_events": False, + "wiki_page_events": False, + "resource_access_token_events": False, + }, ) diff --git a/buildbot_nix/buildbot_nix/models.py b/buildbot_nix/buildbot_nix/models.py index 139d7450b..fefbbfe21 100644 --- a/buildbot_nix/buildbot_nix/models.py +++ b/buildbot_nix/buildbot_nix/models.py @@ -172,6 +172,7 @@ class GitlabConfig(BaseModel): oauth_secret_file: Path | None project_cache_file: Path = Field(default=Path("gitlab-project-cache.json")) + filters: RepoFilters = Field(default_factory=RepoFilters) @property def token(self) -> str: @@ -187,6 +188,11 @@ def oauth_secret(self) -> str: raise InternalError return read_secret_file(self.oauth_secret_file) + model_config = ConfigDict( + extra="forbid", + ignored_types=(property,), + ) + class PostBuildStep(BaseModel): name: str From 49bf4b0e673fb7c901758b7d3580df6a0cd73be7 Mon Sep 17 00:00:00 2001 From: Sam Willcocks Date: Tue, 30 Sep 2025 21:49:44 +0100 Subject: [PATCH 11/13] Revert "fix pyproject license metadata" This reverts commit d911dba57fe730652369f2924fd661c8c074b2a6. --- buildbot_effects/pyproject.toml | 2 +- buildbot_nix/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buildbot_effects/pyproject.toml b/buildbot_effects/pyproject.toml index 7ff6ba6f1..9c810b610 100644 --- a/buildbot_effects/pyproject.toml +++ b/buildbot_effects/pyproject.toml @@ -9,7 +9,7 @@ authors = [ ] description = "CI effects for buildbot-nix" requires-python = ">=3.12" -license = {file = "../LICENSE.md"} +license = {text = "MIT"} classifiers = [ "Programming Language :: Python :: 3", "Development Status :: 5 - Production/Stable", diff --git a/buildbot_nix/pyproject.toml b/buildbot_nix/pyproject.toml index fcd038028..1e29060a7 100644 --- a/buildbot_nix/pyproject.toml +++ b/buildbot_nix/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "buildbot-nix" -license = {file = "../LICENSE.md"} +license = {text = "MIT"} authors = [ { name = "Jörg Thalheim", email = "joerg@thalheim.io" }, ] From e651de6115dae741cc7347236468a953381e5e73 Mon Sep 17 00:00:00 2001 From: Sam Willcocks Date: Tue, 30 Sep 2025 21:56:25 +0100 Subject: [PATCH 12/13] Add missing config default --- buildbot_nix/buildbot_nix/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildbot_nix/buildbot_nix/models.py b/buildbot_nix/buildbot_nix/models.py index fefbbfe21..f64a4f5a9 100644 --- a/buildbot_nix/buildbot_nix/models.py +++ b/buildbot_nix/buildbot_nix/models.py @@ -314,7 +314,7 @@ class BuildbotNixConfig(BaseModel): eval_worker_count: int | None = None gitea: GiteaConfig | None = None github: GitHubConfig | None = None - gitlab: GitlabConfig | None + gitlab: GitlabConfig | None = None pull_based: PullBasedConfig | None outputs_path: Path | None = None post_build_steps: list[PostBuildStep] = [] From 2a94d1f7c2ce93ca99528930ebfa40c1e56c715f Mon Sep 17 00:00:00 2001 From: Sam Willcocks Date: Wed, 1 Oct 2025 12:48:27 +0100 Subject: [PATCH 13/13] fix build canceller --- buildbot_nix/buildbot_nix/build_canceller.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/buildbot_nix/buildbot_nix/build_canceller.py b/buildbot_nix/buildbot_nix/build_canceller.py index d020ce9fd..50d806f72 100644 --- a/buildbot_nix/buildbot_nix/build_canceller.py +++ b/buildbot_nix/buildbot_nix/build_canceller.py @@ -8,9 +8,11 @@ from .projects import GitProject -def branch_key_for_pr(build: dict[str, Any]) -> str: +def branch_key_for_pr(build: dict[str, Any]) -> str | None: """Extract a unique key for PR/change to cancel old builds.""" branch = build.get("branch", "") + if branch is None: + return None # For GitHub/Gitea/GitLab PRs if branch.startswith(("refs/pull/", "refs/merge-requests/")):