From 271e422d6fb23998fda1ebaa4f3063fd83f95c6e Mon Sep 17 00:00:00 2001 From: magic_rb Date: Sun, 29 Dec 2024 14:12:56 +0100 Subject: [PATCH 1/2] Support using SSH to fetch repositories on Gitea Signed-off-by: magic_rb --- buildbot_nix/buildbot_nix/__init__.py | 4 ++++ buildbot_nix/buildbot_nix/gitea_projects.py | 14 +++++++++++++- buildbot_nix/buildbot_nix/github_projects.py | 8 ++++++++ buildbot_nix/buildbot_nix/models.py | 3 +++ buildbot_nix/buildbot_nix/projects.py | 11 +++++++++++ .../buildbot_nix/pull_based/project.py | 9 +++++++++ nix/master.nix | 19 +++++++++++++++++++ 7 files changed, 67 insertions(+), 1 deletion(-) diff --git a/buildbot_nix/buildbot_nix/__init__.py b/buildbot_nix/buildbot_nix/__init__.py index 4fe1cde9c..63c70b02d 100644 --- a/buildbot_nix/buildbot_nix/__init__.py +++ b/buildbot_nix/buildbot_nix/__init__.py @@ -1058,6 +1058,8 @@ def nix_eval_config( submodules=True, haltOnFailure=True, logEnviron=False, + sshPrivateKey=project.private_key_path.read_text() if project.private_key_path else None, + sshKnownHosts=project.known_hosts_path.read_text() if project.known_hosts_path else None, ), ) drv_gcroots_dir = util.Interpolate( @@ -1414,6 +1416,8 @@ def buildbot_effects_config( method="clean", submodules=True, haltOnFailure=True, + sshPrivateKey=project.private_key_path.read_text() if project.private_key_path else None, + sshKnownHosts=project.known_hosts_path.read_text() if project.known_hosts_path else None, ), ) secrets_list = [] diff --git a/buildbot_nix/buildbot_nix/gitea_projects.py b/buildbot_nix/buildbot_nix/gitea_projects.py index a09e741d1..c3fcf4708 100644 --- a/buildbot_nix/buildbot_nix/gitea_projects.py +++ b/buildbot_nix/buildbot_nix/gitea_projects.py @@ -61,7 +61,10 @@ def __init__( def get_project_url(self) -> str: url = urlparse(self.config.instance_url) - return f"{url.scheme}://git:%(secret:{self.config.token_file})s@{url.hostname}/{self.name}" + if self.config.ssh_private_key_file: + return self.data.ssh_url + else: + return f"{url.scheme}://git:%(secret:{self.config.token_file})s@{url.hostname}/{self.name}" def create_change_source(self) -> ChangeSource | None: return None @@ -113,6 +116,15 @@ def belongs_to_org(self) -> bool: # TODO Gitea doesn't include this information return False # self.data["owner"]["type"] == "Organization" + @property + def private_key_path(self) -> Path | None: + return self.config.ssh_private_key_file + + @property + def known_hosts_path(self) -> Path | None: + return self.config.ssh_known_hosts_file + + class GiteaBackend(GitBackend): config: GiteaConfig diff --git a/buildbot_nix/buildbot_nix/github_projects.py b/buildbot_nix/buildbot_nix/github_projects.py index 54a6250a0..0fa8cd568 100644 --- a/buildbot_nix/buildbot_nix/github_projects.py +++ b/buildbot_nix/buildbot_nix/github_projects.py @@ -770,6 +770,14 @@ def topics(self) -> list[str]: def belongs_to_org(self) -> bool: return self.data.owner.ttype == "Organization" + @property + def private_key_path(self) -> Path | None: + return None + + @property + def known_hosts_path(self) -> Path | None: + return None + def refresh_projects( github_token: str, diff --git a/buildbot_nix/buildbot_nix/models.py b/buildbot_nix/buildbot_nix/models.py index f355d9069..ce35e6005 100644 --- a/buildbot_nix/buildbot_nix/models.py +++ b/buildbot_nix/buildbot_nix/models.py @@ -103,6 +103,9 @@ class GiteaConfig(BaseModel): oauth_id: str | None oauth_secret_file: Path | None + ssh_private_key_file: Path | None + ssh_known_hosts_file: Path | None + @property def token(self) -> str: return read_secret_file(self.token_file) diff --git a/buildbot_nix/buildbot_nix/projects.py b/buildbot_nix/buildbot_nix/projects.py index 992f9ba1e..a56949806 100644 --- a/buildbot_nix/buildbot_nix/projects.py +++ b/buildbot_nix/buildbot_nix/projects.py @@ -1,5 +1,6 @@ from abc import ABC, abstractmethod from typing import Any +from pathlib import Path from buildbot.changes.base import ChangeSource from buildbot.config.builder import BuilderConfig @@ -125,3 +126,13 @@ def topics(self) -> list[str]: @abstractmethod def belongs_to_org(self) -> bool: pass + + @property + @abstractmethod + def private_key_path(self) -> Path | None: + pass + + @property + @abstractmethod + def known_hosts_path(self) -> Path | None: + pass diff --git a/buildbot_nix/buildbot_nix/pull_based/project.py b/buildbot_nix/buildbot_nix/pull_based/project.py index e44689335..4c090a3d4 100644 --- a/buildbot_nix/buildbot_nix/pull_based/project.py +++ b/buildbot_nix/buildbot_nix/pull_based/project.py @@ -1,5 +1,6 @@ from typing import Any from urllib.parse import ParseResult, urlparse +from pathlib import Path from buildbot.changes.base import ChangeSource from buildbot.changes.gitpoller import GitPoller @@ -99,3 +100,11 @@ def topics(self) -> list[str]: @property def belongs_to_org(self) -> bool: return False + + @property + def private_key_path(self) -> Path | None: + return None + + @property + def known_hosts_path(self) -> Path | None: + return None diff --git a/nix/master.nix b/nix/master.nix index 497466658..57044583f 100644 --- a/nix/master.nix +++ b/nix/master.nix @@ -368,6 +368,23 @@ in If null, all projects that the buildbot Gitea user has access to, are built. ''; }; + + sshPrivateKeyFile = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + description = '' + If non-null the specified SSH key will be used to fetch all configured repositories. + ''; + }; + + sshKnownHostsFile = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + description = '' + If non-null the specified known hosts file will be matched against when connecting to + repositories over SSH. + ''; + }; }; github = { enable = lib.mkEnableOption "Enable GitHub integration" // { @@ -801,6 +818,8 @@ in instance_url = cfg.gitea.instanceUrl; oauth_id = cfg.gitea.oauthId; topic = cfg.gitea.topic; + ssh_private_key_file = cfg.gitea.sshPrivateKeyFile; + ssh_known_hosts_file = cfg.gitea.sshKnownHostsFile; }; github = if !cfg.github.enable then From 4f7b4257e7f4b5dc528edc3e15869e42770620bb Mon Sep 17 00:00:00 2001 From: magic_rb Date: Wed, 29 Jan 2025 14:51:24 +0100 Subject: [PATCH 2/2] Copy authentication files in `GitLocalPrMerge` Signed-off-by: magic_rb --- buildbot_nix/buildbot_nix/__init__.py | 72 ++++++++++++------- buildbot_nix/buildbot_nix/gitea_projects.py | 4 +- buildbot_nix/buildbot_nix/projects.py | 2 +- .../buildbot_nix/pull_based/project.py | 2 +- 4 files changed, 49 insertions(+), 31 deletions(-) diff --git a/buildbot_nix/buildbot_nix/__init__.py b/buildbot_nix/buildbot_nix/__init__.py index 63c70b02d..8ea82d1fe 100644 --- a/buildbot_nix/buildbot_nix/__init__.py +++ b/buildbot_nix/buildbot_nix/__init__.py @@ -986,6 +986,7 @@ async def run_vc( pr_head = build_props.getProperty("github.head.sha") or build_props.getProperty( "head_sha" ) + auth_workdir = self._get_auth_data_workdir() # Not a PR, fallback to default behavior if merge_base is None or pr_head is None: @@ -1005,33 +1006,42 @@ async def run_vc( self.build.path_module.join(self.workdir, ".git") ) - if not has_git: - await self._dovccmd(["clone", "--recurse-submodules", self.repourl, "."]) + try: + await self._git_auth.download_auth_files_if_needed(auth_workdir) - patched = await self.sourcedirIsPatched() + if not has_git: + await self._dovccmd( + ["clone", "--recurse-submodules", self.repourl, "."] + ) - if patched: - await self._dovccmd(["clean", "-f", "-f", "-d", "-x"]) + patched = await self.sourcedirIsPatched() - await self._dovccmd(["fetch", "-f", "-t", self.repourl, merge_base, pr_head]) + if patched: + await self._dovccmd(["clean", "-f", "-f", "-d", "-x"]) - await self._dovccmd(["checkout", "--detach", "-f", pr_head]) + await self._dovccmd( + ["fetch", "-f", "-t", self.repourl, merge_base, pr_head] + ) - await self._dovccmd( - [ - "-c", - "user.email=buildbot@example.com", - "-c", - "user.name=buildbot", - "merge", - "--no-ff", - "-m", - f"Merge {merge_base} into {pr_head}", - merge_base, - ] - ) - self.updateSourceProperty("got_revision", pr_head) - return await self.parseCommitDescription() + await self._dovccmd(["checkout", "--detach", "-f", pr_head]) + + await self._dovccmd( + [ + "-c", + "user.email=buildbot@example.com", + "-c", + "user.name=buildbot", + "merge", + "--no-ff", + "-m", + f"Merge {merge_base} into {pr_head}", + merge_base, + ] + ) + self.updateSourceProperty("got_revision", pr_head) + return await self.parseCommitDescription() + finally: + await self._git_auth.remove_auth_files_if_needed(auth_workdir) def nix_eval_config( @@ -1058,8 +1068,12 @@ def nix_eval_config( submodules=True, haltOnFailure=True, logEnviron=False, - sshPrivateKey=project.private_key_path.read_text() if project.private_key_path else None, - sshKnownHosts=project.known_hosts_path.read_text() if project.known_hosts_path else None, + sshPrivateKey=project.private_key_path.read_text() + if project.private_key_path + else None, + sshKnownHosts=project.known_hosts_path.read_text() + if project.known_hosts_path + else None, ), ) drv_gcroots_dir = util.Interpolate( @@ -1367,6 +1381,8 @@ def nix_skipped_build_config( collapseRequests=False, env={}, factory=factory, + do_build_if=lambda build: do_register_gcroot_if(build, branch_config_dict) + and outputs_path is not None, ) @@ -1416,8 +1432,12 @@ def buildbot_effects_config( method="clean", submodules=True, haltOnFailure=True, - sshPrivateKey=project.private_key_path.read_text() if project.private_key_path else None, - sshKnownHosts=project.known_hosts_path.read_text() if project.known_hosts_path else None, + sshPrivateKey=project.private_key_path.read_text() + if project.private_key_path + else None, + sshKnownHosts=project.known_hosts_path.read_text() + if project.known_hosts_path + else None, ), ) secrets_list = [] diff --git a/buildbot_nix/buildbot_nix/gitea_projects.py b/buildbot_nix/buildbot_nix/gitea_projects.py index c3fcf4708..034fc3cc0 100644 --- a/buildbot_nix/buildbot_nix/gitea_projects.py +++ b/buildbot_nix/buildbot_nix/gitea_projects.py @@ -63,8 +63,7 @@ def get_project_url(self) -> str: url = urlparse(self.config.instance_url) if self.config.ssh_private_key_file: return self.data.ssh_url - else: - return f"{url.scheme}://git:%(secret:{self.config.token_file})s@{url.hostname}/{self.name}" + return f"{url.scheme}://git:%(secret:{self.config.token_file})s@{url.hostname}/{self.name}" def create_change_source(self) -> ChangeSource | None: return None @@ -125,7 +124,6 @@ def known_hosts_path(self) -> Path | None: return self.config.ssh_known_hosts_file - class GiteaBackend(GitBackend): config: GiteaConfig webhook_secret: str diff --git a/buildbot_nix/buildbot_nix/projects.py b/buildbot_nix/buildbot_nix/projects.py index a56949806..64eba7a5c 100644 --- a/buildbot_nix/buildbot_nix/projects.py +++ b/buildbot_nix/buildbot_nix/projects.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod -from typing import Any from pathlib import Path +from typing import Any from buildbot.changes.base import ChangeSource from buildbot.config.builder import BuilderConfig diff --git a/buildbot_nix/buildbot_nix/pull_based/project.py b/buildbot_nix/buildbot_nix/pull_based/project.py index 4c090a3d4..eaf202075 100644 --- a/buildbot_nix/buildbot_nix/pull_based/project.py +++ b/buildbot_nix/buildbot_nix/pull_based/project.py @@ -1,6 +1,6 @@ +from pathlib import Path from typing import Any from urllib.parse import ParseResult, urlparse -from pathlib import Path from buildbot.changes.base import ChangeSource from buildbot.changes.gitpoller import GitPoller