diff --git a/.env.example b/.env.example index 04010e2..dca580c 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,4 @@ -GITHUB_TOKEN=gho_XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +GITHUB_OR_FORGEJO_TOKEN=gho_XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX LIBVIRT_DEFAULT_URI=qemu:///system # Accept requests with this API token only. diff --git a/.forgejo/workflows/self-test-codeberg.yml b/.forgejo/workflows/self-test-codeberg.yml new file mode 100644 index 0000000..c4304ff --- /dev/null +++ b/.forgejo/workflows/self-test-codeberg.yml @@ -0,0 +1,50 @@ +on: workflow_dispatch + +jobs: + runner-select: + runs-on: codeberg-tiny + outputs: + unique-id: ${{ steps.select.outputs.unique-id }} + selected-runner-label: ${{ steps.select.outputs.selected-runner-label }} + runner-type-label: ${{ steps.select.outputs.runner-type-label }} + is-self-hosted: ${{ steps.select.outputs.is-self-hosted }} + steps: + - uses: actions/checkout@v4 + - name: Runner select + id: select + uses: ./actions/runner-select + with: + forge-is-forgejo: true + queue-api-base-url: http://home.daz.cat:8002 + GITHUB_TOKEN: ${{ github.token }} + # Before updating the GH action runner image for the nightly job, ensure + # that the system has a glibc version that is compatible with the one + # used by the wpt.fyi runners. + github-hosted-runner-label: codeberg-tiny + self-hosted-image-name: base-ubuntu2204 + # You can disable self-hosted runners globally by creating a repository variable named + # NO_SELF_HOSTED_RUNNERS with any non-empty value. + # + NO_SELF_HOSTED_RUNNERS: ${{ vars.NO_SELF_HOSTED_RUNNERS }} + # Any other boolean conditions that disable self-hosted runners go here. + force-github-hosted-runner: ${{ inputs.upload || inputs.force-github-hosted-runner }} + runner-timeout: + needs: + - runner-select + if: ${{ fromJSON(needs.runner-select.outputs.is-self-hosted) }} + runs-on: codeberg-tiny + steps: + - uses: actions/checkout@v4 + - name: Runner timeout + uses: ./actions/runner-timeout + with: + github_token: '${{ secrets.GITHUB_TOKEN }}' + unique-id: '${{ needs.runner-select.outputs.unique-id }}' + + build: + needs: + - runner-select + name: build [${{ needs.runner-select.outputs.unique-id }}] + runs-on: ${{ needs.runner-select.outputs.selected-runner-label }} + steps: + - uses: actions/checkout@v4 diff --git a/.github/workflows/self-test.yml b/.github/workflows/self-test.yml index fe14af1..9b3f595 100644 --- a/.github/workflows/self-test.yml +++ b/.github/workflows/self-test.yml @@ -14,6 +14,8 @@ jobs: id: select uses: ./actions/runner-select with: + forge-is-forgejo: false + queue-api-base-url: https://ci0.servo.org/queue GITHUB_TOKEN: ${{ github.token }} # Before updating the GH action runner image for the nightly job, ensure # that the system has a glibc version that is compatible with the one diff --git a/Cargo.lock b/Cargo.lock index 292b989..59a154d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1246,6 +1246,7 @@ dependencies = [ "toml", "tracing", "tracing-subscriber 0.3.18", + "url", "web", ] @@ -1400,9 +1401,9 @@ dependencies = [ [[package]] name = "percent-encoding" -version = "2.3.1" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pin-project-lite" @@ -1497,7 +1498,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.5.7", + "socket2 0.6.1", "thiserror", "tokio", "tracing", @@ -1534,9 +1535,9 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.7", + "socket2 0.6.1", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -1981,6 +1982,7 @@ dependencies = [ "serde", "toml", "tracing", + "url", ] [[package]] @@ -2517,13 +2519,14 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.4" +version = "2.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] diff --git a/actions/runner-select/action.yml b/actions/runner-select/action.yml index 4221572..3eb2efc 100644 --- a/actions/runner-select/action.yml +++ b/actions/runner-select/action.yml @@ -1,5 +1,11 @@ name: Select Self-hosted Runner inputs: + forge-is-forgejo: + required: true + type: boolean + queue-api-base-url: + required: true + type: string GITHUB_TOKEN: required: true type: string @@ -41,6 +47,19 @@ runs: set -euo pipefail + apt_install() { + # Install distro packages, but only if one or more are not already installed. + # Update the package lists first, to avoid failures when rebaking old images. + if ! dpkg -s "$@" > /dev/null 2>&1; then + apt update + # DEBIAN_FRONTEND needed to avoid hang when installing tshark + DEBIAN_FRONTEND=noninteractive apt install -y "$@" + fi + } + + # Already installed on GitHub ubuntu-24.04, but not Codeberg codeberg-tiny. + apt_install uuid-runtime + # Generate a unique id that allows the workload job to find the runner # we are reserving for it (via runner labels), and allows the timeout # job to find the workload job run (via the job’s friendly name), even @@ -78,7 +97,7 @@ runs: - id: artifact name: Publish artifact with args if: ${{ !fromJSON(steps.init.outputs.disabled) }} - uses: actions/upload-artifact@v4 + uses: ${{ inputs.forge-is-forgejo && 'forgejo/upload-artifact@v4' || 'actions/upload-artifact@v4' }} with: name: servo-ci-runners_${{ steps.init.outputs.unique_id }} path: ${{ steps.init.outputs.artifact_path }} @@ -88,6 +107,7 @@ runs: name: Find a server and reserve a runner shell: bash run: | + queue_api_base_url='${{ inputs.queue-api-base-url }}' github_hosted_runner_label='${{ inputs.github-hosted-runner-label }}' self_hosted_image_name='${{ inputs.self-hosted-image-name }}' disabled='${{ steps.init.outputs.disabled }}' @@ -107,7 +127,6 @@ runs: fall_back_to_github_hosted fi - queue_api_base_url=https://ci0.servo.org/queue # Use the queue API to enqueue this job. enqueue_url=$queue_api_base_url/enqueue\?unique_id=$unique_id\&qualified_repo=${{ github.repository }}\&run_id=${{ github.run_id }} result=$(mktemp) diff --git a/monitor.toml.example b/monitor.toml.example index a87c6d8..e19a57f 100644 --- a/monitor.toml.example +++ b/monitor.toml.example @@ -4,8 +4,14 @@ listen_on = ["::1", "192.168.100.1"] # Prepend this to any internal URL in our own responses. Must end with trailing slash. external_base_url = "http://[::1]:8000/" -# GitHub Actions runner scope (`/repos//` or `/orgs/`). -github_api_scope = "/repos/delan/servo" +# GitHub Actions runner scope, as a full URL including the domain of the forge. For example: +# - `https://api.github.com/repos//` +# - `https://api.github.com/orgs/` +# - `https://codeberg.org/api/v1/repos//` +# - `https://codeberg.org/api/v1/orgs/` +# - `https://codeberg.org/api/v1/user` +github_api_scope_url = "https://api.github.com/repos/delan/servo" +github_api_is_forgejo = false # For tokenless runner select, qualified repos must start with this prefix. allowed_qualified_repo_prefix = "delan/" diff --git a/monitor/Cargo.toml b/monitor/Cargo.toml index b25c817..bef487b 100644 --- a/monitor/Cargo.toml +++ b/monitor/Cargo.toml @@ -31,6 +31,7 @@ tracing = { workspace = true } tracing-subscriber = { workspace = true } web = { workspace = true } rand = "0.9.1" +url = "2.5.7" [dev-dependencies] settings = { workspace = true, features = ["test"] } diff --git a/monitor/settings/Cargo.toml b/monitor/settings/Cargo.toml index 82a0ab7..a775621 100644 --- a/monitor/settings/Cargo.toml +++ b/monitor/settings/Cargo.toml @@ -15,3 +15,4 @@ mktemp = { workspace = true } serde = { workspace = true } toml = { workspace = true } tracing = { workspace = true } +url = { version = "2.5.7", features = ["serde"] } diff --git a/monitor/settings/src/lib.rs b/monitor/settings/src/lib.rs index db45368..c0e4228 100644 --- a/monitor/settings/src/lib.rs +++ b/monitor/settings/src/lib.rs @@ -16,6 +16,7 @@ use std::{ use chrono::TimeDelta; use jane_eyre::eyre::{self, bail}; use serde::Deserialize; +use url::Url; use crate::{profile::Profile, queue::QueueConfig, units::MemorySize}; @@ -53,7 +54,7 @@ pub static TOML: LazyLock = LazyLock::new(|| { #[derive(Default)] pub struct Dotenv { - // GITHUB_TOKEN not used + pub github_or_forgejo_token: String, // LIBVIRT_DEFAULT_URI not used pub monitor_api_token_raw_value: String, pub monitor_api_token_authorization_value: String, @@ -64,7 +65,8 @@ pub struct Dotenv { pub struct Toml { pub listen_on: Vec, pub external_base_url: String, - pub github_api_scope: String, + pub github_api_scope_url: Url, + pub github_api_is_forgejo: bool, pub allowed_qualified_repo_prefix: String, pub github_api_suffix: String, monitor_poll_interval: u64, @@ -94,6 +96,7 @@ impl Dotenv { pub fn load() -> Self { let monitor_api_token = env_string("SERVO_CI_MONITOR_API_TOKEN"); let result = Self { + github_or_forgejo_token: env_string("GITHUB_OR_FORGEJO_TOKEN"), monitor_api_token_raw_value: monitor_api_token.clone(), monitor_api_token_authorization_value: Self::monitor_api_token_authorization_value( &monitor_api_token, @@ -106,6 +109,7 @@ impl Dotenv { #[cfg(any(test, feature = "test"))] fn load_for_tests() -> Self { + let mut github_or_forgejo_token = None; let mut monitor_data_path = None; // TODO: find a way to do this without a temporary file @@ -122,6 +126,7 @@ impl Dotenv { for entry in dotenv::from_path_iter(env_path).expect("Failed to load temporary env file") { let (key, value) = entry.expect("Failed to load entry"); match &*key { + "GITHUB_OR_FORGEJO_TOKEN" => github_or_forgejo_token = Some(value), "SERVO_CI_MONITOR_API_TOKEN" => { /* do nothing (see below) */ } "SERVO_CI_MONITOR_DATA_PATH" => monitor_data_path = Some(value), _ => { /* do nothing */ } @@ -132,6 +137,8 @@ impl Dotenv { let monitor_api_token = "ChangedMe"; let result = Self { + github_or_forgejo_token: github_or_forgejo_token + .expect("Bad contents of monitor.toml.example"), monitor_api_token_raw_value: monitor_api_token.to_owned(), monitor_api_token_authorization_value: Self::monitor_api_token_authorization_value( monitor_api_token, diff --git a/monitor/src/github.rs b/monitor/src/github.rs index 0ef0bbe..fc00e8c 100644 --- a/monitor/src/github.rs +++ b/monitor/src/github.rs @@ -7,7 +7,8 @@ use chrono::{DateTime, FixedOffset}; use cmd_lib::{run_cmd, run_fun}; use jane_eyre::eyre::{self, Context}; use serde::{Deserialize, Serialize}; -use settings::TOML; +use serde_json::json; +use settings::{DOTENV, TOML}; use tracing::trace; #[derive(Clone, Debug, Deserialize, Serialize)] @@ -107,38 +108,123 @@ impl Cache { } fn list_registered_runners() -> eyre::Result> { - let github_api_scope = &TOML.github_api_scope; - let result = run_fun!(gh api -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" - "$github_api_scope/actions/runners" --paginate -q ".runners[]" - | jq -s .)?; + let github_or_forgejo_token = &DOTENV.github_or_forgejo_token; + let github_api_scope_url = &TOML.github_api_scope_url; + let result = if TOML.github_api_is_forgejo { + // FIXME: this leaks the token in logs when the command fails + run_fun!(curl -fsSH "Authorization: token $github_or_forgejo_token" + "$github_api_scope_url/actions/runners" // TODO: pagination? + | jq -er ".runners")? + } else { + run_fun!(GITHUB_TOKEN=$github_or_forgejo_token gh api + -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" + "$github_api_scope_url/actions/runners" --paginate -q ".runners[]" + | jq -s .)? + }; Ok(serde_json::from_str(&result).wrap_err("Failed to parse JSON")?) } pub fn list_registered_runners_for_host() -> eyre::Result> { + let prefix = format!("{}-", TOML.libvirt_runner_guest_prefix()); let suffix = format!("@{}", TOML.github_api_suffix); let result = list_registered_runners()? .into_iter() + .filter(|runner| runner.name.starts_with(&prefix)) .filter(|runner| runner.name.ends_with(&suffix)); Ok(result.collect()) } pub fn register_runner(runner_name: &str, label: &str, work_folder: &str) -> eyre::Result { + let github_or_forgejo_token = &DOTENV.github_or_forgejo_token; + let github_api_scope_url = &TOML.github_api_scope_url; let github_api_suffix = &TOML.github_api_suffix; - let github_api_scope = &TOML.github_api_scope; - let result = run_fun!(gh api --method POST -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" - "$github_api_scope/actions/runners/generate-jitconfig" - -f "name=$runner_name@$github_api_suffix" -F "runner_group_id=1" -f "work_folder=$work_folder" - -f "labels[]=self-hosted" -f "labels[]=X64" -f "labels[]=$label")?; + let result = if TOML.github_api_is_forgejo { + // FIXME: this leaks the token in logs when the command fails + let registration_token = run_fun!(curl -fsSH "Authorization: token $github_or_forgejo_token" + -X POST "$github_api_scope_url/actions/runners/registration-token" + | jq -er .token)?; + + // Hit the internal(?) registration API using JSON instead of protobuf (): + // + // + #[derive(Clone, Debug, Deserialize, Serialize)] + struct RegisterRequest { + name: String, + token: String, + version: String, + labels: Vec, + ephemeral: bool, + } + #[derive(Clone, Debug, Deserialize, Serialize)] + struct ForgejoApiRegisterResponse { + runner: ForgejoApiRegisterResponseRunner, + } + #[derive(Clone, Debug, Deserialize, Serialize)] + struct ForgejoApiRegisterResponseRunner { + id: String, + uuid: String, + token: String, + name: String, + version: String, + labels: Vec, + } + impl ForgejoApiRegisterResponseRunner { + /// Convert the runner to the `.runner` format expected by forgejo-runner. + fn to_forgejo_dot_runner(&self) -> eyre::Result { + let id = usize::from_str_radix(&self.id, 10)?; + let address = TOML.github_api_scope_url.join("/")?.to_string(); + let address = address.strip_suffix("/").expect("Guaranteed by argument"); + Ok(serde_json::to_string(&json!({ + "id": id, + "uuid": self.uuid, + "name": self.name, + "token": self.token, + "address": address, + "labels": self.labels, + }))?) + } + } + let request = RegisterRequest { + name: format!("{runner_name}@{github_api_suffix}"), + token: registration_token, + version: "ServoCI".to_owned(), + labels: vec!["self-hosted".to_owned(), label.to_owned()], + ephemeral: true, + }; + let request = serde_json::to_string(&request)?; + let register_url = + github_api_scope_url.join("/api/actions/runner.v1.RunnerService/Register")?; + let response = run_fun!(curl -fsSH "Content-Type: application/json" + --data-raw $request "$register_url")?; + let response: ForgejoApiRegisterResponse = serde_json::from_str(&response)?; + response.runner.to_forgejo_dot_runner()? + } else { + let response = run_fun!(GITHUB_TOKEN=$github_or_forgejo_token gh api + -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" + --method POST "$github_api_scope_url/actions/runners/generate-jitconfig" + -f "name=$runner_name@$github_api_suffix" -F "runner_group_id=1" -f "work_folder=$work_folder" + -f "labels[]=self-hosted" -f "labels[]=X64" -f "labels[]=$label")?; + let response: ApiGenerateJitconfigResponse = serde_json::from_str(&response)?; + response.encoded_jit_config + }; Ok(result) } pub fn unregister_runner(id: usize) -> eyre::Result<()> { - let github_api_scope = &TOML.github_api_scope; - run_cmd!(gh api --method DELETE -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" - "$github_api_scope/actions/runners/$id")?; + let github_or_forgejo_token = &DOTENV.github_or_forgejo_token; + let github_api_scope_url = &TOML.github_api_scope_url; + if TOML.github_api_is_forgejo { + // FIXME: this leaks the token in logs when the command fails + run_cmd!(curl -fsSH "Authorization: token $github_or_forgejo_token" + -X DELETE "$github_api_scope_url/actions/runners/$id")?; + } else { + run_cmd!(GITHUB_TOKEN=$github_or_forgejo_token gh api + -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" + --method DELETE "$github_api_scope_url/actions/runners/$id")?; + } Ok(()) } @@ -149,13 +235,19 @@ pub fn reserve_runner( reserved_since: SystemTime, reserved_by: &str, ) -> eyre::Result<()> { - let github_api_scope = &TOML.github_api_scope; + let github_or_forgejo_token = &DOTENV.github_or_forgejo_token; + let github_api_scope_url = &TOML.github_api_scope_url; let reserved_since = reserved_since.duration_since(UNIX_EPOCH)?.as_secs(); - run_cmd!(gh api --method POST -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" - "$github_api_scope/actions/runners/$id/labels" - -f "labels[]=reserved-for:$unique_id" - -f "labels[]=reserved-since:$reserved_since" - -f "labels[]=reserved-by:$reserved_by")?; + if TOML.github_api_is_forgejo { + todo!() + } else { + run_cmd!(GITHUB_TOKEN=$github_or_forgejo_token gh api + -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" + --method POST "$github_api_scope_url/actions/runners/$id/labels" + -f "labels[]=reserved-for:$unique_id" + -f "labels[]=reserved-since:$reserved_since" + -f "labels[]=reserved-by:$reserved_by")?; + } Ok(()) } diff --git a/monitor/src/runner.rs b/monitor/src/runner.rs index 3c34906..6e0a68c 100644 --- a/monitor/src/runner.rs +++ b/monitor/src/runner.rs @@ -355,13 +355,9 @@ impl Runner { cfg_if! { if #[cfg(not(test))] { - use monitor::github::ApiGenerateJitconfigResponse; - fn read_github_jitconfig(id: usize) -> eyre::Result { let result = get_runner_data_path(id, Path::new("github-api-registration"))?; - let result: ApiGenerateJitconfigResponse = - serde_json::from_reader(File::open(result)?)?; - Ok(result.encoded_jit_config) + Ok(std::fs::read_to_string(result)?) } fn runner_created_time(id: usize) -> eyre::Result {