From a1538080f94ba64004e077b1f249486f3dde0be3 Mon Sep 17 00:00:00 2001 From: Andy Brummer Date: Tue, 16 Jun 2026 16:06:00 -0500 Subject: [PATCH] =?UTF-8?q?ci:=20defensive=20hardening=20=E2=80=94=20job?= =?UTF-8?q?=20timeouts,=20bounded=20log-dump,=20dispatch=20macOS-only?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tier-0 fixes for the self-hosted-runner stalls hit this session, all config, no external watchdog needed: - timeout-minutes on every build job (linux 60, windows 75, macos 70, benchmarks 60; release.yml mirrors). A hung step can no longer occupy a self-hosted runner indefinitely — GitHub kills the job and frees the runner. Directly prevents the 2h+ wedge a hung vcpkg log-dump caused. - The "Dump vcpkg logs on failure" steps get timeout-minutes: 3 + continue-on-error: true. That diagnostic step is exactly what hung for 2h; a failure handler must never outlive its purpose or block the job. - workflow_dispatch now runs the macOS leg ONLY: linux + windows gain `if: github.event_name != 'workflow_dispatch'`. Dispatch exists solely to validate the billed macOS leg on a branch; previously it also ran redundant Linux/Windows legs that competed for the runners (a cause of today's queue starvation). macos keeps its push|dispatch guard; benchmarks is push+main. Workflows validated (yaml parse + per-job timeout/guard check). Co-Authored-By: Claude Opus 4.8 --- .github/workflows/ci.yml | 23 +++++++++++++++++++++++ .github/workflows/release.yml | 4 ++++ 2 files changed, 27 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bade001..6bbe86d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,15 @@ jobs: # host; the guard installs only if a reimage drops them. # --------------------------------------------------------------------------- linux: + # workflow_dispatch is for validating the (billed) macOS leg only; the + # self-hosted Linux/Windows legs run on push + PR, so skip them on dispatch + # to avoid redundant runs hogging the runners. + if: github.event_name != 'workflow_dispatch' runs-on: [self-hosted, linux, lci-linux] + # Hard ceiling so a hung step (e.g. a wedged vcpkg/dep download) can never + # occupy the self-hosted runner indefinitely — GitHub kills the job and + # frees the runner. Generous vs the ~30-40 min normal Linux build. + timeout-minutes: 60 strategy: fail-fast: false matrix: @@ -77,6 +85,8 @@ jobs: - name: Dump vcpkg logs on failure if: failure() + timeout-minutes: 3 + continue-on-error: true run: | echo "===== vcpkg-manifest-install.log =====" cat build/${{ matrix.preset }}/vcpkg-manifest-install.log 2>/dev/null || true @@ -123,7 +133,11 @@ jobs: # Tools), CMake, and Ninja on PATH on the host. Release only. # --------------------------------------------------------------------------- windows: + if: github.event_name != 'workflow_dispatch' runs-on: [self-hosted, windows, lci-windows] + # Ceiling vs the ~45 min normal MSVC build; prevents a hung vcpkg/log step + # from wedging the Windows runner (as happened: a 2h+ hung log-dump step). + timeout-minutes: 75 name: windows msvc / release steps: - uses: actions/checkout@v4 @@ -140,6 +154,8 @@ jobs: - name: Dump vcpkg logs on failure if: failure() + timeout-minutes: 3 + continue-on-error: true run: | $log = "build/release/vcpkg-manifest-install.log" if (Test-Path $log) { Write-Output "===== $log ====="; Get-Content $log } @@ -188,6 +204,8 @@ jobs: macos: if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' runs-on: macos-latest + # macOS is the only billed leg — cap it so a hang can't run up minutes. + timeout-minutes: 70 name: macos appleclang / release env: VCPKG_BINARY_SOURCES: "clear;x-gha,readwrite" @@ -228,6 +246,8 @@ jobs: # never uploaded, making configure failures undiagnosable. - name: Dump vcpkg logs on failure if: failure() + timeout-minutes: 3 + continue-on-error: true run: | echo "===== vcpkg-manifest-install.log =====" cat build/release/vcpkg-manifest-install.log 2>/dev/null || true @@ -260,6 +280,7 @@ jobs: benchmarks: if: github.event_name == 'push' && github.ref == 'refs/heads/main' runs-on: [self-hosted, linux, lci-linux] + timeout-minutes: 60 name: benchmarks steps: - uses: actions/checkout@v4 @@ -282,6 +303,8 @@ jobs: - name: Dump vcpkg logs on failure if: failure() + timeout-minutes: 3 + continue-on-error: true run: | echo "===== vcpkg-manifest-install.log =====" cat build/release/vcpkg-manifest-install.log 2>/dev/null || true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 639ae5d..de54747 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,6 +14,7 @@ jobs: # these are consumed by the publish job in the same run. build-linux: runs-on: [self-hosted, linux, lci-linux] + timeout-minutes: 60 name: build / lci-linux-amd64 steps: - uses: actions/checkout@v4 @@ -95,6 +96,7 @@ jobs: # needed. Unit suite (lci_tests) gates the package, mirroring build-linux. build-windows: runs-on: [self-hosted, windows, lci-windows] + timeout-minutes: 75 name: build / lci-windows-amd64 steps: - uses: actions/checkout@v4 @@ -149,6 +151,7 @@ jobs: # this artifact (universal binary is a tracked follow-up). build-macos: runs-on: macos-latest + timeout-minutes: 70 name: build / lci-darwin-arm64 env: VCPKG_BINARY_SOURCES: "clear;x-gha,readwrite" @@ -205,6 +208,7 @@ jobs: release: needs: [build-linux, build-windows, build-macos] runs-on: [self-hosted, linux, lci-linux] + timeout-minutes: 30 name: publish steps: - uses: actions/checkout@v4