refactor(scripting)!: Rust-hosted mlua Lua runtime replaces the C++ LuaServer #875

Workflow file for this run

.github/workflows/ci.yml at 2075be7

	# Main CI workflow for pull requests (and manual workflow_dispatch runs).
	#
	# Structure:
	# 1. `changes` — a ~15s path-filter gate. It exists ONLY to skip the
	# sanitizer job on PRs that don't touch native/build-relevant paths; the
	# `test` matrix does NOT depend on it and starts immediately.
	# 2. `test` — the 3-leg build+test matrix (linux-amd64, linux-arm64,
	# darwin-arm64), each running `just bazel-test` (fastbuild, no
	# instrumentation; unit + approval + e2e). The linux-amd64 leg runs the
	# Rust lint gates (formatting + clippy) BEFORE its build+test, as
	# fail-fast gates (clippy compiles the Rust crates natively on the amd64
	# RBE worker — no cross-compile). amd64 carries the heaviest downstream
	# work (Docker smoke, Scout, SARIF), so short-circuiting it on a lint
	# finding saves the most. Linux legs additionally build the Docker image,
	# run the Bazel-built smoke test against it, and feed Docker Scout; amd64
	# alone uploads the SARIF CVE scan. Every leg writes its
	# `just bazel-test` invocation's Build Event Protocol JSON and uploads it
	# as a `bep-<platform>` artifact for offline cache-hit-rate analysis.
	# 3. `sanitizer` — a path-gated ASan+UBSan job running unit + e2e in one
	# invocation (see the scope-rationale comment above the job).
	# 4. `docs` — MkDocs build.
	#
	# Coverage instrumentation lives in `.github/workflows/coverage.yml`, which
	# runs on every push to `main` (post-merge) on a single linux-amd64 runner and
	# uploads lcov to Codecov. PRs do not get a per-PR coverage delta in Codecov;
	# trade-off for faster PR CI.
	#
	# Every step invokes `just <recipe>`. No inline bazel calls — recipes are
	# the contract surface. See the justfile "Bazel — what CI runs" section.
	#
	# No Nix: the build is fully hermetic under Bazel (Nix provisions only the
	# local dev shell, per CLAUDE.md's build-reproducibility invariant). CI
	# runners get their tools from the `ci-setup` composite action (bazelisk +
	# `just`, via `bazelbuild/setup-bazelisk` + `extractions/setup-just`) plus the
	# `gh` CLI, which is preinstalled on GitHub-hosted runners and used directly
	# by the `gh_release_archive` Kakadu fetch (see `ci-setup`'s header comment).
	#
	# Documentation builds run in a separate `docs` job (Python / mkdocs).

	name: ci

	on:
	pull_request:
	types: [opened, synchronize, reopened]
	workflow_dispatch:

	permissions:
	contents: read

	jobs:
	# Path-filter gate for the sanitizer job ONLY — the `test` matrix has no
	# `needs:` on this job and starts immediately, overlapping the ~15s this
	# takes and the ~5m macOS runner queue.
	#
	# Why job-level gating instead of `on.paths` on the sanitizer job
	# itself: `sanitizer` is a REQUIRED status check.
	# With `on.paths`, a PR that doesn't match the filter never triggers the
	# job at all, so its required-check entry sits at "Expected" forever and
	# blocks merge. A job that runs and is merely skipped (via `if:`) reports
	# as skipped, which satisfies a branch-protection ruleset.
	changes:
	name: changes
	runs-on: ubuntu-24.04
	outputs:
	native: ${{ steps.filter.outputs.native }}
	steps:
	- uses: actions/checkout@v6
	with:
	fetch-depth: 0
	lfs: false
	- id: filter
	name: Detect native/build-relevant changes
	shell: bash
	run: \|
	if [ "${{ github.event_name }}" != "pull_request" ]; then
	echo "not a pull_request event (workflow_dispatch) — treating as native=true"
	echo "native=true" >> "$GITHUB_OUTPUT"
	exit 0
	fi
	FILES=$(git diff --name-only "origin/${{ github.base_ref }}...HEAD")
	echo "=== changed files ==="
	echo "$FILES"
	INCLUDE='^(src/\|include/\|test/\|bazel/\|platforms/\|config/\|scripts/\|MODULE\.bazel\|\.bazelrc\|\.bazelversion\|BUILD\.bazel\|justfile\|\.lsan_suppressions\.txt\|\.github/workflows/ci\.yml\|\.github/actions/)'
	MATCHED=$(echo "$FILES" \| grep -E "$INCLUDE" \|\| true)
	if [ -n "$MATCHED" ]; then
	echo "native-relevant change detected:"
	echo "$MATCHED"
	echo "native=true" >> "$GITHUB_OUTPUT"
	else
	echo "no native-relevant change"
	echo "native=false" >> "$GITHUB_OUTPUT"
	fi

	test:
	runs-on: ${{ matrix.runs-on }}
	name: test / ${{ matrix.platform }}
	permissions:
	contents: read
	pull-requests: write # Docker Scout PR comment
	security-events: write # SARIF upload (linux-amd64 only)
	concurrency:
	group: ${{ github.workflow }}-test-${{ matrix.platform }}-${{ github.head_ref \|\| github.run_id }}
	cancel-in-progress: true

	strategy:
	fail-fast: false
	matrix:
	include:
	- platform: linux-amd64
	arch: amd64
	runs-on: ubuntu-24.04
	bazel-platform: "//platforms:linux_x86_64"
	- platform: linux-arm64
	arch: arm64
	runs-on: ubuntu-24.04-arm
	bazel-platform: "//platforms:linux_aarch64"
	- platform: darwin-arm64
	arch: arm64
	runs-on: macos-14
	bazel-platform: "//platforms:darwin_aarch64"
	steps:
	- uses: actions/checkout@v6
	with:
	fetch-depth: 0
	lfs: false # ci-setup restores/pulls LFS via its own cached step

	- name: CI setup (bazelisk, just, LFS, repo cache, RBE flags)
	id: setup
	uses: ./.github/actions/ci-setup
	with:
	lfs: "true"
	target-platform: ${{ matrix.bazel-platform }}
	rbe-runner-endpoint: ${{ vars.REMOTEBUILD_RUNNER_ENDPOINT }}
	rbe-ca-cert: ${{ secrets.REMOTEBUILD_CA_CERT }}
	rbe-client-cert: ${{ secrets.REMOTEBUILD_CLIENT_CERT }}
	rbe-client-key: ${{ secrets.REMOTEBUILD_CLIENT_KEY }}

	# Rust lint gates on the linux-amd64 leg, BEFORE build+test, so a
	# formatting/clippy finding fails the leg in lint time — short-circuiting
	# the leg's heavy downstream steps (build+test, Docker smoke, Scout).
	- if: matrix.arch == 'amd64'
	name: Check Rust formatting
	env:
	GH_TOKEN: ${{ secrets.DASCHBOT_PAT }}
	run: just bazel-rustfmt-check ${{ steps.setup.outputs.flags }}
	- if: matrix.arch == 'amd64'
	name: Run clippy
	env:
	GH_TOKEN: ${{ secrets.DASCHBOT_PAT }}
	run: just bazel-clippy-check ${{ steps.setup.outputs.flags }}

	# Skip on forked PRs and on macOS runners — DOCKER_HUB_TOKEN is
	# unavailable to forks (and macOS doesn't run Docker steps anyway).
	# The smoke test consumes a locally loaded image, so login is only
	# useful for authenticated Docker Hub pulls (rate-limit avoidance)
	# on internal Linux PRs.
	- uses: docker/login-action@v4
	if: runner.os == 'Linux' && (github.event_name != 'pull_request' \|\| github.event.pull_request.head.repo.full_name == github.repository)
	with:
	username: ${{ secrets.DOCKER_USER }}
	password: ${{ secrets.DOCKER_HUB_TOKEN }}

	# REMOTE BUILD EXECUTION:
	#
	# The self-hosted RBE backend (dasch-remotebuild-prod-01, defined in the
	# ops-tf + infra repos) is the single Bazel backend. It serves the
	# Remote Cache + Remote Execution (the runner endpoint, mTLS).
	# http_archive downloads are cached client-side via --repository_cache
	# (.bazelrc) + the restore (`ci-setup`) / end-of-job save cache steps,
	# not via a gRPC remote downloader — Bazel downloads FetchBlob digests
	# from the --remote_cache CAS, so a downloader is only usable when it
	# shares a CAS with the remote cache.
	#
	# `ci-setup` writes the mTLS material (org secrets + variables
	# REMOTEBUILD_*) outside the checkout and emits the Bazel flag string
	# via its `flags` output. Compiles fan out to the x86_64 worker — cross-
	# compiling the arm64/darwin legs — while tests run on the native
	# runner (--strategy=TestRunner=local). A down backend degrades to a
	# local build (--remote_local_fallback, the .bazelrc default). Fork PRs
	# get empty flags (no secrets) and build cold.
	- name: Build + run all tests
	env:
	GH_TOKEN: ${{ secrets.DASCHBOT_PAT }}
	run: just bazel-test ${{ steps.setup.outputs.flags }} --build_event_json_file=bep-${{ matrix.platform }}.json

	# Attach the Build Event Protocol JSON so the action/remote cache hit
	# rate can be analyzed offline. Bazel emits one BuildMetrics event per
	# invocation whose actionSummary.runnerCount[] breaks actions down by
	# runner type (total / remote cache hit / remote / local / internal) —
	# the cache-hit signal, present in the default BEP without
	# --build_event_publish_all_actions. `always()` so a failed build's
	# breakdown (often the most interesting) is still captured.
	- name: Upload build event JSON (cache-hit-rate analysis)
	if: always()
	uses: actions/upload-artifact@v7
	with:
	name: bep-${{ matrix.platform }}
	path: bep-${{ matrix.platform }}.json
	retention-days: 30

	# Docker steps run on Linux only because they `docker load` the image
	# into a local Linux Docker daemon (absent on the macOS runner) — not
	# because the image can't be built on macOS. //src:image is gated on the
	# target OS (`target_compatible_with = ["@platforms//os:linux"]`), which a
	# macOS host can satisfy with `--platforms=linux_*` (see
	# `just bazel-cross-build-image`). With no platform override the default
	# host platform on macOS is darwin, which fails that gate.
	#
	# `bazel-test-smoke` is one Bazel invocation that:
	# 1. Builds //src:image as a transitive `data` dep of
	# //test/e2e:docker_smoke.
	# 2. Runs the smoke test, whose startup code does
	# `docker load -i $SIPI_IMAGE_TAR` — the image lands in the
	# local Docker daemon as `daschswiss/sipi:latest` (matching
	# `oci_load.repo_tags`).
	# 3. Probes HTTP endpoints against the loaded container.
	#
	# Docker Scout reads `local://daschswiss/sipi:latest` left in the
	# daemon by step 2 — no separate build/load step needed.
	- name: Run Docker smoke tests (builds + loads image, then smoke-tests)
	if: runner.os == 'Linux'
	env:
	GH_TOKEN: ${{ secrets.DASCHBOT_PAT }}
	run: just bazel-test-smoke ${{ steps.setup.outputs.flags }}

	# amd64-only: arm64 and amd64 build the same image contents from the
	# same source, so a second Scout comparison adds no signal — and one
	# leg means one PR comment.
	- name: Docker Scout — compare to production
	if: matrix.platform == 'linux-amd64' && github.event_name == 'pull_request'
	uses: docker/scout-action@v1
	with:
	command: compare
	image: local://daschswiss/sipi:latest
	to-env: production
	ignore-unchanged: true
	only-severities: critical,high
	write-comment: true
	github-token: ${{ secrets.GITHUB_TOKEN }}
	organization: daschswiss

	- name: Docker Scout — CVE report (SARIF)
	if: matrix.platform == 'linux-amd64' && github.event_name == 'pull_request'
	uses: docker/scout-action@v1
	with:
	command: cves
	image: local://daschswiss/sipi:latest
	sarif-file: scout-results.sarif
	only-severities: critical,high
	write-comment: false
	organization: daschswiss

	- name: Upload SARIF to GitHub Security
	if: matrix.platform == 'linux-amd64' && github.event_name == 'pull_request'
	uses: github/codeql-action/upload-sarif@v4
	with:
	sarif_file: scout-results.sarif

	# Persist the repository cache even when the job fails — the paired
	# save for ci-setup's restore-only cache step (rationale there).
	- name: Save Bazel repository cache
	if: ${{ !cancelled() && steps.setup.outputs.repo-cache-hit != 'true' }}
	uses: actions/cache/save@v4
	with:
	path: ~/.cache/bazel-repo
	key: ${{ steps.setup.outputs.repo-cache-key }}

	# ASan + UBSan on the full functional test pyramid, split into two parallel
	# path-gated jobs. Catches memory errors (buffer overflow, use-after-free,
	# leaks) and undefined behavior that functional tests miss. This is a merge
	# gate — sanitizer findings fail the PR check. Also runnable on-demand via
	# workflow_dispatch (unconditional — bypasses the `changes` filter).
	#
	# Scope: unit + e2e under ASan/UBSan in one invocation (`just
	# bazel-test-sanitized --config=asan --config=ubsan`). Covering unit — not just e2e — puts the sanitizers over
	# the codec/ICC/metadata paths (the unit round-trip + delivery code) where
	# memory bugs in a media server are most likely. Two suites are
	# intentionally excluded:
	# - `//test/approval` — a byte-exact codec-OUTPUT regression gate, not a
	# memory-safety gate. ASan requires a `-c dbg` instrumented build, whose
	# codec output (notably PNG compression) is not byte-identical to the
	# goldens approved under the shipping build, so it can only ever fail
	# the byte comparison regardless of memory safety. It runs on the
	# normal / coverage gate where its goldens are valid; its emission
	# paths still get ASan coverage via the unit round-trips + e2e
	# delivery.
	# - `docker_smoke` — a packaging test (covered at fastbuild + in
	# publish.yml); instrumenting the OCI image adds no memory-safety
	# signal.
	#
	# `LSAN_OPTIONS` (leak suppressions) and `ASAN_SYMBOLIZER_PATH` are set
	# per-e2e-test by `test/e2e/sipi_e2e_test.bzl`, which data-deps
	# `//:lsan_suppressions` and (under `--config=asan`) `//bazel:llvm-symbolizer`
	# and points the env at their runfiles paths — so both resolve when the test
	# executes on the RBE worker. The symbolizer lets ASan/LSan resolve
	# `sipi+0xOFFSET` frames into function names, needed for the name-based
	# suppressions in `.lsan_suppressions.txt` (`leak:lua*`) to match.
	# `ASAN_OPTIONS log_path` is set by `.bazelrc`'s `test:asan` block to
	# `$TEST_UNDECLARED_OUTPUTS_DIR/asan-e2e`, surfaced by Bazel under
	# `bazel-testlogs/**/test.outputs/asan-e2e.<pid>` for every layer.
	#
	# `--config=asan` / `--config=ubsan` set `--@llvm//config:{asan,ubsan}=true`,
	# which makes the toolchain build + link the compiler-rt ASan/UBSan runtimes
	# from source and put the sanitizer headers on the include path. Runs on
	# linux-x86_64 only (the macOS toolchain args omit the sanitizer header
	# path).
	#
	# One job runs unit + e2e in a single `bazel test` invocation so the
	# instrumented `sipi` tree compiles exactly once and both suites share that
	# one ~5500-file ASan/UBSan compile within a single action graph. Two
	# separate jobs would each compile the full tree (concurrent jobs race, so
	# neither warms the other's remote cache). See
	# `docs/src/development/rbe-write-pressure.md`.
	sanitizer:
	needs: changes
	if: github.event_name == 'workflow_dispatch' \|\| needs.changes.outputs.native == 'true'
	runs-on: ubuntu-24.04
	timeout-minutes: 60
	name: asan-ubsan / amd64
	permissions:
	contents: read
	concurrency:
	group: ${{ github.workflow }}-sanitizer-${{ github.head_ref \|\| github.ref }}
	cancel-in-progress: true
	steps:
	- uses: actions/checkout@v6
	with:
	fetch-depth: 0
	lfs: false

	- name: CI setup (bazelisk, just, LFS, repo cache, RBE flags)
	id: setup
	uses: ./.github/actions/ci-setup
	with:
	lfs: "true"
	rbe-runner-endpoint: ${{ vars.REMOTEBUILD_RUNNER_ENDPOINT }}
	rbe-ca-cert: ${{ secrets.REMOTEBUILD_CA_CERT }}
	rbe-client-cert: ${{ secrets.REMOTEBUILD_CLIENT_CERT }}
	rbe-client-key: ${{ secrets.REMOTEBUILD_CLIENT_KEY }}

	- name: Run unit + e2e tests under ASan + UBSan
	env:
	GH_TOKEN: ${{ secrets.DASCHBOT_PAT }}
	run: just bazel-test-sanitized --config=asan --config=ubsan --build_tag_filters=-no-sanitizer --test_tag_filters=-no-sanitizer ${{ steps.setup.outputs.flags }} --build_event_json_file=bep-sanitizer.json

	- name: Check for sanitizer findings
	if: always()
	run: bash tools/check_sanitizer_findings.sh

	# BEP for cache-hit-rate analysis (see the `test` job's upload step).
	# The instrumented sanitizer build has its own cache behavior worth
	# tracking separately.
	- name: Upload build event JSON (cache-hit-rate analysis)
	if: always()
	uses: actions/upload-artifact@v7
	with:
	name: bep-sanitizer
	retention-days: 30
	path: bep-sanitizer.json

	- name: Upload sanitizer reports
	if: failure()
	uses: actions/upload-artifact@v7
	with:
	name: sanitizer-reports
	retention-days: 30
	path: bazel-testlogs/*/test.outputs/asan-e2e.

	# Persist the repository cache even when the job fails — the paired
	# save for ci-setup's restore-only cache step (rationale there).
	- name: Save Bazel repository cache
	if: ${{ !cancelled() && steps.setup.outputs.repo-cache-hit != 'true' }}
	uses: actions/cache/save@v4
	with:
	path: ~/.cache/bazel-repo
	key: ${{ steps.setup.outputs.repo-cache-key }}

	# Build documentation. Pure Python/MkDocs — standalone LFS handling (no
	# Bazel/RBE needed, so it doesn't use `ci-setup`).
	docs:
	name: docs
	runs-on: ubuntu-latest
	concurrency:
	group: ${{ github.workflow }}-docs-${{ github.head_ref \|\| github.run_id }}
	cancel-in-progress: true
	steps:
	- uses: actions/checkout@v6
	with:
	fetch-depth: 0
	lfs: false
	- name: restore LFS cache
	uses: actions/cache@v4
	with:
	path: .git/lfs
	key: lfs-${{ hashFiles('.gitattributes') }}
	restore-keys: lfs-
	- name: pull LFS objects
	run: \|
	git lfs install
	git lfs pull
	- uses: dasch-swiss/sipi/.github/actions/setup-python@main
	- uses: extractions/setup-just@v3
	- run: just docs-install-requirements
	- run: just docs-build

	# Enforce commit conventions (type allowlist + mandatory scope) on the PR's
	# commits via commitlint-rs. See docs/src/development/commit-conventions.md.
	commit-lint:
	name: commit-lint
	if: github.event_name == 'pull_request'
	runs-on: ubuntu-latest
	concurrency:
	group: ${{ github.workflow }}-commit-lint-${{ github.head_ref \|\| github.run_id }}
	cancel-in-progress: true
	steps:
	- uses: actions/checkout@v6
	with:
	fetch-depth: 0
	lfs: false
	- uses: ./.github/actions/commit-lint
	with:
	base-ref: ${{ github.event.pull_request.base.sha }}

Navigation Menu

refactor(scripting)!: Rust-hosted mlua Lua runtime replaces the C++ LuaServer #875

Workflow file

refactor(scripting)!: Rust-hosted mlua Lua runtime replaces the C++ LuaServer #875

Uh oh!

Workflow file for this run