fix(export): SPEC §82 P0-G — pad tokenizer.ggml.tokens to vocab_size for llama.cpp interop #6247

Workflow file for this run

	# Sovereign CI — calls reusable workflow from paiml/.github
	# Change once in paiml/.github → applies to all repos
	#
	# Jobs provided by sovereign-ci.yml:
	# test: cargo test --lib (self-hosted clean-room)
	# lint: cargo clippy --all-targets -- -D warnings + cargo fmt --check
	# coverage: cargo llvm-cov + codecov upload
	# security: cargo audit (ubuntu-latest, continue-on-error)
	# provenance: SLSA attest-build-provenance
	# gate: aggregates test+lint results
	name: CI

	on:
	push:
	branches: [main, master]
	pull_request:
	branches: [main, master]
	workflow_dispatch:

	concurrency:
	group: ci-${{ github.event.pull_request.number \|\| github.ref }}
	cancel-in-progress: true

	jobs:
	ci:
	uses: paiml/.github/.github/workflows/sovereign-ci.yml@main
	with:
	repo: ${{ github.event.repository.name }}
	# Phase 3 pilot (heavy workload) — build-performance.md §7 Phase 3.
	# APR-MONO monorepo: 879+ compile units, largest dep graph in the fleet.
	# Highest expected sccache hit-rate lift; without it each PR cold-compiles
	# in its per-PR-per-run target dir
	# (`/mnt/nvme-raid0/targets/aprender-ci/<PR>/run-<RUN_ID>`)
	# for ~34min, leaving only ~4min for tests inside the 40min timeout —
	# the entire merge queue saturates.
	#
	# 2026-05-15: target-dir path bumped from `aprender-ci/<PR>` to
	# `aprender-ci/<PR>/run-<RUN_ID>` to break the cancel-corrupt-state
	# race introduced by the prior per-PR fix (paiml/.github#31,
	# 2026-04-23). `concurrency.cancel-in-progress: true` + persistent
	# per-PR mount = the SIGTERM→SIGKILL window of a dying old cargo
	# corrupted `/workspace/target/debug/deps/` for the new run that
	# mounted the same host path. sccache stays on its own mount, so
	# cross-run cache effectiveness is preserved; only cargo-incremental
	# state (small fraction of total compile) is lost per new run.
	#
	# 2026-04-19: temporarily disabled — sovereign-ci:stable container image
	# was missing the `rustc-sccache` wrapper script. Fixed upstream in
	# paiml/infra commit f4fccf9 (PR #66, "use exec script not symlink").
	# 2026-05-12: re-enabled — image verified to ship `/usr/local/bin/rustc-sccache`
	# (sccache 0.14.0), shared cache at `/home/noah/data/sccache` (warm, ~11GB).
	enable_sccache: true
	use_nextest: true
	secrets: inherit

	# APR-MONO: Workspace-wide test (all 75 crates)
	#
	# 2026-05-13: Refactored from GH Actions `container:` syntax (which forces an
	# unconditional `docker pull` with only 3 retries / ~6s total backoff) to
	# explicit `docker run` steps with a 15-attempt linear-backoff pull retry.
	# The previous design conflated "image is required" with "registry must be
	# reachable at pull time" — when `localhost:5000` blipped (registry restart,
	# network reload), the pull failed and the whole job died after ~25s. This
	# refactor preserves the same execution semantics (same image, same volume
	# mounts, same env) but moves the pull into a step the workflow controls,
	# giving us up to ~13 minutes of retry headroom before declaring the
	# registry unreachable. Mirrored in the `mutants` job below.
	workspace-test:
	runs-on: [self-hosted, X64, Linux]
	timeout-minutes: 85 # bumped to match 75min step + 10min overhead
	env:
	IMAGE: localhost:5000/sovereign-ci:stable
	PR_OR_REF: ${{ github.event.pull_request.number \|\| github.ref_name }}
	steps:
	- uses: actions/checkout@v4
	- name: Pull sovereign-ci image (with retry + local-cache fallback)
	# Self-hosted runner's local Docker registry at localhost:5000 is
	# occasionally restarting OR experiencing extended outages
	# (paiml/infra ops). Two layers of resilience:
	# 1. Check the local Docker daemon cache first — the image was
	# successfully pulled on a prior run, so it's almost certainly
	# still in the cache (Docker doesn't GC images unless `prune`
	# is run). If present, skip the pull entirely; this makes the
	# workflow registry-outage-tolerant.
	# 2. Otherwise, try to pull with 15-attempt linear-backoff retry
	# (~13min total) — plenty for any normal restart cycle.
	# The local-cache path accepts slight staleness as the price of
	# registry-outage tolerance. paiml/infra:machines/intel/sovereign-
	# ci/rebuild.sh rebuilds the stable tag nightly so any drift gets
	# corrected within 24h on the next successful pull.
	run: \|
	if docker image inspect "$IMAGE" > /dev/null 2>&1; then
	echo "Image $IMAGE already cached locally — skipping pull"
	echo "(local cache is registry-outage-tolerant; nightly rebuild keeps it fresh)"
	exit 0
	fi
	max_attempts=15
	delay=4
	for i in $(seq 1 $max_attempts); do
	if docker pull "$IMAGE"; then
	echo "Image pulled successfully on attempt $i"
	exit 0
	fi
	if [ $i -eq $max_attempts ]; then
	echo "::error::Registry localhost:5000 unreachable after $max_attempts attempts (~13min) AND image not in local cache"
	echo "::error::Suggests paiml/infra runner-side registry restart + initial image seed needed"
	exit 1
	fi
	echo "Pull attempt $i/$max_attempts failed; sleeping ${delay}s"
	sleep "$delay"
	delay=$((delay + 6)) # linear backoff: 4,10,16,22,28,34,...
	done
	- name: Workspace lib tests (25,300+)
	# Excluded: aprender-gpu (cuBLAS), aprender-cuda-edge (CUDA), aprender-compute (SIMD SIGSEGV at exit)
	# Timeout: 75min (was 55, was 40).
	# 2026-05-15: bumped to 75min after the P0 per-run target-dir fix
	# (#1693) eliminated cargo-incremental cross-run warmth. Cold
	# compiles now happen on every run; sccache covers codegen
	# (~80% hit rate on warm cache) but cargo's metadata + linking +
	# test binaries still cost ~40-50min cold. Under runner-pool
	# saturation (7+ concurrent CI runs) we observed 55min hits
	# exactly at the timeout — runs 25919246467 / 25919258460 /
	# sibling PRs failed simultaneously at 55:00.0.
	timeout-minutes: 75
	run: \|
	docker run --rm \
	-e CI -e GITHUB_ACTIONS -e GITHUB_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -e GITHUB_EVENT_NAME -e GITHUB_WORKFLOW \
	-v "${GITHUB_WORKSPACE}:/workspace" \
	-v "/mnt/nvme-raid0/cargo-ci/registry/${PR_OR_REF}:/usr/local/cargo/registry" \
	-v "/mnt/nvme-raid0/targets/aprender-ci/${PR_OR_REF}/run-${GITHUB_RUN_ID}:/workspace/target" \
	-v "/home/noah/data/sccache:/sccache" \
	-w /workspace \
	-e CARGO_TARGET_DIR=/workspace/target \
	-e RUSTC_WRAPPER=rustc-sccache \
	-e SCCACHE_DIR=/sccache \
	-e CARGO_INCREMENTAL=0 \
	-e CARGO_BUILD_JOBS=8 \
	-e CARGO_PROFILE_TEST_DEBUG=line-tables-only \
	-e CARGO_PROFILE_DEV_DEBUG=line-tables-only \
	"$IMAGE" \
	cargo test --workspace --lib --exclude aprender-gpu --exclude aprender-cuda-edge --exclude aprender-compute
	- name: Compute tests (tolerate SIGSEGV at exit — all tests pass but harness crashes on cleanup)
	run: \|
	docker run --rm \
	-e CI -e GITHUB_ACTIONS -e GITHUB_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -e GITHUB_EVENT_NAME -e GITHUB_WORKFLOW \
	-v "${GITHUB_WORKSPACE}:/workspace" \
	-v "/mnt/nvme-raid0/cargo-ci/registry/${PR_OR_REF}:/usr/local/cargo/registry" \
	-v "/mnt/nvme-raid0/targets/aprender-ci/${PR_OR_REF}/run-${GITHUB_RUN_ID}:/workspace/target" \
	-v "/home/noah/data/sccache:/sccache" \
	-w /workspace \
	-e CARGO_TARGET_DIR=/workspace/target \
	-e RUSTC_WRAPPER=rustc-sccache \
	-e SCCACHE_DIR=/sccache \
	-e CARGO_INCREMENTAL=0 \
	-e CARGO_BUILD_JOBS=8 \
	"$IMAGE" \
	bash -c 'cargo test -p aprender-compute --lib 2>&1 \| tee /tmp/compute-test.log; grep -q "test result.*0 failed" /tmp/compute-test.log'
	- name: Integration tests
	run: \|
	docker run --rm \
	-e CI -e GITHUB_ACTIONS -e GITHUB_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -e GITHUB_EVENT_NAME -e GITHUB_WORKFLOW \
	-v "${GITHUB_WORKSPACE}:/workspace" \
	-v "/mnt/nvme-raid0/cargo-ci/registry/${PR_OR_REF}:/usr/local/cargo/registry" \
	-v "/mnt/nvme-raid0/targets/aprender-ci/${PR_OR_REF}/run-${GITHUB_RUN_ID}:/workspace/target" \
	-v "/home/noah/data/sccache:/sccache" \
	-w /workspace \
	-e CARGO_TARGET_DIR=/workspace/target \
	-e RUSTC_WRAPPER=rustc-sccache \
	-e SCCACHE_DIR=/sccache \
	-e CARGO_INCREMENTAL=0 \
	-e CARGO_BUILD_JOBS=8 \
	"$IMAGE" \
	bash -c 'cargo test -p aprender-core --test monorepo_invariants && cargo test -p aprender-core --test readme_contract && cargo test -p apr-cli --test cli_commands'
	- name: Build.rs crate-root escape check (v0.31.1 yank guard)
	# Static Poka-Yoke: flags build.rs files that panic on files outside
	# CARGO_MANIFEST_DIR, which break `cargo install` from crates.io.
	# See scripts/check_build_rs_paths.sh for the full rationale.
	run: \|
	docker run --rm \
	-e CI -e GITHUB_ACTIONS -e GITHUB_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -e GITHUB_EVENT_NAME -e GITHUB_WORKFLOW \
	-v "${GITHUB_WORKSPACE}:/workspace" \
	-w /workspace \
	"$IMAGE" \
	bash scripts/check_build_rs_paths.sh
	- name: Fix file ownership (container runs as root, runner as noah:1000)
	if: always()
	run: \|
	# Five-whys: Docker container creates files as root on bind-mounted
	# workspace. Runner (noah:1000) can't git-clean them on next run
	# → checkout fails → CI breaks. This runs inside the container
	# (as root) restoring host ownership for subsequent bare-metal jobs.
	docker run --rm \
	-e CI -e GITHUB_ACTIONS -e GITHUB_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -e GITHUB_EVENT_NAME -e GITHUB_WORKFLOW \
	-v "${GITHUB_WORKSPACE}:/workspace" \
	-v "/mnt/nvme-raid0/cargo-ci/registry/${PR_OR_REF}:/usr/local/cargo/registry" \
	-v "/mnt/nvme-raid0/targets/aprender-ci/${PR_OR_REF}/run-${GITHUB_RUN_ID}:/workspace/target" \
	"$IMAGE" \
	bash -c 'chown -R 1000:1000 /workspace \|\| true; chown -R 1000:1000 /usr/local/cargo/registry \|\| true; chown -R 1000:1000 /workspace/target \|\| true'

	# Top-level gate: satisfies org ruleset "Green Main" which requires check named "gate".
	# The reusable workflow produces "ci / gate" but rulesets need exact match on "gate".
	gate:
	runs-on: [self-hosted, X64, Linux]
	needs: [ci, workspace-test]
	if: always()
	steps:
	- name: Check required jobs
	run: \|
	if [ "${{ needs.ci.result }}" != "success" ]; then
	echo "ci failed: ${{ needs.ci.result }}"
	exit 1
	fi
	if [ "${{ needs.workspace-test.result }}" != "success" ]; then
	echo "workspace-test failed: ${{ needs.workspace-test.result }}"
	exit 1
	fi
	echo "All required jobs passed"

	# Refactored to explicit docker run for the same registry-flake reason
	# documented above the workspace-test job.
	mutants:
	runs-on: [self-hosted, X64, Linux]
	continue-on-error: true
	timeout-minutes: 120
	needs: [gate]
	if: github.event_name == 'push' && github.ref == 'refs/heads/main'
	env:
	IMAGE: localhost:5000/sovereign-ci:stable
	steps:
	- uses: actions/checkout@v4
	- name: Pull sovereign-ci image (with retry + local-cache fallback)
	# Same two-layer resilience as workspace-test — see that job for full context.
	run: \|
	if docker image inspect "$IMAGE" > /dev/null 2>&1; then
	echo "Image $IMAGE already cached locally — skipping pull"
	exit 0
	fi
	max_attempts=15
	delay=4
	for i in $(seq 1 $max_attempts); do
	if docker pull "$IMAGE"; then
	echo "Image pulled successfully on attempt $i"
	exit 0
	fi
	if [ $i -eq $max_attempts ]; then
	echo "::error::Registry localhost:5000 unreachable after $max_attempts attempts AND image not in local cache"
	exit 1
	fi
	echo "Pull attempt $i/$max_attempts failed; sleeping ${delay}s"
	sleep "$delay"
	delay=$((delay + 6))
	done
	- name: Install cargo-mutants
	run: \|
	docker run --rm \
	-e CI -e GITHUB_ACTIONS -e GITHUB_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -e GITHUB_EVENT_NAME -e GITHUB_WORKFLOW \
	-v "${GITHUB_WORKSPACE}:/workspace" \
	-w /workspace \
	"$IMAGE" \
	cargo install cargo-mutants --locked
	- name: Run mutation testing
	continue-on-error: true
	run: \|
	docker run --rm \
	-e CI -e GITHUB_ACTIONS -e GITHUB_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -e GITHUB_EVENT_NAME -e GITHUB_WORKFLOW \
	-v "${GITHUB_WORKSPACE}:/workspace" \
	-w /workspace \
	"$IMAGE" \
	cargo mutants --no-times --timeout 300 --in-place -- --lib
	- name: Upload mutation results
	uses: actions/upload-artifact@v7
	with:
	name: mutation-results
	path: mutants.out/

Navigation Menu

Uh oh!

fix(export): SPEC §82 P0-G — pad tokenizer.ggml.tokens to vocab_size for llama.cpp interop #6247

Workflow file

fix(export): SPEC §82 P0-G — pad tokenizer.ggml.tokens to vocab_size for llama.cpp interop #6247

Uh oh!

Workflow file for this run