Add resumable validated workflow and tarball-first scheme runbook

- Add `buildabench_workshop.run_validated_workflow`, a packaged resumable CLI that orchestrates `env_agent`, `synth_task`, `validate_task`, and `check_validated_tasks` end-to-end.
- Implement interruption-safe resume behavior with checkpoint files (`tasks.jsonl`, `validated_tasks.jsonl`, `check_results.jsonl`, `workflow.log`) and a `--force-fresh` path for clean reruns.
- Ensure the workflow runner passes the selected container explicitly to `validate_task` (`--container ...`) so tarball-based/manual runs stay consistent even when container names are overridden.
- Add/extend tests in `tests/test_run_validated_workflow.py`, including coverage that validates container propagation to `validate_task`.
- Update `.github/workflows/synth-eval-scheme-interpreter.yml` to execute the scheme-interpreter CI path in an isolated temporary git repo and clean it up after the job.
- Simplify `README.md` workflow instructions to a tarballed-git-repo-first path (no `mktemp` workspace-copy flow), while preserving manual step-by-step commands.
- Update `AGENTS.md` runbook with BOA launch guidance for `qwen3_coder_30b_a3b_instruct_fp8` and required vLLM tool-calling flags (`--enable-auto-tool-choice`, `--tool-call-parser qwen3_xml`, `--enable-prefix-caching`) on port `8000`.
- Keep generated artifacts out of version control and retain only source/workflow/documentation updates required for the validated pipeline.

Author: arjunguha

.github/workflows/synth-eval-scheme-interpreter.yml

@@ -17,43 +17,68 @@ jobs:
   synth-eval:
     runs-on: self-hosted
     env:
+      # This workflow assumes a vLLM server is already running on BOA
+      # and reachable over the 192.168.50/24 VPN.
       OPENAI_MODEL: openai/boa
-      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-      OPENAI_API_BASE: ${{ secrets.OPENAI_API_BASE }}
-      CONTAINER_NAME: env_agent__scheme_interpreter_${{ github.run_id }}
+      OPENAI_API_KEY: sk-local
+      OPENAI_API_BASE: http://192.168.50.7:8080/v1
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
 
       - name: Add Codex CLI to PATH
         run: echo "/home/ci/codexcli/node_modules/.bin" >> "$GITHUB_PATH"
 
+      - name: Prepare isolated scheme interpreter repo
+        run: |
+          set -euo pipefail
+
+          WORK_REPO="$RUNNER_TEMP/scheme_interpreter_repo_${GITHUB_RUN_ID}_${GITHUB_RUN_ATTEMPT}"
+          mkdir -p "$WORK_REPO"
+          cp -a test_projects/scheme_interpreter/. "$WORK_REPO/"
+
+          git -C "$WORK_REPO" init
+          git -C "$WORK_REPO" config user.name "CI"
+          git -C "$WORK_REPO" config user.email "ci@example.invalid"
+          git -C "$WORK_REPO" add .
+          git -C "$WORK_REPO" commit -m "Initial scheme interpreter fixture snapshot"
+
+          echo "SCHEME_REPO=$WORK_REPO" >> "$GITHUB_ENV"
+          echo "TASKS_JSONL=$RUNNER_TEMP/ci_tasks_${GITHUB_RUN_ID}_${GITHUB_RUN_ATTEMPT}.jsonl" >> "$GITHUB_ENV"
+          echo "VALIDATED_JSONL=$RUNNER_TEMP/ci_validated_tasks_${GITHUB_RUN_ID}_${GITHUB_RUN_ATTEMPT}.jsonl" >> "$GITHUB_ENV"
+          echo "CONTAINER_NAME=env_agent__scheme_interpreter_${GITHUB_RUN_ID}_${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_ENV"
+
       - name: Build container via env_agent
         run: |
           uv run python3 -m buildabench_workshop.env_agent \
-            --repo test_projects/scheme_interpreter \
-            --tips-path test_projects/scheme_interpreter/env_agent_tips.txt \
+            --repo "$SCHEME_REPO" \
+            --tips-path "$SCHEME_REPO/env_agent_tips.txt" \
             --agent codex \
             --container "$CONTAINER_NAME"
 
+      - name: Verify BOA vLLM endpoint
+        run: |
+          curl -fsS "$OPENAI_API_BASE/models"
+
       - name: Synthesize 3 tasks
         run: |
           uv run python3 -m buildabench_workshop.synth_task \
             --json \
             --num-candidates 3 \
-            --model openai/boa \
-            test_projects/scheme_interpreter \
+            --model "$OPENAI_MODEL" \
+            "$SCHEME_REPO" \
             "src/scheme_interpreter/*.py" \
             "tests/*.py" \
-            > test_projects/scheme_interpreter/ci_tasks.jsonl
+            > "$TASKS_JSONL"
 
       - name: Print synthesized tasks
         run: |
           uv run python3 - <<'PY'
           import json
+          import os
           from pathlib import Path
 
-          tasks_path = Path("test_projects/scheme_interpreter/ci_tasks.jsonl")
+          tasks_path = Path(os.environ["TASKS_JSONL"])
           for line in tasks_path.read_text().splitlines():
             if not line.strip():
               continue
@@ -77,8 +102,8 @@ jobs:
           import os
           from pathlib import Path
 
-          tasks_path = Path("test_projects/scheme_interpreter/ci_tasks.jsonl")
-          validated_path = Path("test_projects/scheme_interpreter/ci_validated_tasks.jsonl")
+          tasks_path = Path(os.environ["TASKS_JSONL"])
+          validated_path = Path(os.environ["VALIDATED_JSONL"])
           container = os.environ["CONTAINER_NAME"]
           with validated_path.open("w") as out:
             for line in tasks_path.read_text().splitlines():
@@ -97,10 +122,11 @@ jobs:
           import json
           import subprocess
           import sys
+          import os
           from pathlib import Path
 
-          tasks_path = Path("test_projects/scheme_interpreter/ci_tasks.jsonl")
-          validated_path = Path("test_projects/scheme_interpreter/ci_validated_tasks.jsonl")
+          tasks_path = Path(os.environ["TASKS_JSONL"])
+          validated_path = Path(os.environ["VALIDATED_JSONL"])
 
           failures = 0
           passes = 0
@@ -144,7 +170,9 @@ jobs:
             sys.exit(1)
           PY
 
-      - name: Cleanup container
+      - name: Cleanup container and isolated repo
         if: always()
         run: |
           podman image rm -f "$CONTAINER_NAME" || true
+          rm -rf "${SCHEME_REPO:-}" || true
+          rm -f "${TASKS_JSONL:-}" "${VALIDATED_JSONL:-}" || true

AGENTS.md

@@ -0,0 +1,237 @@
+# AGENTS.md
+
+## BOA + LiteLLM Runbook (Scheme Interpreter CI Path)
+
+This runbook documents how to run the Scheme interpreter synth/eval flow against a BOA-hosted vLLM model via LiteLLM, while ensuring BOA only has one active vLLM server.
+
+### 1. Verify BOA access and current vLLM state
+
+Check BOA server alias from MCP, then inspect running vLLM processes and listeners:
+
+```bash
+# On BOA
+ps -eo pid,ppid,cmd | grep -E 'vllm serve|VLLM::EngineCore' | grep -v grep
+ss -ltnp | grep -E ':8000|:8005|:8080' || true
+```
+
+Goal: exactly one active vLLM server listener.
+
+### 2. Ensure there is at most one vLLM server on BOA
+
+If multiple vLLM instances exist, stop extras. Keep only the intended `boa` server.
+
+```bash
+# On BOA (example: stop old qwen3_coder_30b_a3b_instruct_fp8 launch)
+pkill -f 'vllm serve ~arjun/models/qwen3_coder_30b_a3b_instruct_fp8' || true
+```
+
+Re-check:
+
+```bash
+ps -eo pid,ppid,cmd | grep -E 'vllm serve|VLLM::EngineCore' | grep -v grep
+ss -ltnp | grep -E ':8000|:8005|:8080' || true
+```
+
+### 3. Pick the BOA port/model naming that LiteLLM expects
+
+For `litellm.guha-anderson.com` routing to model group `boa`, use:
+
+- `--served-model-name boa`
+- `--host 0.0.0.0`
+- `--port 8000`
+- `--enable-auto-tool-choice`
+- `--tool-call-parser qwen3_xml`
+- `--enable-prefix-caching`
+
+(Using port `8080` worked for direct VPN calls but did not route through LiteLLM in our validated run.)
+
+### 4. Launch vLLM on BOA in background with log file
+
+Use `nohup` and a dedicated log:
+
+```bash
+# On BOA
+nohup env CUDA_VISIBLE_DEVICES=3 \
+  uvx --from vllm==0.15.0 vllm serve ~arjun/models/qwen3_coder_30b_a3b_instruct_fp8 \
+  --host 0.0.0.0 \
+  --port 8000 \
+  --served-model-name boa \
+  --enable-auto-tool-choice \
+  --tool-call-parser qwen3_xml \
+  --enable-prefix-caching \
+  > ~/vllm_boa_8000.log 2>&1 < /dev/null &
+```
+
+Wait for readiness:
+
+```bash
+# On BOA
+curl -s --max-time 3 http://localhost:8000/v1/models
+tail -n 120 ~/vllm_boa_8000.log
+```
+
+### 5. Verify connectivity from this host
+
+Direct VPN path (fallback):
+
+```bash
+curl -sS --max-time 8 http://192.168.50.7:8000/v1/models
+```
+
+LiteLLM path (preferred for this workflow):
+
+```bash
+curl -sS --max-time 20 https://litellm.guha-anderson.com/v1/chat/completions \
+  -H "Authorization: Bearer $OPENAI_API_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{"model":"boa","messages":[{"role":"user","content":"Reply with OK only."}],"max_tokens":8}'
+```
+
+### 6. Run full validated workflow from this repo
+
+Always run against an isolated working copy so the checked-in fixture under
+`test_projects/` stays unchanged:
+
+```bash
+WORK_REPO=$(mktemp -d /tmp/scheme_interpreter_work.XXXXXX)
+cp -a test_projects/scheme_interpreter/. "$WORK_REPO/"
+git -C "$WORK_REPO" init
+git -C "$WORK_REPO" config user.name "workflow"
+git -C "$WORK_REPO" config user.email "workflow@example.invalid"
+git -C "$WORK_REPO" add .
+git -C "$WORK_REPO" commit -m "Initial scheme interpreter fixture snapshot"
+```
+
+Then use the packaged resumable runner:
+
+```bash
+OPENAI_API_BASE=https://litellm.guha-anderson.com \
+OPENAI_API_KEY=$OPENAI_API_KEY \
+uv run python3 -m buildabench_workshop.run_validated_workflow \
+  --env-tips-path "$WORK_REPO/env_agent_tips.txt" \
+  --validate-tips-path "$WORK_REPO/validate_task_tips.txt" \
+  --agent codex \
+  --model openai/boa \
+  --num-candidates 3 \
+  --state-dir workflow_state/scheme_interpreter \
+  "$WORK_REPO" \
+  "src/scheme_interpreter/*.py" \
+  "tests/*.py"
+```
+
+What resume means in this script:
+
+1. If container already exists, it skips `env_agent`.
+2. If `tasks.jsonl` already has `N` tasks, it only asks `synth_task` for the remaining `num_candidates - N`.
+3. If `validated_tasks.jsonl` already contains some task IDs, it only runs `validate_task` for missing IDs.
+4. If `check_results.jsonl` already contains checked task IDs, it only runs `check_validated_tasks` for unchecked IDs.
+
+Force a fresh run:
+
+```bash
+OPENAI_API_BASE=https://litellm.guha-anderson.com \
+OPENAI_API_KEY=$OPENAI_API_KEY \
+uv run python3 -m buildabench_workshop.run_validated_workflow \
+  --force-fresh \
+  --env-tips-path "$WORK_REPO/env_agent_tips.txt" \
+  --validate-tips-path "$WORK_REPO/validate_task_tips.txt" \
+  --agent codex \
+  --model openai/boa \
+  --num-candidates 3 \
+  --state-dir workflow_state/scheme_interpreter \
+  "$WORK_REPO" \
+  "src/scheme_interpreter/*.py" \
+  "tests/*.py"
+```
+
+State files created in `--state-dir`:
+
+1. `tasks.jsonl`
+2. `validated_tasks.jsonl`
+3. `check_results.jsonl`
+4. `workflow.log`
+
+Cleanup:
+
+```bash
+rm -rf "$WORK_REPO"
+```
+
+### 7. Manual validated workflow (human step-by-step)
+
+If you want to run the pipeline manually and inspect each stage:
+
+```bash
+export OPENAI_API_BASE=https://litellm.guha-anderson.com
+export OPENAI_API_KEY=$OPENAI_API_KEY
+export CONTAINER_NAME=env_agent__scheme_interpreter_manual
+export WORK_REPO=$(mktemp -d /tmp/scheme_interpreter_manual.XXXXXX)
+cp -a test_projects/scheme_interpreter/. "$WORK_REPO/"
+git -C "$WORK_REPO" init
+git -C "$WORK_REPO" config user.name "workflow"
+git -C "$WORK_REPO" config user.email "workflow@example.invalid"
+git -C "$WORK_REPO" add .
+git -C "$WORK_REPO" commit -m "Initial scheme interpreter fixture snapshot"
+```
+
+1. Build container only if missing:
+
+```bash
+podman image exists "$CONTAINER_NAME" || \
+uv run python3 -m buildabench_workshop.env_agent \
+  --repo "$WORK_REPO" \
+  --tips-path "$WORK_REPO/env_agent_tips.txt" \
+  --agent codex \
+  --container "$CONTAINER_NAME"
+```
+
+2. Synthesize tasks:
+
+```bash
+uv run python3 -m buildabench_workshop.synth_task \
+  --json \
+  --num-candidates 3 \
+  --model openai/boa \
+  "$WORK_REPO" \
+  "src/scheme_interpreter/*.py" \
+  "tests/*.py" \
+  > tasks.jsonl
+```
+
+3. Validate tasks sequentially:
+
+```bash
+start=$(( $(wc -l < validated_tasks.jsonl 2>/dev/null || echo 0) + 1 ))
+end=$(wc -l < tasks.jsonl)
+for i in $(seq "$start" "$end"); do
+  sed -n "${i}p" tasks.jsonl | \
+    uv run python3 -m buildabench_workshop.validate_task \
+      --tips-path "$WORK_REPO/validate_task_tips.txt" \
+      --agent codex \
+      --input-json \
+      --output-json \
+      >> validated_tasks.jsonl
+done
+```
+
+4. LLM-free validation:
+
+```bash
+uv run python3 -m buildabench_workshop.check_validated_tasks validated_tasks.jsonl
+```
+
+5. Cleanup the isolated repo and container:
+
+```bash
+podman image rm -f "$CONTAINER_NAME" || true
+rm -rf "$WORK_REPO"
+```
+### 8. Quick diagnostics if LiteLLM fails for `model=boa`
+
+1. Confirm BOA server is on `:8000` and model name is exactly `boa`.
+2. Confirm only one BOA vLLM server is running.
+3. Validate BOA local endpoint:
+   - `curl http://localhost:8000/v1/models`
+4. Validate from this host over VPN:
+   - `curl http://192.168.50.7:8000/v1/models`
+5. Retry LiteLLM completion call with `model=boa`.

Makefile

@@ -1,4 +1,7 @@
-.PHONY: test
+.PHONY: test dep
+
+dep:
+	uv sync
 
 test:
 	uv run -m pytest tests -v