tests

Phase 21: held-out eval corpus + heuristic baseline + LLM debug toggle #11

	name: tests

	on:
	push:
	branches: [main]
	pull_request:
	branches: [main]

	# Cancel in-progress runs on the same ref when a new commit lands.
	concurrency:
	group: tests-${{ github.ref }}
	cancel-in-progress: true

	jobs:
	pytest:
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	python-version: ["3.11", "3.12"]

	steps:
	- uses: actions/checkout@v4

	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}
	cache: pip

	- name: Install dependencies
	run: \|
	python -m pip install --upgrade pip
	# Trim heavy / platform-specific deps that aren't needed for unit tests.
	# Prophet is optional in tests (covered by skipif), but install if it builds quickly.
	pip install -r requirements.txt
	pip install pytest

	- name: Run pytest
	env:
	# Force tests to use ephemeral DB & disabled channels regardless of host env.
	SLACK_WEBHOOK: ""
	SMTP_HOST: ""
	GEMINI_API_KEY: ""
	run: pytest -v

	# Eval harness — runs the agent against the 55-case corpus on the
	# heuristic backend and fails if pass rate drops > 2pp below the
	# pinned baseline. Only run on one Python version since it's slow-ish.
	- name: Run agent evals
	if: matrix.python-version == '3.11'
	env:
	GEMINI_API_KEY: ""
	run: \|
	python -m analyst.evals \
	--backend heuristic \
	--baseline tests/fixtures/eval_baseline.json \
	--min-pass-rate 0.95 \
	--json eval_report.json

	- name: Upload eval report
	if: matrix.python-version == '3.11' && always()
	uses: actions/upload-artifact@v4
	with:
	name: eval-report
	path: eval_report.json
	if-no-files-found: ignore

Navigation Menu