|
| 1 | +"""CLI entry point — runs the eval corpus and exits with a non-zero status if |
| 2 | +the pass rate falls below `--min-pass-rate` (or below the baseline). |
| 3 | +
|
| 4 | +Examples: |
| 5 | +
|
| 6 | + python -m analyst.evals |
| 7 | + python -m analyst.evals --backend llm |
| 8 | + python -m analyst.evals --tag retention --tag products |
| 9 | + python -m analyst.evals --json eval_results.json |
| 10 | + python -m analyst.evals --min-pass-rate 0.95 |
| 11 | + python -m analyst.evals --baseline tests/fixtures/eval_baseline.json |
| 12 | +""" |
| 13 | +from __future__ import annotations |
| 14 | + |
| 15 | +import argparse |
| 16 | +import json |
| 17 | +import sys |
| 18 | +from pathlib import Path |
| 19 | +from typing import List |
| 20 | + |
| 21 | +import pandas as pd |
| 22 | + |
| 23 | +from analyst.evals import ALL_CASES, EvalReport, load_cases, run_all |
| 24 | + |
| 25 | + |
| 26 | +REPO_ROOT = Path(__file__).resolve().parents[2] |
| 27 | +DEFAULT_DATA = REPO_ROOT / "analyst" / "sample_data" / "retail_orders.csv" |
| 28 | + |
| 29 | + |
| 30 | +def _build_parser() -> argparse.ArgumentParser: |
| 31 | + p = argparse.ArgumentParser( |
| 32 | + prog="python -m analyst.evals", |
| 33 | + description="Run the LLM agent eval harness against a sample dataset.", |
| 34 | + ) |
| 35 | + p.add_argument( |
| 36 | + "--data", type=Path, default=DEFAULT_DATA, |
| 37 | + help=f"CSV/Parquet to run evals against (default: {DEFAULT_DATA.name}).", |
| 38 | + ) |
| 39 | + p.add_argument( |
| 40 | + "--backend", choices=["heuristic", "llm", "auto"], default="heuristic", |
| 41 | + help="Agent planning backend. CI should use 'heuristic' for determinism.", |
| 42 | + ) |
| 43 | + p.add_argument( |
| 44 | + "--tag", action="append", default=None, |
| 45 | + help="Filter by tag (repeatable). OR semantics across tags.", |
| 46 | + ) |
| 47 | + p.add_argument( |
| 48 | + "--id", action="append", default=None, |
| 49 | + help="Filter by case id (repeatable).", |
| 50 | + ) |
| 51 | + p.add_argument( |
| 52 | + "--json", type=Path, default=None, |
| 53 | + help="Write the full report to this file as JSON.", |
| 54 | + ) |
| 55 | + p.add_argument( |
| 56 | + "--min-pass-rate", type=float, default=0.0, |
| 57 | + help="Exit non-zero if pass rate falls below this threshold.", |
| 58 | + ) |
| 59 | + p.add_argument( |
| 60 | + "--baseline", type=Path, default=None, |
| 61 | + help=("Compare to a previous run's JSON. Exit non-zero if pass_rate " |
| 62 | + "drops by more than 0.02."), |
| 63 | + ) |
| 64 | + p.add_argument( |
| 65 | + "--quiet", action="store_true", help="Suppress per-case logs.", |
| 66 | + ) |
| 67 | + return p |
| 68 | + |
| 69 | + |
| 70 | +def _print_report(report: EvalReport, *, quiet: bool = False) -> None: |
| 71 | + print(f"\n=== eval report (backend={report.backend}) ===") |
| 72 | + print(f"pass_rate : {report.pass_rate * 100:.1f}% " |
| 73 | + f"({report.n_passed}/{report.n_cases})") |
| 74 | + print(f"mean_overall : {report.mean_overall:.3f}") |
| 75 | + print(f" tool_match : {report.mean_tool_match:.3f}") |
| 76 | + print(f" args_match : {report.mean_args_match:.3f}") |
| 77 | + print(f" no_forbidden: {report.mean_no_forbidden:.3f}") |
| 78 | + print(f" success_match:{report.mean_success_match:.3f}") |
| 79 | + |
| 80 | + if report.per_tool: |
| 81 | + print("\nper-tool:") |
| 82 | + for tool, stats in sorted(report.per_tool.items(), |
| 83 | + key=lambda kv: kv[1]["pass_rate"]): |
| 84 | + print(f" {tool:<22s} {stats['pass_rate']*100:5.1f}% " |
| 85 | + f"(n={int(stats['n'])}, mean={stats['mean_overall']:.3f})") |
| 86 | + |
| 87 | + if not quiet: |
| 88 | + failures = [c for c in report.cases if not c.passed] |
| 89 | + if failures: |
| 90 | + print(f"\nfailures ({len(failures)}):") |
| 91 | + for r in failures: |
| 92 | + print(f" ✗ {r.case_id} {r.question[:70]}") |
| 93 | + for reason in r.failed_reasons: |
| 94 | + print(f" • {reason}") |
| 95 | + |
| 96 | + |
| 97 | +def _maybe_compare_baseline(report: EvalReport, baseline_path: Path) -> int: |
| 98 | + if not baseline_path.exists(): |
| 99 | + print(f"\n(no baseline at {baseline_path} — skipping comparison)") |
| 100 | + return 0 |
| 101 | + baseline = json.loads(baseline_path.read_text()) |
| 102 | + base_pass = float(baseline.get("pass_rate", 0.0)) |
| 103 | + delta = report.pass_rate - base_pass |
| 104 | + print(f"\nbaseline pass_rate : {base_pass*100:.1f}%") |
| 105 | + print(f"current pass_rate : {report.pass_rate*100:.1f}%") |
| 106 | + print(f"delta : {delta*100:+.1f}pp") |
| 107 | + if delta < -0.02: # >2 percentage point regression |
| 108 | + print("✗ REGRESSION — pass rate dropped by more than 2pp") |
| 109 | + return 2 |
| 110 | + return 0 |
| 111 | + |
| 112 | + |
| 113 | +def main(argv: list[str] | None = None) -> int: |
| 114 | + args = _build_parser().parse_args(argv) |
| 115 | + |
| 116 | + if not args.data.exists(): |
| 117 | + print(f"error: data file not found: {args.data}", file=sys.stderr) |
| 118 | + return 1 |
| 119 | + |
| 120 | + df = pd.read_csv(args.data) if args.data.suffix == ".csv" else pd.read_parquet(args.data) |
| 121 | + print(f"loaded {len(df):,} rows from {args.data.name}") |
| 122 | + |
| 123 | + cases = load_cases(tags=args.tag, ids=args.id) |
| 124 | + if not cases: |
| 125 | + print("error: no cases matched the filters", file=sys.stderr) |
| 126 | + return 1 |
| 127 | + print(f"running {len(cases)} cases on backend={args.backend}") |
| 128 | + |
| 129 | + report = run_all(cases, df, backend=args.backend) |
| 130 | + _print_report(report, quiet=args.quiet) |
| 131 | + |
| 132 | + if args.json: |
| 133 | + args.json.parent.mkdir(parents=True, exist_ok=True) |
| 134 | + args.json.write_text(json.dumps(report.to_dict(), default=str, indent=2)) |
| 135 | + print(f"\nwrote {args.json}") |
| 136 | + |
| 137 | + exit_code = 0 |
| 138 | + if args.baseline: |
| 139 | + exit_code = max(exit_code, _maybe_compare_baseline(report, args.baseline)) |
| 140 | + |
| 141 | + if report.pass_rate < args.min_pass_rate: |
| 142 | + print(f"\n✗ pass rate {report.pass_rate*100:.1f}% < min " |
| 143 | + f"{args.min_pass_rate*100:.1f}%") |
| 144 | + exit_code = max(exit_code, 1) |
| 145 | + elif args.min_pass_rate > 0: |
| 146 | + print(f"\n✓ pass rate {report.pass_rate*100:.1f}% ≥ min " |
| 147 | + f"{args.min_pass_rate*100:.1f}%") |
| 148 | + |
| 149 | + return exit_code |
| 150 | + |
| 151 | + |
| 152 | +if __name__ == "__main__": |
| 153 | + raise SystemExit(main()) |
0 commit comments