|
| 1 | +"""Validate an external verified-examples corpus against the revenue contract. |
| 2 | +
|
| 3 | +Run: |
| 4 | + uv run python examples/revenue_agent/setup_db.py # once — builds the DuckDB |
| 5 | + uv run python examples/revenue_agent/verify_examples.py |
| 6 | +
|
| 7 | +What this shows |
| 8 | +--------------- |
| 9 | +``verified_examples.yml`` is an *external* corpus of question -> SQL pairs — the |
| 10 | +kind your analytics agent's lessons-learned MR flow produces. The framework never |
| 11 | +owns that file; ``validate_examples`` only re-checks each ``sql`` against the |
| 12 | +contract, using the SAME two-layer Validator that gates live agent queries: |
| 13 | +
|
| 14 | + * Layer 1 (static) — allowed tables, forbidden ops, required filters, no SELECT * |
| 15 | + * Layer 2 (dry run) — a live DuckDB ``EXPLAIN``, so schema drift (a dropped or |
| 16 | + renamed column) is caught even when the static contract |
| 17 | + cannot see it |
| 18 | +
|
| 19 | +Each example lands in exactly one status — ``valid`` (contract-checked and |
| 20 | +passed), ``violation`` (a check rejected it), ``unverified`` (engine-planned but |
| 21 | +policy not statically checked; see the note below), or ``unchecked`` (no verdict) |
| 22 | +— with two flags, ``contract_checked`` and ``engine_checked``, recording *what* |
| 23 | +was verified. ``report.ok`` is True only when every example is ``valid``. Extra |
| 24 | +YAML keys (``type``, ``verified_by``, ``last_verified``) are preserved untouched |
| 25 | +in ``.metadata``. |
| 26 | +
|
| 27 | +Two real uses of the same call: |
| 28 | + * MR gate — validate the corpus in CI before a human reviews; ``sys.exit(1)`` |
| 29 | + when ``not report.ok``. |
| 30 | + * Drift sweep — re-run against a changed contract; ``report.violations`` are the |
| 31 | + examples the change (or schema drift) just broke. |
| 32 | +
|
| 33 | +Note: the decision-B engine fallback (asking the engine to parse SQL sqlglot |
| 34 | +cannot) never triggers here — DuckDB and sqlglot both parse standard SQL. It |
| 35 | +earns its keep on engines sqlglot does not model, e.g. Denodo/VDP. |
| 36 | +""" |
| 37 | + |
| 38 | +from __future__ import annotations |
| 39 | + |
| 40 | +import sys |
| 41 | +from pathlib import Path |
| 42 | + |
| 43 | +import yaml |
| 44 | + |
| 45 | +from agentic_data_contracts import DataContract |
| 46 | +from agentic_data_contracts.adapters.duckdb import DuckDBAdapter |
| 47 | +from agentic_data_contracts.semantic.yaml_source import YamlSource |
| 48 | +from agentic_data_contracts.validation import VerifiedExample, validate_examples |
| 49 | + |
| 50 | +EXAMPLE_DIR = Path(__file__).parent |
| 51 | + |
| 52 | + |
| 53 | +def main() -> None: |
| 54 | + contract = DataContract.from_yaml(EXAMPLE_DIR / "contract.yml") |
| 55 | + semantic = YamlSource(EXAMPLE_DIR / "semantic.yml") |
| 56 | + |
| 57 | + db_path = EXAMPLE_DIR / "sample_data.duckdb" |
| 58 | + sys.path.insert(0, str(EXAMPLE_DIR)) |
| 59 | + from setup_db import ensure_sample_db # type: ignore[import] |
| 60 | + |
| 61 | + ensure_sample_db(str(db_path)) |
| 62 | + sys.path.pop(0) |
| 63 | + adapter = DuckDBAdapter(str(db_path)) |
| 64 | + |
| 65 | + # You own this load step — the framework never reads your corpus for you. |
| 66 | + raw = yaml.safe_load((EXAMPLE_DIR / "verified_examples.yml").read_text()) |
| 67 | + examples = [VerifiedExample.from_dict(row) for row in raw] |
| 68 | + |
| 69 | + report = validate_examples( |
| 70 | + examples, |
| 71 | + contract, |
| 72 | + dialect=adapter.dialect, # so Layer 1 parses in the engine's dialect |
| 73 | + explain_adapter=adapter, # enables the live DuckDB EXPLAIN dry run |
| 74 | + semantic_source=semantic, |
| 75 | + ) |
| 76 | + |
| 77 | + print("=== Verified-examples validation (live, DuckDB EXPLAIN) ===\n") |
| 78 | + for r in report.results: |
| 79 | + label = r.example.id or r.example.question or "<unnamed>" |
| 80 | + flags = ( |
| 81 | + f"contract_checked={r.contract_checked} engine_checked={r.engine_checked}" |
| 82 | + ) |
| 83 | + print(f"[{r.status.upper():9}] {label} ({flags})") |
| 84 | + for reason in r.reasons: |
| 85 | + print(f" reason: {reason}") |
| 86 | + for warning in r.warnings: |
| 87 | + print(f" warning: {warning}") |
| 88 | + |
| 89 | + print("\n--- report.summary() (ready to post as an MR comment) ---") |
| 90 | + print(report.summary()) |
| 91 | + |
| 92 | + print( |
| 93 | + f"\nGate: ok={report.ok} " |
| 94 | + f"({len(report.valid)} valid, {len(report.violations)} violation(s), " |
| 95 | + f"{len(report.unchecked)} unchecked, " |
| 96 | + f"{len(report.unverified_compliance)} plannable-but-unverified)" |
| 97 | + ) |
| 98 | + # In CI you would gate the merge on this: if not report.ok: sys.exit(1) |
| 99 | + |
| 100 | + |
| 101 | +if __name__ == "__main__": |
| 102 | + main() |
0 commit comments