generate_scores.py

#!/usr/bin/env python3
import argparse
import json
from pathlib import Path


def discover_evaluation_files(repo_root: Path) -> list[Path]:
    experiment_files = sorted(repo_root.glob("experiments/*/evaluation_stats.json"))
    if experiment_files:
        return experiment_files
    # Backward compatibility for legacy root-level experiment folders.
    return sorted(repo_root.glob("*/evaluation_stats.json"))


def format_mean(value: object) -> str:
    if isinstance(value, (int, float)):
        return f"{value:.4f}"
    return "-"


def build_markdown_table(rows: list[dict], metric_names: list[str]) -> str:
    headers = ["experiment", *metric_names]
    divider = ["---"] * len(headers)

    lines = [
        "| " + " | ".join(headers) + " |",
        "| " + " | ".join(divider) + " |",
    ]

    for row in rows:
        line = [row["experiment"]]
        for metric in metric_names:
            line.append(format_mean(row["means"].get(metric)))
        lines.append("| " + " | ".join(line) + " |")

    return "\n".join(lines)


def main() -> None:
    parser = argparse.ArgumentParser(
        description=(
            "Collect metrics_summary mean values from all evaluation_stats.json "
            "files and export a markdown table."
        )
    )
    parser.add_argument(
        "--repo-root",
        default=".",
        help="Repository root to scan (default: current directory).",
    )
    parser.add_argument(
        "--output",
        default="Scores.MD",
        help="Path to output markdown file (default: Scores.MD).",
    )
    args = parser.parse_args()

    repo_root = Path(args.repo_root).resolve()
    output_path = Path(args.output)
    if not output_path.is_absolute():
        output_path = repo_root / output_path

    rows: list[dict] = []
    metric_names: set[str] = set()
    skipped: list[str] = []

    for stats_path in discover_evaluation_files(repo_root):
        experiment = stats_path.parent.name
        try:
            with stats_path.open("r", encoding="utf-8") as f:
                payload = json.load(f)
        except (json.JSONDecodeError, OSError):
            skipped.append(experiment)
            continue

        metrics_summary = payload.get("metrics_summary")
        if not isinstance(metrics_summary, dict):
            skipped.append(experiment)
            continue

        means = {}
        for metric_name, summary in metrics_summary.items():
            if isinstance(summary, dict):
                means[metric_name] = summary.get("mean")
            else:
                means[metric_name] = None

        metric_names.update(means.keys())
        rows.append({"experiment": experiment, "means": means})

    ordered_metrics = sorted(metric_names)
    rows.sort(key=lambda row: row["experiment"])

    lines = ["# Scores", "", "## metrics_summary (mean)", ""]
    lines.append(build_markdown_table(rows, ordered_metrics))

    if skipped:
        lines.extend(
            [
                "",
                "## Skipped",
                "",
                "These experiments were skipped because `metrics_summary` is missing or invalid:",
                "",
            ]
        )
        for experiment in sorted(skipped):
            lines.append(f"- {experiment}")

    output_path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8")
    print(f"Wrote {output_path}")


if __name__ == "__main__":
    main()