test.py

#!/usr/bin/env python3
"""
Delimit Compatibility Matrix — test runner.

Downloads OpenAPI specs from real public repositories and runs
`delimit lint` against each one. Produces a JSON results file
and a static HTML page suitable for GitHub Pages deployment.
"""

import json
import subprocess
import os
import sys
import datetime
import tempfile
import shutil
import urllib.request
import urllib.error

REPOS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "repos.json")
RESULTS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "results.json")
HTML_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "index.html")
DELIMIT_CLI = os.environ.get("DELIMIT_CLI", "delimit")


def fetch_stars(owner: str, repo: str) -> int | None:
    """Fetch star count via GitHub API. Returns None on failure."""
    token = os.environ.get("GITHUB_TOKEN", "")
    url = f"https://api.github.com/repos/{owner}/{repo}"
    req = urllib.request.Request(url)
    req.add_header("Accept", "application/vnd.github.v3+json")
    if token:
        req.add_header("Authorization", f"Bearer {token}")
    try:
        with urllib.request.urlopen(req, timeout=10) as resp:
            data = json.loads(resp.read())
            return data.get("stargazers_count")
    except Exception:
        return None


def download_spec(owner: str, repo: str, spec_path: str, dest: str) -> bool:
    """Download a spec file from GitHub, trying main then master branch."""
    for branch in ("main", "master"):
        url = f"https://raw.githubusercontent.com/{owner}/{repo}/{branch}/{spec_path}"
        try:
            req = urllib.request.Request(url)
            with urllib.request.urlopen(req, timeout=15) as resp:
                with open(dest, "wb") as f:
                    f.write(resp.read())
                # Verify we got something meaningful (not a 404 HTML page)
                size = os.path.getsize(dest)
                if size > 50:
                    return True
        except (urllib.error.URLError, urllib.error.HTTPError, OSError):
            continue
    return False


def run_lint(spec_file: str) -> dict:
    """Run delimit lint on a spec (self-diff) and return structured results."""
    try:
        cmd = DELIMIT_CLI.split() + ["lint", spec_file, spec_file, "--json"]
        proc = subprocess.run(
            cmd,
            capture_output=True,
            text=True,
            timeout=60,
        )
        try:
            data = json.loads(proc.stdout)
        except json.JSONDecodeError:
            data = {}

        output_text = proc.stdout or proc.stderr or ""
        # Detect engine crash vs clean lint result
        is_crash = "Traceback" in output_text or "ETIMEDOUT" in output_text
        decision = data.get("decision", "error" if is_crash else "unknown")

        return {
            "exit_code": proc.returncode,
            "passed": proc.returncode == 0,
            "decision": decision,
            "is_engine_error": is_crash,
            "total_changes": data.get("summary", {}).get("total_changes"),
            "breaking_changes": data.get("summary", {}).get("breaking_changes"),
            "output_snippet": output_text[:300],
        }
    except subprocess.TimeoutExpired:
        return {
            "exit_code": -1,
            "passed": False,
            "decision": "timeout",
            "is_engine_error": True,
            "total_changes": None,
            "breaking_changes": None,
            "output_snippet": "Lint timed out after 60s",
        }
    except FileNotFoundError:
        return {
            "exit_code": -2,
            "passed": False,
            "decision": "cli_not_found",
            "is_engine_error": True,
            "total_changes": None,
            "breaking_changes": None,
            "output_snippet": f"CLI not found: {DELIMIT_CLI}",
        }


def format_stars(count: int | None) -> str:
    """Format star count for display."""
    if count is None:
        return "N/A"
    if count >= 1000:
        return f"{count / 1000:.1f}K"
    return str(count)


def generate_html(results: list, run_date: str) -> str:
    """Generate the static HTML results page."""
    total = len(results)
    passed = sum(1 for r in results if r["lint"]["passed"])
    found = sum(1 for r in results if r["spec_found"])
    not_found = total - found

    rows = []
    for r in results:
        repo = r["repo"]
        stars = format_stars(r["stars"])
        desc = r.get("description", "")
        repo_url = f"https://github.com/{r['owner']}/{r['repo_name']}"

        if not r["spec_found"]:
            status = '<span class="badge badge-gray">Spec Not Found</span>'
            spec_cell = '<span class="muted">Not found</span>'
        elif r["lint"]["passed"]:
            status = '<span class="badge badge-green">Pass</span>'
            spec_cell = f'<code>{r["spec"]}</code>'
        elif r["lint"].get("is_engine_error"):
            decision = r["lint"].get("decision", "error")
            status = f'<span class="badge badge-yellow">{decision.title()}</span>'
            spec_cell = f'<code>{r["spec"]}</code>'
        else:
            decision = r["lint"].get("decision", "fail")
            status = f'<span class="badge badge-red">{decision.title()}</span>'
            spec_cell = f'<code>{r["spec"]}</code>'

        rows.append(f"""
        <tr>
          <td><a href="{repo_url}" target="_blank">{repo}</a><br><span class="muted">{desc}</span></td>
          <td class="center">{stars}</td>
          <td>{spec_cell}</td>
          <td class="center">{status}</td>
        </tr>""")

    rows_html = "\n".join(rows)

    return f"""<!DOCTYPE html>
<html lang="en">
<head>
  <meta charset="UTF-8">
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <title>Delimit Compatibility Matrix</title>
  <style>
    :root {{
      --bg: #0d1117;
      --surface: #161b22;
      --border: #30363d;
      --text: #e6edf3;
      --muted: #8b949e;
      --green: #238636;
      --green-text: #3fb950;
      --red: #da3633;
      --blue: #1f6feb;
      --blue-text: #58a6ff;
      --gray: #484f58;
    }}
    * {{ box-sizing: border-box; margin: 0; padding: 0; }}
    body {{
      font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Helvetica, Arial, sans-serif;
      background: var(--bg);
      color: var(--text);
      line-height: 1.6;
    }}
    .container {{
      max-width: 960px;
      margin: 0 auto;
      padding: 40px 20px;
    }}
    h1 {{
      font-size: 28px;
      font-weight: 600;
      margin-bottom: 8px;
    }}
    .subtitle {{
      color: var(--muted);
      font-size: 16px;
      margin-bottom: 32px;
    }}
    .stats {{
      display: flex;
      gap: 16px;
      margin-bottom: 32px;
      flex-wrap: wrap;
    }}
    .stat {{
      background: var(--surface);
      border: 1px solid var(--border);
      border-radius: 8px;
      padding: 16px 24px;
      flex: 1;
      min-width: 140px;
    }}
    .stat-value {{
      font-size: 32px;
      font-weight: 700;
    }}
    .stat-value.green {{ color: var(--green-text); }}
    .stat-label {{
      color: var(--muted);
      font-size: 13px;
      text-transform: uppercase;
      letter-spacing: 0.5px;
    }}
    table {{
      width: 100%;
      border-collapse: collapse;
      background: var(--surface);
      border: 1px solid var(--border);
      border-radius: 8px;
      overflow: hidden;
    }}
    th {{
      text-align: left;
      padding: 12px 16px;
      font-size: 13px;
      font-weight: 600;
      color: var(--muted);
      text-transform: uppercase;
      letter-spacing: 0.5px;
      border-bottom: 1px solid var(--border);
      background: var(--bg);
    }}
    td {{
      padding: 12px 16px;
      border-bottom: 1px solid var(--border);
      font-size: 14px;
      vertical-align: middle;
    }}
    tr:last-child td {{
      border-bottom: none;
    }}
    td.center, th.center {{
      text-align: center;
    }}
    a {{
      color: var(--blue-text);
      text-decoration: none;
    }}
    a:hover {{
      text-decoration: underline;
    }}
    code {{
      font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace;
      font-size: 12px;
      color: var(--muted);
      background: var(--bg);
      padding: 2px 6px;
      border-radius: 4px;
    }}
    .badge {{
      display: inline-block;
      padding: 3px 10px;
      border-radius: 12px;
      font-size: 12px;
      font-weight: 600;
      text-transform: uppercase;
      letter-spacing: 0.3px;
    }}
    .badge-green {{
      background: rgba(35, 134, 54, 0.2);
      color: var(--green-text);
    }}
    .badge-red {{
      background: rgba(218, 54, 51, 0.2);
      color: #f85149;
    }}
    .badge-yellow {{
      background: rgba(210, 153, 34, 0.2);
      color: #d29922;
    }}
    .badge-gray {{
      background: rgba(72, 79, 88, 0.3);
      color: var(--muted);
    }}
    .muted {{
      color: var(--muted);
      font-size: 12px;
    }}
    .footer {{
      margin-top: 40px;
      padding-top: 20px;
      border-top: 1px solid var(--border);
      color: var(--muted);
      font-size: 13px;
      display: flex;
      justify-content: space-between;
      align-items: center;
      flex-wrap: wrap;
      gap: 8px;
    }}
    .footer a {{
      color: var(--blue-text);
    }}
    @media (max-width: 640px) {{
      .stats {{ flex-direction: column; }}
      code {{ word-break: break-all; }}
    }}
  </style>
</head>
<body>
  <div class="container">
    <h1>Delimit Compatibility Matrix</h1>
    <p class="subtitle">
      Automated daily testing of <code>delimit lint</code> against real-world OpenAPI specs
      from popular open-source projects.
    </p>

    <div class="stats">
      <div class="stat">
        <div class="stat-value">{total}</div>
        <div class="stat-label">Repos Tested</div>
      </div>
      <div class="stat">
        <div class="stat-value green">{passed}/{found}</div>
        <div class="stat-label">Specs Passing</div>
      </div>
      <div class="stat">
        <div class="stat-value">{not_found}</div>
        <div class="stat-label">Specs Not Found</div>
      </div>
    </div>

    <table>
      <thead>
        <tr>
          <th>Repository</th>
          <th class="center">Stars</th>
          <th>Spec Path</th>
          <th class="center">Status</th>
        </tr>
      </thead>
      <tbody>
        {rows_html}
      </tbody>
    </table>

    <div class="footer">
      <span>Last run: {run_date} UTC</span>
      <span>
        Powered by <a href="https://github.com/delimit-ai/delimit-action">Delimit</a>
        &mdash; API governance for CI/CD
      </span>
    </div>
  </div>
</body>
</html>
"""


def main():
    with open(REPOS_FILE) as f:
        repos = json.load(f)

    cli = shutil.which(DELIMIT_CLI)
    if not cli:
        # Fall back to npx
        os.environ["DELIMIT_CLI"] = DELIMIT_CLI
        print(f"Using CLI: {DELIMIT_CLI}")
    else:
        print(f"Using CLI: {cli}")

    run_date = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M")
    results = []

    for entry in repos:
        owner = entry["owner"]
        repo = entry["repo"]
        spec_path = entry["spec"]
        desc = entry.get("description", "")
        full_name = f"{owner}/{repo}"

        print(f"\n--- {full_name} ---")

        # Fetch stars
        stars = fetch_stars(owner, repo)
        print(f"  Stars: {format_stars(stars)}")

        # Download spec
        with tempfile.NamedTemporaryFile(suffix=os.path.splitext(spec_path)[1], delete=False) as tmp:
            tmp_path = tmp.name

        try:
            found = download_spec(owner, repo, spec_path, tmp_path)
            if not found:
                print(f"  Spec not found at {spec_path}")
                results.append({
                    "owner": owner,
                    "repo_name": repo,
                    "repo": full_name,
                    "spec": spec_path,
                    "description": desc,
                    "stars": stars,
                    "spec_found": False,
                    "lint": {
                        "exit_code": -1,
                        "passed": False,
                        "decision": "spec_not_found",
                        "total_changes": None,
                        "breaking_changes": None,
                        "output_snippet": "",
                    },
                })
                continue

            size = os.path.getsize(tmp_path)
            print(f"  Spec downloaded: {size:,} bytes")

            # Run lint
            lint_result = run_lint(tmp_path)
            print(f"  Lint: {'PASS' if lint_result['passed'] else 'FAIL'} (exit {lint_result['exit_code']})")

            results.append({
                "owner": owner,
                "repo_name": repo,
                "repo": full_name,
                "spec": spec_path,
                "description": desc,
                "stars": stars,
                "spec_found": True,
                "lint": lint_result,
            })
        finally:
            if os.path.exists(tmp_path):
                os.unlink(tmp_path)

    # Write results JSON
    output = {
        "run_date": run_date,
        "total": len(results),
        "specs_found": sum(1 for r in results if r["spec_found"]),
        "passed": sum(1 for r in results if r["lint"]["passed"]),
        "results": results,
    }
    with open(RESULTS_FILE, "w") as f:
        json.dump(output, f, indent=2)
    print(f"\nResults written to {RESULTS_FILE}")

    # Generate HTML
    html = generate_html(results, run_date)
    with open(HTML_FILE, "w") as f:
        f.write(html)
    print(f"HTML written to {HTML_FILE}")

    # Summary
    found = output["specs_found"]
    passed = output["passed"]
    total = output["total"]
    print(f"\n=== Summary: {passed}/{found} specs passed, {total - found} specs not found ===")

    # Exit with failure only if a found spec fails lint
    # Only count policy violations as failures, not engine errors
    policy_failures = sum(
        1 for r in results
        if r["spec_found"]
        and not r["lint"]["passed"]
        and not r["lint"].get("is_engine_error")
    )
    engine_errors = sum(
        1 for r in results
        if r["spec_found"]
        and not r["lint"]["passed"]
        and r["lint"].get("is_engine_error")
    )
    if engine_errors:
        print(f"  ({engine_errors} engine errors — not counted as failures)")
    sys.exit(1 if policy_failures > 0 else 0)


if __name__ == "__main__":
    main()