Skip to content

Commit c7de400

Browse files
committed
feat: /benchmark runs full dogbench pipeline + auto-submit
1 parent b843480 commit c7de400

1 file changed

Lines changed: 13 additions & 14 deletions

File tree

cli.py

Lines changed: 13 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -8494,20 +8494,19 @@ def _show_credits(self):
84948494
print(" 🟡 Cancelled. No credits added.")
84958495

84968496
def _handle_benchmark(self, args: str = ""):
8497-
"""Run dogbench token benchmark."""
8498-
try:
8499-
from dogbench_pkg import build_telemetry, estimate_hermes_baseline, read_collar_session
8500-
s = read_collar_session()
8501-
if not s["prompt_tokens"]:
8502-
self._console_print("[yellow]No session data yet. Chat first, then benchmark.[/]")
8503-
return
8504-
t = build_telemetry(self.model, s["prompt_tokens"], s["completion_tokens"],
8505-
baseline_tokens=estimate_hermes_baseline(s["prompt_tokens"]))
8506-
self._console_print(t.to_json())
8507-
except ImportError:
8508-
self._console_print("[yellow]dogbench not installed. Run: pip install dogbench[/]")
8509-
except Exception as e:
8510-
self._console_print(f"[red]Benchmark failed: {e}[/]")
8497+
"""Run dogbench token benchmark and auto-submit results."""
8498+
import subprocess, os
8499+
dogbench_dir = os.path.expanduser("~/dogbench")
8500+
if not os.path.isdir(dogbench_dir):
8501+
self._console_print("[yellow]dogbench not installed. Run: git clone https://github.com/specdog/dogbench.git ~/dogbench[/]")
8502+
return
8503+
no_submit = "--no-submit" in args
8504+
cmd = f"cd {dogbench_dir} && git pull origin main && ./dogbench --json" + (" --no-submit" if no_submit else "")
8505+
self._console_print("[dim]Running benchmark...[/]")
8506+
result = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=300)
8507+
self._console_print(result.stdout)
8508+
if result.stderr:
8509+
self._console_print(f"[red]{result.stderr[-500:]}[/]")
85118510

85128511
def _show_insights(self, command: str = "/insights"):
85138512
"""Show usage insights and analytics from session history."""

0 commit comments

Comments
 (0)