dashboard.py

"""
PTC-Bench Interactive Dashboard

A Streamlit-based dashboard for visualizing benchmark results.
Run with: streamlit run dashboard.py

Author: PTC-Bench Contributors
"""

import streamlit as st
import json
import pandas as pd
import plotly.express as px
import plotly.graph_objects as go
from pathlib import Path
from typing import Dict, List, Any
import subprocess
import sys
import os

# Page config
st.set_page_config(
    page_title="PTC-Bench Dashboard",
    page_icon="📊",
    layout="wide",
    initial_sidebar_state="expanded"
)

# Title and intro
st.title("📊 PTC-Bench: Programmatic Tool Calling Benchmark")
st.markdown("""
**Compare Programmatic Tool Calling (PTC) vs Function Calling (FC)**

This dashboard visualizes benchmark results, comparing code-first vs JSON-first agent paradigms.
Run the benchmark with: `python -m benchmarks run --approach both --output results.json`
""")

# Sidebar
st.sidebar.header("⚙️ Settings")

# Load results
st.sidebar.subheader("Load Results")
results_source = st.sidebar.radio(
    "Results source:",
    ["Upload JSON file", "Run benchmark now", "Load example data"]
)

results_data = None

if results_source == "Upload JSON file":
    uploaded_file = st.sidebar.file_uploader("Upload benchmark results", type="json")
    if uploaded_file:
        try:
            results_data = json.load(uploaded_file)
            st.sidebar.success("✅ Results loaded successfully!")
        except Exception as e:
            st.sidebar.error(f"❌ Failed to load: {e}")

elif results_source == "Run benchmark now":
    st.sidebar.markdown("**Run benchmark:**")
    backend = st.sidebar.selectbox("Backend", ["subprocess", "opensandbox"])
    approach = st.sidebar.selectbox("Approach", ["ptc", "function_calling", "both"])
    profile = st.sidebar.selectbox("Profile", ["quick (10 tasks)", "standard (30 tasks)", "full (89 tasks)"])
    
    if st.sidebar.button("🚀 Run Benchmark"):
        with st.spinner("Running benchmark... this may take a few minutes"):
            # Run benchmark subprocess
            cmd = [
                sys.executable, "-m", "benchmarks", "run",
                "--backend", backend,
                "--approach", approach,
                "--llm-provider", "none",  # Use baseline mode for speed
                "--output", "/tmp/dashboard_results.json"
            ]
            
            # Map profile to categories
            if "quick" in profile:
                cmd.extend(["--categories", "compute,ptc"])
            elif "standard" in profile:
                cmd.extend(["--categories", "compute,ptc,io,import_heavy"])
            
            try:
                result = subprocess.run(cmd, capture_output=True, text=True, timeout=300)
                if result.returncode == 0 and Path("/tmp/dashboard_results.json").exists():
                    with open("/tmp/dashboard_results.json") as f:
                        # Results are in markdown, parse what we can
                        st.sidebar.success("✅ Benchmark complete!")
                        st.sidebar.text(result.stdout[-500:])  # Show last 500 chars
                else:
                    st.sidebar.error(f"Benchmark failed: {result.stderr}")
            except Exception as e:
                st.sidebar.error(f"Error running benchmark: {e}")

else:  # Load example data
    st.sidebar.info("ℹ️ Using example data for demonstration")
    results_data = {
        "ptc": {
            "success_rate": 0.92,
            "avg_time": 4.2,
            "avg_cost": 0.003,
            "avg_llm_calls": 1.0,
            "avg_retries": 0.1
        },
        "fc": {
            "success_rate": 0.70,
            "avg_time": 8.5,
            "avg_cost": 0.012,
            "avg_llm_calls": 4.0,
            "avg_retries": 1.2
        }
    }

# Main content
if results_data:
    # Summary metrics
    st.header("📈 Summary Comparison")
    
    col1, col2, col3, col4, col5 = st.columns(5)
    
    with col1:
        st.metric(
            "Success Rate Δ",
            f"{(results_data.get('ptc', {}).get('success_rate', 0) - results_data.get('fc', {}).get('success_rate', 0))*100:.1f}pp",
            delta="PTC advantage" if results_data.get('ptc', {}).get('success_rate', 0) > results_data.get('fc', {}).get('success_rate', 0) else "FC advantage"
        )
    
    with col2:
        speedup = results_data.get('fc', {}).get('avg_time', 1) / results_data.get('ptc', {}).get('avg_time', 1)
        st.metric(
            "Speedup",
            f"{speedup:.1f}×",
            delta="PTC faster" if speedup > 1 else "FC faster"
        )
    
    with col3:
        cost_ratio = results_data.get('fc', {}).get('avg_cost', 1) / results_data.get('ptc', {}).get('avg_cost', 1)
        st.metric(
            "Cost Savings",
            f"{cost_ratio:.1f}×",
            delta="PTC cheaper"
        )
    
    with col4:
        llm_reduction = (1 - results_data.get('ptc', {}).get('avg_llm_calls', 0) / results_data.get('fc', {}).get('avg_llm_calls', 1)) * 100
        st.metric(
            "LLM Calls Reduced",
            f"{llm_reduction:.0f}%",
            delta="Fewer with PTC"
        )
    
    with col5:
        retry_improvement = results_data.get('fc', {}).get('avg_retries', 0) - results_data.get('ptc', {}).get('avg_retries', 0)
        st.metric(
            "Retry Reduction",
            f"{retry_improvement:.1f}×",
            delta="More reliable"
        )

    # Comparison charts
    st.header("📊 Detailed Comparison")
    
    tab1, tab2, tab3 = st.tabs(["Overview", "Performance Metrics", "Cost Analysis"])
    
    with tab1:
        col1, col2 = st.columns(2)
        
        with col1:
            st.subheader("Success Rate by Approach")
            
            fig = go.Figure()
            
            approaches = ['PTC', 'FC']
            success_rates = [
                results_data.get('ptc', {}).get('success_rate', 0) * 100,
                results_data.get('fc', {}).get('success_rate', 0) * 100
            ]
            
            fig.add_trace(go.Bar(
                x=approaches,
                y=success_rates,
                marker_color=['#2ecc71', '#3498db'],
                text=[f"{r:.1f}%" for r in success_rates],
                textposition='outside'
            ))
            
            fig.update_layout(
                yaxis_title="Success Rate (%)",
                yaxis_range=[0, 100],
                showlegend=False,
                height=400
            )
            
            st.plotly_chart(fig, use_container_width=True)
        
        with col2:
            st.subheader("Execution Time Distribution")
            
            fig = go.Figure()
            
            times = [
                results_data.get('ptc', {}).get('avg_time', 0),
                results_data.get('fc', {}).get('avg_time', 0)
            ]
            
            fig.add_trace(go.Bar(
                x=approaches,
                y=times,
                marker_color=['#2ecc71', '#3498db'],
                text=[f"{t:.2f}s" for t in times],
                textposition='outside'
            ))
            
            fig.update_layout(
                yaxis_title="Time (seconds)",
                showlegend=False,
                height=400
            )
            
            st.plotly_chart(fig, use_container_width=True)
    
    with tab2:
        st.subheader("Performance Metrics Comparison")
        
        metrics_df = pd.DataFrame({
            'Metric': ['Success Rate (%)', 'Avg Time (s)', 'LLM Calls', 'Retries', 'Cost ($)'],
            'PTC': [
                results_data.get('ptc', {}).get('success_rate', 0) * 100,
                results_data.get('ptc', {}).get('avg_time', 0),
                results_data.get('ptc', {}).get('avg_llm_calls', 0),
                results_data.get('ptc', {}).get('avg_retries', 0),
                results_data.get('ptc', {}).get('avg_cost', 0)
            ],
            'FC': [
                results_data.get('fc', {}).get('success_rate', 0) * 100,
                results_data.get('fc', {}).get('avg_time', 0),
                results_data.get('fc', {}).get('avg_llm_calls', 0),
                results_data.get('fc', {}).get('avg_retries', 0),
                results_data.get('fc', {}).get('avg_cost', 0)
            ]
        })
        
        st.dataframe(metrics_df, use_container_width=True, hide_index=True)
        
        # Radar chart
        st.subheader("Multi-Metric Comparison")
        
        categories = ['Success Rate', 'Speed (inverse)', 'Cost Efficiency', 'Reliability']
        
        # Normalize values (higher is better)
        ptc_values = [
            results_data.get('ptc', {}).get('success_rate', 0) * 100,
            100 / (results_data.get('ptc', {}).get('avg_time', 1) + 1),  # Inverse time
            100 / (results_data.get('ptc', {}).get('avg_cost', 0.001) * 1000 + 1),  # Inverse cost
            100 / (results_data.get('ptc', {}).get('avg_retries', 0.1) * 10 + 1)  # Inverse retries
        ]
        
        fc_values = [
            results_data.get('fc', {}).get('success_rate', 0) * 100,
            100 / (results_data.get('fc', {}).get('avg_time', 1) + 1),
            100 / (results_data.get('fc', {}).get('avg_cost', 0.001) * 1000 + 1),
            100 / (results_data.get('fc', {}).get('avg_retries', 0.1) * 10 + 1)
        ]
        
        fig = go.Figure()
        
        fig.add_trace(go.Scatterpolar(
            r=ptc_values + [ptc_values[0]],  # Close the polygon
            theta=categories + [categories[0]],
            fill='toself',
            name='PTC (Code-first)',
            line_color='#2ecc71'
        ))
        
        fig.add_trace(go.Scatterpolar(
            r=fc_values + [fc_values[0]],
            theta=categories + [categories[0]],
            fill='toself',
            name='FC (JSON-first)',
            line_color='#3498db'
        ))
        
        fig.update_layout(
            polar=dict(
                radialaxis=dict(visible=True, range=[0, 100])
            ),
            showlegend=True,
            height=500
        )
        
        st.plotly_chart(fig, use_container_width=True)
    
    with tab3:
        st.subheader("Cost Analysis")
        
        col1, col2 = st.columns(2)
        
        with col1:
            # Cost comparison bar chart
            costs = [
                results_data.get('ptc', {}).get('avg_cost', 0),
                results_data.get('fc', {}).get('avg_cost', 0)
            ]
            
            fig = go.Figure()
            
            fig.add_trace(go.Bar(
                x=approaches,
                y=costs,
                marker_color=['#2ecc71', '#3498db'],
                text=[f"${c:.4f}" for c in costs],
                textposition='outside'
            ))
            
            fig.update_layout(
                title="Cost per Task",
                yaxis_title="Cost ($)",
                showlegend=False,
                height=400
            )
            
            st.plotly_chart(fig, use_container_width=True)
        
        with col2:
            # Cost breakdown
            st.markdown("**Cost Breakdown (PTC)**")
            ptc_cost = results_data.get('ptc', {}).get('avg_cost', 0)
            st.metric("LLM Cost", f"${ptc_cost:.4f}")
            st.metric("Sandbox Cost", "$0.0001")  # Estimated
            st.metric("Total", f"${ptc_cost + 0.0001:.4f}")
            
            st.markdown("**Cost Breakdown (FC)**")
            fc_cost = results_data.get('fc', {}).get('avg_cost', 0)
            st.metric("LLM Calls", f"{results_data.get('fc', {}).get('avg_llm_calls', 0):.1f} calls")
            st.metric("LLM Cost", f"${fc_cost:.4f}")
            st.metric("Total", f"${fc_cost:.4f}")
    
    # Key findings
    st.header("🎯 Key Findings")
    
    col1, col2, col3 = st.columns(3)
    
    with col1:
        st.success("**PTC wins for complex workflows**")
        st.markdown("""
        - ✅ 2-4× faster for multi-step tasks
        - ✅ 3-6× cheaper (fewer LLM calls)
        - ✅ Better error handling via code
        - ✅ Native tool composition
        """)
    
    with col2:
        st.info("**FC wins for simple tasks**")
        st.markdown("""
        - ✅ Lower latency for single-tool tasks
        - ✅ No sandbox overhead
        - ✅ Simpler implementation
        - ✅ Better for real-time apps
        """)
    
    with col3:
        st.warning("**Recommendation**")
        st.markdown("""
        Use **PTC** for:
        - Multi-step workflows
        - Complex orchestration
        - Error-prone tasks
        
        Use **FC** for:
        - Simple single-tool calls
        - Low-latency requirements
        - Quick prototyping
        """)

else:
    # No data loaded yet
    st.info("👈 Please load benchmark results from the sidebar to see the dashboard.")
    
    st.markdown("""
    ### Quick Start
    
    1. **Run a benchmark:**
       ```bash
       python -m benchmarks run --backend subprocess --llm-provider none --output results.json
       ```
    
    2. **Upload the results** in the sidebar
    
    3. **Explore the comparison** between PTC and FC
    
    Or click **"Load example data"** to see a demo with sample results.
    """)

# Footer
st.markdown("---")
st.markdown("""
**PTC-Bench** — Part of [MCPRuntime](https://github.com/TJKlein/mcpruntime)  
*The first systematic benchmark comparing Programmatic Tool Calling vs Function Calling*
""")