Merge pull request #1 from darkresearch/e/clean-benchmark

Author: edgarpavlovsky

.ai-rules.md

@@ -0,0 +1,66 @@
+# AI Assistant Rules for Fireteam
+
+This file contains rules for AI coding assistants (Cursor, Claude, Warp, GitHub Copilot, etc.)
+
+## Python Version: 3.12+ ONLY
+
+**CRITICAL**: This project requires Python 3.12 or higher.
+
+- ✅ Use: `python3.12` or higher
+- ❌ Never use: Python 3.9, 3.10, or 3.11
+- Dependencies like `claude-agent-sdk>=0.1.4` require Python 3.10+, and we standardize on 3.12+
+
+### Checking Python Version
+```bash
+python3.12 --version  # Should show Python 3.12.x or higher
+```
+
+## Dependency Management: Use `uv`
+
+**CRITICAL**: Always use `uv` for Python package management.
+
+- ✅ Use: `uv pip install`, `uv add`, `uv venv`
+- ❌ Never use: `pip install`, `pip3 install`, standard pip commands
+
+### Why `uv`?
+- 10-100x faster than pip
+- Better dependency resolution
+- Drop-in replacement for pip
+- Production-ready and maintained by Astral (creators of Ruff)
+
+### Common Commands
+```bash
+# Create virtual environment
+uv venv
+
+# Activate virtual environment (macOS/Linux)
+source venv/bin/activate
+
+# Install all dependencies
+uv pip install -r requirements.txt
+
+# Install a single package
+uv pip install <package-name>
+
+# Add a new dependency (updates requirements.txt)
+uv add <package-name>
+
+# Sync to exact versions
+uv pip sync requirements.txt
+```
+
+## Installing `uv`
+
+If `uv` is not installed:
+```bash
+# macOS/Linux
+curl -LsSf https://astral.sh/uv/install.sh | sh
+
+# Or with Homebrew
+brew install uv
+```
+
+## Summary
+
+1. **Python**: Always 3.12+
+2. **Packages**: Always use `uv`, never `pip`

.cursorrules

@@ -0,0 +1,28 @@
+# Fireteam Development Rules
+
+## Python Version
+- Always use Python 3.12 or higher
+- Never use Python 3.9, 3.10, or 3.11
+- When creating virtual environments, use: `python3.12 -m venv venv` or `uv venv`
+
+## Dependency Management
+- Always use `uv` for Python dependency management
+- Never use `pip`, `pip3`, or `pip install` directly
+- Install dependencies with: `uv pip install <package>`
+- Sync dependencies with: `uv pip sync requirements.txt`
+- Add dependencies with: `uv add <package>`
+
+## Example Commands
+```bash
+# Create virtual environment
+uv venv
+
+# Install dependencies
+uv pip install -r requirements.txt
+
+# Add a new dependency
+uv add <package-name>
+
+# Sync dependencies
+uv pip sync requirements.txt
+```

.env.example

@@ -8,3 +8,6 @@ SUDO_PASSWORD=claude
 # Git configuration (optional overrides)
 # GIT_USER_NAME=Your Name
 # GIT_USER_EMAIL=your.email@example.com
+
+# Anthropic
+ANTHROPIC_API_KEY=""

.github/workflows/test.yml

@@ -0,0 +1,142 @@
+name: Tests
+
+on:
+  pull_request:
+    branches: [ main ]
+  push:
+    branches: [ main ]  # Only run on direct pushes to main
+
+jobs:
+  fast-tests:
+    name: Fast Tests (Unit + Lightweight)
+    runs-on: ubuntu-latest
+    
+    steps:
+    - uses: actions/checkout@v4
+    
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+    
+    - name: Install uv
+      run: curl -LsSf https://astral.sh/uv/install.sh | sh
+    
+    - name: Create virtual environment
+      run: uv venv
+    
+    - name: Install dependencies
+      run: |
+        source .venv/bin/activate
+        uv pip install -r requirements.txt
+    
+    - name: Run all fast tests
+      run: |
+        source .venv/bin/activate
+        pytest tests/ -m "not slow and not e2e and not integration" -v --tb=short
+
+  e2e-tests:
+    name: End-to-End Tests (API)
+    runs-on: ubuntu-latest
+    timeout-minutes: 20  # Fail fast if tests hang
+    # Run on main branch and e/* branches for testing
+    if: |
+      github.ref == 'refs/heads/main' || 
+      startsWith(github.ref, 'refs/heads/e/') ||
+      startsWith(github.head_ref, 'e/')
+    
+    steps:
+    - uses: actions/checkout@v4
+    
+    - name: Set up Node.js
+      uses: actions/setup-node@v4
+      with:
+        node-version: '20'
+    
+    - name: Install Claude CLI
+      run: |
+        npm install -g @anthropic-ai/claude-code
+        echo "Claude CLI installed at: $(which claude)"
+        claude --version
+    
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+    
+    - name: Install uv
+      run: curl -LsSf https://astral.sh/uv/install.sh | sh
+    
+    - name: Create virtual environment
+      run: uv venv
+    
+    - name: Install dependencies
+      run: |
+        source .venv/bin/activate
+        uv pip install -r requirements.txt
+    
+    - name: Run E2E tests
+      timeout-minutes: 15  # Per-step timeout
+      env:
+        ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+        PYTHONUNBUFFERED: "1"  # Force immediate output
+      run: |
+        source .venv/bin/activate
+        echo "Starting e2e tests at $(date)"
+        pytest tests/ -m "e2e" -v --tb=short -s --log-cli-level=INFO
+        echo "E2E tests completed at $(date)"
+    
+    - name: Upload logs on failure
+      if: failure()
+      uses: actions/upload-artifact@v4
+      with:
+        name: e2e-test-logs
+        path: |
+          /tmp/fireteam-test-*/
+          tests/**/*.log
+        retention-days: 7
+
+  integration-tests:
+    name: Terminal-bench Integration
+    runs-on: ubuntu-latest
+    # Temporarily disabled - needs debugging
+    if: false
+    
+    steps:
+    - uses: actions/checkout@v4
+    
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+    
+    - name: Set up Docker
+      uses: docker/setup-buildx-action@v3
+    
+    - name: Install uv
+      run: curl -LsSf https://astral.sh/uv/install.sh | sh
+    
+    - name: Install terminal-bench
+      run: uv tool install terminal-bench
+    
+    - name: Create virtual environment
+      run: uv venv
+    
+    - name: Install dependencies
+      run: |
+        source .venv/bin/activate
+        uv pip install -r requirements.txt
+    
+    - name: Install Fireteam adapter
+      run: |
+        source .venv/bin/activate
+        cd benchmark
+        uv pip install -e .
+    
+    - name: Run terminal-bench integration test
+      env:
+        ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+      run: |
+        source .venv/bin/activate
+        pytest tests/ -m "integration" -v --tb=short
+

.gitignore

@@ -34,3 +34,6 @@ Thumbs.db
 
 # Logs
 logs/
+
+# Benchmark runs
+runs/

CLAUDE.md

@@ -0,0 +1,30 @@
+# Claude AI Assistant Rules for Fireteam
+
+## Python Version Requirements
+- **REQUIRED**: Use Python 3.12 or higher for all operations
+- **NEVER** use Python 3.9, 3.10, or 3.11
+- When checking Python version, ensure it's 3.12+: `python3.12 --version`
+
+## Dependency Management
+- **REQUIRED**: Use `uv` for all Python dependency management
+- **NEVER** use `pip`, `pip3`, or standard pip commands
+- `uv` is a fast, modern Python package installer and resolver
+
+### Common Operations
+```bash
+# Install dependencies from requirements.txt
+uv pip install -r requirements.txt
+
+# Install a single package
+uv pip install <package-name>
+
+# Create virtual environment with uv
+uv venv
+
+# Sync dependencies (install exact versions from lockfile)
+uv pip sync requirements.txt
+```
+
+## Why These Rules?
+- Python 3.12+: Required by `claude-agent-sdk>=0.1.4` and provides better performance
+- `uv`: 10-100x faster than pip, better dependency resolution, production-ready