diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..5c95f6f --- /dev/null +++ b/.env.example @@ -0,0 +1,4 @@ +OPENAI_API_KEY=your_openai_api_key_here +DEEPSEEK_API_KEY=your_deepseek_api_key_here +# Optional overrides +# BASE_URL=https://api.openai.com/v1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f43a647..849ba7b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,7 +26,7 @@ jobs: run: | # Skip GPU tests as GitHub Actions runners don't have CUDA # To run GPU tests locally: pytest tests/ -v -m "gpu" - pytest tests/ -v --tb=short -m "not slow and not gpu and not integration" + python -m pytest tests/ -v --tb=short -m "not slow and not gpu and not integration" lint: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 55ae470..e5c5c0e 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,19 @@ dist/ */.DS_Store *.DS_Store +# Evaluation/Profiling ignores +*.prof +evaluation/sandbox/results/* +!evaluation/sandbox/results/.gitkeep +!evaluation/sandbox/results/run_local_eval.py + +# Sandbox SIF images +*.sif + +# Sandbox Cache +evaluation/sandbox/cache/ + +# Slurm and Apptainer temporary files +*.err +*.out +build-temp-*/ diff --git a/README.md b/README.md index 69c8521..85f8c2a 100644 --- a/README.md +++ b/README.md @@ -1,228 +1,110 @@ -
- ContextPilot Logo +# ContextPilot: L7 Middleware Proxy for Multi-Agent Workflows -

ContextPilot: Fast Long-Context Inference via Context Reuse

+![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg) +![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg) +![Status: Research Prototype](https://img.shields.io/badge/status-research_prototype-blue.svg) - [![Python](https://img.shields.io/badge/python-≥3.10-blue)](https://www.python.org/) - [![PyPI](https://img.shields.io/pypi/v/contextpilot)](https://pypi.org/project/contextpilot/) - [![License](https://img.shields.io/badge/license-Apache%202.0-green)](LICENSE) +## Overview +ContextPilot is an advanced Layer-7 middleware proxy designed to neutralize the **"Orchestration Tax"** in multi-agent LLM deployments. By seamlessly intercepting API traffic between agents and inference endpoints, ContextPilot applies extreme context compression and Cache-Homogenization techniques. This forces divergent, bloated agent requests to realign with the strict radix-tree structures of state-of-the-art KV-Caches (like vLLM), dramatically reducing network I/O, API costs, and VRAM consumption on edge devices. -

4–12× cache hits | 1.5–3× faster prefill | ~36% token savings across vLLM, SGLang, RAG, AI Agents, and more.

+## Academic Attribution & Contributions +This repository serves as the empirical apparatus for an MSc Dissertation, built upon a foundational library engineered by senior PhD researchers. -
+* **Foundational Work (Prior PhD Research):** The core ContextPilot mechanism—including Prefix Cache Indexing, Exact Lexical Deduplication, and Context Reordering—was originally conceptualized as an intrusive, low-level kernel patch targeting datacenter compute clusters. +* **MSc Project Contributions:** This dissertation engineered the transition to edge computing and multi-agent workflows. The novel contributions include: + * **L7 Middleware Gateway Architecture:** Transformed the intrusive kernel patch into a completely transparent, dual-interface (ZMQ + HTTP) reverse-proxy, ensuring framework-agnostic interception. + * **Dynamic Skill Filtering & Semantic Pruning:** Engineered the `SkillAwareContextPlugin` to neutralize dictionary bloat and the `DynamicPruningPlugin` (using NLP embeddings like `all-MiniLM-L6-v2`) to transcend exact-match limitations. + * **Black-Box Shadow Telemetry:** Engineered mathematical projections for closed-source APIs (like DeepSeek) where low-level hypervisor cache access is impossible. + * **Edge Device Viability (Synergistic Quantization):** Proved the system works on extreme edge hardware (16GB GPUs) by pairing it with 4-bit AWQ quantized models to prevent Out-Of-Memory errors. --------------------------------------------------------------------------------- +## Core Architecture (The 5 Plugins) +ContextPilot achieves cache homogenization through a pipeline of 5 decoupling plugins: -| [**Documentation**](https://efficientcontext.github.io/contextpilot-docs/) | [**Examples**](examples/) | [**Benchmarks**](https://efficientcontext.github.io/contextpilot-docs/reference/benchmarks) | [**Docker**](https://efficientcontext.github.io/contextpilot-docs/getting_started/docker) | [**Paper**](https://arxiv.org/abs/2511.03475) | +* **`ContextDedupPlugin`**: Deduplicates redundant conversational history across multi-turn agent execution by substituting repeated messages with lightweight reference hints (e.g., `[Reference to Turn N]`). On black-box APIs it runs in shadow mode, reporting theoretical savings without mutating the payload. +* **`DynamicPruningPlugin`**: Leverages `all-MiniLM-L6-v2` to aggressively prune historically irrelevant semantic noise, applying a dynamic cutoff threshold to protect critical context while maximizing compression. +* **`SkillAwareContextPlugin`**: Filters monolithic tool registries, dropping unused functions dynamically based on an oracle or predictive router to ensure bloated JSON schemas do not invalidate prefixes. +* **`ContextReorderPlugin`**: Deterministically sorts system prompts, tool schemas, and few-shot examples to guarantee rigid prefix alignment for the backend KV-Cache. +* **`KVCacheLookupPlugin`**: Subscribes to worker cache events over ZeroMQ (ZMQ) and maintains a shadow Radix tree per worker, routing each request to the endpoint with the longest cached prefix. -## News +### Architectural Diagram +![ContextPilot Architecture](assets/architecture.png) +*Dual-Interface architecture of the Middleware Token Proxy, illustrating the high-speed ZMQ IPC backbone for native agents and the transparent HTTP Compatibility Gateway.* -- [2026/03] ContextPilot now can run on **macOS / Apple Silicon** via [llama.cpp](docs/guides/mac_llama_cpp.md). -- [2026/02] ContextPilot v0.3.2 released, supporting [PageIndex](https://github.com/VectifyAI/PageIndex) and [Mem0](https://github.com/mem0ai/mem0). -- [2026/01] ContextPilot has been accepted to MLSys 2026 🎉! See you in Bellevue, WA, USA. +### Edge Deployment & VRAM Optimization +![VRAM Allocation Comparison](assets/vram_chart.png) +*Comparison of VRAM allocation across FP16 Baseline, AWQ Baseline, and AWQ+Proxy, highlighting the elimination of Out-Of-Memory (OOM) failures.* -## About +## Empirical Performance Data -Long-context workloads (RAG, memory chat, tool-augmented agents) prepend many context blocks. Across requests, these blocks often overlap but get reordered or duplicated, changing token prefixes and triggering cache misses and redundant KV recomputation. Common examples include (1) Trending Topic QA, (2) Closed-Domain Long-Context QA, (3) Batched Long-Context Inference, (4) multi-turn conversations with long-term memory and many more. +### Cache Homogenization (DeepSeek V4 Pro) +By systematically excising task-specific tool noise and deduplicating prefixes, ContextPilot achieves massive secondary cache hits on backend inference engines. -ContextPilot sits between context assembly and inference to maximize prefix reuse and remove duplicates: +| Benchmark | Baseline Cache Hits | Proxy Cache Hits | Architectural Phenomenon | +| :--- | :--- | :--- | :--- | +| **BigCodeBench** | 457,344 | 289,152 | Client-Side Bandwidth Conservation | +| **MCP-Atlas** | 18,688 | 66,304 | Cache-Homogenization via Prefix Alignment | -1. **Higher throughput & cache hits** — boosts prefill throughput and prefix cache hit ratio via context reuse. -2. **Drop-in solutions** — works with [PageIndex](https://github.com/VectifyAI/PageIndex), [Mem0](https://github.com/mem0ai/mem0), [LMCache](https://github.com/LMCache/LMCache), and backends like [vLLM](https://github.com/vllm-project/vllm) / [SGLang](https://github.com/sgl-project/sglang) / [llama.cpp](docs/guides/mac_llama_cpp.md). -3. **No compromise in reasoning quality** — can even improve with extremely long contexts. -4. **Widely tested** — validated across diverse RAG and agentic workloads. +### Compression & Accuracy (ELM GPT-5.5) +The Full Triple Pipeline demonstrates that hash-based deduplication and semantic-based pruning stack without degrading accuracy (McNemar $p = 0.28$). -It maintains a **Context Index** of cached content, then per request applies **Reorder** (align shared blocks into a common prefix) and/or **Deduplicate** (replace repeats with reference hints), plus **cache-aware scheduling** to maximize prefix sharing. The optimized prompt is sent via the OpenAI-compatible API; `POST /evict` keeps the index synced when KV cache is reclaimed. See its design overview below. +| Pipeline Configuration | Paired Baseline | Pass@1 | History Saved (Chars / %) | Tools Reduced | +| :--- | :--- | :--- | :--- | :--- | +| **Semantic Pruning Only** | 62.4% | 63.2% | 76,380 (5.33%) | 70% | +| **Full Triple Pipeline** | 62.2% | 63.2% | 168,720 (11.78%)† | 70% | -
-ContextPilot Architecture -
+† Includes a 6.45% referential-deduplication component reported as a theoretical shadow-mode projection. -> For more design details, see [Paper](https://arxiv.org/abs/2511.03475) and [Documentation](https://efficientcontext.github.io/contextpilot-docs/). +## Quick Start / Installation +Clone the repository and install the proxy server and all associated dependencies locally. -## Performance at a Glance - -ContextPilot is validated across three representative settings: single-node academic RAG, multi-node production MoE inference, and multi-turn memory-augmented chat. In every case it delivers significant speedups with comparable answer quality. - -**Qwen3-32B on 4×A6000** — single-node academic RAG with a 32B model on consumer GPUs. - -| Benchmark | Method | Prefill TP (tok/s) | Cache Hit | F1 (%) | -|-----------|--------|--------------------|-----------|--------| -| MultihopRAG | SGLang | 7,290 | 4.64% | 60.42 | -| | **SGLang + ContextPilot** | **14,214** | **33.97%** | **64.39** | -| NarrativeQA | SGLang | 7,921 | 5.91% | 28.41 | -| | **SGLang + ContextPilot** | **12,117** | **20.82%** | **29.64** | - -**DeepSeek-R1-671B on 16×H20** — production-scale 671B MoE inference on a multi-node GPU cluster. - -| Benchmark | Method | Prefill TP (tok/s) | Cache Hit | F1 (%) | -|-----------|--------|--------------------|-----------|--------| -| MultihopRAG | SGLang | 9,636 | 5.12% | 64.15 | -| | **SGLang + ContextPilot** | **17,498** | **60.37%** | **64.68** | -| NarrativeQA | SGLang | 8,687 | 6.08% | 40.20 | -| | **SGLang + ContextPilot** | **13,201** | **38.24%** | **41.08** | - -**Qwen3-4B on 1×A6000** — multi-turn memory chat with [Mem0](https://github.com/mem0ai/mem0) on the [LoCoMo](https://github.com/snap-research/locomo) benchmark. - -| Context Size | Method | TTFT (s) | LLM Judge | -|--------------|--------|----------|-----------| -| 100 memories | SGLang | 0.1012 | 0.437 | -| | **SGLang + ContextPilot** | **0.0554** | 0.420 | - ->ContextPilot results in mem0 table are without context annotation — an optional feature that adds original importance ranking to reordered context blocks, which can further improve answer quality (see [Paper](https://arxiv.org/abs/2511.03475)). - -**Llama-3.2-1B on Apple M3 (MacBook Air, 16 GB)** — MultihopRAG on Apple Silicon with llama.cpp, no GPU server required. - -| Method | Avg Latency (ms) | -|--------|-----------------| -| llama.cpp | 3,315 | -| **llama.cpp + ContextPilot** | **1,378** | - -Settings: `Llama-3.2-1B-Instruct-Q4_K_M.gguf`, Metal offload (`-ngl 99`), `--cache-reuse 256`, `--parallel 4`, context 32768 tokens. See the [Mac + llama.cpp guide](docs/guides/mac_llama_cpp.md). - -## Installation - -**Requirements:** Python >= 3.10 - ---- - -### vLLM / SGLang - -ContextPilot works with both CPU and GPU backends for building the context index. The `[gpu]` extra enables GPU-accelerated distance computation (via `cupy-cuda12x`) and is faster for large batches; without it, ContextPilot falls back to the CPU backend automatically. - -**From PyPI** — the vLLM and SGLang hooks are installed automatically: -```bash -pip install contextpilot # CPU index computation -pip install "contextpilot[gpu]" # GPU index computation (CUDA 12.x) -``` - -**From source** — run `install_hook` manually after install, since editable installs do not copy the `.pth` file to site-packages: ```bash git clone https://github.com/EfficientContext/ContextPilot.git cd ContextPilot -pip install -e . # CPU -pip install -e ".[gpu]" # GPU (CUDA 12.x) -python -m contextpilot.install_hook # one-time: enables automatic vLLM / SGLang integration +git checkout feature/msc-cache-optimization +python -m venv venv +source venv/bin/activate +pip install -e . +pip install -r requirements.txt ``` -The `install_hook` step writes a `.pth` file into your site-packages so the vLLM and SGLang hooks load automatically at Python startup — no code changes required. To uninstall: `python -m contextpilot.install_hook --remove`. +## Reproducibility Guide ---- +The empirical validation pipelines for **BigCodeBench** and **MCP-Atlas** (using DeepSeek V4 Pro and OpenAI/ELM GPT-5.5) have been fully refactored into a parameter-driven, production-ready interface. -### Mac / Apple Silicon — llama.cpp +### Standard Local Execution +Six unified entry points are provided in the root directory. They all support the following standard arguments: +- `--limit `: Restrict the evaluation to `N` tasks (useful for fast dry-runs). +- `--plugins `: Comma-separated string to selectively toggle optimizations (e.g., `dedup,dynamic,skill` or simply `all`). -**From PyPI:** +**Example: Fast Dry-Run Testing (5 Tasks)** +Run a rapid verification of the proxy server, the dynamic pruning logic, and the LLM endpoint connection: ```bash -pip install contextpilot -xcode-select --install # one-time: provides clang++ to compile the native hook +./eval_openai_bigcodebench.sh --limit 5 --plugins dynamic ``` -**From source:** +**Example: Full Benchmark Execution (All Plugins Stacked)** +Execute the complete empirical evaluation pipeline with maximum concurrency: ```bash -git clone https://github.com/EfficientContext/ContextPilot.git -cd ContextPilot -pip install -e . -xcode-select --install # one-time: provides clang++ to compile the native hook -``` - -> **Why `xcode-select`?** The llama.cpp integration uses a small C++ shared library injected into `llama-server` via `DYLD_INSERT_LIBRARIES`. It is compiled automatically on first use and requires `clang++` from Xcode Command Line Tools. - ---- - -More [detailed installation instructions](https://efficientcontext.github.io/contextpilot-docs/getting_started/installation) are available in the docs. - -Docker images are also available for both all-in-one and standalone deployment. See the [Docker guide](https://efficientcontext.github.io/contextpilot-docs/getting_started/docker). - -## Getting Started - -### Quick Start with Context Ordering - -Add **one call** (`cp_instance.optimize()`) before inference to rearrange context blocks so that shared content aligns into a common prefix, enabling cache reuse. An importance ranking in the prompt preserves accuracy. - -| Mode | When to Use | How It Works | -|------|-------------|--------------| -| **Online** | Multi-turn (e.g., chatbot + [Mem0](https://github.com/mem0ai/mem0)) | Tracks previously cached blocks; moves overlapping ones to the prefix each turn | -| **Offline** | Batch / single-shot | Globally reorders and schedules all requests for maximum prefix sharing | - -Both modes work with any OpenAI-compatible endpoint (vLLM, SGLang, etc.) — no changes to your inference deployment. They support both direct API calls (shown below) and HTTP server deployment (see the [online usage guide](https://efficientcontext.github.io/contextpilot-docs/guides/online_usage)). - ---- - -#### Accelerating Online Inference - -Multi-turn chatbot with Mem0 or RAG where each turn's context blocks partially overlap. `cp_instance.optimize()` moves shared blocks to the prefix so the engine reuses cached KV states. - -```python -from openai import OpenAI -# Step 1: Import ContextPilot -import contextpilot as cp - -client = OpenAI(base_url="http://localhost:30000/v1", api_key="EMPTY") -# Step 2: Create a ContextPilot instance -cp_instance = cp.ContextPilot(use_gpu=False) - -for query in queries: - contexts = get_contexts(query) # Mem0, Retriever, ... - # Step 3: Optimize context ordering and build ready-to-use messages - messages = cp_instance.optimize(contexts, query) - - response = client.chat.completions.create( - model="Qwen/Qwen3-4B", - messages=messages, - ) - print(f"Q: {query}\nA: {response.choices[0].message.content}\n") +./eval_deepseek_mcpatlas.sh --plugins all ``` -> **Note:** When the engine evicts KV-cache entries under memory pressure, ContextPilot's index can go stale. Set `CONTEXTPILOT_INDEX_URL` when launching [SGLang or vLLM](https://efficientcontext.github.io/contextpilot-docs/guides/online_usage#inference-engine-integration) to enable automatic eviction sync. For distributed setups, see [Distributed Setup](https://efficientcontext.github.io/contextpilot-docs/getting_started/installation#distributed-setup). - ---- - -#### Accelerating Offline Inference - -Batch of requests with overlapping context blocks. `cp_instance.optimize_batch()` globally reorders blocks and schedules execution order so queries with similar contexts run consecutively, maximizing cache reuse. See the [offline usage guide](https://efficientcontext.github.io/contextpilot-docs/guides/offline_usage) for details. Offline mode can also be deployed as an HTTP server without eviction sync — see [Stateless Mode](https://efficientcontext.github.io/contextpilot-docs/guides/online_usage#stateless-mode). - -```python -import asyncio -import openai -# Step 1: Import ContextPilot -import contextpilot as cp - -BASE_URL = "http://localhost:30000/v1" -# Step 2: Create a ContextPilot instance -cp_instance = cp.ContextPilot(use_gpu=False) - -all_contexts = [get_contexts(q) for q in queries] # Mem0, Retriever, ... -# Step 3: Optimize — reorder, schedule, and build prompts in one call -messages_batch, order = cp_instance.optimize_batch(all_contexts, queries) - -# Send all requests concurrently -async def generate_all(): - ac = openai.AsyncOpenAI(base_url=BASE_URL, api_key="EMPTY") - return await asyncio.gather(*[ac.chat.completions.create( - model="Qwen/Qwen3-4B", messages=m - ) for m in messages_batch]) - -for resp, idx in zip(asyncio.run(generate_all()), order): - print(f"Q: {queries[idx]}\nA: {resp.choices[0].message.content}\n") +**Example: Ablation & Quantization Studies** +Execute the specialized dissertation studies: +```bash +./eval_ablation_dynamic_pruning.sh --limit 10 +./eval_local_vram_quantization.sh --limit 10 ``` -For a detailed walkthrough with concrete examples, see the [Quick Start Guide](https://efficientcontext.github.io/contextpilot-docs/getting_started/quickstart). For more fine-grained control, you can also use `cp_instance.reorder()` and `cp_instance.deduplicate()` directly — see the [API reference](https://efficientcontext.github.io/contextpilot-docs/reference/api) and [multi-turn deduplication guide](https://efficientcontext.github.io/contextpilot-docs/guides/multi_turn). - -### Adoption Examples +### Telemetry Analysis +When the pipeline completes, the proxy will gracefully flush its telemetry buffers and print the final metric teardown directly to your standard output. Check the terminal logs to view the total characters saved, tools filtered, and most importantly, the `prompt_cache_hit_tokens` successfully returned by the backend engine! -See many useful adoption examples: [Mem0 integration](https://efficientcontext.github.io/contextpilot-docs/guides/mem0), [PageIndex RAG](https://efficientcontext.github.io/contextpilot-docs/guides/pageindex), [offline batch scheduling](https://efficientcontext.github.io/contextpilot-docs/guides/offline_usage), and [multi-turn deduplication](https://efficientcontext.github.io/contextpilot-docs/guides/multi_turn). +### Academic Markers (HPC / SLURM Execution) +For academic markers attempting to reproduce the exact empirical environment on the university `Teaching` cluster, all original `#SBATCH` wrapper scripts have been archived in the `evaluation/slurm_launchers/` directory. -## Citation -```bibtex -@inproceedings{contextpilot2026, - title = {ContextPilot: Fast Long-Context Inference via Context Reuse}, - author = {Jiang, Yinsicheng and Huang, Yeqi and Cheng, Liang and Deng, Cheng and Sun, Xuan and Mai, Luo}, - booktitle = {Proceedings of the 9th Conference on Machine Learning and Systems (MLSys 2026)}, - year = {2026}, - url = {https://arxiv.org/abs/2511.03475} -} +To run via SLURM: +```bash +cd evaluation/slurm_launchers/ +sbatch submit_test_all_plugins_elm.slurm ``` -## Contributing -We welcome and value all contributions! Please feel free to submit issues and pull requests. diff --git a/assets/architecture.png b/assets/architecture.png new file mode 100644 index 0000000..64ddf10 Binary files /dev/null and b/assets/architecture.png differ diff --git a/assets/vram_chart.png b/assets/vram_chart.png new file mode 100644 index 0000000..9a35de4 Binary files /dev/null and b/assets/vram_chart.png differ diff --git a/contextpilot/context_index/compute_distance_cpu.py b/contextpilot/context_index/compute_distance_cpu.py index e3f10b0..621c1d8 100755 --- a/contextpilot/context_index/compute_distance_cpu.py +++ b/contextpilot/context_index/compute_distance_cpu.py @@ -272,7 +272,7 @@ def compute_distance_matrix_cpu_optimized(contexts: List[List[int]], start = time.time() chunk_ids, original_positions, lengths, offsets = prepare_contexts_for_cpu(contexts) prep_time = time.time() - start - print(f"✓ Prepared in {prep_time:.1f}s") + print(f"+ Prepared in {prep_time:.1f}s") # Generate batches of pair indices print(f"\nGenerating pair batches...") @@ -290,7 +290,7 @@ def compute_distance_matrix_cpu_optimized(contexts: List[List[int]], if current_batch: batches.append(current_batch) - print(f"✓ Generated {len(batches):,} batches") + print(f"+ Generated {len(batches):,} batches") # Prepare arguments for workers worker_args = [ @@ -306,13 +306,13 @@ def compute_distance_matrix_cpu_optimized(contexts: List[List[int]], start_time = time.time() processed = 0 - with Pool(num_workers) as pool: - for batch_results in pool.imap_unordered(compute_batch_worker, worker_args): + if num_workers == 1: + # Bypass multiprocessing Pool entirely to save initialization overhead + for args in worker_args: + batch_results = compute_batch_worker(args) for i, j, dist in batch_results: - # Convert (i, j) to condensed index condensed_idx = n * i - i * (i + 1) // 2 + j - i - 1 condensed_distances[condensed_idx] = dist - processed += 1 # Progress update @@ -326,6 +326,27 @@ def compute_distance_matrix_cpu_optimized(contexts: List[List[int]], f"Rate: {rate:,.0f} pairs/sec | " f"Elapsed: {elapsed:.1f}s | " f"ETA: {eta:.1f}s ({eta/60:.1f} min)") + else: + with Pool(num_workers) as pool: + for batch_results in pool.imap_unordered(compute_batch_worker, worker_args): + for i, j, dist in batch_results: + # Convert (i, j) to condensed index + condensed_idx = n * i - i * (i + 1) // 2 + j - i - 1 + condensed_distances[condensed_idx] = dist + + processed += 1 + + # Progress update + if processed % 100000 == 0 or processed == num_pairs: + elapsed = time.time() - start_time + rate = processed / elapsed if elapsed > 0 else 0 + eta = (num_pairs - processed) / rate if rate > 0 else 0 + progress_pct = processed / num_pairs * 100 + + print(f" {processed:,}/{num_pairs:,} ({progress_pct:.1f}%) | " + f"Rate: {rate:,.0f} pairs/sec | " + f"Elapsed: {elapsed:.1f}s | " + f"ETA: {eta:.1f}s ({eta/60:.1f} min)") compute_time = time.time() - start_time total_time = compute_time + prep_time diff --git a/contextpilot/server/http_server.py b/contextpilot/server/http_server.py index 6aca488..45fbac1 100644 --- a/contextpilot/server/http_server.py +++ b/contextpilot/server/http_server.py @@ -229,6 +229,9 @@ class DeduplicateRequest(BaseModel): async def lifespan(app: FastAPI): """Lifespan context manager for startup/shutdown.""" global _aiohttp_session + global _total_prompt_cache_hit_tokens + + _total_prompt_cache_hit_tokens = 0 # Initialize config from environment variables _init_config() @@ -239,11 +242,15 @@ async def lifespan(app: FastAPI): logger.info(f" max_tokens: {_max_tokens}") logger.info(f" infer_api_url: {_infer_api_url}") - _aiohttp_session = aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=3600)) + _aiohttp_session = aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=3600), trust_env=True) yield if _aiohttp_session: await _aiohttp_session.close() logger.info("ContextPilot Index Server shutting down...") + + print("\n=== Final Telemetry Summary ===") + print(f"Total Prompt Cache Hit Tokens: {_total_prompt_cache_hit_tokens}") + # (Note: chars_saved_percentage and tools_filtered_percentage will be output here when plugins are fully integrated) app = FastAPI( @@ -930,11 +937,9 @@ async def proxy_completions(request: Request): _index.track_request(request_id) # Pass request_id to inference engine so it can use the same ID for request tracking - # Engine will notify ContextPilot via /evict callback when this request is evicted + # We don't inject request_id into body anymore to avoid breaking strict APIs. if request_id: - body["rid"] = request_id # SGLang - body["request_id"] = request_id # vLLM - logger.info(f"Proxy: forwarding request with request_id={request_id}") + logger.info(f"Proxy: tracking request with request_id={request_id}") else: logger.info("Proxy: forwarding request without rid (no ContextPilot tracking)") @@ -942,9 +947,26 @@ async def proxy_completions(request: Request): api_url = f"{infer_api_url}/v1/completions" logger.debug(f"Proxying to {api_url}") - async with _aiohttp_session.post(api_url, json=body) as response: + # Extract headers to forward + headers = dict(request.headers) + headers.pop("host", None) + headers.pop("content-length", None) + headers["accept-encoding"] = "gzip, deflate" + + async with _aiohttp_session.post(api_url, json=body, headers=headers) as response: result = await response.json() + global _total_prompt_cache_hit_tokens + if response.status == 200 and isinstance(result, dict): + usage = result.get("usage", {}) + if isinstance(usage, dict): + # Legacy DeepSeek V3 format + if "prompt_cache_hit_tokens" in usage: + _total_prompt_cache_hit_tokens += int(usage["prompt_cache_hit_tokens"]) + # DeepSeek V4 / OpenAI standard format + elif "prompt_tokens_details" in usage and isinstance(usage["prompt_tokens_details"], dict): + _total_prompt_cache_hit_tokens += int(usage["prompt_tokens_details"].get("cached_tokens", 0)) + # Token tracking is handled by the inference engine via CONTEXTPILOT_INDEX_URL # The engine calls /evict after its internal cache eviction @@ -999,17 +1021,39 @@ async def proxy_engine(path: str, request: Request): # Inject rid for SGLang cache tracking (same logic as proxy_completions) request_id = body.pop("request_id", None) or body.get("rid", None) + + # Pop custom proxy parameters so upstream OpenAI doesn't reject them + body.pop("user_id", None) + body.pop("parent_id", None) + body.pop("_required_skills", None) + if not request_id: request_id = f"req-{uuid.uuid4().hex[:12]}" logger.debug(f"Auto-assigned request_id={request_id}") if _index: _index.track_request(request_id) - if request_id: - body["rid"] = request_id - body["request_id"] = request_id - async with _aiohttp_session.post(target_url, json=body) as response: + # Extract headers to forward (excluding hop-by-hop headers that aiohttp manages) + headers = dict(request.headers) + # Remove proxy-specific headers + headers.pop("host", None) + headers.pop("content-length", None) + headers["accept-encoding"] = "gzip, deflate" + + async with _aiohttp_session.post(target_url, json=body, headers=headers) as response: result = await response.json() + + global _total_prompt_cache_hit_tokens + if response.status == 200 and isinstance(result, dict): + usage = result.get("usage", {}) + if isinstance(usage, dict): + # Legacy DeepSeek V3 format + if "prompt_cache_hit_tokens" in usage: + _total_prompt_cache_hit_tokens += int(usage["prompt_cache_hit_tokens"]) + # DeepSeek V4 / OpenAI standard format + elif "prompt_tokens_details" in usage and isinstance(usage["prompt_tokens_details"], dict): + _total_prompt_cache_hit_tokens += int(usage["prompt_tokens_details"].get("cached_tokens", 0)) + return JSONResponse(content=result, status_code=response.status) except aiohttp.ClientError as e: diff --git a/contextpilot/server/live_index.py b/contextpilot/server/live_index.py index cbaeb31..ea6df7b 100644 --- a/contextpilot/server/live_index.py +++ b/contextpilot/server/live_index.py @@ -172,15 +172,15 @@ def build_and_schedule(self, contexts: List[List[int]], print("\n1. Building static index...") self.initial_result = self.fit_transform(contexts) - print(f" ✓ Built tree with {self.initial_result.stats['total_nodes']} nodes") - print(f" ✓ Leaf nodes: {self.initial_result.stats['leaf_nodes']}") + print(f" + Built tree with {self.initial_result.stats['total_nodes']} nodes") + print(f" + Leaf nodes: {self.initial_result.stats['leaf_nodes']}") # Step 2: Inter-context scheduling print("\n2. Scheduling contexts for optimal execution...") scheduled_reordered, scheduled_originals, final_mapping, groups = \ self.inter_scheduler.schedule_contexts(self.initial_result) - print(f" ✓ Created {len(groups)} execution groups") + print(f" + Created {len(groups)} execution groups") self.scheduled_result = { 'reordered_contexts': scheduled_reordered, @@ -197,8 +197,8 @@ def build_and_schedule(self, contexts: List[List[int]], num_input_contexts=len(contexts) ) - print(f" ✓ Initialized {len(self.metadata)} nodes with metadata") - print(f" ✓ Auto-assigned {len(request_id_mapping)} request IDs") + print(f" + Initialized {len(self.metadata)} nodes with metadata") + print(f" + Auto-assigned {len(request_id_mapping)} request IDs") # Add request_id mapping to result (dict and ordered list) self.scheduled_result['request_id_mapping'] = request_id_mapping @@ -208,7 +208,7 @@ def build_and_schedule(self, contexts: List[List[int]], self.is_live = True print("\n" + "=" * 80) - print("✓ INDEX IS NOW LIVE - Ready for dynamic operations") + print("+ INDEX IS NOW LIVE - Ready for dynamic operations") print("=" * 80 + "\n") return self.scheduled_result @@ -534,8 +534,8 @@ def build_incremental(self, contexts: List[List[int]], # No match - will build new index for these unmatched_contexts.append((i, context)) - print(f" ✓ Found {len(matched_contexts)} contexts with matches") - print(f" ✓ Found {len(unmatched_contexts)} contexts without matches") + print(f" + Found {len(matched_contexts)} contexts with matches") + print(f" + Found {len(unmatched_contexts)} contexts without matches") # Prepare result arrays (will fill in order) request_ids = [None] * len(contexts) @@ -585,7 +585,7 @@ def build_incremental(self, contexts: List[List[int]], ) temp_result = temp_index.fit_transform(unmatched_only) - print(f" ✓ Built temp index with {temp_result.stats['total_nodes']} nodes") + print(f" + Built temp index with {temp_result.stats['total_nodes']} nodes") # Step 4: Merge temp index into global index print("\n4. Merging temp index into global index...") @@ -606,16 +606,16 @@ def build_incremental(self, contexts: List[List[int]], context_info.append((orig_idx, merged_request_ids[i], merged_search_paths[i])) merged_count = len(unmatched_contexts) - print(f" ✓ Merged {merged_count} new subtrees under global root") + print(f" + Merged {merged_count} new subtrees under global root") # Step 5: Schedule execution order print("\n5. Scheduling execution order for cache reuse...") scheduled_order = self._schedule_incremental(context_info) groups = self._group_by_path_prefix(context_info) - print(f" ✓ Scheduled {len(scheduled_order)} contexts into {len(groups)} groups") + print(f" + Scheduled {len(scheduled_order)} contexts into {len(groups)} groups") print("\n" + "=" * 80) - print(f"✓ INCREMENTAL BUILD COMPLETE") + print(f"+ INCREMENTAL BUILD COMPLETE") print(f" Matched & inserted: {len(matched_contexts)}") print(f" Built & merged: {merged_count}") print("=" * 80 + "\n") @@ -906,15 +906,15 @@ def schedule_only(self, contexts: List[List[int]]) -> Dict: print("\n1. Building static index...") result = self.fit_transform(contexts) - print(f" ✓ Built tree with {result.stats['total_nodes']} nodes") - print(f" ✓ Leaf nodes: {result.stats['leaf_nodes']}") + print(f" + Built tree with {result.stats['total_nodes']} nodes") + print(f" + Leaf nodes: {result.stats['leaf_nodes']}") # Step 2: Inter-context scheduling print("\n2. Scheduling contexts for optimal execution...") scheduled_reordered, scheduled_originals, final_mapping, groups = \ self.inter_scheduler.schedule_contexts(result) - print(f" ✓ Created {len(groups)} execution groups") + print(f" + Created {len(groups)} execution groups") # Return results without going live (stateless) scheduled_result = { @@ -931,7 +931,7 @@ def schedule_only(self, contexts: List[List[int]]) -> Dict: } print("\n" + "=" * 80) - print("✓ BATCH SCHEDULED (Stateless - no cache tracking)") + print("+ BATCH SCHEDULED (Stateless - no cache tracking)") print("=" * 80 + "\n") return scheduled_result diff --git a/docs/guides/multi_turn.md b/docs/guides/multi_turn.md index 021582c..461d623 100644 --- a/docs/guides/multi_turn.md +++ b/docs/guides/multi_turn.md @@ -256,10 +256,10 @@ print(f"New docs: {result['new_docs']}") # [2] | Operation | `/reorder` | `/deduplicate` | |-----------|----------|----------------| -| Index build | ✓ | ✗ | -| Clustering | ✓ | ✗ | -| Search | ✓ | ✗ | -| Deduplication | ✓ | ✓ | +| Index build | + | ✗ | +| Clustering | + | ✗ | +| Search | + | ✗ | +| Deduplication | + | + | | **Latency** | ~50-200ms | ~1-5ms | For multi-turn conversations, Turn 2+ typically doesn't need index operations — just deduplication against conversation history. The `/deduplicate` endpoint is **10-100x faster**. diff --git a/eval_ablation_dynamic_pruning.sh b/eval_ablation_dynamic_pruning.sh new file mode 100755 index 0000000..91edcf7 --- /dev/null +++ b/eval_ablation_dynamic_pruning.sh @@ -0,0 +1,53 @@ +#!/bin/bash +export PYTHONPATH="$(pwd):$PYTHONPATH" +export NO_PROXY="localhost,127.0.0.1,::1" +export no_proxy="localhost,127.0.0.1,::1" + +LIMIT=0 +while [[ $# -gt 0 ]]; do + case $1 in + --limit) LIMIT="$2"; shift 2 ;; + *) echo "Unknown parameter: $1"; exit 1 ;; + esac +done + +echo "=============================================" +echo " Dynamic Pruning Hyperparameter Ablation" +echo " Limit: $LIMIT" +echo "=============================================" + +echo "[1/3] Booting ContextPilot Proxy Server..." +python -m contextpilot.server.http_server --port 8000 --infer-api-url "https://api.openai.com" > proxy_ablation.log 2>&1 & +PROXY_PID=$! +sleep 5 + +THRESHOLDS=(0.1 0.2 0.4 0.5) + +echo "[2/3] Running Python Ablation Iterations..." +for THRESHOLD in "${THRESHOLDS[@]}"; do + echo "--- Running Threshold: $THRESHOLD ---" + python evaluation/benchmarks/run_bigcodebench.py \ + --model gpt-5.5 \ + --api_base "https://api.openai.com/v1" \ + --concurrency 5 \ + --limit "$LIMIT" \ + --plugins "all" \ + --eval_mode with_plugin \ + --threshold "$THRESHOLD" + + mv evaluation/benchmarks/results_with_plugin_gpt-5.5.jsonl evaluation/benchmarks/results_ablation_${THRESHOLD}_with_plugin_gpt-5.5.jsonl +done + +echo "Shutting down Proxy Server to flush telemetry..." +kill -INT $PROXY_PID +sleep 2 +cat proxy_ablation.log + +echo "[3/3] Sandbox Evaluation..." +for THRESHOLD in "${THRESHOLDS[@]}"; do + echo "--- Evaluating Sandbox for Threshold: $THRESHOLD ---" + cp evaluation/benchmarks/results_ablation_${THRESHOLD}_with_plugin_gpt-5.5.jsonl evaluation/benchmarks/elm_samples_full.jsonl + cd evaluation/benchmarks && bash run_sandbox_eval_full.sh && cd ../.. +done + +echo "Pipeline Complete!" diff --git a/eval_deepseek_bigcodebench.sh b/eval_deepseek_bigcodebench.sh new file mode 100755 index 0000000..fdf1c07 --- /dev/null +++ b/eval_deepseek_bigcodebench.sh @@ -0,0 +1,49 @@ +#!/bin/bash +export PYTHONPATH="$(pwd):$PYTHONPATH" +export NO_PROXY="localhost,127.0.0.1,::1" +export no_proxy="localhost,127.0.0.1,::1" + +LIMIT=0 +PLUGINS="all" + +while [[ $# -gt 0 ]]; do + case $1 in + --limit) LIMIT="$2"; shift 2 ;; + --plugins) PLUGINS="$2"; shift 2 ;; + *) echo "Unknown parameter: $1"; exit 1 ;; + esac +done + +export OPENAI_API_KEY="${DEEPSEEK_API_KEY:-dummy-key}" + +echo "=============================================" +echo " DeepSeek (BigCodeBench) Evaluation" +echo " Plugins: $PLUGINS | Limit: $LIMIT" +echo "=============================================" + +echo "[1/3] Booting ContextPilot Proxy Server..." +python -m contextpilot.server.http_server --port 8000 --infer-api-url "https://api.deepseek.com" > proxy_deepseek_bigcodebench.log 2>&1 & +PROXY_PID=$! +sleep 5 + +echo "[2/3] Running Python Evaluation Script..." +python evaluation/benchmarks/run_bigcodebench.py \ + --model deepseek-v4-pro \ + --api_base "https://api.deepseek.com/v1" \ + --api_key "$OPENAI_API_KEY" \ + --concurrency 20 \ + --limit "$LIMIT" \ + --plugins "$PLUGINS" \ + --eval_mode all + +echo "Shutting down Proxy Server to flush telemetry..." +kill -INT $PROXY_PID +sleep 2 +cat proxy_deepseek_bigcodebench.log + +echo "[3/3] Sandbox Evaluation..." +cp evaluation/benchmarks/results_with_plugin_deepseek-v4-pro.jsonl evaluation/benchmarks/elm_samples_full.jsonl +cd evaluation/benchmarks && bash run_sandbox_eval_full.sh +cd ../.. + +echo "Pipeline Complete!" diff --git a/eval_deepseek_mcpatlas.sh b/eval_deepseek_mcpatlas.sh new file mode 100755 index 0000000..461dadb --- /dev/null +++ b/eval_deepseek_mcpatlas.sh @@ -0,0 +1,44 @@ +#!/bin/bash +export PYTHONPATH="$(pwd):$PYTHONPATH" +export NO_PROXY="localhost,127.0.0.1,::1" +export no_proxy="localhost,127.0.0.1,::1" + +LIMIT=0 +PLUGINS="all" + +while [[ $# -gt 0 ]]; do + case $1 in + --limit) LIMIT="$2"; shift 2 ;; + --plugins) PLUGINS="$2"; shift 2 ;; + *) echo "Unknown parameter: $1"; exit 1 ;; + esac +done + +export OPENAI_API_KEY="${DEEPSEEK_API_KEY:-dummy-key}" + +echo "=============================================" +echo " DeepSeek (MCP-Atlas) Evaluation" +echo " Plugins: $PLUGINS | Limit: $LIMIT" +echo "=============================================" + +echo "[1/2] Booting ContextPilot Proxy Server..." +python -m contextpilot.server.http_server --port 8000 --infer-api-url "https://api.deepseek.com" > proxy_deepseek_mcpatlas.log 2>&1 & +PROXY_PID=$! +sleep 5 + +echo "[2/2] Running Python Evaluation Script..." +python evaluation/benchmarks/run_mcpatlas.py \ + --model deepseek-v4-pro \ + --api_base "https://api.deepseek.com/v1" \ + --api_key "$OPENAI_API_KEY" \ + --concurrency 20 \ + --limit "$LIMIT" \ + --plugins "$PLUGINS" \ + --eval_mode all + +echo "Shutting down Proxy Server to flush telemetry..." +kill -INT $PROXY_PID +sleep 2 +cat proxy_deepseek_mcpatlas.log + +echo "Pipeline Complete!" diff --git a/eval_local_vram_quantization.sh b/eval_local_vram_quantization.sh new file mode 100755 index 0000000..1cfce7f --- /dev/null +++ b/eval_local_vram_quantization.sh @@ -0,0 +1,60 @@ +#!/bin/bash +export PYTHONPATH="$(pwd):$PYTHONPATH" +export NO_PROXY="localhost,127.0.0.1,::1" +export no_proxy="localhost,127.0.0.1,::1" + +LIMIT=0 +while [[ $# -gt 0 ]]; do + case $1 in + --limit) LIMIT="$2"; shift 2 ;; + *) echo "Unknown parameter: $1"; exit 1 ;; + esac +done + +echo "==========================================================" +echo " ContextPilot Quantized AWQ Evaluation (Local vLLM)" +echo " Limit: $LIMIT" +echo "==========================================================" + +export TMPDIR="$(pwd)/tmp" +export TMP="$(pwd)/tmp" +mkdir -p "$TMPDIR" + +PORT=$(shuf -i 15000-20000 -n 1) +MODEL_NAME="Qwen/Qwen2.5-7B-Instruct-AWQ" + +echo "[1/3] Booting Local vLLM Engine on port $PORT..." +python -m vllm.entrypoints.openai.api_server \ + --model "$MODEL_NAME" \ + --port "$PORT" \ + --gpu-memory-utilization 0.8 \ + --quantization awq \ + --max-model-len 4096 \ + --enforce-eager \ + --enable-auto-tool-choice \ + --tool-call-parser hermes \ + > vllm_quantized.log 2>&1 & +VLLM_PID=$! +sleep 30 # Give vLLM time to load weights into VRAM + +echo "[2/3] Booting ContextPilot Proxy Server..." +python -m contextpilot.server.http_server --port 8000 --infer-api-url "http://localhost:$PORT" > proxy_quantized.log 2>&1 & +PROXY_PID=$! +sleep 5 + +echo "[3/3] Running Python Evaluation Script..." +python evaluation/benchmarks/run_mcpatlas.py \ + --model "$MODEL_NAME" \ + --api_base "http://localhost:$PORT/v1" \ + --concurrency 5 \ + --limit "$LIMIT" \ + --plugins "all" \ + --eval_mode all + +echo "Shutting down services..." +kill -INT $PROXY_PID +kill -INT $VLLM_PID +sleep 2 + +cat proxy_quantized.log +echo "Pipeline Complete!" diff --git a/eval_openai_bigcodebench.sh b/eval_openai_bigcodebench.sh new file mode 100755 index 0000000..5ca3e2c --- /dev/null +++ b/eval_openai_bigcodebench.sh @@ -0,0 +1,46 @@ +#!/bin/bash +export PYTHONPATH="$(pwd):$PYTHONPATH" +export NO_PROXY="localhost,127.0.0.1,::1" +export no_proxy="localhost,127.0.0.1,::1" + +LIMIT=0 +PLUGINS="all" + +while [[ $# -gt 0 ]]; do + case $1 in + --limit) LIMIT="$2"; shift 2 ;; + --plugins) PLUGINS="$2"; shift 2 ;; + *) echo "Unknown parameter: $1"; exit 1 ;; + esac +done + +echo "=============================================" +echo " OpenAI (BigCodeBench) Evaluation" +echo " Plugins: $PLUGINS | Limit: $LIMIT" +echo "=============================================" + +echo "[1/3] Booting ContextPilot Proxy Server..." +python -m contextpilot.server.http_server --port 8000 --infer-api-url "https://api.openai.com" > proxy_openai_bigcodebench.log 2>&1 & +PROXY_PID=$! +sleep 5 + +echo "[2/3] Running Python Evaluation Script..." +python evaluation/benchmarks/run_bigcodebench.py \ + --model gpt-5.5 \ + --api_base "https://api.openai.com/v1" \ + --concurrency 5 \ + --limit "$LIMIT" \ + --plugins "$PLUGINS" \ + --eval_mode all + +echo "Shutting down Proxy Server to flush telemetry..." +kill -INT $PROXY_PID +sleep 2 +cat proxy_openai_bigcodebench.log + +echo "[3/3] Sandbox Evaluation..." +cp evaluation/benchmarks/results_with_plugin_gpt-5.5.jsonl evaluation/benchmarks/elm_samples_full.jsonl +cd evaluation/benchmarks && bash run_sandbox_eval_full.sh +cd ../.. + +echo "Pipeline Complete!" diff --git a/eval_openai_mcpatlas.sh b/eval_openai_mcpatlas.sh new file mode 100755 index 0000000..a21b065 --- /dev/null +++ b/eval_openai_mcpatlas.sh @@ -0,0 +1,41 @@ +#!/bin/bash +export PYTHONPATH="$(pwd):$PYTHONPATH" +export NO_PROXY="localhost,127.0.0.1,::1" +export no_proxy="localhost,127.0.0.1,::1" + +LIMIT=0 +PLUGINS="all" + +while [[ $# -gt 0 ]]; do + case $1 in + --limit) LIMIT="$2"; shift 2 ;; + --plugins) PLUGINS="$2"; shift 2 ;; + *) echo "Unknown parameter: $1"; exit 1 ;; + esac +done + +echo "=============================================" +echo " OpenAI (MCP-Atlas) Evaluation" +echo " Plugins: $PLUGINS | Limit: $LIMIT" +echo "=============================================" + +echo "[1/2] Booting ContextPilot Proxy Server..." +python -m contextpilot.server.http_server --port 8000 --infer-api-url "https://api.openai.com" > proxy_openai_mcpatlas.log 2>&1 & +PROXY_PID=$! +sleep 5 + +echo "[2/2] Running Python Evaluation Script..." +python evaluation/benchmarks/run_mcpatlas.py \ + --model gpt-5.5 \ + --api_base "https://api.openai.com/v1" \ + --concurrency 5 \ + --limit "$LIMIT" \ + --plugins "$PLUGINS" \ + --eval_mode all + +echo "Shutting down Proxy Server to flush telemetry..." +kill -INT $PROXY_PID +sleep 2 +cat proxy_openai_mcpatlas.log + +echo "Pipeline Complete!" diff --git a/evaluation/benchmark_data/baseline_metrics.json b/evaluation/benchmark_data/baseline_metrics.json new file mode 100644 index 0000000..4e9f6a6 --- /dev/null +++ b/evaluation/benchmark_data/baseline_metrics.json @@ -0,0 +1,4 @@ +{ + "peak_gpu_cache_usage_perc": 0.0019054971951409927, + "peak_gpu_cache_usage_human": "0.19%" +} \ No newline at end of file diff --git a/evaluation/benchmark_data/metrics_dump.txt b/evaluation/benchmark_data/metrics_dump.txt new file mode 100644 index 0000000..a0094f8 --- /dev/null +++ b/evaluation/benchmark_data/metrics_dump.txt @@ -0,0 +1,605 @@ +# HELP python_gc_objects_collected_total Objects collected during gc +# TYPE python_gc_objects_collected_total counter +python_gc_objects_collected_total{generation="0"} 17065.0 +python_gc_objects_collected_total{generation="1"} 1837.0 +python_gc_objects_collected_total{generation="2"} 1217.0 +# HELP python_gc_objects_uncollectable_total Uncollectable objects found during GC +# TYPE python_gc_objects_uncollectable_total counter +python_gc_objects_uncollectable_total{generation="0"} 0.0 +python_gc_objects_uncollectable_total{generation="1"} 0.0 +python_gc_objects_uncollectable_total{generation="2"} 0.0 +# HELP python_gc_collections_total Number of times this generation was collected +# TYPE python_gc_collections_total counter +python_gc_collections_total{generation="0"} 2109.0 +python_gc_collections_total{generation="1"} 191.0 +python_gc_collections_total{generation="2"} 10.0 +# HELP python_info Python platform information +# TYPE python_info gauge +python_info{implementation="CPython",major="3",minor="12",patchlevel="3",version="3.12.3"} 1.0 +# HELP process_virtual_memory_bytes Virtual memory size in bytes. +# TYPE process_virtual_memory_bytes gauge +process_virtual_memory_bytes 7.79462656e+09 +# HELP process_resident_memory_bytes Resident memory size in bytes. +# TYPE process_resident_memory_bytes gauge +process_resident_memory_bytes 1.475756032e+09 +# HELP process_start_time_seconds Start time of the process since unix epoch in seconds. +# TYPE process_start_time_seconds gauge +process_start_time_seconds 1.78385584263e+09 +# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds. +# TYPE process_cpu_seconds_total counter +process_cpu_seconds_total 20.43 +# HELP process_open_fds Number of open file descriptors. +# TYPE process_open_fds gauge +process_open_fds 47.0 +# HELP process_max_fds Maximum number of open file descriptors. +# TYPE process_max_fds gauge +process_max_fds 131072.0 +# HELP vllm:estimated_flops_per_gpu_total Estimated number of floating point operations per GPU (for Model Flops Utilization calculations). +# TYPE vllm:estimated_flops_per_gpu_total counter +vllm:estimated_flops_per_gpu_total{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +# HELP vllm:estimated_flops_per_gpu_created Estimated number of floating point operations per GPU (for Model Flops Utilization calculations). +# TYPE vllm:estimated_flops_per_gpu_created gauge +vllm:estimated_flops_per_gpu_created{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.7838559278043773e+09 +# HELP vllm:estimated_read_bytes_per_gpu_total Estimated number of bytes read from memory per GPU (for Model Flops Utilization calculations). +# TYPE vllm:estimated_read_bytes_per_gpu_total counter +vllm:estimated_read_bytes_per_gpu_total{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +# HELP vllm:estimated_read_bytes_per_gpu_created Estimated number of bytes read from memory per GPU (for Model Flops Utilization calculations). +# TYPE vllm:estimated_read_bytes_per_gpu_created gauge +vllm:estimated_read_bytes_per_gpu_created{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.7838559278044033e+09 +# HELP vllm:estimated_write_bytes_per_gpu_total Estimated number of bytes written to memory per GPU (for Model Flops Utilization calculations). +# TYPE vllm:estimated_write_bytes_per_gpu_total counter +vllm:estimated_write_bytes_per_gpu_total{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +# HELP vllm:estimated_write_bytes_per_gpu_created Estimated number of bytes written to memory per GPU (for Model Flops Utilization calculations). +# TYPE vllm:estimated_write_bytes_per_gpu_created gauge +vllm:estimated_write_bytes_per_gpu_created{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.7838559278044224e+09 +# HELP vllm:num_requests_running Number of requests in model execution batches. +# TYPE vllm:num_requests_running gauge +vllm:num_requests_running{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +# HELP vllm:num_requests_waiting Number of requests waiting to be processed. +# TYPE vllm:num_requests_waiting gauge +vllm:num_requests_waiting{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +# HELP vllm:num_requests_waiting_by_reason Number of waiting requests by reason. Reason labels: 'capacity' = waiting for scheduling capacity; 'deferred' = deferred by transient constraints (LoRA budget, KV transfer, blocked status). Sum of all reasons equals vllm:num_requests_waiting. +# TYPE vllm:num_requests_waiting_by_reason gauge +vllm:num_requests_waiting_by_reason{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",reason="capacity"} 0.0 +vllm:num_requests_waiting_by_reason{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",reason="deferred"} 0.0 +# HELP vllm:engine_sleep_state Engine sleep state; awake = 0 means engine is sleeping; awake = 1 means engine is awake; weights_offloaded = 1 means sleep level 1; discard_all = 1 means sleep level 2. +# TYPE vllm:engine_sleep_state gauge +vllm:engine_sleep_state{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",sleep_state="awake"} 1.0 +vllm:engine_sleep_state{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",sleep_state="weights_offloaded"} 0.0 +vllm:engine_sleep_state{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",sleep_state="discard_all"} 0.0 +# HELP vllm:kv_cache_usage_perc KV-cache usage. 1 means 100 percent usage. +# TYPE vllm:kv_cache_usage_perc gauge +vllm:kv_cache_usage_perc{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +# HELP vllm:prefix_cache_queries_total Prefix cache queries, in terms of number of queried tokens. +# TYPE vllm:prefix_cache_queries_total counter +vllm:prefix_cache_queries_total{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 24824.0 +# HELP vllm:prefix_cache_queries_created Prefix cache queries, in terms of number of queried tokens. +# TYPE vllm:prefix_cache_queries_created gauge +vllm:prefix_cache_queries_created{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.783855927807422e+09 +# HELP vllm:prefix_cache_hits_total Prefix cache hits, in terms of number of cached tokens. +# TYPE vllm:prefix_cache_hits_total counter +vllm:prefix_cache_hits_total{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 6848.0 +# HELP vllm:prefix_cache_hits_created Prefix cache hits, in terms of number of cached tokens. +# TYPE vllm:prefix_cache_hits_created gauge +vllm:prefix_cache_hits_created{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.783855927807437e+09 +# HELP vllm:external_prefix_cache_queries_total External prefix cache queries from KV connector cross-instance cache sharing, in terms of number of queried tokens. +# TYPE vllm:external_prefix_cache_queries_total counter +vllm:external_prefix_cache_queries_total{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +# HELP vllm:external_prefix_cache_queries_created External prefix cache queries from KV connector cross-instance cache sharing, in terms of number of queried tokens. +# TYPE vllm:external_prefix_cache_queries_created gauge +vllm:external_prefix_cache_queries_created{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.7838559278074527e+09 +# HELP vllm:external_prefix_cache_hits_total External prefix cache hits from KV connector cross-instance cache sharing, in terms of number of cached tokens. +# TYPE vllm:external_prefix_cache_hits_total counter +vllm:external_prefix_cache_hits_total{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +# HELP vllm:external_prefix_cache_hits_created External prefix cache hits from KV connector cross-instance cache sharing, in terms of number of cached tokens. +# TYPE vllm:external_prefix_cache_hits_created gauge +vllm:external_prefix_cache_hits_created{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.7838559278074691e+09 +# HELP vllm:mm_cache_queries_total Multi-modal cache queries, in terms of number of queried items. +# TYPE vllm:mm_cache_queries_total counter +vllm:mm_cache_queries_total{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +# HELP vllm:mm_cache_queries_created Multi-modal cache queries, in terms of number of queried items. +# TYPE vllm:mm_cache_queries_created gauge +vllm:mm_cache_queries_created{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.7838559278074834e+09 +# HELP vllm:mm_cache_hits_total Multi-modal cache hits, in terms of number of cached items. +# TYPE vllm:mm_cache_hits_total counter +vllm:mm_cache_hits_total{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +# HELP vllm:mm_cache_hits_created Multi-modal cache hits, in terms of number of cached items. +# TYPE vllm:mm_cache_hits_created gauge +vllm:mm_cache_hits_created{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.7838559278074975e+09 +# HELP vllm:num_preemptions_total Cumulative number of preemption from the engine. +# TYPE vllm:num_preemptions_total counter +vllm:num_preemptions_total{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +# HELP vllm:num_preemptions_created Cumulative number of preemption from the engine. +# TYPE vllm:num_preemptions_created gauge +vllm:num_preemptions_created{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.7838559278075118e+09 +# HELP vllm:prompt_tokens_total Number of prefill tokens processed. +# TYPE vllm:prompt_tokens_total counter +vllm:prompt_tokens_total{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 24824.0 +# HELP vllm:prompt_tokens_created Number of prefill tokens processed. +# TYPE vllm:prompt_tokens_created gauge +vllm:prompt_tokens_created{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.783855927807526e+09 +# HELP vllm:prompt_tokens_by_source_total Number of prompt tokens by source. +# TYPE vllm:prompt_tokens_by_source_total counter +vllm:prompt_tokens_by_source_total{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",source="local_compute"} 17976.0 +vllm:prompt_tokens_by_source_total{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",source="local_cache_hit"} 6848.0 +vllm:prompt_tokens_by_source_total{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",source="external_kv_transfer"} 0.0 +# HELP vllm:prompt_tokens_by_source_created Number of prompt tokens by source. +# TYPE vllm:prompt_tokens_by_source_created gauge +vllm:prompt_tokens_by_source_created{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",source="local_compute"} 1.7838559278075457e+09 +vllm:prompt_tokens_by_source_created{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",source="local_cache_hit"} 1.783855927807553e+09 +vllm:prompt_tokens_by_source_created{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",source="external_kv_transfer"} 1.78385592780756e+09 +# HELP vllm:prompt_tokens_cached_total Number of cached prompt tokens (local + external). +# TYPE vllm:prompt_tokens_cached_total counter +vllm:prompt_tokens_cached_total{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 6848.0 +# HELP vllm:prompt_tokens_cached_created Number of cached prompt tokens (local + external). +# TYPE vllm:prompt_tokens_cached_created gauge +vllm:prompt_tokens_cached_created{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.783855927807574e+09 +# HELP vllm:generation_tokens_total Number of generation tokens processed. +# TYPE vllm:generation_tokens_total counter +vllm:generation_tokens_total{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 5121.0 +# HELP vllm:generation_tokens_created Number of generation tokens processed. +# TYPE vllm:generation_tokens_created gauge +vllm:generation_tokens_created{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.783855927807589e+09 +# HELP vllm:request_success_total Count of successfully processed requests. +# TYPE vllm:request_success_total counter +vllm:request_success_total{engine="0",finished_reason="stop",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_success_total{engine="0",finished_reason="length",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_success_total{engine="0",finished_reason="abort",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_success_total{engine="0",finished_reason="error",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_success_total{engine="0",finished_reason="repetition",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +# HELP vllm:request_success_created Count of successfully processed requests. +# TYPE vllm:request_success_created gauge +vllm:request_success_created{engine="0",finished_reason="stop",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.7838559278076181e+09 +vllm:request_success_created{engine="0",finished_reason="length",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.7838559278076267e+09 +vllm:request_success_created{engine="0",finished_reason="abort",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.7838559278076344e+09 +vllm:request_success_created{engine="0",finished_reason="error",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.783855927807641e+09 +vllm:request_success_created{engine="0",finished_reason="repetition",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.7838559278076499e+09 +# HELP vllm:request_prompt_tokens Number of prefill tokens processed. +# TYPE vllm:request_prompt_tokens histogram +vllm:request_prompt_tokens_bucket{engine="0",le="1.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_prompt_tokens_bucket{engine="0",le="2.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_prompt_tokens_bucket{engine="0",le="5.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_prompt_tokens_bucket{engine="0",le="10.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_prompt_tokens_bucket{engine="0",le="20.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_prompt_tokens_bucket{engine="0",le="50.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_prompt_tokens_bucket{engine="0",le="100.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_prompt_tokens_bucket{engine="0",le="200.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 11.0 +vllm:request_prompt_tokens_bucket{engine="0",le="500.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 99.0 +vllm:request_prompt_tokens_bucket{engine="0",le="1000.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_prompt_tokens_bucket{engine="0",le="2000.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_prompt_tokens_bucket{engine="0",le="+Inf",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_prompt_tokens_count{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_prompt_tokens_sum{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 24824.0 +# HELP vllm:request_prompt_tokens_created Number of prefill tokens processed. +# TYPE vllm:request_prompt_tokens_created gauge +vllm:request_prompt_tokens_created{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.7838559278076909e+09 +# HELP vllm:request_generation_tokens Number of generation tokens processed. +# TYPE vllm:request_generation_tokens histogram +vllm:request_generation_tokens_bucket{engine="0",le="1.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_generation_tokens_bucket{engine="0",le="2.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_generation_tokens_bucket{engine="0",le="5.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_generation_tokens_bucket{engine="0",le="10.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_generation_tokens_bucket{engine="0",le="20.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 3.0 +vllm:request_generation_tokens_bucket{engine="0",le="50.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 65.0 +vllm:request_generation_tokens_bucket{engine="0",le="100.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 96.0 +vllm:request_generation_tokens_bucket{engine="0",le="200.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 98.0 +vllm:request_generation_tokens_bucket{engine="0",le="500.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_generation_tokens_bucket{engine="0",le="1000.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_generation_tokens_bucket{engine="0",le="2000.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_generation_tokens_bucket{engine="0",le="+Inf",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_generation_tokens_count{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_generation_tokens_sum{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 5121.0 +# HELP vllm:request_generation_tokens_created Number of generation tokens processed. +# TYPE vllm:request_generation_tokens_created gauge +vllm:request_generation_tokens_created{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.7838559278077445e+09 +# HELP vllm:iteration_tokens_total Histogram of number of tokens per engine_step. +# TYPE vllm:iteration_tokens_total histogram +vllm:iteration_tokens_total_bucket{engine="0",le="1.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 32.0 +vllm:iteration_tokens_total_bucket{engine="0",le="8.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1015.0 +vllm:iteration_tokens_total_bucket{engine="0",le="16.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1015.0 +vllm:iteration_tokens_total_bucket{engine="0",le="32.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1016.0 +vllm:iteration_tokens_total_bucket{engine="0",le="64.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1019.0 +vllm:iteration_tokens_total_bucket{engine="0",le="128.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1020.0 +vllm:iteration_tokens_total_bucket{engine="0",le="256.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1099.0 +vllm:iteration_tokens_total_bucket{engine="0",le="512.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1109.0 +vllm:iteration_tokens_total_bucket{engine="0",le="1024.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1109.0 +vllm:iteration_tokens_total_bucket{engine="0",le="2048.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1110.0 +vllm:iteration_tokens_total_bucket{engine="0",le="4096.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1110.0 +vllm:iteration_tokens_total_bucket{engine="0",le="8192.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1110.0 +vllm:iteration_tokens_total_bucket{engine="0",le="16384.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1110.0 +vllm:iteration_tokens_total_bucket{engine="0",le="+Inf",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1110.0 +vllm:iteration_tokens_total_count{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1110.0 +vllm:iteration_tokens_total_sum{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 23097.0 +# HELP vllm:iteration_tokens_total_created Histogram of number of tokens per engine_step. +# TYPE vllm:iteration_tokens_total_created gauge +vllm:iteration_tokens_total_created{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.78385592780778e+09 +# HELP vllm:request_max_num_generation_tokens Histogram of maximum number of requested generation tokens. +# TYPE vllm:request_max_num_generation_tokens histogram +vllm:request_max_num_generation_tokens_bucket{engine="0",le="1.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_max_num_generation_tokens_bucket{engine="0",le="2.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_max_num_generation_tokens_bucket{engine="0",le="5.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_max_num_generation_tokens_bucket{engine="0",le="10.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_max_num_generation_tokens_bucket{engine="0",le="20.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 3.0 +vllm:request_max_num_generation_tokens_bucket{engine="0",le="50.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 65.0 +vllm:request_max_num_generation_tokens_bucket{engine="0",le="100.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 96.0 +vllm:request_max_num_generation_tokens_bucket{engine="0",le="200.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 98.0 +vllm:request_max_num_generation_tokens_bucket{engine="0",le="500.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_max_num_generation_tokens_bucket{engine="0",le="1000.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_max_num_generation_tokens_bucket{engine="0",le="2000.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_max_num_generation_tokens_bucket{engine="0",le="+Inf",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_max_num_generation_tokens_count{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_max_num_generation_tokens_sum{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 5121.0 +# HELP vllm:request_max_num_generation_tokens_created Histogram of maximum number of requested generation tokens. +# TYPE vllm:request_max_num_generation_tokens_created gauge +vllm:request_max_num_generation_tokens_created{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.7838559278078184e+09 +# HELP vllm:request_params_n Histogram of the n request parameter. +# TYPE vllm:request_params_n histogram +vllm:request_params_n_bucket{engine="0",le="1.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_params_n_bucket{engine="0",le="2.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_params_n_bucket{engine="0",le="5.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_params_n_bucket{engine="0",le="10.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_params_n_bucket{engine="0",le="20.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_params_n_bucket{engine="0",le="+Inf",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_params_n_count{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_params_n_sum{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +# HELP vllm:request_params_n_created Histogram of the n request parameter. +# TYPE vllm:request_params_n_created gauge +vllm:request_params_n_created{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.783855927807849e+09 +# HELP vllm:request_params_max_tokens Histogram of the max_tokens request parameter. +# TYPE vllm:request_params_max_tokens histogram +vllm:request_params_max_tokens_bucket{engine="0",le="1.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_params_max_tokens_bucket{engine="0",le="2.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_params_max_tokens_bucket{engine="0",le="5.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_params_max_tokens_bucket{engine="0",le="10.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_params_max_tokens_bucket{engine="0",le="20.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_params_max_tokens_bucket{engine="0",le="50.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_params_max_tokens_bucket{engine="0",le="100.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_params_max_tokens_bucket{engine="0",le="200.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_params_max_tokens_bucket{engine="0",le="500.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_params_max_tokens_bucket{engine="0",le="1000.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_params_max_tokens_bucket{engine="0",le="2000.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_params_max_tokens_bucket{engine="0",le="+Inf",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_params_max_tokens_count{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_params_max_tokens_sum{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 384776.0 +# HELP vllm:request_params_max_tokens_created Histogram of the max_tokens request parameter. +# TYPE vllm:request_params_max_tokens_created gauge +vllm:request_params_max_tokens_created{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.7838559278078763e+09 +# HELP vllm:time_to_first_token_seconds Histogram of time to first token in seconds. +# TYPE vllm:time_to_first_token_seconds histogram +vllm:time_to_first_token_seconds_bucket{engine="0",le="0.001",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="0.005",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="0.01",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="0.02",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="0.04",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 95.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="0.06",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 95.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="0.08",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 95.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="0.1",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 95.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="0.25",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="0.5",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="0.75",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="1.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="2.5",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="5.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="7.5",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="10.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="20.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="40.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="80.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="160.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="640.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="2560.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="+Inf",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:time_to_first_token_seconds_count{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:time_to_first_token_seconds_sum{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 3.496826410293579 +# HELP vllm:time_to_first_token_seconds_created Histogram of time to first token in seconds. +# TYPE vllm:time_to_first_token_seconds_created gauge +vllm:time_to_first_token_seconds_created{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.783855927807912e+09 +# HELP vllm:inter_token_latency_seconds Histogram of inter-token latency in seconds. +# TYPE vllm:inter_token_latency_seconds histogram +vllm:inter_token_latency_seconds_bucket{engine="0",le="0.01",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 3732.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="0.025",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 5021.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="0.05",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 5021.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="0.075",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 5021.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="0.1",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 5021.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="0.15",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 5021.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="0.2",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 5021.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="0.3",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 5021.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="0.4",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 5021.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="0.5",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 5021.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="0.75",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 5021.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="1.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 5021.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="2.5",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 5021.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="5.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 5021.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="7.5",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 5021.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="10.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 5021.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="20.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 5021.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="40.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 5021.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="80.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 5021.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="+Inf",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 5021.0 +vllm:inter_token_latency_seconds_count{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 5021.0 +vllm:inter_token_latency_seconds_sum{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 49.891806724946946 +# HELP vllm:inter_token_latency_seconds_created Histogram of inter-token latency in seconds. +# TYPE vllm:inter_token_latency_seconds_created gauge +vllm:inter_token_latency_seconds_created{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.7838559278079534e+09 +# HELP vllm:request_time_per_output_token_seconds Histogram of time_per_output_token_seconds per request. +# TYPE vllm:request_time_per_output_token_seconds histogram +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.01",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 70.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.025",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.05",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.075",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.1",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.15",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.2",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.3",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.4",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.5",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.75",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="1.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="2.5",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="5.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="7.5",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="10.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="20.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="40.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="80.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="+Inf",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_time_per_output_token_seconds_count{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_time_per_output_token_seconds_sum{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.9928265589591149 +# HELP vllm:request_time_per_output_token_seconds_created Histogram of time_per_output_token_seconds per request. +# TYPE vllm:request_time_per_output_token_seconds_created gauge +vllm:request_time_per_output_token_seconds_created{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.7838559278079915e+09 +# HELP vllm:e2e_request_latency_seconds Histogram of e2e request latency in seconds. +# TYPE vllm:e2e_request_latency_seconds histogram +vllm:e2e_request_latency_seconds_bucket{engine="0",le="0.3",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 25.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="0.5",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 56.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="0.8",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 89.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="1.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 96.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="1.5",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 98.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="2.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 98.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="2.5",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 98.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="5.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="10.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="15.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="20.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="30.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="40.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="50.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="60.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="120.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="240.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="480.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="960.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="1920.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="7680.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="+Inf",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:e2e_request_latency_seconds_count{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:e2e_request_latency_seconds_sum{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 53.38213324546814 +# HELP vllm:e2e_request_latency_seconds_created Histogram of e2e request latency in seconds. +# TYPE vllm:e2e_request_latency_seconds_created gauge +vllm:e2e_request_latency_seconds_created{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.7838559278080328e+09 +# HELP vllm:request_queue_time_seconds Histogram of time spent in WAITING phase for request. +# TYPE vllm:request_queue_time_seconds histogram +vllm:request_queue_time_seconds_bucket{engine="0",le="0.3",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="0.5",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="0.8",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="1.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="1.5",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="2.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="2.5",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="5.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="10.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="15.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="20.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="30.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="40.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="50.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="60.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="120.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="240.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="480.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="960.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="1920.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="7680.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="+Inf",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_queue_time_seconds_count{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_queue_time_seconds_sum{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0005439361557364464 +# HELP vllm:request_queue_time_seconds_created Histogram of time spent in WAITING phase for request. +# TYPE vllm:request_queue_time_seconds_created gauge +vllm:request_queue_time_seconds_created{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.7838559278080688e+09 +# HELP vllm:request_inference_time_seconds Histogram of time spent in RUNNING phase for request. +# TYPE vllm:request_inference_time_seconds histogram +vllm:request_inference_time_seconds_bucket{engine="0",le="0.3",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 27.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="0.5",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 58.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="0.8",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 89.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="1.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 96.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="1.5",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 98.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="2.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 98.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="2.5",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 98.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="5.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="10.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="15.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="20.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="30.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="40.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="50.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="60.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="120.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="240.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="480.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="960.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="1920.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="7680.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="+Inf",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_inference_time_seconds_count{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_inference_time_seconds_sum{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 52.23948321421631 +# HELP vllm:request_inference_time_seconds_created Histogram of time spent in RUNNING phase for request. +# TYPE vllm:request_inference_time_seconds_created gauge +vllm:request_inference_time_seconds_created{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.7838559278081043e+09 +# HELP vllm:request_prefill_time_seconds Histogram of time spent in PREFILL phase for request. +# TYPE vllm:request_prefill_time_seconds histogram +vllm:request_prefill_time_seconds_bucket{engine="0",le="0.3",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="0.5",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="0.8",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="1.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="1.5",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="2.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="2.5",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="5.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="10.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="15.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="20.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="30.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="40.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="50.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="60.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="120.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="240.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="480.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="960.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="1920.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="7680.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="+Inf",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_prefill_time_seconds_count{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_prefill_time_seconds_sum{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 2.347676489269361 +# HELP vllm:request_prefill_time_seconds_created Histogram of time spent in PREFILL phase for request. +# TYPE vllm:request_prefill_time_seconds_created gauge +vllm:request_prefill_time_seconds_created{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.7838559278081868e+09 +# HELP vllm:request_decode_time_seconds Histogram of time spent in DECODE phase for request. +# TYPE vllm:request_decode_time_seconds histogram +vllm:request_decode_time_seconds_bucket{engine="0",le="0.3",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 31.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="0.5",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 65.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="0.8",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 89.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="1.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 96.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="1.5",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 98.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="2.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 98.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="2.5",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 99.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="5.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="10.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="15.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="20.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="30.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="40.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="50.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="60.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="120.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="240.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="480.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="960.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="1920.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="7680.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="+Inf",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_decode_time_seconds_count{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_decode_time_seconds_sum{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 49.891806724946946 +# HELP vllm:request_decode_time_seconds_created Histogram of time spent in DECODE phase for request. +# TYPE vllm:request_decode_time_seconds_created gauge +vllm:request_decode_time_seconds_created{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.7838559278082242e+09 +# HELP vllm:request_prefill_kv_computed_tokens Histogram of new KV tokens computed during prefill (excluding cached tokens). +# TYPE vllm:request_prefill_kv_computed_tokens histogram +vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="1.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="2.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="5.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="10.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="20.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.0 +vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="50.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 4.0 +vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="100.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 4.0 +vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="200.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 75.0 +vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="500.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 99.0 +vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="1000.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="2000.0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="+Inf",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_prefill_kv_computed_tokens_count{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 100.0 +vllm:request_prefill_kv_computed_tokens_sum{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 17976.0 +# HELP vllm:request_prefill_kv_computed_tokens_created Histogram of new KV tokens computed during prefill (excluding cached tokens). +# TYPE vllm:request_prefill_kv_computed_tokens_created gauge +vllm:request_prefill_kv_computed_tokens_created{engine="0",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 1.7838559278082678e+09 +# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig +# TYPE vllm:cache_config_info gauge +vllm:cache_config_info{_block_size_resolved="True",block_size="16",cache_dtype="auto",calculate_kv_scales="False",enable_prefix_caching="True",engine="0",gpu_memory_utilization="0.8",hash_block_size="None",is_attention_free="False",kv_cache_dtype_skip_layers="[]",kv_cache_max_concurrency="475.6015625",kv_cache_memory_bytes="None",kv_cache_size_tokens="1948064",kv_offloading_backend="native",kv_offloading_size="None",kv_sharing_fast_prefill="False",mamba_block_size="None",mamba_cache_dtype="auto",mamba_cache_mode="none",mamba_page_size_padded="None",mamba_ssm_cache_dtype="auto",num_cpu_blocks="None",num_gpu_blocks="121754",num_gpu_blocks_override="None",prefix_caching_hash_algo="sha256",sliding_window="None",user_specified_block_size="False",user_specified_mamba_block_size="False"} 1.0 +# HELP http_requests_total Total number of requests by method, status and handler. +# TYPE http_requests_total counter +http_requests_total{handler="/v1/chat/completions",method="POST",status="2xx"} 100.0 +# HELP http_requests_created Total number of requests by method, status and handler. +# TYPE http_requests_created gauge +http_requests_created{handler="/v1/chat/completions",method="POST",status="2xx"} 1.7838559408861754e+09 +# HELP http_request_size_bytes Content length of incoming requests by handler. Only value of header is respected. Otherwise ignored. No percentile calculated. +# TYPE http_request_size_bytes summary +http_request_size_bytes_count{handler="/v1/chat/completions"} 100.0 +http_request_size_bytes_sum{handler="/v1/chat/completions"} 77630.0 +# HELP http_request_size_bytes_created Content length of incoming requests by handler. Only value of header is respected. Otherwise ignored. No percentile calculated. +# TYPE http_request_size_bytes_created gauge +http_request_size_bytes_created{handler="/v1/chat/completions"} 1.783855940886198e+09 +# HELP http_response_size_bytes Content length of outgoing responses by handler. Only value of header is respected. Otherwise ignored. No percentile calculated. +# TYPE http_response_size_bytes summary +http_response_size_bytes_count{handler="/v1/chat/completions"} 100.0 +http_response_size_bytes_sum{handler="/v1/chat/completions"} 88359.0 +# HELP http_response_size_bytes_created Content length of outgoing responses by handler. Only value of header is respected. Otherwise ignored. No percentile calculated. +# TYPE http_response_size_bytes_created gauge +http_response_size_bytes_created{handler="/v1/chat/completions"} 1.7838559408862255e+09 +# HELP http_request_duration_highr_seconds Latency with many buckets but no API specific labels. Made for more accurate percentile calculations. +# TYPE http_request_duration_highr_seconds histogram +http_request_duration_highr_seconds_bucket{le="0.01"} 0.0 +http_request_duration_highr_seconds_bucket{le="0.025"} 0.0 +http_request_duration_highr_seconds_bucket{le="0.05"} 0.0 +http_request_duration_highr_seconds_bucket{le="0.075"} 0.0 +http_request_duration_highr_seconds_bucket{le="0.1"} 0.0 +http_request_duration_highr_seconds_bucket{le="0.25"} 13.0 +http_request_duration_highr_seconds_bucket{le="0.5"} 56.0 +http_request_duration_highr_seconds_bucket{le="0.75"} 84.0 +http_request_duration_highr_seconds_bucket{le="1.0"} 96.0 +http_request_duration_highr_seconds_bucket{le="1.5"} 98.0 +http_request_duration_highr_seconds_bucket{le="2.0"} 98.0 +http_request_duration_highr_seconds_bucket{le="2.5"} 98.0 +http_request_duration_highr_seconds_bucket{le="3.0"} 100.0 +http_request_duration_highr_seconds_bucket{le="3.5"} 100.0 +http_request_duration_highr_seconds_bucket{le="4.0"} 100.0 +http_request_duration_highr_seconds_bucket{le="4.5"} 100.0 +http_request_duration_highr_seconds_bucket{le="5.0"} 100.0 +http_request_duration_highr_seconds_bucket{le="7.5"} 100.0 +http_request_duration_highr_seconds_bucket{le="10.0"} 100.0 +http_request_duration_highr_seconds_bucket{le="30.0"} 100.0 +http_request_duration_highr_seconds_bucket{le="60.0"} 100.0 +http_request_duration_highr_seconds_bucket{le="+Inf"} 100.0 +http_request_duration_highr_seconds_count 100.0 +http_request_duration_highr_seconds_sum 53.51098665478639 +# HELP http_request_duration_highr_seconds_created Latency with many buckets but no API specific labels. Made for more accurate percentile calculations. +# TYPE http_request_duration_highr_seconds_created gauge +http_request_duration_highr_seconds_created 1.7838559281001766e+09 +# HELP http_request_duration_seconds Latency with only few buckets by handler. Made to be only used if aggregation by handler is important. +# TYPE http_request_duration_seconds histogram +http_request_duration_seconds_bucket{handler="/v1/chat/completions",le="0.1",method="POST"} 0.0 +http_request_duration_seconds_bucket{handler="/v1/chat/completions",le="0.5",method="POST"} 56.0 +http_request_duration_seconds_bucket{handler="/v1/chat/completions",le="1.0",method="POST"} 96.0 +http_request_duration_seconds_bucket{handler="/v1/chat/completions",le="+Inf",method="POST"} 100.0 +http_request_duration_seconds_count{handler="/v1/chat/completions",method="POST"} 100.0 +http_request_duration_seconds_sum{handler="/v1/chat/completions",method="POST"} 53.51098665478639 +# HELP http_request_duration_seconds_created Latency with only few buckets by handler. Made to be only used if aggregation by handler is important. +# TYPE http_request_duration_seconds_created gauge +http_request_duration_seconds_created{handler="/v1/chat/completions",method="POST"} 1.7838559408862545e+09 +# HELP vllm:tool_call_parser_invocations_total Total number of ToolParser invocations. Non-streaming increments once per choice; streaming increments once per delta. +# TYPE vllm:tool_call_parser_invocations_total counter +vllm:tool_call_parser_invocations_total{mode="streaming",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",outcome="tool_call",request_type="chat_completions"} 0.0 +vllm:tool_call_parser_invocations_total{mode="streaming",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",outcome="tool_call",request_type="responses"} 0.0 +vllm:tool_call_parser_invocations_total{mode="streaming",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",outcome="tool_call",request_type="other"} 0.0 +vllm:tool_call_parser_invocations_total{mode="streaming",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",outcome="no_tool_call",request_type="chat_completions"} 0.0 +vllm:tool_call_parser_invocations_total{mode="streaming",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",outcome="no_tool_call",request_type="responses"} 0.0 +vllm:tool_call_parser_invocations_total{mode="streaming",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",outcome="no_tool_call",request_type="other"} 0.0 +vllm:tool_call_parser_invocations_total{mode="non_streaming",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",outcome="tool_call",request_type="chat_completions"} 99.0 +vllm:tool_call_parser_invocations_total{mode="non_streaming",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",outcome="tool_call",request_type="responses"} 0.0 +vllm:tool_call_parser_invocations_total{mode="non_streaming",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",outcome="tool_call",request_type="other"} 0.0 +vllm:tool_call_parser_invocations_total{mode="non_streaming",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",outcome="no_tool_call",request_type="chat_completions"} 1.0 +vllm:tool_call_parser_invocations_total{mode="non_streaming",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",outcome="no_tool_call",request_type="responses"} 0.0 +vllm:tool_call_parser_invocations_total{mode="non_streaming",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",outcome="no_tool_call",request_type="other"} 0.0 +# HELP vllm:tool_call_parser_invocations_created Total number of ToolParser invocations. Non-streaming increments once per choice; streaming increments once per delta. +# TYPE vllm:tool_call_parser_invocations_created gauge +vllm:tool_call_parser_invocations_created{mode="streaming",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",outcome="tool_call",request_type="chat_completions"} 1.783855928100832e+09 +vllm:tool_call_parser_invocations_created{mode="streaming",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",outcome="tool_call",request_type="responses"} 1.7838559281008468e+09 +vllm:tool_call_parser_invocations_created{mode="streaming",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",outcome="tool_call",request_type="other"} 1.7838559281008568e+09 +vllm:tool_call_parser_invocations_created{mode="streaming",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",outcome="no_tool_call",request_type="chat_completions"} 1.783855928100866e+09 +vllm:tool_call_parser_invocations_created{mode="streaming",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",outcome="no_tool_call",request_type="responses"} 1.7838559281008773e+09 +vllm:tool_call_parser_invocations_created{mode="streaming",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",outcome="no_tool_call",request_type="other"} 1.7838559281008916e+09 +vllm:tool_call_parser_invocations_created{mode="non_streaming",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",outcome="tool_call",request_type="chat_completions"} 1.7838559281009007e+09 +vllm:tool_call_parser_invocations_created{mode="non_streaming",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",outcome="tool_call",request_type="responses"} 1.7838559281009085e+09 +vllm:tool_call_parser_invocations_created{mode="non_streaming",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",outcome="tool_call",request_type="other"} 1.7838559281009164e+09 +vllm:tool_call_parser_invocations_created{mode="non_streaming",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",outcome="no_tool_call",request_type="chat_completions"} 1.7838559281009247e+09 +vllm:tool_call_parser_invocations_created{mode="non_streaming",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",outcome="no_tool_call",request_type="responses"} 1.7838559281009333e+09 +vllm:tool_call_parser_invocations_created{mode="non_streaming",model_name="Qwen/Qwen2.5-7B-Instruct-AWQ",outcome="no_tool_call",request_type="other"} 1.7838559281009417e+09 diff --git a/evaluation/benchmark_data/plugin_metrics.json b/evaluation/benchmark_data/plugin_metrics.json new file mode 100644 index 0000000..0a3a61a --- /dev/null +++ b/evaluation/benchmark_data/plugin_metrics.json @@ -0,0 +1,4 @@ +{ + "peak_gpu_cache_usage_perc": 0.000936321897612391, + "peak_gpu_cache_usage_human": "0.09%" +} \ No newline at end of file diff --git a/evaluation/benchmarks/run_ablation_elm.py b/evaluation/benchmarks/run_ablation_elm.py new file mode 100644 index 0000000..8603ca4 --- /dev/null +++ b/evaluation/benchmarks/run_ablation_elm.py @@ -0,0 +1,195 @@ +import argparse +import asyncio +import json +import logging +import os +import re + +from datasets import load_dataset +from openai import AsyncOpenAI + +from refactored_plugins.dedup import ContextDedupPlugin +from refactored_plugins.dynamic_pruning import DynamicPruningPlugin +from refactored_plugins.skill_index import SkillAwareContextPlugin + +logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s") +logger = logging.getLogger(__name__) + +# Create a registry of 10 dummy tools to trigger the Skill plugin +DUMMY_TOOL_REGISTRY = { + f"tool_{i}": { + "type": "function", + "function": { + "name": f"tool_{i}", + "description": f"Dummy tool number {i}" + } + } + for i in range(1, 11) +} + +async def process_task(task, client, semaphore, output_file, turn_1_id, mode, model_name): + """ + Processes a single BigCodeBench task through our ELM API. + """ + async with semaphore: + task_id = task.get("task_id", "unknown_task") + prompt = task.get("complete_prompt", task.get("instruction", "No prompt found.")) + + # Mock heavy agent request with redundant history and bloated tools + request = { + "user_id": "evaluator_1", + "parent_id": turn_1_id, + "_required_skills": ["tool_1", "tool_3", "tool_7"], # Require only 3 tools out of 10 + "messages": [ + {"role": "system", "content": "You are a senior python developer. Always wrap your code in ```python blocks."}, + {"role": "user", "content": "Please help me write some code."}, + {"role": "assistant", "content": "Of course! I can help you with that."}, + {"role": "user", "content": prompt} + ], + "tools": list(DUMMY_TOOL_REGISTRY.values()) + } + if mode == "with_plugin": + # Send extra_body for ContextPilot proxy to intercept + request["_required_skills"] = ["tool_1", "tool_3", "tool_7"] + + # 1. Apply ALL THREE plugins on the client before sending + request = await dedup_plugin.process(request) + request = await dynamic_plugin.process(request) + request = await skill_plugin.process(request) + + api_kwargs = { + "model": model_name, + "messages": request["messages"], + "tools": request["tools"] + } + + api_kwargs["extra_body"] = { + "user_id": request.get("user_id"), + "parent_id": request.get("parent_id"), + "_required_skills": request.get("_required_skills") + } + else: + api_kwargs = { + "model": model_name, + "messages": request["messages"], + "tools": request["tools"] + } + + try: + logger.info(f"[{mode}] Sending task {task_id}...") + response = await client.chat.completions.create(**api_kwargs) + response_content = response.choices[0].message.content + except Exception as e: + logger.error(f"[{mode}] API Error for {task_id}: {str(e)}") + response_content = "" + + # Extract code block using regex + extracted_code = "" + if response_content: + match = re.search(r"```python\s*(.*?)\s*```", response_content, re.DOTALL) + if match: + extracted_code = match.group(1).strip() + else: + # Fallback if the LLM didn't use the markdown block + extracted_code = response_content.strip() + + # Append result to JSONL + with open(output_file, "a", encoding="utf-8") as f: + f.write(json.dumps({"task_id": task_id, "solution": extracted_code}) + "\n") + + logger.info(f"[{mode}] Finished {task_id}") + +async def run_evaluation(mode, args, tasks): + # Route BOTH baseline and with_plugin through the proxy + client = AsyncOpenAI(api_key=args.api_key, base_url="http://localhost:8000/v1") + + # We use a dummy turn_1_id for simulation + turn_1_id = "test-turn-1-id" + + output_file = os.path.join(os.path.dirname(__file__), f"results_ablation_{args.threshold}_{mode}_{args.model}.jsonl") + if os.path.exists(output_file): + os.remove(output_file) + + # Use configurable Semaphore to allow high concurrency + semaphore = asyncio.Semaphore(args.concurrency) + + # Instantiate plugins + global dedup_plugin, dynamic_plugin, skill_plugin + dedup_plugin = ContextDedupPlugin(shadow_mode=True) + dynamic_plugin = DynamicPruningPlugin(similarity_threshold=args.threshold) + skill_plugin = SkillAwareContextPlugin(DUMMY_TOOL_REGISTRY) + + # SEED THE TRACKER FOR TELEMETRY: + # Inject the "Turn 1" system prompt and history into the dedup plugin's memory. + turn_1_messages = [ + {"role": "system", "content": "You are a senior python developer. Always wrap your code in ```python blocks."}, + {"role": "user", "content": "Please help me write some code."}, + {"role": "assistant", "content": "Of course! I can help you with that."} + ] + msg_ids = [dedup_plugin._get_id(m["content"]) for m in turn_1_messages] + dedup_plugin.tracker.deduplicate(request_id=turn_1_id, docs=msg_ids, parent_request_id=None) + + coroutines = [process_task(t, client, semaphore, output_file, turn_1_id, mode, args.model) for t in tasks] + await asyncio.gather(*coroutines) + + print(f"\n=== Evaluation Complete for mode: {mode} (Threshold {args.threshold}) ===") + print(f"Results saved to {output_file}") + + if mode == "with_plugin": + print(f"\n=== ContextPilot Client Telemetry (ALL PLUGINS) [Threshold {args.threshold}] ===") + dedup_metrics = dedup_plugin.get_plugin_metrics() + dynamic_metrics = dynamic_plugin.get_plugin_metrics() + skill_metrics = skill_plugin.get_plugin_metrics() + + # Calculate theoretical stacked savings + total_orig = dynamic_metrics['total_original_chars'] + total_saved = dedup_metrics['total_chars_saved'] + dynamic_metrics['total_chars_saved'] + combined_pct = (total_saved / total_orig * 100) if total_orig > 0 else 0.0 + + print(f"[Dedup] Chars Saved: {dedup_metrics['total_chars_saved']} / {dedup_metrics['total_original_chars']} ({dedup_metrics['chars_saved_percentage']:.2f}%)") + print(f"[Dynamic Pruning] Chars Saved: {dynamic_metrics['total_chars_saved']} / {dynamic_metrics['total_original_chars']} ({dynamic_metrics['chars_saved_percentage']:.2f}%)") + print(f"[COMBINED THEORETICAL SAVINGS]: {total_saved} / {total_orig} ({combined_pct:.2f}%)") + print(f"[Skill] Tools Filtered: {skill_metrics['total_tools_filtered']} / {skill_metrics.get('total_original_tools', 'N/A')} ({skill_metrics.get('tools_filtered_percentage', 0):.2f}%)") + + +async def main(): + parser = argparse.ArgumentParser(description="BigCodeBench ELM API Runner (Ablation)") + parser.add_argument("--model", default="gpt-5.5", help="Model name to evaluate") + parser.add_argument("--api_base", default=os.environ.get("BASE_URL", "https://api.openai.com/v1"), help="Baseline ELM API Base URL") + parser.add_argument("--api_key", default=os.environ.get("OPENAI_API_KEY", "dummy-elm-key"), help="API Key") + parser.add_argument("--concurrency", type=int, default=1, help="Number of concurrent requests") + parser.add_argument("--limit", type=int, default=0, help="Limit number of tasks to run (0 for all)") + parser.add_argument("--eval_mode", choices=["baseline", "with_plugin", "all"], default="all", help="Evaluation mode") + parser.add_argument("--threshold", type=float, default=0.3, help="Similarity threshold for Dynamic Pruning") + args = parser.parse_args() + + # Load BigCodeBench dataset + logger.info("Loading BigCodeBench dataset...") + try: + dataset = load_dataset("bigcode/bigcodebench", split="train") + except Exception as e: + logger.warning(f"Failed to load split='train'. Trying standard default split. Error: {e}") + try: + dataset = load_dataset("bigcode/bigcodebench", split="v0.1.2") + except Exception: + dataset = load_dataset("bigcode/bigcodebench", split="v0.1.0_240822") + + # Select all tasks for full evaluation + tasks = list(dataset) + if args.limit > 0: + tasks = tasks[:args.limit] + logger.info(f"Loaded {len(tasks)} tasks (LIMITED) for evaluation.") + else: + logger.info(f"Loaded {len(tasks)} tasks for full evaluation.") + + if args.eval_mode == "all": + modes = ["baseline", "with_plugin"] + else: + modes = [args.eval_mode] + + for mode in modes: + logger.info(f"\n--- Starting Evaluation: {mode} (Threshold: {args.threshold}) ---") + await run_evaluation(mode, args, tasks) + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/evaluation/benchmarks/run_bigcodebench.py b/evaluation/benchmarks/run_bigcodebench.py new file mode 100644 index 0000000..b0bdb73 --- /dev/null +++ b/evaluation/benchmarks/run_bigcodebench.py @@ -0,0 +1,196 @@ +import argparse +import asyncio +import json +import logging +import os +import re + +from datasets import load_dataset +from openai import AsyncOpenAI + +from refactored_plugins.dedup import ContextDedupPlugin +from refactored_plugins.dynamic_pruning import DynamicPruningPlugin +from refactored_plugins.skill_index import SkillAwareContextPlugin + +logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s") +logger = logging.getLogger(__name__) + +# Create a registry of 10 dummy tools to trigger the Skill plugin +DUMMY_TOOL_REGISTRY = { + f"tool_{i}": { + "type": "function", + "function": { + "name": f"tool_{i}", + "description": f"Dummy tool number {i}" + } + } + for i in range(1, 11) +} + +async def process_task(task, client, semaphore, output_file, turn_1_id, mode, model_name, active_plugins): + """ + Processes a single BigCodeBench task through our API. + """ + async with semaphore: + task_id = task.get("task_id", "unknown_task") + prompt = task.get("complete_prompt", task.get("instruction", "No prompt found.")) + + # Mock heavy agent request with redundant history and bloated tools + request = { + "user_id": "evaluator_1", + "parent_id": turn_1_id, + "_required_skills": ["tool_1", "tool_3", "tool_7"], # Require only 3 tools out of 10 + "messages": [ + {"role": "system", "content": "You are a senior python developer. Always wrap your code in ```python blocks."}, + {"role": "user", "content": "Please help me write some code."}, + {"role": "assistant", "content": "Of course! I can help you with that."}, + {"role": "user", "content": prompt} + ], + "tools": list(DUMMY_TOOL_REGISTRY.values()) + } + if mode == "with_plugin": + # Send extra_body for ContextPilot proxy to intercept + request["_required_skills"] = ["tool_1", "tool_3", "tool_7"] + + # Apply plugins selectively + if "dedup" in active_plugins or "all" in active_plugins: + request = await dedup_plugin.process(request) + if "dynamic" in active_plugins or "all" in active_plugins: + request = await dynamic_plugin.process(request) + if "skill" in active_plugins or "all" in active_plugins: + request = await skill_plugin.process(request) + + api_kwargs = { + "model": model_name, + "messages": request["messages"], + "tools": request["tools"] + } + + api_kwargs["extra_body"] = { + "user_id": request.get("user_id"), + "parent_id": request.get("parent_id"), + "_required_skills": request.get("_required_skills") + } + else: + api_kwargs = { + "model": model_name, + "messages": request["messages"], + "tools": request["tools"] + } + + try: + logger.info(f"[{mode}] Sending task {task_id}...") + response = await client.chat.completions.create(**api_kwargs) + response_content = response.choices[0].message.content + except Exception as e: + logger.error(f"[{mode}] API Error for {task_id}: {str(e)}") + response_content = "" + + # Extract code block using regex + extracted_code = "" + if response_content: + match = re.search(r"```python\s*(.*?)\s*```", response_content, re.DOTALL) + if match: + extracted_code = match.group(1).strip() + else: + extracted_code = response_content.strip() + + # Append result to JSONL + with open(output_file, "a", encoding="utf-8") as f: + f.write(json.dumps({"task_id": task_id, "solution": extracted_code}) + "\n") + + logger.info(f"[{mode}] Finished {task_id}") + +async def run_evaluation(mode, args, tasks): + client = AsyncOpenAI(api_key=args.api_key, base_url="http://localhost:8000/v1" if mode == "with_plugin" else args.api_base) + + turn_1_id = "test-turn-1-id" + output_file = os.path.join(os.path.dirname(__file__), f"results_{mode}_{args.model}.jsonl") + if os.path.exists(output_file): + os.remove(output_file) + + semaphore = asyncio.Semaphore(args.concurrency) + active_plugins = [p.strip() for p in args.plugins.split(",")] + + # Instantiate plugins + global dedup_plugin, dynamic_plugin, skill_plugin + dedup_plugin = ContextDedupPlugin(shadow_mode=True) + dynamic_plugin = DynamicPruningPlugin(similarity_threshold=args.threshold) + skill_plugin = SkillAwareContextPlugin(DUMMY_TOOL_REGISTRY) + + # SEED THE TRACKER FOR TELEMETRY + turn_1_messages = [ + {"role": "system", "content": "You are a senior python developer. Always wrap your code in ```python blocks."}, + {"role": "user", "content": "Please help me write some code."}, + {"role": "assistant", "content": "Of course! I can help you with that."} + ] + msg_ids = [dedup_plugin._get_id(m["content"]) for m in turn_1_messages] + dedup_plugin.tracker.deduplicate(request_id=turn_1_id, docs=msg_ids, parent_request_id=None) + + coroutines = [process_task(t, client, semaphore, output_file, turn_1_id, mode, args.model, active_plugins) for t in tasks] + await asyncio.gather(*coroutines) + + print(f"\n=== Evaluation Complete for mode: {mode} ===") + print(f"Results saved to {output_file}") + + if mode == "with_plugin": + print("\n=== ContextPilot Client Telemetry ===") + dedup_metrics = dedup_plugin.get_plugin_metrics() + dynamic_metrics = dynamic_plugin.get_plugin_metrics() + skill_metrics = skill_plugin.get_plugin_metrics() + + total_saved = 0 + total_orig = dynamic_metrics['total_original_chars'] + + if "dedup" in active_plugins or "all" in active_plugins: + print(f"[Dedup] Chars Saved: {dedup_metrics['total_chars_saved']} / {dedup_metrics['total_original_chars']} ({dedup_metrics['chars_saved_percentage']:.2f}%)") + total_saved += dedup_metrics['total_chars_saved'] + + if "dynamic" in active_plugins or "all" in active_plugins: + print(f"[Dynamic Pruning] Chars Saved: {dynamic_metrics['total_chars_saved']} / {dynamic_metrics['total_original_chars']} ({dynamic_metrics['chars_saved_percentage']:.2f}%)") + total_saved += dynamic_metrics['total_chars_saved'] + + if "skill" in active_plugins or "all" in active_plugins: + print(f"[Skill] Tools Filtered: {skill_metrics['total_tools_filtered']} / {skill_metrics.get('total_original_tools', 'N/A')} ({skill_metrics.get('tools_filtered_percentage', 0):.2f}%)") + + if total_orig > 0 and ("dedup" in active_plugins or "all" in active_plugins) and ("dynamic" in active_plugins or "all" in active_plugins): + combined_pct = (total_saved / total_orig * 100) + print(f"[COMBINED THEORETICAL SAVINGS]: {total_saved} / {total_orig} ({combined_pct:.2f}%)") + +async def main(): + parser = argparse.ArgumentParser(description="BigCodeBench Evaluation Script") + parser.add_argument("--model", default="gpt-5.5", help="Model name to evaluate") + parser.add_argument("--api_base", default=os.environ.get("BASE_URL", "https://api.openai.com/v1"), help="Baseline API Base URL") + parser.add_argument("--api_key", default=os.environ.get("OPENAI_API_KEY", "dummy-key"), help="API Key") + parser.add_argument("--concurrency", type=int, default=1, help="Number of concurrent requests") + parser.add_argument("--limit", type=int, default=0, help="Limit number of tasks to run (0 for all)") + parser.add_argument("--eval_mode", choices=["baseline", "with_plugin", "all"], default="all", help="Evaluation mode") + parser.add_argument("--plugins", default="all", help="Comma-separated list of plugins (dedup,dynamic,skill,all)") + parser.add_argument("--threshold", type=float, default=0.3, help="Threshold for DynamicPruningPlugin") + args = parser.parse_args() + + logger.info("Loading BigCodeBench dataset...") + try: + dataset = load_dataset("bigcode/bigcodebench", split="train") + except Exception as e: + logger.warning(f"Failed to load split='train'. Trying standard default split. Error: {e}") + try: + dataset = load_dataset("bigcode/bigcodebench", split="v0.1.2") + except Exception: + dataset = load_dataset("bigcode/bigcodebench", split="v0.1.0_240822") + + tasks = list(dataset) + if args.limit > 0: + tasks = tasks[:args.limit] + logger.info(f"Loaded {len(tasks)} tasks (LIMITED) for evaluation.") + else: + logger.info(f"Loaded {len(tasks)} tasks for full evaluation.") + + modes = ["baseline", "with_plugin"] if args.eval_mode == "all" else [args.eval_mode] + + for mode in modes: + logger.info(f"\n--- Starting Evaluation: {mode} ---") + await run_evaluation(mode, args, tasks) + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/evaluation/benchmarks/run_bigcodebench_all_plugins.py b/evaluation/benchmarks/run_bigcodebench_all_plugins.py new file mode 100644 index 0000000..ff52063 --- /dev/null +++ b/evaluation/benchmarks/run_bigcodebench_all_plugins.py @@ -0,0 +1,194 @@ +import argparse +import asyncio +import json +import logging +import os +import re + +from datasets import load_dataset +from openai import AsyncOpenAI + +from refactored_plugins.dedup import ContextDedupPlugin +from refactored_plugins.dynamic_pruning import DynamicPruningPlugin +from refactored_plugins.skill_index import SkillAwareContextPlugin + +logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s") +logger = logging.getLogger(__name__) + +# Create a registry of 10 dummy tools to trigger the Skill plugin +DUMMY_TOOL_REGISTRY = { + f"tool_{i}": { + "type": "function", + "function": { + "name": f"tool_{i}", + "description": f"Dummy tool number {i}" + } + } + for i in range(1, 11) +} + +async def process_task(task, client, semaphore, output_file, turn_1_id, mode, model_name): + """ + Processes a single BigCodeBench task through our ELM API. + """ + async with semaphore: + task_id = task.get("task_id", "unknown_task") + prompt = task.get("complete_prompt", task.get("instruction", "No prompt found.")) + + # Mock heavy agent request with redundant history and bloated tools + request = { + "user_id": "evaluator_1", + "parent_id": turn_1_id, + "_required_skills": ["tool_1", "tool_3", "tool_7"], # Require only 3 tools out of 10 + "messages": [ + {"role": "system", "content": "You are a senior python developer. Always wrap your code in ```python blocks."}, + {"role": "user", "content": "Please help me write some code."}, + {"role": "assistant", "content": "Of course! I can help you with that."}, + {"role": "user", "content": prompt} + ], + "tools": list(DUMMY_TOOL_REGISTRY.values()) + } + if mode == "with_plugin": + # Send extra_body for ContextPilot proxy to intercept + request["_required_skills"] = ["tool_1", "tool_3", "tool_7"] + + # 1. Apply ALL THREE plugins on the client before sending + request = await dedup_plugin.process(request) + request = await dynamic_plugin.process(request) + request = await skill_plugin.process(request) + + api_kwargs = { + "model": model_name, + "messages": request["messages"], + "tools": request["tools"] + } + + api_kwargs["extra_body"] = { + "user_id": request.get("user_id"), + "parent_id": request.get("parent_id"), + "_required_skills": request.get("_required_skills") + } + else: + api_kwargs = { + "model": model_name, + "messages": request["messages"], + "tools": request["tools"] + } + + try: + logger.info(f"[{mode}] Sending task {task_id}...") + response = await client.chat.completions.create(**api_kwargs) + response_content = response.choices[0].message.content + except Exception as e: + logger.error(f"[{mode}] API Error for {task_id}: {str(e)}") + response_content = "" + + # Extract code block using regex + extracted_code = "" + if response_content: + match = re.search(r"```python\s*(.*?)\s*```", response_content, re.DOTALL) + if match: + extracted_code = match.group(1).strip() + else: + # Fallback if the LLM didn't use the markdown block + extracted_code = response_content.strip() + + # Append result to JSONL + with open(output_file, "a", encoding="utf-8") as f: + f.write(json.dumps({"task_id": task_id, "solution": extracted_code}) + "\n") + + logger.info(f"[{mode}] Finished {task_id}") + +async def run_evaluation(mode, args, tasks): + # Route BOTH baseline and with_plugin through the proxy + client = AsyncOpenAI(api_key=args.api_key, base_url="http://localhost:8000/v1") + + # We use a dummy turn_1_id for simulation + turn_1_id = "test-turn-1-id" + + output_file = os.path.join(os.path.dirname(__file__), f"results_all_plugins_{mode}_{args.model}.jsonl") + if os.path.exists(output_file): + os.remove(output_file) + + # Use configurable Semaphore to allow high concurrency + semaphore = asyncio.Semaphore(args.concurrency) + + # Instantiate plugins + global dedup_plugin, dynamic_plugin, skill_plugin + dedup_plugin = ContextDedupPlugin(shadow_mode=True) + dynamic_plugin = DynamicPruningPlugin(similarity_threshold=0.3) + skill_plugin = SkillAwareContextPlugin(DUMMY_TOOL_REGISTRY) + + # SEED THE TRACKER FOR TELEMETRY: + # Inject the "Turn 1" system prompt and history into the dedup plugin's memory. + turn_1_messages = [ + {"role": "system", "content": "You are a senior python developer. Always wrap your code in ```python blocks."}, + {"role": "user", "content": "Please help me write some code."}, + {"role": "assistant", "content": "Of course! I can help you with that."} + ] + msg_ids = [dedup_plugin._get_id(m["content"]) for m in turn_1_messages] + dedup_plugin.tracker.deduplicate(request_id=turn_1_id, docs=msg_ids, parent_request_id=None) + + coroutines = [process_task(t, client, semaphore, output_file, turn_1_id, mode, args.model) for t in tasks] + await asyncio.gather(*coroutines) + + print(f"\n=== Evaluation Complete for mode: {mode} ===") + print(f"Results saved to {output_file}") + + if mode == "with_plugin": + print("\n=== ContextPilot Client Telemetry (ALL PLUGINS) ===") + dedup_metrics = dedup_plugin.get_plugin_metrics() + dynamic_metrics = dynamic_plugin.get_plugin_metrics() + skill_metrics = skill_plugin.get_plugin_metrics() + + # Calculate theoretical stacked savings + total_orig = dynamic_metrics['total_original_chars'] + total_saved = dedup_metrics['total_chars_saved'] + dynamic_metrics['total_chars_saved'] + combined_pct = (total_saved / total_orig * 100) if total_orig > 0 else 0.0 + + print(f"[Dedup] Chars Saved: {dedup_metrics['total_chars_saved']} / {dedup_metrics['total_original_chars']} ({dedup_metrics['chars_saved_percentage']:.2f}%)") + print(f"[Dynamic Pruning] Chars Saved: {dynamic_metrics['total_chars_saved']} / {dynamic_metrics['total_original_chars']} ({dynamic_metrics['chars_saved_percentage']:.2f}%)") + print(f"[COMBINED THEORETICAL SAVINGS]: {total_saved} / {total_orig} ({combined_pct:.2f}%)") + print(f"[Skill] Tools Filtered: {skill_metrics['total_tools_filtered']} / {skill_metrics.get('total_original_tools', 'N/A')} ({skill_metrics.get('tools_filtered_percentage', 0):.2f}%)") + + +async def main(): + parser = argparse.ArgumentParser(description="BigCodeBench ELM API Runner") + parser.add_argument("--model", default="gpt-5.5", help="Model name to evaluate") + parser.add_argument("--api_base", default=os.environ.get("BASE_URL", "https://api.openai.com/v1"), help="Baseline ELM API Base URL") + parser.add_argument("--api_key", default=os.environ.get("OPENAI_API_KEY", "dummy-elm-key"), help="API Key") + parser.add_argument("--concurrency", type=int, default=1, help="Number of concurrent requests") + parser.add_argument("--limit", type=int, default=0, help="Limit number of tasks to run (0 for all)") + parser.add_argument("--eval_mode", choices=["baseline", "with_plugin", "all"], default="all", help="Evaluation mode") + args = parser.parse_args() + + # Load BigCodeBench dataset + logger.info("Loading BigCodeBench dataset...") + try: + dataset = load_dataset("bigcode/bigcodebench", split="train") + except Exception as e: + logger.warning(f"Failed to load split='train'. Trying standard default split. Error: {e}") + try: + dataset = load_dataset("bigcode/bigcodebench", split="v0.1.2") + except Exception: + dataset = load_dataset("bigcode/bigcodebench", split="v0.1.0_240822") + + # Select all tasks for full evaluation + tasks = list(dataset) + if args.limit > 0: + tasks = tasks[:args.limit] + logger.info(f"Loaded {len(tasks)} tasks (LIMITED) for evaluation.") + else: + logger.info(f"Loaded {len(tasks)} tasks for full evaluation.") + + if args.eval_mode == "all": + modes = ["baseline", "with_plugin"] + else: + modes = [args.eval_mode] + + for mode in modes: + logger.info(f"\n--- Starting Evaluation: {mode} ---") + await run_evaluation(mode, args, tasks) + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/evaluation/benchmarks/run_bigcodebench_deepseek_all.py b/evaluation/benchmarks/run_bigcodebench_deepseek_all.py new file mode 100644 index 0000000..ff52063 --- /dev/null +++ b/evaluation/benchmarks/run_bigcodebench_deepseek_all.py @@ -0,0 +1,194 @@ +import argparse +import asyncio +import json +import logging +import os +import re + +from datasets import load_dataset +from openai import AsyncOpenAI + +from refactored_plugins.dedup import ContextDedupPlugin +from refactored_plugins.dynamic_pruning import DynamicPruningPlugin +from refactored_plugins.skill_index import SkillAwareContextPlugin + +logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s") +logger = logging.getLogger(__name__) + +# Create a registry of 10 dummy tools to trigger the Skill plugin +DUMMY_TOOL_REGISTRY = { + f"tool_{i}": { + "type": "function", + "function": { + "name": f"tool_{i}", + "description": f"Dummy tool number {i}" + } + } + for i in range(1, 11) +} + +async def process_task(task, client, semaphore, output_file, turn_1_id, mode, model_name): + """ + Processes a single BigCodeBench task through our ELM API. + """ + async with semaphore: + task_id = task.get("task_id", "unknown_task") + prompt = task.get("complete_prompt", task.get("instruction", "No prompt found.")) + + # Mock heavy agent request with redundant history and bloated tools + request = { + "user_id": "evaluator_1", + "parent_id": turn_1_id, + "_required_skills": ["tool_1", "tool_3", "tool_7"], # Require only 3 tools out of 10 + "messages": [ + {"role": "system", "content": "You are a senior python developer. Always wrap your code in ```python blocks."}, + {"role": "user", "content": "Please help me write some code."}, + {"role": "assistant", "content": "Of course! I can help you with that."}, + {"role": "user", "content": prompt} + ], + "tools": list(DUMMY_TOOL_REGISTRY.values()) + } + if mode == "with_plugin": + # Send extra_body for ContextPilot proxy to intercept + request["_required_skills"] = ["tool_1", "tool_3", "tool_7"] + + # 1. Apply ALL THREE plugins on the client before sending + request = await dedup_plugin.process(request) + request = await dynamic_plugin.process(request) + request = await skill_plugin.process(request) + + api_kwargs = { + "model": model_name, + "messages": request["messages"], + "tools": request["tools"] + } + + api_kwargs["extra_body"] = { + "user_id": request.get("user_id"), + "parent_id": request.get("parent_id"), + "_required_skills": request.get("_required_skills") + } + else: + api_kwargs = { + "model": model_name, + "messages": request["messages"], + "tools": request["tools"] + } + + try: + logger.info(f"[{mode}] Sending task {task_id}...") + response = await client.chat.completions.create(**api_kwargs) + response_content = response.choices[0].message.content + except Exception as e: + logger.error(f"[{mode}] API Error for {task_id}: {str(e)}") + response_content = "" + + # Extract code block using regex + extracted_code = "" + if response_content: + match = re.search(r"```python\s*(.*?)\s*```", response_content, re.DOTALL) + if match: + extracted_code = match.group(1).strip() + else: + # Fallback if the LLM didn't use the markdown block + extracted_code = response_content.strip() + + # Append result to JSONL + with open(output_file, "a", encoding="utf-8") as f: + f.write(json.dumps({"task_id": task_id, "solution": extracted_code}) + "\n") + + logger.info(f"[{mode}] Finished {task_id}") + +async def run_evaluation(mode, args, tasks): + # Route BOTH baseline and with_plugin through the proxy + client = AsyncOpenAI(api_key=args.api_key, base_url="http://localhost:8000/v1") + + # We use a dummy turn_1_id for simulation + turn_1_id = "test-turn-1-id" + + output_file = os.path.join(os.path.dirname(__file__), f"results_all_plugins_{mode}_{args.model}.jsonl") + if os.path.exists(output_file): + os.remove(output_file) + + # Use configurable Semaphore to allow high concurrency + semaphore = asyncio.Semaphore(args.concurrency) + + # Instantiate plugins + global dedup_plugin, dynamic_plugin, skill_plugin + dedup_plugin = ContextDedupPlugin(shadow_mode=True) + dynamic_plugin = DynamicPruningPlugin(similarity_threshold=0.3) + skill_plugin = SkillAwareContextPlugin(DUMMY_TOOL_REGISTRY) + + # SEED THE TRACKER FOR TELEMETRY: + # Inject the "Turn 1" system prompt and history into the dedup plugin's memory. + turn_1_messages = [ + {"role": "system", "content": "You are a senior python developer. Always wrap your code in ```python blocks."}, + {"role": "user", "content": "Please help me write some code."}, + {"role": "assistant", "content": "Of course! I can help you with that."} + ] + msg_ids = [dedup_plugin._get_id(m["content"]) for m in turn_1_messages] + dedup_plugin.tracker.deduplicate(request_id=turn_1_id, docs=msg_ids, parent_request_id=None) + + coroutines = [process_task(t, client, semaphore, output_file, turn_1_id, mode, args.model) for t in tasks] + await asyncio.gather(*coroutines) + + print(f"\n=== Evaluation Complete for mode: {mode} ===") + print(f"Results saved to {output_file}") + + if mode == "with_plugin": + print("\n=== ContextPilot Client Telemetry (ALL PLUGINS) ===") + dedup_metrics = dedup_plugin.get_plugin_metrics() + dynamic_metrics = dynamic_plugin.get_plugin_metrics() + skill_metrics = skill_plugin.get_plugin_metrics() + + # Calculate theoretical stacked savings + total_orig = dynamic_metrics['total_original_chars'] + total_saved = dedup_metrics['total_chars_saved'] + dynamic_metrics['total_chars_saved'] + combined_pct = (total_saved / total_orig * 100) if total_orig > 0 else 0.0 + + print(f"[Dedup] Chars Saved: {dedup_metrics['total_chars_saved']} / {dedup_metrics['total_original_chars']} ({dedup_metrics['chars_saved_percentage']:.2f}%)") + print(f"[Dynamic Pruning] Chars Saved: {dynamic_metrics['total_chars_saved']} / {dynamic_metrics['total_original_chars']} ({dynamic_metrics['chars_saved_percentage']:.2f}%)") + print(f"[COMBINED THEORETICAL SAVINGS]: {total_saved} / {total_orig} ({combined_pct:.2f}%)") + print(f"[Skill] Tools Filtered: {skill_metrics['total_tools_filtered']} / {skill_metrics.get('total_original_tools', 'N/A')} ({skill_metrics.get('tools_filtered_percentage', 0):.2f}%)") + + +async def main(): + parser = argparse.ArgumentParser(description="BigCodeBench ELM API Runner") + parser.add_argument("--model", default="gpt-5.5", help="Model name to evaluate") + parser.add_argument("--api_base", default=os.environ.get("BASE_URL", "https://api.openai.com/v1"), help="Baseline ELM API Base URL") + parser.add_argument("--api_key", default=os.environ.get("OPENAI_API_KEY", "dummy-elm-key"), help="API Key") + parser.add_argument("--concurrency", type=int, default=1, help="Number of concurrent requests") + parser.add_argument("--limit", type=int, default=0, help="Limit number of tasks to run (0 for all)") + parser.add_argument("--eval_mode", choices=["baseline", "with_plugin", "all"], default="all", help="Evaluation mode") + args = parser.parse_args() + + # Load BigCodeBench dataset + logger.info("Loading BigCodeBench dataset...") + try: + dataset = load_dataset("bigcode/bigcodebench", split="train") + except Exception as e: + logger.warning(f"Failed to load split='train'. Trying standard default split. Error: {e}") + try: + dataset = load_dataset("bigcode/bigcodebench", split="v0.1.2") + except Exception: + dataset = load_dataset("bigcode/bigcodebench", split="v0.1.0_240822") + + # Select all tasks for full evaluation + tasks = list(dataset) + if args.limit > 0: + tasks = tasks[:args.limit] + logger.info(f"Loaded {len(tasks)} tasks (LIMITED) for evaluation.") + else: + logger.info(f"Loaded {len(tasks)} tasks for full evaluation.") + + if args.eval_mode == "all": + modes = ["baseline", "with_plugin"] + else: + modes = [args.eval_mode] + + for mode in modes: + logger.info(f"\n--- Starting Evaluation: {mode} ---") + await run_evaluation(mode, args, tasks) + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/evaluation/benchmarks/run_bigcodebench_deepseek_dynamic.py b/evaluation/benchmarks/run_bigcodebench_deepseek_dynamic.py new file mode 100644 index 0000000..3fb0b21 --- /dev/null +++ b/evaluation/benchmarks/run_bigcodebench_deepseek_dynamic.py @@ -0,0 +1,175 @@ +import argparse +import asyncio +import json +import logging +import os +import re + +from datasets import load_dataset +from openai import AsyncOpenAI + +from refactored_plugins.dynamic_pruning import DynamicPruningPlugin +from refactored_plugins.skill_index import SkillAwareContextPlugin + +logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s") +logger = logging.getLogger(__name__) + +# Create a registry of 10 dummy tools to trigger the Skill plugin +DUMMY_TOOL_REGISTRY = { + f"tool_{i}": { + "type": "function", + "function": { + "name": f"tool_{i}", + "description": f"Dummy tool number {i}" + } + } + for i in range(1, 11) +} + +async def process_task(task, client, semaphore, output_file, turn_1_id, mode, model_name): + """ + Processes a single BigCodeBench task through our ELM API (bypassing or routing to proxy). + """ + async with semaphore: + task_id = task.get("task_id", "unknown_task") + prompt = task.get("complete_prompt", task.get("instruction", "No prompt found.")) + + # Mock heavy agent request with redundant history and bloated tools + request = { + "user_id": "evaluator_1", + "parent_id": turn_1_id, + "_required_skills": ["tool_1", "tool_3", "tool_7"], # Require only 3 tools out of 10 + "messages": [ + {"role": "system", "content": "You are a senior python developer. Always wrap your code in ```python blocks."}, + {"role": "user", "content": "Please help me write some code."}, + {"role": "assistant", "content": "Of course! I can help you with that."}, + {"role": "user", "content": prompt} + ], + "tools": list(DUMMY_TOOL_REGISTRY.values()) + } + if mode == "with_plugin": + # Send extra_body for ContextPilot proxy to intercept + request["_required_skills"] = ["tool_1", "tool_3", "tool_7"] + + # 1. Apply plugins on the client before sending + request = await dynamic_plugin.process(request) + request = await skill_plugin.process(request) + + api_kwargs = { + "model": model_name, + "messages": request["messages"], + "tools": request["tools"] + } + + api_kwargs["extra_body"] = { + "user_id": request.get("user_id"), + "parent_id": request.get("parent_id"), + "_required_skills": request.get("_required_skills") + } + else: + api_kwargs = { + "model": model_name, + "messages": request["messages"], + "tools": request["tools"] + } + + try: + logger.info(f"[{mode}] Sending task {task_id}...") + response = await client.chat.completions.create(**api_kwargs) + response_content = response.choices[0].message.content + except Exception as e: + logger.error(f"[{mode}] API Error for {task_id}: {str(e)}") + response_content = "" + + # Extract code block using regex + extracted_code = "" + if response_content: + match = re.search(r"```python\s*(.*?)\s*```", response_content, re.DOTALL) + if match: + extracted_code = match.group(1).strip() + else: + # Fallback if the LLM didn't use the markdown block + extracted_code = response_content.strip() + + # Append result to JSONL + with open(output_file, "a", encoding="utf-8") as f: + f.write(json.dumps({"task_id": task_id, "solution": extracted_code}) + "\n") + + logger.info(f"[{mode}] Finished {task_id}") + +async def run_evaluation(mode, args, tasks): + # Route BOTH baseline and with_plugin through the proxy to intercept prompt_cache_hit_tokens + client = AsyncOpenAI(api_key=args.api_key, base_url="http://localhost:8000/v1") + + # We use a dummy turn_1_id for simulation + turn_1_id = "test-turn-1-id" + + output_file = os.path.join(os.path.dirname(__file__), f"results_dynamic_skill_{mode}_{args.model}.jsonl") + if os.path.exists(output_file): + os.remove(output_file) + + # Use configurable Semaphore to allow high concurrency + semaphore = asyncio.Semaphore(args.concurrency) + + # Instantiate plugins + from refactored_plugins.dynamic_pruning import DynamicPruningPlugin + from refactored_plugins.skill_index import SkillAwareContextPlugin + global dynamic_plugin, skill_plugin + dynamic_plugin = DynamicPruningPlugin(similarity_threshold=0.3) + skill_plugin = SkillAwareContextPlugin(DUMMY_TOOL_REGISTRY) + + coroutines = [process_task(t, client, semaphore, output_file, turn_1_id, mode, args.model) for t in tasks] + await asyncio.gather(*coroutines) + + print(f"\n=== Evaluation Complete for mode: {mode} ===") + print(f"Results saved to {output_file}") + + if mode == "with_plugin": + print("\n=== ContextPilot Client Telemetry ===") + dynamic_metrics = dynamic_plugin.get_plugin_metrics() + skill_metrics = skill_plugin.get_plugin_metrics() + print(f"[Dynamic Pruning] Chars Saved: {dynamic_metrics['total_chars_saved']} / {dynamic_metrics['total_original_chars']} ({dynamic_metrics['chars_saved_percentage']:.2f}%)") + print(f"[Skill] Tools Filtered: {skill_metrics['total_tools_filtered']} / {skill_metrics.get('total_original_tools', 'N/A')} ({skill_metrics.get('tools_filtered_percentage', 0):.2f}%)") + + +async def main(): + parser = argparse.ArgumentParser(description="BigCodeBench ELM API Runner") + parser.add_argument("--model", default="gpt-5.5", help="Model name to evaluate") + parser.add_argument("--api_base", default=os.environ.get("BASE_URL", "https://api.openai.com/v1"), help="Baseline ELM API Base URL") + parser.add_argument("--api_key", default=os.environ.get("OPENAI_API_KEY", "dummy-elm-key"), help="API Key") + parser.add_argument("--concurrency", type=int, default=1, help="Number of concurrent requests") + parser.add_argument("--limit", type=int, default=0, help="Limit number of tasks to run (0 for all)") + parser.add_argument("--eval_mode", choices=["baseline", "with_plugin", "all"], default="all", help="Evaluation mode") + args = parser.parse_args() + + # Load BigCodeBench dataset + logger.info("Loading BigCodeBench dataset...") + try: + dataset = load_dataset("bigcode/bigcodebench", split="train") + except Exception as e: + logger.warning(f"Failed to load split='train'. Trying standard default split. Error: {e}") + # Fallback to the common default split format if 'train' split does not exist + try: + dataset = load_dataset("bigcode/bigcodebench", split="v0.1.2") + except Exception: + dataset = load_dataset("bigcode/bigcodebench", split="v0.1.0_240822") + + # Select all tasks for full evaluation + tasks = list(dataset) + if args.limit > 0: + tasks = tasks[:args.limit] + logger.info(f"Loaded {len(tasks)} tasks (LIMITED) for evaluation.") + else: + logger.info(f"Loaded {len(tasks)} tasks for full evaluation.") + + if args.eval_mode == "all": + modes = ["baseline", "with_plugin"] + else: + modes = [args.eval_mode] + + for mode in modes: + logger.info(f"\n--- Starting Evaluation: {mode} ---") + await run_evaluation(mode, args, tasks) + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/evaluation/benchmarks/run_bigcodebench_dynamic_skill.py b/evaluation/benchmarks/run_bigcodebench_dynamic_skill.py new file mode 100644 index 0000000..3fb0b21 --- /dev/null +++ b/evaluation/benchmarks/run_bigcodebench_dynamic_skill.py @@ -0,0 +1,175 @@ +import argparse +import asyncio +import json +import logging +import os +import re + +from datasets import load_dataset +from openai import AsyncOpenAI + +from refactored_plugins.dynamic_pruning import DynamicPruningPlugin +from refactored_plugins.skill_index import SkillAwareContextPlugin + +logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s") +logger = logging.getLogger(__name__) + +# Create a registry of 10 dummy tools to trigger the Skill plugin +DUMMY_TOOL_REGISTRY = { + f"tool_{i}": { + "type": "function", + "function": { + "name": f"tool_{i}", + "description": f"Dummy tool number {i}" + } + } + for i in range(1, 11) +} + +async def process_task(task, client, semaphore, output_file, turn_1_id, mode, model_name): + """ + Processes a single BigCodeBench task through our ELM API (bypassing or routing to proxy). + """ + async with semaphore: + task_id = task.get("task_id", "unknown_task") + prompt = task.get("complete_prompt", task.get("instruction", "No prompt found.")) + + # Mock heavy agent request with redundant history and bloated tools + request = { + "user_id": "evaluator_1", + "parent_id": turn_1_id, + "_required_skills": ["tool_1", "tool_3", "tool_7"], # Require only 3 tools out of 10 + "messages": [ + {"role": "system", "content": "You are a senior python developer. Always wrap your code in ```python blocks."}, + {"role": "user", "content": "Please help me write some code."}, + {"role": "assistant", "content": "Of course! I can help you with that."}, + {"role": "user", "content": prompt} + ], + "tools": list(DUMMY_TOOL_REGISTRY.values()) + } + if mode == "with_plugin": + # Send extra_body for ContextPilot proxy to intercept + request["_required_skills"] = ["tool_1", "tool_3", "tool_7"] + + # 1. Apply plugins on the client before sending + request = await dynamic_plugin.process(request) + request = await skill_plugin.process(request) + + api_kwargs = { + "model": model_name, + "messages": request["messages"], + "tools": request["tools"] + } + + api_kwargs["extra_body"] = { + "user_id": request.get("user_id"), + "parent_id": request.get("parent_id"), + "_required_skills": request.get("_required_skills") + } + else: + api_kwargs = { + "model": model_name, + "messages": request["messages"], + "tools": request["tools"] + } + + try: + logger.info(f"[{mode}] Sending task {task_id}...") + response = await client.chat.completions.create(**api_kwargs) + response_content = response.choices[0].message.content + except Exception as e: + logger.error(f"[{mode}] API Error for {task_id}: {str(e)}") + response_content = "" + + # Extract code block using regex + extracted_code = "" + if response_content: + match = re.search(r"```python\s*(.*?)\s*```", response_content, re.DOTALL) + if match: + extracted_code = match.group(1).strip() + else: + # Fallback if the LLM didn't use the markdown block + extracted_code = response_content.strip() + + # Append result to JSONL + with open(output_file, "a", encoding="utf-8") as f: + f.write(json.dumps({"task_id": task_id, "solution": extracted_code}) + "\n") + + logger.info(f"[{mode}] Finished {task_id}") + +async def run_evaluation(mode, args, tasks): + # Route BOTH baseline and with_plugin through the proxy to intercept prompt_cache_hit_tokens + client = AsyncOpenAI(api_key=args.api_key, base_url="http://localhost:8000/v1") + + # We use a dummy turn_1_id for simulation + turn_1_id = "test-turn-1-id" + + output_file = os.path.join(os.path.dirname(__file__), f"results_dynamic_skill_{mode}_{args.model}.jsonl") + if os.path.exists(output_file): + os.remove(output_file) + + # Use configurable Semaphore to allow high concurrency + semaphore = asyncio.Semaphore(args.concurrency) + + # Instantiate plugins + from refactored_plugins.dynamic_pruning import DynamicPruningPlugin + from refactored_plugins.skill_index import SkillAwareContextPlugin + global dynamic_plugin, skill_plugin + dynamic_plugin = DynamicPruningPlugin(similarity_threshold=0.3) + skill_plugin = SkillAwareContextPlugin(DUMMY_TOOL_REGISTRY) + + coroutines = [process_task(t, client, semaphore, output_file, turn_1_id, mode, args.model) for t in tasks] + await asyncio.gather(*coroutines) + + print(f"\n=== Evaluation Complete for mode: {mode} ===") + print(f"Results saved to {output_file}") + + if mode == "with_plugin": + print("\n=== ContextPilot Client Telemetry ===") + dynamic_metrics = dynamic_plugin.get_plugin_metrics() + skill_metrics = skill_plugin.get_plugin_metrics() + print(f"[Dynamic Pruning] Chars Saved: {dynamic_metrics['total_chars_saved']} / {dynamic_metrics['total_original_chars']} ({dynamic_metrics['chars_saved_percentage']:.2f}%)") + print(f"[Skill] Tools Filtered: {skill_metrics['total_tools_filtered']} / {skill_metrics.get('total_original_tools', 'N/A')} ({skill_metrics.get('tools_filtered_percentage', 0):.2f}%)") + + +async def main(): + parser = argparse.ArgumentParser(description="BigCodeBench ELM API Runner") + parser.add_argument("--model", default="gpt-5.5", help="Model name to evaluate") + parser.add_argument("--api_base", default=os.environ.get("BASE_URL", "https://api.openai.com/v1"), help="Baseline ELM API Base URL") + parser.add_argument("--api_key", default=os.environ.get("OPENAI_API_KEY", "dummy-elm-key"), help="API Key") + parser.add_argument("--concurrency", type=int, default=1, help="Number of concurrent requests") + parser.add_argument("--limit", type=int, default=0, help="Limit number of tasks to run (0 for all)") + parser.add_argument("--eval_mode", choices=["baseline", "with_plugin", "all"], default="all", help="Evaluation mode") + args = parser.parse_args() + + # Load BigCodeBench dataset + logger.info("Loading BigCodeBench dataset...") + try: + dataset = load_dataset("bigcode/bigcodebench", split="train") + except Exception as e: + logger.warning(f"Failed to load split='train'. Trying standard default split. Error: {e}") + # Fallback to the common default split format if 'train' split does not exist + try: + dataset = load_dataset("bigcode/bigcodebench", split="v0.1.2") + except Exception: + dataset = load_dataset("bigcode/bigcodebench", split="v0.1.0_240822") + + # Select all tasks for full evaluation + tasks = list(dataset) + if args.limit > 0: + tasks = tasks[:args.limit] + logger.info(f"Loaded {len(tasks)} tasks (LIMITED) for evaluation.") + else: + logger.info(f"Loaded {len(tasks)} tasks for full evaluation.") + + if args.eval_mode == "all": + modes = ["baseline", "with_plugin"] + else: + modes = [args.eval_mode] + + for mode in modes: + logger.info(f"\n--- Starting Evaluation: {mode} ---") + await run_evaluation(mode, args, tasks) + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/evaluation/benchmarks/run_bigcodebench_elm.py b/evaluation/benchmarks/run_bigcodebench_elm.py new file mode 100644 index 0000000..92b9b7d --- /dev/null +++ b/evaluation/benchmarks/run_bigcodebench_elm.py @@ -0,0 +1,147 @@ +import asyncio +import json +import logging +import os +import re + +# pip install datasets +from datasets import load_dataset +from openai import AsyncOpenAI + +# Set PYTHONPATH in the environment before running +from refactored_plugins.skill_index import SkillAwareContextPlugin +from refactored_plugins.dedup import ContextDedupPlugin + +logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s") +logger = logging.getLogger(__name__) + +# Create a registry of 10 dummy tools to trigger the Skill plugin +DUMMY_TOOL_REGISTRY = { + f"tool_{i}": { + "type": "function", + "function": { + "name": f"tool_{i}", + "description": f"Dummy tool number {i}" + } + } + for i in range(1, 11) +} + +async def process_task(task, skill_plugin, dedup_plugin, client, semaphore, output_file, turn_1_id): + """ + Processes a single BigCodeBench task through our ContextPilot plugins and ELM API. + """ + async with semaphore: + task_id = task.get("task_id", "unknown_task") + # BigCodeBench prompts are usually in 'complete_prompt' or 'instruction' + prompt = task.get("complete_prompt", task.get("instruction", "No prompt found.")) + + # Mock heavy agent request with redundant history and bloated tools + request = { + "user_id": "evaluator_1", + "parent_id": turn_1_id, + "_required_skills": ["tool_1", "tool_3", "tool_7"], # Require only 3 tools out of 10 + "messages": [ + {"role": "system", "content": "You are a senior python developer. Always wrap your code in ```python blocks."}, + {"role": "user", "content": "Please help me write some code."}, + {"role": "assistant", "content": "Of course! I can help you with that."}, + {"role": "user", "content": prompt} + ], + "tools": list(DUMMY_TOOL_REGISTRY.values()) + } + + # Pass through ContextPilot local plugins + optimized_request = await dedup_plugin.process(request) + optimized_request = await skill_plugin.process(optimized_request) + + # Prepare ELM API request (OpenAI-compatible) + api_kwargs = { + "model": "gpt-5.5", + "messages": optimized_request.get("messages", []) + } + if "tools" in optimized_request and optimized_request["tools"]: + api_kwargs["tools"] = optimized_request["tools"] + + try: + logger.info(f"Sending optimized task {task_id} to ELM API...") + response = await client.chat.completions.create(**api_kwargs) + response_content = response.choices[0].message.content + except Exception as e: + logger.error(f"API Error for {task_id}: {str(e)}") + response_content = "" + + # Extract code block using regex + extracted_code = "" + if response_content: + match = re.search(r"```python\s*(.*?)\s*```", response_content, re.DOTALL) + if match: + extracted_code = match.group(1).strip() + else: + # Fallback if the LLM didn't use the markdown block + extracted_code = response_content.strip() + + # Append result to JSONL + with open(output_file, "a", encoding="utf-8") as f: + f.write(json.dumps({"task_id": task_id, "solution": extracted_code}) + "\n") + + logger.info(f"Finished {task_id}") + +async def main(): + api_key = os.environ.get("OPENAI_API_KEY", "dummy-elm-key") + base_url = os.environ.get("BASE_URL", "https://api.openai.com/v1") + + client = AsyncOpenAI(api_key=api_key, base_url=base_url) + + skill_plugin = SkillAwareContextPlugin(tool_registry=DUMMY_TOOL_REGISTRY) + dedup_plugin = ContextDedupPlugin() + + # Pre-warm Dedup plugin with the initial messages to simulate conversation history + turn_1 = { + "user_id": "evaluator_1", + "messages": [ + {"role": "system", "content": "You are a senior python developer. Always wrap your code in ```python blocks."}, + {"role": "user", "content": "Please help me write some code."}, + {"role": "assistant", "content": "Of course! I can help you with that."} + ] + } + turn_1_res = await dedup_plugin.process(turn_1) + turn_1_id = turn_1_res.get("current_id") + + # Load BigCodeBench dataset + logger.info("Loading BigCodeBench dataset...") + try: + dataset = load_dataset("bigcode/bigcodebench", split="train") + except Exception as e: + logger.warning(f"Failed to load split='train'. Trying standard default split. Error: {e}") + # Fallback to the common default split format if 'train' split does not exist + try: + dataset = load_dataset("bigcode/bigcodebench", split="v0.1.2") + except Exception: + dataset = load_dataset("bigcode/bigcodebench", split="v0.1.0_240822") + + # Select first 5 tasks for a smoke test + tasks = list(dataset)[:5] + logger.info(f"Loaded {len(tasks)} tasks for smoke test.") + + output_file = os.path.join(os.path.dirname(__file__), "elm_samples.jsonl") + if os.path.exists(output_file): + os.remove(output_file) + + # Use a Semaphore with 1 to process sequentially and avoid early rate limits + semaphore = asyncio.Semaphore(1) + + coroutines = [process_task(t, skill_plugin, dedup_plugin, client, semaphore, output_file, turn_1_id) for t in tasks] + await asyncio.gather(*coroutines) + + print("\n=== Phase 2 Dataset Smoke Test Complete ===") + print(f"Results saved to {output_file}") + + print("\n=== Combined Cost-Savings Telemetry ===") + metrics = { + "skill_plugin_metrics": skill_plugin.get_plugin_metrics(), + "dedup_plugin_metrics": dedup_plugin.get_plugin_metrics() + } + print(json.dumps(metrics, indent=2)) + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/evaluation/benchmarks/run_bigcodebench_elm_full.py b/evaluation/benchmarks/run_bigcodebench_elm_full.py new file mode 100644 index 0000000..dea0e35 --- /dev/null +++ b/evaluation/benchmarks/run_bigcodebench_elm_full.py @@ -0,0 +1,195 @@ +import argparse +import asyncio +import json +import logging +import os +import re + +from datasets import load_dataset +from openai import AsyncOpenAI + +from refactored_plugins.dedup import ContextDedupPlugin +from refactored_plugins.skill_index import SkillAwareContextPlugin + +logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s") +logger = logging.getLogger(__name__) + +# Create a registry of 10 dummy tools to trigger the Skill plugin +DUMMY_TOOL_REGISTRY = { + f"tool_{i}": { + "type": "function", + "function": { + "name": f"tool_{i}", + "description": f"Dummy tool number {i}" + } + } + for i in range(1, 11) +} + +async def process_task(task, client, semaphore, output_file, turn_1_id, mode, model_name, seed=None): + """ + Processes a single BigCodeBench task through our ELM API (bypassing or routing to proxy). + """ + async with semaphore: + task_id = task.get("task_id", "unknown_task") + prompt = task.get("complete_prompt", task.get("instruction", "No prompt found.")) + + # Mock heavy agent request with redundant history and bloated tools + request = { + "user_id": "evaluator_1", + "parent_id": turn_1_id, + "_required_skills": ["tool_1", "tool_3", "tool_7"], # Require only 3 tools out of 10 + "messages": [ + {"role": "system", "content": "You are a senior python developer. Always wrap your code in ```python blocks."}, + {"role": "user", "content": "Please help me write some code."}, + {"role": "assistant", "content": "Of course! I can help you with that."}, + {"role": "user", "content": prompt} + ], + "tools": list(DUMMY_TOOL_REGISTRY.values()) + } + if mode == "with_plugin": + # Send extra_body for ContextPilot proxy to intercept + request["_required_skills"] = ["tool_1", "tool_3", "tool_7"] + + # 1. Apply plugins on the client before sending + request = await dedup_plugin.process(request) + request = await skill_plugin.process(request) + + # The proxy needs user_id and parent_id for cache tracking if implemented, + # though our http_server currently just forwards. + # But we must ensure the tools array is updated properly in api_kwargs! + + api_kwargs = { + "model": model_name, + "messages": request["messages"], + "tools": request["tools"] + } + + api_kwargs["extra_body"] = { + "user_id": request.get("user_id"), + "parent_id": request.get("parent_id"), + "_required_skills": request.get("_required_skills") + } + else: + api_kwargs = { + "model": model_name, + "messages": request["messages"], + "tools": request["tools"] + } + + if seed is not None: + api_kwargs["seed"] = seed + + try: + logger.info(f"[{mode}] Sending task {task_id}...") + response = await client.chat.completions.create(**api_kwargs) + response_content = response.choices[0].message.content + except Exception as e: + logger.error(f"[{mode}] API Error for {task_id}: {str(e)}") + response_content = "" + + # Extract code block using regex + extracted_code = "" + if response_content: + match = re.search(r"```python\s*(.*?)\s*```", response_content, re.DOTALL) + if match: + extracted_code = match.group(1).strip() + else: + # Fallback if the LLM didn't use the markdown block + extracted_code = response_content.strip() + + # Append result to JSONL + with open(output_file, "a", encoding="utf-8") as f: + f.write(json.dumps({"task_id": task_id, "solution": extracted_code}) + "\n") + + logger.info(f"[{mode}] Finished {task_id}") + +async def run_evaluation(mode, args, tasks): + # Route BOTH baseline and with_plugin through the proxy to intercept prompt_cache_hit_tokens + client = AsyncOpenAI(api_key=args.api_key, base_url="http://localhost:8000/v1") + + # We use a dummy turn_1_id for simulation + turn_1_id = "test-turn-1-id" + + seed_suffix = f"_seed{args.seed}" if args.seed is not None else "" + output_file = os.path.join(os.path.dirname(__file__), f"results_{mode}_{args.model}{seed_suffix}.jsonl") + if os.path.exists(output_file): + os.remove(output_file) + + # Use configurable Semaphore to allow high concurrency + semaphore = asyncio.Semaphore(args.concurrency) + + # Instantiate plugins + from refactored_plugins.dedup import ContextDedupPlugin + from refactored_plugins.skill_index import SkillAwareContextPlugin + global dedup_plugin, skill_plugin + dedup_plugin = ContextDedupPlugin(shadow_mode=True) + skill_plugin = SkillAwareContextPlugin(DUMMY_TOOL_REGISTRY) + + # SEED THE TRACKER FOR TELEMETRY: + # Inject the "Turn 1" system prompt and history into the dedup plugin's memory. + # Without this, the tracker thinks test-turn-1-id is empty, resulting in 0 chars saved! + turn_1_messages = [ + {"role": "system", "content": "You are a senior python developer. Always wrap your code in ```python blocks."}, + {"role": "user", "content": "Please help me write some code."}, + {"role": "assistant", "content": "Of course! I can help you with that."} + ] + msg_ids = [dedup_plugin._get_id(m["content"]) for m in turn_1_messages] + dedup_plugin.tracker.deduplicate(request_id=turn_1_id, docs=msg_ids, parent_request_id=None) + + coroutines = [process_task(t, client, semaphore, output_file, turn_1_id, mode, args.model, args.seed) for t in tasks] + await asyncio.gather(*coroutines) + + print(f"\n=== Evaluation Complete for mode: {mode} ===") + print(f"Results saved to {output_file}") + + if mode == "with_plugin": + print("\n=== ContextPilot Client Telemetry ===") + dedup_metrics = dedup_plugin.get_plugin_metrics() + skill_metrics = skill_plugin.get_plugin_metrics() + print(f"[Dedup] Chars Saved: {dedup_metrics['total_chars_saved']} / {dedup_metrics['total_original_chars']} ({dedup_metrics['chars_saved_percentage']:.2f}%)") + print(f"[Skill] Tools Filtered: {skill_metrics['total_tools_filtered']} / {skill_metrics.get('total_original_tools', 'N/A')} ({skill_metrics.get('tools_filtered_percentage', 0):.2f}%)") + + +async def main(): + parser = argparse.ArgumentParser(description="BigCodeBench ELM API Runner") + parser.add_argument("--model", default="gpt-5.5", help="Model name to evaluate") + parser.add_argument("--api_base", default=os.environ.get("BASE_URL", "https://api.openai.com/v1"), help="Baseline ELM API Base URL") + parser.add_argument("--api_key", default=os.environ.get("OPENAI_API_KEY", "dummy-elm-key"), help="API Key") + parser.add_argument("--concurrency", type=int, default=1, help="Number of concurrent requests") + parser.add_argument("--limit", type=int, default=0, help="Limit number of tasks to run (0 for all)") + parser.add_argument("--eval_mode", choices=["baseline", "with_plugin", "all"], default="all", help="Evaluation mode") + parser.add_argument("--seed", type=int, default=None, help="Random seed for the LLM (None = not set)") + args = parser.parse_args() + + # Load BigCodeBench dataset + logger.info("Loading BigCodeBench dataset...") + try: + dataset = load_dataset("bigcode/bigcodebench", split="train") + except Exception as e: + logger.warning(f"Failed to load split='train'. Trying standard default split. Error: {e}") + # Fallback to the common default split format if 'train' split does not exist + try: + dataset = load_dataset("bigcode/bigcodebench", split="v0.1.2") + except Exception: + dataset = load_dataset("bigcode/bigcodebench", split="v0.1.0_240822") + + # Select all tasks for full evaluation + tasks = list(dataset) + if args.limit > 0: + tasks = tasks[:args.limit] + logger.info(f"Loaded {len(tasks)} tasks (LIMITED) for evaluation.") + else: + logger.info(f"Loaded {len(tasks)} tasks for full evaluation.") + + if args.eval_mode == "all": + modes = ["baseline", "with_plugin"] + else: + modes = [args.eval_mode] + + for mode in modes: + logger.info(f"\n--- Starting Evaluation: {mode} ---") + await run_evaluation(mode, args, tasks) + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/evaluation/benchmarks/run_elm_eval.py b/evaluation/benchmarks/run_elm_eval.py new file mode 100644 index 0000000..eb7530c --- /dev/null +++ b/evaluation/benchmarks/run_elm_eval.py @@ -0,0 +1,150 @@ +import asyncio +import os +import json +import logging +from typing import Any, Dict +from openai import AsyncOpenAI + +# Set PYTHONPATH in the environment before running if needed +from refactored_plugins.skill_index import SkillAwareContextPlugin +from refactored_plugins.dedup import ContextDedupPlugin + +logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s") + +# Dummy tool registry for SkillAwareContextPlugin to filter from +DUMMY_TOOL_REGISTRY = { + "python_repl": { + "type": "function", + "function": { + "name": "python_repl", + "description": "Executes Python code in a sandboxed environment" + } + }, + "web_search": { + "type": "function", + "function": { + "name": "web_search", + "description": "Searches the web for up-to-date documentation" + } + }, + "file_writer": { + "type": "function", + "function": { + "name": "file_writer", + "description": "Writes code to a file" + } + } +} + +async def evaluate_task(task_data: Dict[str, Any], + skill_plugin: SkillAwareContextPlugin, + dedup_plugin: ContextDedupPlugin, + client: AsyncOpenAI) -> Dict[str, Any]: + """ + Optimizes a request using Phase 1 plugins and sends it to the ELM API. + """ + # 1. ContextDedupPlugin (strip redundant history) + optimized_data = await dedup_plugin.process(task_data) + + # 2. SkillAwareContextPlugin (strip redundant tools based on _required_skills) + optimized_data = await skill_plugin.process(optimized_data) + + # 3. Call ELM API via OpenAI client + api_kwargs = { + "model": "gpt-5.5", + "messages": optimized_data.get("messages", []) + } + + # Only pass tools if the plugin injected any + if "tools" in optimized_data and optimized_data["tools"]: + api_kwargs["tools"] = optimized_data["tools"] + + try: + response = await client.chat.completions.create(**api_kwargs) + message = response.choices[0].message + if message.tool_calls: + # If the model decides to invoke tools, format the tool calls details + calls = [] + for tc in message.tool_calls: + # Safely parse arguments if present + args = {} + if tc.function.arguments: + try: + args = json.loads(tc.function.arguments) + except Exception: + args = tc.function.arguments + calls.append({ + "id": tc.id, + "name": tc.function.name, + "arguments": args + }) + response_content = f"Tool Calls Triggered:\n{json.dumps(calls, indent=2)}" + else: + response_content = message.content + except Exception as e: + response_content = f"API Error: {str(e)}" + response = None + + # 4. Gather Telemetry + telemetry = { + "skill_plugin_metrics": skill_plugin.get_plugin_metrics(), + "dedup_plugin_metrics": dedup_plugin.get_plugin_metrics() + } + + return { + "response": response, + "response_content": response_content, + "telemetry": telemetry, + "optimized_payload": optimized_data + } + +async def main(): + api_key = os.environ.get("OPENAI_API_KEY", "dummy-elm-key") + base_url = os.environ.get("BASE_URL", "https://api.openai.com/v1") + + client = AsyncOpenAI(api_key=api_key, base_url=base_url) + + skill_plugin = SkillAwareContextPlugin(tool_registry=DUMMY_TOOL_REGISTRY) + dedup_plugin = ContextDedupPlugin() + + print("=== Phase 2: ELM API Evaluator ===") + + # To demonstrate deduplication savings, we first run a mock Turn 1 to prime the history + turn_1 = { + "user_id": "evaluator_1", + "messages": [ + {"role": "system", "content": "You are an expert Python engineer taking the BigCodeBench evaluation."}, + {"role": "user", "content": "Write a script to compute the fast inverse square root."}, + {"role": "assistant", "content": "Here is the implementation: `def q_rsqrt(number): ...`"} + ] + } + turn_1_res = await dedup_plugin.process(turn_1) + parent_id = turn_1_res.get("current_id") + + # Mock BigCodeBench Turn 2 Task (includes redundant history from Turn 1) + mock_task = { + "user_id": "evaluator_1", + "parent_id": parent_id, + "_required_skills": ["python_repl", "file_writer"], + "messages": [ + {"role": "system", "content": "You are an expert Python engineer taking the BigCodeBench evaluation."}, + {"role": "user", "content": "Write a script to compute the fast inverse square root."}, + {"role": "assistant", "content": "Here is the implementation: `def q_rsqrt(number): ...`"}, + {"role": "user", "content": "Now, execute this code in the python_repl to verify it handles float(0.15625) correctly."} + ] + } + + print(f"\n[1] Starting API Request Evaluation...") + result = await evaluate_task(mock_task, skill_plugin, dedup_plugin, client) + + print("\n[2] Optimized Payload Sent to ELM API:") + print(json.dumps(result["optimized_payload"], indent=2)) + + print("\n[3] ELM API Response:") + print(result["response_content"]) + + print("\n[4] Cost-Savings Telemetry:") + print(json.dumps(result["telemetry"], indent=2)) + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/evaluation/benchmarks/run_mcpatlas.py b/evaluation/benchmarks/run_mcpatlas.py new file mode 100644 index 0000000..fccf319 --- /dev/null +++ b/evaluation/benchmarks/run_mcpatlas.py @@ -0,0 +1,211 @@ +import argparse +import asyncio +import json +import logging +import os +import time + +from datasets import load_dataset +from openai import AsyncOpenAI + +from refactored_plugins.dedup import ContextDedupPlugin +from refactored_plugins.dynamic_pruning import DynamicPruningPlugin +from refactored_plugins.skill_index import SkillAwareContextPlugin + +logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s") +logger = logging.getLogger(__name__) + +async def process_task(task, client, semaphore, mode, model_name, active_plugins, threshold): + async with semaphore: + task_id = task.get("id", str(time.time())) + + prompt = task.get("query", task.get("user_prompt", "Please use the appropriate tool.")) + available_tools = task.get("tools", []) + if isinstance(available_tools, str): + try: + available_tools = json.loads(available_tools) + except: + available_tools = [] + + formatted_tools = [] + for t in available_tools: + if isinstance(t, dict) and "name" in t: + raw_params = t.get("parameters", {}) + if not isinstance(raw_params, dict): raw_params = {} + openai_params = {"type": "object", "properties": {}, "required": []} + for p_name, p_info in raw_params.items(): + if not isinstance(p_info, dict): continue + p_type = p_info.get("type", "string") + if isinstance(p_type, str): + p_type_lower = p_type.lower() + if "str" in p_type_lower: p_type = "string" + elif "int" in p_type_lower: p_type = "integer" + elif "float" in p_type_lower: p_type = "number" + elif "bool" in p_type_lower: p_type = "boolean" + elif "dict" in p_type_lower: p_type = "object" + elif "list" in p_type_lower: p_type = "array" + else: p_type = "string" + else: p_type = "string" + + prop = {"type": p_type} + if "description" in p_info: prop["description"] = str(p_info["description"]) + openai_params["properties"][p_name] = prop + + if "default" not in p_info: + if "optional" not in str(p_info.get("type", "")).lower(): + openai_params["required"].append(p_name) + + formatted_tools.append({ + "type": "function", + "function": { + "name": t.get("name"), + "description": t.get("description", ""), + "parameters": openai_params + } + }) + elif isinstance(t, dict) and "type" in t and t["type"] == "function": + formatted_tools.append(t) + + ground_truth_tool = None + answers = task.get("answers", []) + if isinstance(answers, str): + try: answers = json.loads(answers) + except: answers = [] + if answers and isinstance(answers, list) and len(answers) > 0: + if isinstance(answers[0], dict): ground_truth_tool = answers[0].get("name") + else: ground_truth_tool = answers[0] + if not ground_truth_tool: ground_truth_tool = task.get("expected_tool", "unknown_tool") + + request = { + "messages": [ + {"role": "system", "content": "You are a helpful assistant with access to tools. Always use tools when appropriate."}, + {"role": "user", "content": prompt} + ], + "tools": formatted_tools + } + + skill_plugin = None + dynamic_plugin = None + if mode == "with_plugin" and len(formatted_tools) > 0: + if "skill" in active_plugins or "all" in active_plugins: + registry = {t["function"]["name"]: t for t in formatted_tools} + skill_plugin = SkillAwareContextPlugin(registry) + request["_required_skills"] = [ground_truth_tool] + request = await skill_plugin.process(request) + + if "dynamic" in active_plugins or "all" in active_plugins: + dynamic_plugin = DynamicPruningPlugin(similarity_threshold=threshold) + request = await dynamic_plugin.process(request) + + api_kwargs = { + "model": model_name, + "messages": request["messages"], + "tools": request["tools"], + "extra_body": { + "_required_skills": request.get("_required_skills") + } + } + else: + api_kwargs = { + "model": model_name, + "messages": request["messages"] + } + if len(formatted_tools) > 0: + api_kwargs["tools"] = formatted_tools + + selected_tool = None + try: + logger.info(f"[{mode}] Sending task {task_id}...") + response = await client.chat.completions.create(**api_kwargs) + message = response.choices[0].message + if message.tool_calls and len(message.tool_calls) > 0: + selected_tool = message.tool_calls[0].function.name + else: + selected_tool = "No tool called" + except Exception as e: + logger.error(f"[{mode}] API Error for {task_id}: {str(e)}") + selected_tool = "Error" + + is_correct = (selected_tool == ground_truth_tool) + logger.info(f"[{mode}] Finished {task_id} - Selected: {selected_tool}, Expected: {ground_truth_tool}, Correct: {is_correct}") + + metrics = {} + if skill_plugin: metrics["skill"] = skill_plugin.get_plugin_metrics() + if dynamic_plugin: metrics["dynamic"] = dynamic_plugin.get_plugin_metrics() + + return is_correct, metrics + +async def run_evaluation(mode, args, tasks): + client = AsyncOpenAI(api_key=args.api_key, base_url="http://localhost:8000/v1" if mode == "with_plugin" else args.api_base) + semaphore = asyncio.Semaphore(args.concurrency) + active_plugins = [p.strip() for p in args.plugins.split(",")] + + coroutines = [process_task(t, client, semaphore, mode, args.model, active_plugins, args.threshold) for t in tasks] + results = await asyncio.gather(*coroutines) + + correct_count = sum(r[0] for r in results) + total_count = len(results) + accuracy = (correct_count / total_count * 100) if total_count > 0 else 0 + + print(f"\n=== Evaluation Complete for mode: {mode} ===") + print(f"Total Tasks: {total_count}") + print(f"Correct Tool Selection: {correct_count}") + print(f"Tool-Selection Accuracy: {accuracy:.2f}%\n") + + if mode == "with_plugin": + print("=== ContextPilot Client Telemetry ===") + if "skill" in active_plugins or "all" in active_plugins: + total_orig = sum(r[1]["skill"]["total_original_tools"] for r in results if r[1] and "skill" in r[1]) + total_filt = sum(r[1]["skill"]["total_tools_filtered"] for r in results if r[1] and "skill" in r[1]) + perc = (total_filt / total_orig * 100) if total_orig > 0 else 0 + print(f"[Skill] Tools Filtered: {total_filt} / {total_orig} ({perc:.2f}%)") + if "dynamic" in active_plugins or "all" in active_plugins: + total_orig = sum(r[1]["dynamic"]["total_original_chars"] for r in results if r[1] and "dynamic" in r[1]) + total_filt = sum(r[1]["dynamic"]["total_chars_saved"] for r in results if r[1] and "dynamic" in r[1]) + perc = (total_filt / total_orig * 100) if total_orig > 0 else 0 + print(f"[Dynamic Pruning] Chars Saved: {total_filt} / {total_orig} ({perc:.2f}%)") + +async def main(): + parser = argparse.ArgumentParser(description="MCP-Atlas Toolkit Evaluation") + parser.add_argument("--model", default="gpt-5.5", help="Model name to evaluate") + parser.add_argument("--api_base", default="https://api.openai.com/v1", help="API Base URL") + parser.add_argument("--api_key", default="dummy-key", help="API Key") + parser.add_argument("--concurrency", type=int, default=1, help="Number of concurrent requests") + parser.add_argument("--limit", type=int, default=0, help="Limit number of tasks to run (0 for all)") + parser.add_argument("--eval_mode", choices=["baseline", "with_plugin", "all"], default="all", help="Evaluation mode") + parser.add_argument("--plugins", default="all", help="Comma-separated list of plugins (dedup,dynamic,skill,all)") + parser.add_argument("--threshold", type=float, default=0.3, help="Threshold for DynamicPruningPlugin") + args = parser.parse_args() + + logger.info("Loading Tool-Use dataset...") + try: + dataset = load_dataset("Salesforce/xlam-function-calling-60k", split="train[:500]") + except Exception as e: + logger.warning(f"Failed to load standard dataset. Generating dummy tasks. Error: {e}") + dataset = [ + { + "id": f"task_{i}", + "query": f"What is the weather in city {i}?", + "tools": [ + {"name": "get_weather", "description": "Get weather for a city"}, + {"name": "get_time", "description": "Get current time"} + ], + "expected_tool": "get_weather" + } + for i in range(10) + ] + + tasks = list(dataset) + if args.limit > 0: + tasks = tasks[:args.limit] + logger.info(f"Loaded {len(tasks)} tasks (LIMITED) for evaluation.") + else: + logger.info(f"Loaded {len(tasks)} tasks for full evaluation.") + + modes = ["baseline", "with_plugin"] if args.eval_mode == "all" else [args.eval_mode] + for mode in modes: + logger.info(f"\n--- Starting Evaluation: {mode} ---") + await run_evaluation(mode, args, tasks) + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/evaluation/benchmarks/run_mcpatlas_eval.py b/evaluation/benchmarks/run_mcpatlas_eval.py new file mode 100644 index 0000000..308f53b --- /dev/null +++ b/evaluation/benchmarks/run_mcpatlas_eval.py @@ -0,0 +1,249 @@ +import argparse +import asyncio +import json +import logging +import os +import time +import random + +from datasets import load_dataset +from openai import AsyncOpenAI + +from refactored_plugins.skill_index import SkillAwareContextPlugin + +logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s") +logger = logging.getLogger(__name__) + +async def process_task(task, client, semaphore, mode, model_name, distractor_ratio, global_tool_pool=None): + async with semaphore: + task_id = task.get("id", str(time.time())) + + # We will assume a standard format where 'query' is user prompt, 'tools' is list of tools, + # and 'answers' contains the ground truth function name. + prompt = task.get("query", task.get("user_prompt", "Please use the appropriate tool.")) + available_tools = task.get("tools", []) + if isinstance(available_tools, str): + try: + available_tools = json.loads(available_tools) + except: + available_tools = [] + + ground_truth_tool = None + answers = task.get("answers", []) + if isinstance(answers, str): + try: + answers = json.loads(answers) + except: + answers = [] + + if answers and isinstance(answers, list) and len(answers) > 0: + if isinstance(answers[0], dict): + ground_truth_tool = answers[0].get("name") + else: + ground_truth_tool = answers[0] + + if not ground_truth_tool: + ground_truth_tool = task.get("expected_tool", "unknown_tool") + + if global_tool_pool: + import random + random.seed(42) + distractor_candidates = [t for t in global_tool_pool if t.get("name") != ground_truth_tool and t.get("name") not in [ex.get("name") for ex in available_tools if isinstance(ex, dict)]] + sampled_distractors = random.sample(distractor_candidates, min(50, len(distractor_candidates))) + available_tools.extend(sampled_distractors) + + # Format the tools for OpenAI API + formatted_tools = [] + for t in available_tools: + if isinstance(t, dict) and "name" in t: + raw_params = t.get("parameters", {}) + if not isinstance(raw_params, dict): + raw_params = {} + + openai_params = {"type": "object", "properties": {}, "required": []} + for p_name, p_info in raw_params.items(): + if not isinstance(p_info, dict): + continue + + p_type = p_info.get("type", "string") + if isinstance(p_type, str): + p_type_lower = p_type.lower() + if "str" in p_type_lower: p_type = "string" + elif "int" in p_type_lower: p_type = "integer" + elif "float" in p_type_lower: p_type = "number" + elif "bool" in p_type_lower: p_type = "boolean" + elif "dict" in p_type_lower: p_type = "object" + elif "list" in p_type_lower: p_type = "array" + else: p_type = "string" + else: + p_type = "string" + + prop = {"type": p_type} + if "description" in p_info: + prop["description"] = str(p_info["description"]) + + openai_params["properties"][p_name] = prop + + if "default" not in p_info: + raw_type_str = str(p_info.get("type", "")).lower() + if "optional" not in raw_type_str: + openai_params["required"].append(p_name) + + formatted_tools.append({ + "type": "function", + "function": { + "name": t.get("name"), + "description": t.get("description", ""), + "parameters": openai_params + } + }) + elif isinstance(t, dict) and "type" in t and t["type"] == "function": + formatted_tools.append(t) + + request = { + "messages": [ + {"role": "system", "content": "You are a helpful assistant with access to tools. Always use tools when appropriate."}, + {"role": "user", "content": prompt} + ], + "tools": formatted_tools + } + + skill_plugin = None + if mode == "with_plugin" and len(formatted_tools) > 0: + # Dynamically instantiate a plugin instance per task to avoid async race conditions on registry + registry = {t["function"]["name"]: t for t in formatted_tools} + skill_plugin = SkillAwareContextPlugin(registry) + + # For evaluation, we simulate that the router correctly predicted the ground truth tool + if distractor_ratio is not None: + all_tools = [t["function"]["name"] for t in formatted_tools] + distractors = [t for t in all_tools if t != ground_truth_tool] + k = round(distractor_ratio * len(distractors)) + random.seed(42) + retained_distractors = random.sample(distractors, k) if k > 0 else [] + request["_required_skills"] = [ground_truth_tool] + retained_distractors + else: + request["_required_skills"] = [ground_truth_tool] + + request = await skill_plugin.process(request) + + api_kwargs = { + "model": model_name, + "messages": request["messages"], + "tools": request["tools"] + } + else: + api_kwargs = { + "model": model_name, + "messages": request["messages"] + } + if len(formatted_tools) > 0: + api_kwargs["tools"] = formatted_tools + + selected_tool = None + try: + logger.info(f"[{mode}] Sending task {task_id}...") + response = await client.chat.completions.create(**api_kwargs) + + message = response.choices[0].message + if message.tool_calls and len(message.tool_calls) > 0: + selected_tool = message.tool_calls[0].function.name + else: + selected_tool = "No tool called" + + except Exception as e: + logger.error(f"[{mode}] API Error for {task_id}: {str(e)}") + selected_tool = "Error" + + is_correct = (selected_tool == ground_truth_tool) + logger.info(f"[{mode}] Finished {task_id} - Selected: {selected_tool}, Expected: {ground_truth_tool}, Correct: {is_correct}") + + metrics = skill_plugin.get_plugin_metrics() if skill_plugin else None + return is_correct, metrics + +async def run_evaluation(mode, args, tasks, global_tool_pool=None): + # Both baseline and with_plugin use the provided api_base + client = AsyncOpenAI(api_key=args.api_key, base_url=args.api_base) + semaphore = asyncio.Semaphore(args.concurrency) + + coroutines = [process_task(t, client, semaphore, mode, args.model, args.distractor_ratio, global_tool_pool) for t in tasks] + results = await asyncio.gather(*coroutines) + + correct_count = sum(r[0] for r in results) + total_count = len(results) + accuracy = (correct_count / total_count * 100) if total_count > 0 else 0 + + print(f"\n=== Evaluation Complete for mode: {mode} ===") + print(f"Total Tasks: {total_count}") + print(f"Correct Tool Selection: {correct_count}") + print(f"Tool-Selection Accuracy: {accuracy:.2f}%\n") + + if mode == "with_plugin": + total_orig = sum(r[1]["total_original_tools"] for r in results if r[1]) + total_filt = sum(r[1]["total_tools_filtered"] for r in results if r[1]) + perc = (total_filt / total_orig * 100) if total_orig > 0 else 0 + print("=== ContextPilot Client Telemetry ===") + print(f"[Skill] Tools Filtered: {total_filt} / {total_orig} ({perc:.2f}%)") + +async def main(): + parser = argparse.ArgumentParser(description="MCP-Atlas Toolkit Evaluation") + parser.add_argument("--model", default="gpt-5.5", help="Model name to evaluate") + parser.add_argument("--api_base", default="https://api.openai.com/v1", help="API Base URL") + parser.add_argument("--api_key", default="dummy-elm-key", help="API Key") + parser.add_argument("--concurrency", type=int, default=1, help="Number of concurrent requests") + parser.add_argument("--limit", type=int, default=0, help="Limit number of tasks to run (0 for all)") + parser.add_argument("--eval_mode", choices=["baseline", "with_plugin", "all"], default="all", help="Evaluation mode") + parser.add_argument("--distractor_ratio", type=float, default=None, help="Ratio of distractor tools to retain in skill filtering") + args = parser.parse_args() + + logger.info("Loading Tool-Use dataset...") + try: + # Load a 500-task slice of the dataset to keep evaluation time and cost manageable + dataset = load_dataset("Salesforce/xlam-function-calling-60k", split="train[:500]") + except Exception as e: + logger.warning(f"Failed to load standard dataset. Generating dummy tasks. Error: {e}") + dataset = [ + { + "id": f"task_{i}", + "query": f"What is the weather in city {i}?", + "tools": [ + {"name": "get_weather", "description": "Get weather for a city"}, + {"name": "get_time", "description": "Get current time"}, + {"name": "calculate_sum", "description": "Calculate sum of numbers"}, + {"name": "search_web", "description": "Search the web"}, + {"name": "send_email", "description": "Send an email"}, + ], + "expected_tool": "get_weather" + } + for i in range(10) + ] + + global_tool_schemas = {} + for task in dataset: + raw_tools = task.get("tools", []) + if isinstance(raw_tools, str): + try: raw_tools = json.loads(raw_tools) + except: raw_tools = [] + for t in raw_tools: + if isinstance(t, dict) and "name" in t: + global_tool_schemas[t["name"]] = t + global_tool_pool = list(global_tool_schemas.values()) + + tasks = list(dataset) + if args.limit > 0: + tasks = tasks[:args.limit] + logger.info(f"Loaded {len(tasks)} tasks (LIMITED) for evaluation.") + else: + logger.info(f"Loaded {len(tasks)} tasks for full evaluation.") + + if args.eval_mode == "all": + modes = ["baseline", "with_plugin"] + else: + modes = [args.eval_mode] + + for mode in modes: + logger.info(f"\n--- Starting Evaluation: {mode} ---") + await run_evaluation(mode, args, tasks, global_tool_pool) + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/evaluation/benchmarks/run_sandbox_eval.ps1 b/evaluation/benchmarks/run_sandbox_eval.ps1 new file mode 100644 index 0000000..a85236f --- /dev/null +++ b/evaluation/benchmarks/run_sandbox_eval.ps1 @@ -0,0 +1,29 @@ +$source = "D:\AI4Coding\ContextPilot\evaluation\benchmarks\elm_samples.jsonl" +$destDir = "D:\AI4Coding\ContextPilot\evaluation\sandbox\results" +$destination = Join-Path $destDir "elm_samples.jsonl" + +Write-Host "Preparing Sandbox Environment..." + +# Ensure the results directory exists +if (-not (Test-Path -Path $destDir)) { + New-Item -ItemType Directory -Force -Path $destDir | Out-Null +} + +# Copy the samples to the volume-mounted folder +Copy-Item -Path $source -Destination $destination -Force +Write-Host "Successfully copied elm_samples.jsonl to the Docker volume mount ($destDir)." + +Write-Host "`nChecking if bigcodebench-sandbox is running..." +$containerStatus = docker inspect -f '{{.State.Running}}' bigcodebench-sandbox 2>$null +if ($containerStatus -ne "true") { + Write-Host "Container is not running. Starting it now..." + docker start bigcodebench-sandbox +} else { + Write-Host "Container is already running." +} + +Write-Host "`nExecuting BigCodeBench Evaluation securely inside the Docker Sandbox..." +# Run the evaluation command inside the existing running container +docker exec bigcodebench-sandbox bigcodebench.evaluate --samples /app/results/elm_samples.jsonl + +Write-Host "`nEvaluation Complete!" diff --git a/evaluation/benchmarks/run_sandbox_eval.sh b/evaluation/benchmarks/run_sandbox_eval.sh new file mode 100644 index 0000000..49461c6 --- /dev/null +++ b/evaluation/benchmarks/run_sandbox_eval.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Define paths (Git Bash / WSL compatible Windows paths or native Windows paths work with cp) +SOURCE="D:/AI4Coding/ContextPilot/evaluation/benchmarks/elm_samples.jsonl" +DEST_DIR="D:/AI4Coding/ContextPilot/evaluation/sandbox/results" +DEST="$DEST_DIR/elm_samples.jsonl" + +echo "Preparing Sandbox Environment..." + +# Ensure the results directory exists +mkdir -p "$DEST_DIR" + +# Copy the samples to the volume-mounted folder +cp "$SOURCE" "$DEST" +echo "Successfully copied elm_samples.jsonl to the Docker volume mount ($DEST_DIR)." + +echo -e "\nChecking if bigcodebench-sandbox is running..." +if [ "$(docker inspect -f '{{.State.Running}}' bigcodebench-sandbox 2>/dev/null)" != "true" ]; then + echo "Container is not running. Starting it now..." + docker start bigcodebench-sandbox +else + echo "Container is already running." +fi + +echo -e "\nExecuting BigCodeBench Evaluation securely inside the Docker Sandbox..." +# Run the evaluation command inside the existing running container +docker exec bigcodebench-sandbox bigcodebench.evaluate --samples /app/results/elm_samples.jsonl + +echo -e "\nEvaluation Complete!" diff --git a/evaluation/benchmarks/run_sandbox_eval_cluster.sh b/evaluation/benchmarks/run_sandbox_eval_cluster.sh new file mode 100755 index 0000000..41e45bd --- /dev/null +++ b/evaluation/benchmarks/run_sandbox_eval_cluster.sh @@ -0,0 +1,109 @@ +#!/bin/bash +# ============================================================================= +# run_sandbox_eval_cluster.sh +# Adapted from run_sandbox_eval.ps1 for Edinburgh Informatics cluster +# Uses Apptainer (instead of Docker) to run BigCodeBench evaluation +# ============================================================================= +set -euo pipefail + +# --- Configuration ----------------------------------------------------------- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +SANDBOX_DIR="$SCRIPT_DIR/../sandbox" +SIF_IMAGE="$SANDBOX_DIR/my_project.sif" +RESULTS_DIR="$SANDBOX_DIR/results" +SAMPLES_FILE="$SCRIPT_DIR/elm_samples.jsonl" +DEST_FILE="$RESULTS_DIR/elm_samples.jsonl" + +# BigCodeBench evaluation parameters +# SPLIT: "complete" or "instruct" (BigCodeBench task format) +SPLIT="${BCB_SPLIT:-complete}" +# SUBSET: "full" or "hard" +SUBSET="${BCB_SUBSET:-full}" + +echo "=============================================" +echo " ContextPilot Sandbox Evaluation (Cluster)" +echo "=============================================" +echo "" +echo "Configuration:" +echo " SIF Image: $SIF_IMAGE" +echo " Results Dir: $RESULTS_DIR" +echo " Samples File: $SAMPLES_FILE" +echo " Split: $SPLIT" +echo " Subset: $SUBSET" +echo "" + +# --- Step 1: Validate Prerequisites ----------------------------------------- +echo "[1/4] Validating prerequisites..." + +if [ ! -f "$SIF_IMAGE" ]; then + echo "ERROR: SIF image not found at $SIF_IMAGE" + echo "Build it first with:" + echo " cd $SANDBOX_DIR && apptainer build --fakeroot my_project.sif recipe.def" + exit 1 +fi + +if [ ! -f "$SAMPLES_FILE" ]; then + echo "ERROR: elm_samples.jsonl not found at $SAMPLES_FILE" + echo "" + echo "You need to generate it first by running run_bigcodebench_elm.py." + echo "This script calls the LLM API (requires OPENAI_API_KEY) to produce code" + echo "solutions, which are then evaluated inside the sandbox." + echo "" + echo "To generate samples:" + echo " cd $PROJECT_ROOT" + echo " export PYTHONPATH=\"$PROJECT_ROOT:\$PYTHONPATH\"" + echo " export OPENAI_API_KEY='your-api-key'" + echo " python3 evaluation/benchmarks/run_bigcodebench_elm.py" + echo "" + exit 1 +fi + +echo " ✓ SIF image found ($(du -h "$SIF_IMAGE" | cut -f1))" +echo " ✓ Samples file found ($(wc -l < "$SAMPLES_FILE") samples)" +echo "" + +# --- Step 2: Prepare Sandbox Environment ------------------------------------ +echo "[2/4] Preparing sandbox environment..." + +mkdir -p "$RESULTS_DIR" +cp "$SAMPLES_FILE" "$DEST_FILE" +echo " ✓ Copied elm_samples.jsonl to $RESULTS_DIR" +echo "" + +# --- Step 3: Run BigCodeBench Evaluation Inside Apptainer -------------------- +echo "[3/4] Executing BigCodeBench evaluation inside Apptainer sandbox..." +echo " Command: bigcodebench.evaluate $SPLIT $SUBSET --samples /app/results/elm_samples.jsonl --execution local --pass_k 1" +echo "" + +# Bind-mount the results directory into the container at /app/results +# --no-home: don't mount home directory (isolation) +# --bind: mount results directory so we can read input and write output +# --execution local: run tests locally inside container (remote Gradio rejects partial sample sets) +# --pass_k 1: compute only Pass@1 (we have 1 sample per task) +CACHE_DIR="$SANDBOX_DIR/cache" +mkdir -p "$CACHE_DIR" + +apptainer exec \ + --no-home \ + --writable-tmpfs \ + --bind "$RESULTS_DIR:/app/results" \ + --bind "$CACHE_DIR:/app/cache" \ + --env "HF_HOME=/app/cache" \ + --env "HF_DATASETS_CACHE=/app/cache/datasets" \ + --env "TMPDIR=/app/cache" \ + --env "XDG_CACHE_HOME=/app/cache" \ + "$SIF_IMAGE" \ + python3 /app/results/run_local_eval.py --samples /app/results/elm_samples.jsonl + +echo "" + +# --- Step 4: Report Results -------------------------------------------------- +echo "[4/4] Evaluation complete!" +echo "" +echo "Results saved to: $RESULTS_DIR" +ls -la "$RESULTS_DIR" +echo "" +echo "=============================================" +echo " Evaluation Finished Successfully" +echo "=============================================" diff --git a/evaluation/benchmarks/run_sandbox_eval_full.sh b/evaluation/benchmarks/run_sandbox_eval_full.sh new file mode 100755 index 0000000..4ff5d7d --- /dev/null +++ b/evaluation/benchmarks/run_sandbox_eval_full.sh @@ -0,0 +1,109 @@ +#!/bin/bash +# ============================================================================= +# run_sandbox_eval_cluster.sh +# Adapted from run_sandbox_eval.ps1 for Edinburgh Informatics cluster +# Uses Apptainer (instead of Docker) to run BigCodeBench evaluation +# ============================================================================= +set -euo pipefail + +# --- Configuration ----------------------------------------------------------- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +SANDBOX_DIR="$SCRIPT_DIR/../sandbox" +SIF_IMAGE="$SANDBOX_DIR/my_project.sif" +RESULTS_DIR="$SANDBOX_DIR/results" +SAMPLES_FILE="$SCRIPT_DIR/elm_samples_full.jsonl" +DEST_FILE="$RESULTS_DIR/elm_samples_full.jsonl" + +# BigCodeBench evaluation parameters +# SPLIT: "complete" or "instruct" (BigCodeBench task format) +SPLIT="${BCB_SPLIT:-complete}" +# SUBSET: "full" or "hard" +SUBSET="${BCB_SUBSET:-full}" + +echo "=============================================" +echo " ContextPilot Sandbox Evaluation (Cluster)" +echo "=============================================" +echo "" +echo "Configuration:" +echo " SIF Image: $SIF_IMAGE" +echo " Results Dir: $RESULTS_DIR" +echo " Samples File: $SAMPLES_FILE" +echo " Split: $SPLIT" +echo " Subset: $SUBSET" +echo "" + +# --- Step 1: Validate Prerequisites ----------------------------------------- +echo "[1/4] Validating prerequisites..." + +if [ ! -f "$SIF_IMAGE" ]; then + echo "ERROR: SIF image not found at $SIF_IMAGE" + echo "Build it first with:" + echo " cd $SANDBOX_DIR && apptainer build --fakeroot my_project.sif recipe.def" + exit 1 +fi + +if [ ! -f "$SAMPLES_FILE" ]; then + echo "ERROR: elm_samples_full.jsonl not found at $SAMPLES_FILE" + echo "" + echo "You need to generate it first by running run_bigcodebench_elm.py." + echo "This script calls the LLM API (requires OPENAI_API_KEY) to produce code" + echo "solutions, which are then evaluated inside the sandbox." + echo "" + echo "To generate samples:" + echo " cd $PROJECT_ROOT" + echo " export PYTHONPATH=\"$PROJECT_ROOT:\$PYTHONPATH\"" + echo " export OPENAI_API_KEY='your-api-key'" + echo " python3 evaluation/benchmarks/run_bigcodebench_elm.py" + echo "" + exit 1 +fi + +echo " ✓ SIF image found ($(du -h "$SIF_IMAGE" | cut -f1))" +echo " ✓ Samples file found ($(wc -l < "$SAMPLES_FILE") samples)" +echo "" + +# --- Step 2: Prepare Sandbox Environment ------------------------------------ +echo "[2/4] Preparing sandbox environment..." + +mkdir -p "$RESULTS_DIR" +cp "$SAMPLES_FILE" "$DEST_FILE" +echo " ✓ Copied elm_samples_full.jsonl to $RESULTS_DIR" +echo "" + +# --- Step 3: Run BigCodeBench Evaluation Inside Apptainer -------------------- +echo "[3/4] Executing BigCodeBench evaluation inside Apptainer sandbox..." +echo " Command: bigcodebench.evaluate $SPLIT $SUBSET --samples /app/results/elm_samples_full.jsonl --execution local --pass_k 1" +echo "" + +# Bind-mount the results directory into the container at /app/results +# --no-home: don't mount home directory (isolation) +# --bind: mount results directory so we can read input and write output +# --execution local: run tests locally inside container (remote Gradio rejects partial sample sets) +# --pass_k 1: compute only Pass@1 (we have 1 sample per task) +CACHE_DIR="$SANDBOX_DIR/cache" +mkdir -p "$CACHE_DIR" + +apptainer exec \ + --no-home \ + --writable-tmpfs \ + --bind "$RESULTS_DIR:/app/results" \ + --bind "$CACHE_DIR:/app/cache" \ + --env "HF_HOME=/app/cache" \ + --env "HF_DATASETS_CACHE=/app/cache/datasets" \ + --env "TMPDIR=/app/cache" \ + --env "XDG_CACHE_HOME=/app/cache" \ + "$SIF_IMAGE" \ + python3 /app/results/run_local_eval.py --samples /app/results/elm_samples_full.jsonl + +echo "" + +# --- Step 4: Report Results -------------------------------------------------- +echo "[4/4] Evaluation complete!" +echo "" +echo "Results saved to: $RESULTS_DIR" +ls -la "$RESULTS_DIR" +echo "" +echo "=============================================" +echo " Evaluation Finished Successfully" +echo "=============================================" diff --git a/evaluation/core_merge/mock_proxy.py b/evaluation/core_merge/mock_proxy.py new file mode 100644 index 0000000..56be93c --- /dev/null +++ b/evaluation/core_merge/mock_proxy.py @@ -0,0 +1,150 @@ +import asyncio +import json +import logging +import sys +from typing import Any, Dict, List + +# Import all 4 plugins from refactored_plugins +from refactored_plugins.skill_index import SkillAwareContextPlugin +from refactored_plugins.dedup import ContextDedupPlugin +from refactored_plugins.reorder import ContextReorderPlugin +from refactored_plugins.kv_lookup import KVCacheLookupPlugin + +# Configure simple logging +logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s") + +class MockProxy: + """ + Simulates the core Token Proxy Middleware pipeline by chaining all Phase 1 plugins. + """ + def __init__(self): + # 1. SkillAwareContextPlugin (dummy tool registry) + dummy_tool_registry = { + "math": { + "type": "function", + "function": { + "name": "math_tool", + "description": "Performs mathematical calculations" + } + }, + "weather": { + "type": "function", + "function": { + "name": "weather_tool", + "description": "Gets the current weather" + } + } + } + self.skill_plugin = SkillAwareContextPlugin(tool_registry=dummy_tool_registry) + + # 2. ContextDedupPlugin + self.dedup_plugin = ContextDedupPlugin() + + # 3. ContextReorderPlugin + # We specify use_gpu=False for the mock test to avoid requiring torch/CUDA. + self.reorder_plugin = ContextReorderPlugin(use_gpu=False) + + # 4. KVCacheLookupPlugin (dummy ZMQ endpoints) + dummy_endpoints = ["tcp://localhost:5557", "tcp://localhost:5558"] + self.kv_lookup_plugin = KVCacheLookupPlugin(endpoints=dummy_endpoints) + + async def process_batch(self, request_batch: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """ + Executes the exact chain of execution for a batch of requests. + """ + # 1. SkillAwareContextPlugin (processes individual requests) + batch_after_skill = [] + for req in request_batch: + res = await self.skill_plugin.process(req) + batch_after_skill.append(res) + + # 2. ContextDedupPlugin (processes individual requests) + batch_after_dedup = [] + user_to_last_id = {} + for req in batch_after_skill: + user_id = req.get("user_id") + if user_id in user_to_last_id: + req["parent_id"] = user_to_last_id[user_id] + + res = await self.dedup_plugin.process(req) + + if user_id and "current_id" in res: + user_to_last_id[user_id] = res["current_id"] + + batch_after_dedup.append(res) + + # 3. ContextReorderPlugin (processes the entire batch) + batch_after_reorder = await self.reorder_plugin.process(batch_after_dedup) + + # 4. KVCacheLookupPlugin (processes individual requests) + final_batch = [] + for req in batch_after_reorder: + res = await self.kv_lookup_plugin.process(req) + final_batch.append(res) + + return final_batch + + def get_all_metrics(self) -> Dict[str, Dict[str, float]]: + """Aggregates telemetry from all plugins.""" + return { + "skill_plugin": self.skill_plugin.get_plugin_metrics(), + "dedup_plugin": self.dedup_plugin.get_plugin_metrics(), + "reorder_plugin": self.reorder_plugin.get_plugin_metrics(), + "kv_lookup_plugin": self.kv_lookup_plugin.get_plugin_metrics(), + } + +async def main(): + # Create a complex mock batch of 3 OpenAI requests simulating: + # 1. Multi-turn conversation (redundant history) + # 2. Dynamic Tool/Skill filtering + # 3. Overlapping system prompts for Prefix Sharing + mock_batch = [ + { + "user_id": "user_1", + "_required_skills": ["math"], + "messages": [ + {"role": "system", "content": "You are an AI assistant. Answer accurately and be concise."}, + {"role": "user", "content": "Hello! I am preparing for my exams."}, + {"role": "assistant", "content": "Hello! I can help you study. What subject?"}, + {"role": "user", "content": "Calculate 15 * 32 for my math homework."} + ] + }, + { + "user_id": "user_1", + "_required_skills": ["weather"], + "messages": [ + {"role": "system", "content": "You are an AI assistant. Answer accurately and be concise."}, + {"role": "user", "content": "Hello! I am preparing for my exams."}, + {"role": "assistant", "content": "Hello! I can help you study. What subject?"}, + {"role": "user", "content": "Actually, skip studying. What is the weather outside?"} + ] + }, + { + "user_id": "user_2", + "_required_skills": ["math", "weather", "invalid_skill"], + "messages": [ + {"role": "system", "content": "You are an AI assistant. Answer accurately and be concise."}, + {"role": "user", "content": "I need help with math and weather!"} + ] + } + ] + + print("=== Phase 1: Core Merge - Mock Proxy Pipeline ===\n") + print("[1] Initializing Mock Proxy & 4 Plugins...") + proxy = MockProxy() + + print("\n[2] Executing process_batch()...") + optimized_batch = await proxy.process_batch(mock_batch) + + print("\n=== Optimized Batch Output ===") + print(json.dumps(optimized_batch, indent=2)) + + print("\n=== Telemetry Metrics ===") + metrics = proxy.get_all_metrics() + print(json.dumps(metrics, indent=2)) + + # Force exit to cleanly terminate lingering background ZMQ tasks created by KVCacheLookupPlugin + sys.exit(0) + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/evaluation/profiling/dummy_agent.py b/evaluation/profiling/dummy_agent.py new file mode 100644 index 0000000..07281b6 --- /dev/null +++ b/evaluation/profiling/dummy_agent.py @@ -0,0 +1,55 @@ +import json +import re +import time +import random + +def heavy_json_ops(): + """Simulates API payload handling (serialization/deserialization).""" + # Create a reasonably complex nested structure + payload = { + "metadata": {"version": "1.0", "timestamp": time.time()}, + "agents": [ + {"id": i, "name": f"Agent_{i}", "history": ["observation" * 10 for _ in range(20)]} + for i in range(50) + ], + "configuration": {f"key_{i}": "value" * 50 for i in range(100)} + } + + # Burn CPU parsing and serializing + for _ in range(1000): # Reduced from 10k to keep dummy run time reasonable (approx few seconds) + s = json.dumps(payload) + _ = json.loads(s) + +def heavy_regex_ops(): + """Simulates prompt formatting and log parsing.""" + # Large block of text + base_text = "The quick brown fox jumps over the lazy dog. " * 500 + + # Search for patterns and manipulate strings + patterns = [r"\b\w{5}\b", r"fox.*?dog", r"[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+"] + + for _ in range(50): + # Regex search + for p in patterns: + re.findall(p, base_text) + + # String concatenation (O(n^2) behavior in some languages, but Python is optimized) + # Still burns time for large N + _ = "".join([base_text[i:i+10] for i in range(0, len(base_text), 2)]) + +def agent_turn(turn_id): + print(f"Executing Agent Turn {turn_id}...") + heavy_json_ops() + heavy_regex_ops() + +def main(): + start_time = time.time() + # Simulate 50 agent turns + for i in range(50): + agent_turn(i) + + end_time = time.time() + print(f"\nSimulation complete in {end_time - start_time:.2f} seconds.") + +if __name__ == "__main__": + main() diff --git a/evaluation/profiling/run_profile.ps1 b/evaluation/profiling/run_profile.ps1 new file mode 100644 index 0000000..cd6fec9 --- /dev/null +++ b/evaluation/profiling/run_profile.ps1 @@ -0,0 +1,11 @@ +# Ensure dependencies are installed +Write-Host "Installing snakeviz..." -ForegroundColor Cyan +pip install snakeviz --quiet + +# Run the script with cProfile +Write-Host "Running dummy_agent.py with cProfile..." -ForegroundColor Cyan +python -m cProfile -o agent_profile.prof dummy_agent.py + +# Visualize the results +Write-Host "Launching snakeviz for visualization..." -ForegroundColor Cyan +snakeviz agent_profile.prof diff --git a/evaluation/profiling/run_profile.sh b/evaluation/profiling/run_profile.sh new file mode 100644 index 0000000..bb10e44 --- /dev/null +++ b/evaluation/profiling/run_profile.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Ensure dependencies are installed +echo "Installing snakeviz..." +pip install snakeviz --quiet + +# Run the script with cProfile +# -o agent_profile.prof: outputs the binary profile data to a file +echo "Running dummy_agent.py with cProfile..." +python -m cProfile -o agent_profile.prof dummy_agent.py + +# Visualize the results +# This will open a browser tab with a flame graph/icicle graph +echo "Launching snakeviz for visualization..." +snakeviz agent_profile.prof diff --git a/evaluation/sandbox/Dockerfile b/evaluation/sandbox/Dockerfile new file mode 100644 index 0000000..e8ab197 --- /dev/null +++ b/evaluation/sandbox/Dockerfile @@ -0,0 +1,25 @@ +# Use a slim Python 3.10 base image +FROM python:3.10-slim + +# Install system dependencies that might be needed for common code execution +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + +# Create a non-root user for security +RUN useradd -m -s /bin/bash evaluser + +# Set working directory +WORKDIR /app + +# Install bigcodebench +RUN pip install --no-cache-dir bigcodebench transformers torch + +# Create results directory and set ownership +RUN mkdir -p /app/results && chown -R evaluser:evaluser /app + +# Switch to the non-root user +USER evaluser + +# Set entrypoint to bash for interactive testing/persistence +ENTRYPOINT ["/bin/bash"] diff --git a/evaluation/sandbox/docker-compose.yml b/evaluation/sandbox/docker-compose.yml new file mode 100644 index 0000000..fef038b --- /dev/null +++ b/evaluation/sandbox/docker-compose.yml @@ -0,0 +1,21 @@ +version: '3.8' + +services: + bigcodebench-sandbox: + build: + context: . + dockerfile: Dockerfile + container_name: bigcodebench-sandbox + # Security: Disable network access to prevent data exfiltration or external attacks + network_mode: "none" + deploy: + resources: + limits: + cpus: '1.0' + memory: 2G + volumes: + # Mount results directory to extract benchmark data + - ./results:/app/results + working_dir: /app + stdin_open: true # docker run -i + tty: true # docker run -t diff --git a/evaluation/sandbox/recipe.def b/evaluation/sandbox/recipe.def new file mode 100644 index 0000000..4d609d9 --- /dev/null +++ b/evaluation/sandbox/recipe.def @@ -0,0 +1,42 @@ +Bootstrap: docker +From: python:3.10-slim + +%post + # 1. Pin the container's temporary directory + export TMPDIR=/disk/scratch/s2206068/tmp + export PIP_CACHE_DIR=/disk/scratch/s2206068/pip_cache + + apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + wget \ + && rm -rf /var/lib/apt/lists/* + + useradd -m -s /bin/bash evaluser + + # 2. Core fix: + # Install the heavy dependencies (torch and transformers) separately first, + # since they ship as prebuilt binary wheels and do not need to be compiled. + pip install --no-cache-dir torch transformers + + # 3. Final fix: + # Fetch and install the evaluation requirements + wget https://raw.githubusercontent.com/bigcode-project/bigcodebench/main/Requirements/requirements-eval.txt + + # Install requirements-eval normally (allows build isolation for scikit-image) + pip install --no-cache-dir -r requirements-eval.txt + + # Install bigcodebench with --no-build-isolation. + # This forces pip to inherit the existing global environment variables + # instead of spawning a sub-venv that would drop them. + pip install --no-cache-dir --no-build-isolation bigcodebench + + # Force downgrade numpy, pandas, and protobuf to avoid binary incompatibility + pip install --no-cache-dir "numpy<1.23.0" "pandas<2.1.0" "protobuf<3.20" + + mkdir -p /app/results && chown -R evaluser:evaluser /app + +%environment + export LC_ALL=C + +%runscript + exec /bin/bash diff --git a/evaluation/sandbox/results/.gitkeep b/evaluation/sandbox/results/.gitkeep new file mode 100644 index 0000000..98770b1 --- /dev/null +++ b/evaluation/sandbox/results/.gitkeep @@ -0,0 +1 @@ +# Keep this directory tracked by Git, so the Docker mount target exists. diff --git a/evaluation/sandbox/results/run_local_eval.py b/evaluation/sandbox/results/run_local_eval.py new file mode 100644 index 0000000..f14685b --- /dev/null +++ b/evaluation/sandbox/results/run_local_eval.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +""" +Custom local evaluator for BigCodeBench smoke-test subset. +Runs test cases for only the tasks present in our samples file, +bypassing the full-dataset assertion in bigcodebench.evaluate. + +Usage (inside Apptainer container): + python3 /app/results/run_local_eval.py --samples /app/results/elm_samples.jsonl +""" +import argparse +import json +import os +import sys +import traceback +import multiprocessing +import unittest +from typing import Dict, Any + +# Attempt to import BigCodeBench data loader +try: + from bigcodebench.data import get_bigcodebench +except ImportError: + print("ERROR: bigcodebench package not available. Run inside the SIF container.") + sys.exit(1) + + +def run_test(task_id: str, solution_code: str, test_code: str, timeout: int = 30) -> Dict[str, Any]: + """ + Execute a solution + test harness in a subprocess with timeout. + Returns a dict with pass/fail status and optional error message. + """ + def _worker(solution_code: str, test_code: str, result_queue): + try: + exec_globals = {} + # Execute the solution code + exec(solution_code, exec_globals) + # Execute the test harness (defines the unittest.TestCase class) + exec(test_code, exec_globals) + + # Find all unittest.TestCase classes defined in the exec_globals + suite = unittest.TestSuite() + loader = unittest.TestLoader() + test_cases_found = False + + for name, value in list(exec_globals.items()): + if isinstance(value, type) and issubclass(value, unittest.TestCase) and value is not unittest.TestCase: + suite.addTests(loader.loadTestsFromTestCase(value)) + test_cases_found = True + + if not test_cases_found: + # Fallback for raw asserts + result_queue.put({"passed": True, "error": None}) + return + + # Run the loaded unit tests + runner = unittest.TextTestRunner(verbosity=0) + result = runner.run(suite) + + if result.wasSuccessful(): + result_queue.put({"passed": True, "error": None}) + else: + # Gather failure traceback summaries + errors = [] + for test, tb_text in result.failures + result.errors: + last_line = tb_text.strip().splitlines()[-1] if tb_text.strip() else "Unknown failure" + errors.append(f"{test.id().split('.')[-1]}: {last_line}") + result_queue.put({"passed": False, "error": "; ".join(errors)}) + + except Exception as e: + result_queue.put({"passed": False, "error": f"{type(e).__name__}: {str(e)}"}) + + result_queue = multiprocessing.Queue() + proc = multiprocessing.Process(target=_worker, args=(solution_code, test_code, result_queue)) + proc.start() + proc.join(timeout=timeout) + + if proc.is_alive(): + proc.kill() + proc.join() + return {"passed": False, "error": "TimeoutError: Execution exceeded time limit"} + + if not result_queue.empty(): + return result_queue.get() + else: + return {"passed": False, "error": "Process exited without result (possible crash)"} + + +def main(): + parser = argparse.ArgumentParser(description="Local BigCodeBench subset evaluator") + parser.add_argument("--samples", required=True, help="Path to JSONL samples file") + parser.add_argument("--timeout", type=int, default=30, help="Per-task timeout in seconds") + args = parser.parse_args() + + # Load samples + print(f"Loading samples from {args.samples}...") + samples = [] + with open(args.samples, "r") as f: + for line in f: + line = line.strip() + if line: + samples.append(json.loads(line)) + + print(f"Loaded {len(samples)} samples") + + # Load BigCodeBench problems (ground truth test cases) + print("Loading BigCodeBench dataset for test cases...") + problems = get_bigcodebench() + print(f"Loaded {len(problems)} problems from BigCodeBench") + + # Evaluate each sample + results = {} + passed_count = 0 + total_count = len(samples) + + print(f"\n{'='*60}") + print(f" Running evaluation on {total_count} tasks") + print(f"{'='*60}\n") + + for i, sample in enumerate(samples): + task_id = sample["task_id"] + solution = sample["solution"] + + if task_id not in problems: + print(f"[{i+1}/{total_count}] {task_id}: SKIP (not found in dataset)") + results[task_id] = {"status": "skip", "error": "Task not found in dataset"} + continue + + problem = problems[task_id] + test_code = problem.get("test", "") + + if not test_code: + print(f"[{i+1}/{total_count}] {task_id}: SKIP (no test code)") + results[task_id] = {"status": "skip", "error": "No test code available"} + continue + + # Run the test + result = run_test(task_id, solution, test_code, timeout=args.timeout) + + if result["passed"]: + passed_count += 1 + status_str = "PASS ✓" + else: + status_str = f"FAIL ✗ ({result['error']})" + + print(f"[{i+1}/{total_count}] {task_id}: {status_str}") + results[task_id] = { + "status": "pass" if result["passed"] else "fail", + "error": result["error"] + } + + # Calculate Pass@1 + pass_at_1 = passed_count / total_count if total_count > 0 else 0.0 + + print(f"\n{'='*60}") + print(f" RESULTS SUMMARY") + print(f"{'='*60}") + print(f" Total tasks: {total_count}") + print(f" Passed: {passed_count}") + print(f" Failed: {total_count - passed_count}") + print(f" Pass@1: {pass_at_1:.1%} ({passed_count}/{total_count})") + print(f"{'='*60}\n") + + # Save results JSON + output_path = args.samples.replace(".jsonl", "_eval_results.json") + eval_output = { + "pass_at_1": pass_at_1, + "total": total_count, + "passed": passed_count, + "failed": total_count - passed_count, + "details": results + } + with open(output_path, "w") as f: + json.dump(eval_output, f, indent=2) + print(f"Results saved to {output_path}") + +if __name__ == '__main__': + main() diff --git a/evaluation/slurm_launchers/submit_distractor_sweep_mcpatlas.slurm b/evaluation/slurm_launchers/submit_distractor_sweep_mcpatlas.slurm new file mode 100644 index 0000000..29714cc --- /dev/null +++ b/evaluation/slurm_launchers/submit_distractor_sweep_mcpatlas.slurm @@ -0,0 +1,36 @@ +#!/bin/bash +#SBATCH --partition=Teaching +#SBATCH --gres=gpu:1 +#SBATCH --mem=16G +#SBATCH --time=24:00:00 +#SBATCH --output=distractor_sweep_mcpatlas_%j.out +#SBATCH --error=distractor_sweep_mcpatlas_%j.err + +echo "=============================================" +echo " ContextPilot MCP-Atlas Distractor Sweep" +echo "=============================================" + +export PYTHONPATH="$(pwd):$PYTHONPATH" +export NO_PROXY="localhost,127.0.0.1,::1" +export no_proxy="localhost,127.0.0.1,::1" + +ELM_BASE_URL="${BASE_URL:-https://api.openai.com/v1}" +PROXY_TARGET_URL="${ELM_BASE_URL%/v1}" + +echo "[0/3] Installing dependencies..." +pip install -r requirements.txt +pip install fastapi uvicorn pydantic aiohttp datasets + + + +echo "[2/3] Running Sweep..." +for R in 0.0 0.25 0.5 0.75 1.0; do + echo "--- Distractor Ratio: $R ---" + python evaluation/benchmarks/run_mcpatlas_eval.py --model gpt-5.5 --api_base "$ELM_BASE_URL" --api_key "$OPENAI_API_KEY" --concurrency 1 --eval_mode with_plugin --distractor_ratio $R +done + + + +echo "=============================================" +echo " Pipeline Complete!" +echo "=============================================" diff --git a/evaluation/slurm_launchers/submit_eval_only.slurm b/evaluation/slurm_launchers/submit_eval_only.slurm new file mode 100644 index 0000000..eade688 --- /dev/null +++ b/evaluation/slurm_launchers/submit_eval_only.slurm @@ -0,0 +1,13 @@ +#!/bin/bash +#SBATCH --partition=Teaching +#SBATCH --gres=gpu:1 +#SBATCH --mem=16G +#SBATCH --time=24:00:00 +#SBATCH --output=eval_only_%j.out +#SBATCH --error=eval_only_%j.err + +echo "Running Sandbox evaluation..." +cd evaluation/benchmarks +bash run_sandbox_eval_full.sh + +echo "Re-evaluation finished!" diff --git a/evaluation/slurm_launchers/submit_final_ablation_elm.slurm b/evaluation/slurm_launchers/submit_final_ablation_elm.slurm new file mode 100644 index 0000000..86c1c6f --- /dev/null +++ b/evaluation/slurm_launchers/submit_final_ablation_elm.slurm @@ -0,0 +1,57 @@ +#!/bin/bash +#SBATCH --partition=Teaching +#SBATCH --gres=gpu:1 +#SBATCH --mem=16G +#SBATCH --time=24:00:00 +#SBATCH --output=final_ablation_elm_%j.out +#SBATCH --error=final_ablation_elm_%j.err + +echo "=============================================" +echo " Dynamic Pruning Ablation Evaluation (FULL)" +echo "=============================================" + +export PYTHONPATH="$(pwd):$PYTHONPATH" +export NO_PROXY="localhost,127.0.0.1,::1" +export no_proxy="localhost,127.0.0.1,::1" + +# Install missing dependencies on compute node before starting the proxy +echo "[0/3] Installing dependencies..." +pip install -r requirements.txt +pip install fastapi uvicorn pydantic aiohttp sentence-transformers + +# 1. Boot the local ContextPilot Proxy Server in the background +echo "[1/3] Booting ContextPilot Proxy Server on port 8000..." +ELM_BASE_URL="${BASE_URL:-https://api.openai.com/v1}" +PROXY_TARGET_URL="${ELM_BASE_URL%/v1}" +python -m contextpilot.server.http_server --port 8000 --infer-api-url "$PROXY_TARGET_URL" > proxy_final_ablation.log 2>&1 & +PROXY_PID=$! +sleep 5 + +THRESHOLDS=(0.6 0.7) + +echo "[2/3] Running Ablation Scripts (FULL)..." +for THRESHOLD in "${THRESHOLDS[@]}"; do + echo "--- Running Threshold: $THRESHOLD ---" + python evaluation/benchmarks/run_ablation_elm.py --model gpt-5.5 --concurrency 5 --eval_mode with_plugin --threshold $THRESHOLD +done + +echo "Shutting down Proxy Server to flush telemetry..." +kill -INT $PROXY_PID +sleep 2 + +echo "[3/3] Executing Apptainer Sandbox Evaluations..." +cd evaluation/benchmarks + +for THRESHOLD in "${THRESHOLDS[@]}"; do + echo "--- Evaluating Threshold: $THRESHOLD ---" + cp results_ablation_${THRESHOLD}_with_plugin_gpt-5.5.jsonl elm_samples_full.jsonl + bash run_sandbox_eval_full.sh + mv ../sandbox/results/elm_samples_full_eval_results.json ../sandbox/results/final_ablation_${THRESHOLD}_eval_results.json +done + +rm elm_samples_full.jsonl +cd ../.. + +echo "=============================================" +echo " Final Ablation Complete!" +echo "=============================================" diff --git a/evaluation/slurm_launchers/submit_final_all_plugins_elm.slurm b/evaluation/slurm_launchers/submit_final_all_plugins_elm.slurm new file mode 100644 index 0000000..8dfb9e3 --- /dev/null +++ b/evaluation/slurm_launchers/submit_final_all_plugins_elm.slurm @@ -0,0 +1,60 @@ +#!/bin/bash +#SBATCH --partition=Teaching +#SBATCH --gres=gpu:1 +#SBATCH --mem=16G +#SBATCH --time=24:00:00 +#SBATCH --output=final_all_plugins_%j.out +#SBATCH --error=final_all_plugins_%j.err + +echo "=============================================" +echo " All Plugins A/B Test Evaluation (FULL)" +echo "=============================================" + +export PYTHONPATH="$(pwd):$PYTHONPATH" +export NO_PROXY="localhost,127.0.0.1,::1" +export no_proxy="localhost,127.0.0.1,::1" + +# Install missing dependencies on compute node before starting the proxy +echo "[0/4] Installing dependencies..." +pip install -r requirements.txt +pip install fastapi uvicorn pydantic aiohttp sentence-transformers + +# 1. Boot the local ContextPilot Proxy Server in the background +echo "[1/4] Booting ContextPilot Proxy Server on port 8000..." +# Determine ELM API Base URL (stripping trailing /v1 if present for the proxy config) +ELM_BASE_URL="${BASE_URL:-https://api.openai.com/v1}" +PROXY_TARGET_URL="${ELM_BASE_URL%/v1}" +python -m contextpilot.server.http_server --port 8000 --infer-api-url "$PROXY_TARGET_URL" > proxy_final_all_plugins.log 2>&1 & +PROXY_PID=$! + +# Wait briefly for the server to be ready +sleep 5 + +# 2. Run the newly refactored A/B generation pipeline (FULL) +echo "[2/4] Running A/B API Evaluation Pipeline (Generation) FULL SUITE..." +python evaluation/benchmarks/run_bigcodebench_all_plugins.py --model gpt-5.5 --concurrency 5 + +echo "[3/4] Shutting down Proxy Server to flush telemetry..." +kill -INT $PROXY_PID +sleep 2 + +# 3. Sandbox Evaluation for Baseline +echo "[4/4] Executing Apptainer Sandbox Evaluations..." +cd evaluation/benchmarks + +echo "--- Evaluating Baseline Results ---" +cp results_all_plugins_baseline_gpt-5.5.jsonl elm_samples_full.jsonl +bash run_sandbox_eval_full.sh +mv ../sandbox/results/elm_samples_full_eval_results.json ../sandbox/results/final_all_plugins_baseline_eval_results.json + +echo "--- Evaluating With-Plugin Results ---" +cp results_all_plugins_with_plugin_gpt-5.5.jsonl elm_samples_full.jsonl +bash run_sandbox_eval_full.sh +mv ../sandbox/results/elm_samples_full_eval_results.json ../sandbox/results/final_all_plugins_with_plugin_eval_results.json + +rm elm_samples_full.jsonl + +cd ../.. +echo "=============================================" +echo " Test Pipeline Complete!" +echo "=============================================" diff --git a/evaluation/slurm_launchers/submit_final_bigcodebench_elm_seeds.slurm b/evaluation/slurm_launchers/submit_final_bigcodebench_elm_seeds.slurm new file mode 100755 index 0000000..491a9aa --- /dev/null +++ b/evaluation/slurm_launchers/submit_final_bigcodebench_elm_seeds.slurm @@ -0,0 +1,66 @@ +#!/bin/bash +#SBATCH --partition=Teaching +#SBATCH --gres=gpu:1 +#SBATCH --mem=16G +#SBATCH --time=48:00:00 +#SBATCH --output=final_eval_seeds_%j.out +#SBATCH --error=final_eval_seeds_%j.err + +echo "=============================================" +echo " ContextPilot Final A/B Evaluation (ELM GPT-5.5) - 3 Seeds" +echo "=============================================" + +export PYTHONPATH="$(pwd):$PYTHONPATH" +export NO_PROXY="localhost,127.0.0.1,::1" +export no_proxy="localhost,127.0.0.1,::1" + +echo "[0/4] Installing dependencies..." +pip install -r requirements.txt +pip install fastapi uvicorn pydantic aiohttp + +# 1. Boot the local ContextPilot Proxy Server in the background +echo "[1/4] Booting ContextPilot Proxy Server on port 8000..." +ELM_BASE_URL="${BASE_URL:-https://api.openai.com/v1}" +PROXY_TARGET_URL="${ELM_BASE_URL%/v1}" +python -m contextpilot.server.http_server --port 8000 --infer-api-url "$PROXY_TARGET_URL" > proxy_server.log 2>&1 & +PROXY_PID=$! + +sleep 5 + +# 2. Run the newly refactored A/B generation pipeline for seeds 0, 1, 2 +echo "[2/4] Running A/B API Evaluation Pipeline (Generation)..." +for SEED in 0 1 2; do + echo "--- Running Generation Pipeline for SEED ${SEED} ---" + python evaluation/benchmarks/run_bigcodebench_elm_full.py --model gpt-5.5 --concurrency 1 --eval_mode all --seed ${SEED} +done + +echo "[3/4] Shutting down Proxy Server to flush telemetry..." +kill -INT $PROXY_PID +sleep 2 + +# 3. Sandbox Evaluation for Baseline +echo "[4/4] Executing Apptainer Sandbox Evaluations..." + +cd evaluation/benchmarks + +for SEED in 0 1 2; do + echo "--- Evaluating Baseline Results (SEED ${SEED}) ---" + cp results_baseline_gpt-5.5_seed${SEED}.jsonl elm_samples_full.jsonl + bash run_sandbox_eval_full.sh + # Rename results so they aren't overwritten + mv ../sandbox/results/elm_samples_full_eval_results.json ../sandbox/results/baseline_eval_results_seed${SEED}.json + + echo "--- Evaluating With-Plugin Results (SEED ${SEED}) ---" + cp results_with_plugin_gpt-5.5_seed${SEED}.jsonl elm_samples_full.jsonl + bash run_sandbox_eval_full.sh + # Rename results + mv ../sandbox/results/elm_samples_full_eval_results.json ../sandbox/results/with_plugin_eval_results_seed${SEED}.json +done + +# Cleanup the temporary copy +rm -f elm_samples_full.jsonl + +cd ../.. +echo "=============================================" +echo " Final 3-Seed Evaluation Pipeline Complete!" +echo "=============================================" diff --git a/evaluation/slurm_launchers/submit_final_deepseek_eval.slurm b/evaluation/slurm_launchers/submit_final_deepseek_eval.slurm new file mode 100644 index 0000000..08abcd4 --- /dev/null +++ b/evaluation/slurm_launchers/submit_final_deepseek_eval.slurm @@ -0,0 +1,74 @@ +#!/bin/bash +#SBATCH --partition=Teaching +#SBATCH --gres=gpu:1 +#SBATCH --mem=16G +#SBATCH --time=24:00:00 +#SBATCH --output=final_deepseek_eval_%j.out +#SBATCH --error=final_deepseek_eval_%j.err + +echo "=============================================" +echo " ContextPilot A/B Test DeepSeek (FULL)" +echo "=============================================" + +export PYTHONPATH="$(pwd):$PYTHONPATH" +export NO_PROXY="localhost,127.0.0.1,::1" +export no_proxy="localhost,127.0.0.1,::1" + +export BASE_URL="https://api.deepseek.com/v1" +export OPENAI_API_KEY=$DEEPSEEK_API_KEY + +echo "[0/4] Installing dependencies..." +pip install -r requirements.txt +pip install fastapi uvicorn pydantic aiohttp + +PROXY_TARGET_URL="${BASE_URL%/v1}" + +echo "[1/4] Booting ContextPilot Proxy Server for Baseline..." +python -m contextpilot.server.http_server --port 8000 --infer-api-url "$PROXY_TARGET_URL" > proxy_final_deepseek_server_baseline.log 2>&1 & +PROXY_PID=$! +sleep 5 + +echo "[2/4] Running Baseline Evaluation (FULL)..." +python evaluation/benchmarks/run_bigcodebench_elm_full.py --model deepseek-v4-pro --api_base https://api.deepseek.com/v1 --api_key $OPENAI_API_KEY --concurrency 20 --eval_mode baseline + +echo "Shutting down Proxy Server to flush Baseline telemetry..." +kill -INT $PROXY_PID +sleep 2 + +echo "=== Baseline Proxy Server Telemetry Logs ===" +cat proxy_final_deepseek_server_baseline.log + +echo "[3/4] Booting ContextPilot Proxy Server for With-Plugin..." +python -m contextpilot.server.http_server --port 8000 --infer-api-url "$PROXY_TARGET_URL" > proxy_final_deepseek_server_plugin.log 2>&1 & +PROXY_PID=$! +sleep 5 + +echo "[4/4] Running With-Plugin Evaluation (FULL)..." +python evaluation/benchmarks/run_bigcodebench_elm_full.py --model deepseek-v4-pro --api_base https://api.deepseek.com/v1 --api_key $OPENAI_API_KEY --concurrency 20 --eval_mode with_plugin + +echo "Shutting down Proxy Server to flush With-Plugin telemetry..." +kill -INT $PROXY_PID +sleep 2 + +echo "=== With-Plugin Proxy Server Telemetry Logs ===" +cat proxy_final_deepseek_server_plugin.log + +echo "Executing Apptainer Sandbox Evaluations..." +cd evaluation/benchmarks + +echo "--- Evaluating Baseline Results ---" +cp results_baseline_deepseek-v4-pro.jsonl elm_samples_full.jsonl +bash run_sandbox_eval_full.sh +mv ../sandbox/results/elm_samples_full_eval_results.json ../sandbox/results/baseline_deepseek_eval_results.json + +echo "--- Evaluating With-Plugin Results ---" +cp results_with_plugin_deepseek-v4-pro.jsonl elm_samples_full.jsonl +bash run_sandbox_eval_full.sh +mv ../sandbox/results/elm_samples_full_eval_results.json ../sandbox/results/with_plugin_deepseek_eval_results.json + +rm elm_samples_full.jsonl +cd ../.. + +echo "=============================================" +echo " Final Evaluation Pipeline Complete!" +echo "=============================================" diff --git a/evaluation/slurm_launchers/submit_final_deepseek_pruning.slurm b/evaluation/slurm_launchers/submit_final_deepseek_pruning.slurm new file mode 100644 index 0000000..77111e9 --- /dev/null +++ b/evaluation/slurm_launchers/submit_final_deepseek_pruning.slurm @@ -0,0 +1,72 @@ +#!/bin/bash +#SBATCH --partition=Teaching +#SBATCH --gres=gpu:1 +#SBATCH --mem=16G +#SBATCH --time=24:00:00 +#SBATCH --output=final_deepseek_pruning_%j.out +#SBATCH --error=final_deepseek_pruning_%j.err + +echo "=============================================" +echo " DeepSeek Dynamic Pruning Evaluation (FULL)" +echo "=============================================" + +export PYTHONPATH="$(pwd):$PYTHONPATH" +export NO_PROXY="localhost,127.0.0.1,::1" +export no_proxy="localhost,127.0.0.1,::1" + +export BASE_URL="https://api.deepseek.com/v1" +export OPENAI_API_KEY=$DEEPSEEK_API_KEY + +echo "[0/6] Installing dependencies..." +pip install -r requirements.txt +pip install fastapi uvicorn pydantic aiohttp sentence-transformers + +PROXY_TARGET_URL="${BASE_URL%/v1}" + +echo "[1/6] Booting ContextPilot Proxy Server for Dynamic Pruning..." +python -m contextpilot.server.http_server --port 8000 --infer-api-url "$PROXY_TARGET_URL" > proxy_final_deepseek_dynamic.log 2>&1 & +PROXY_PID=$! +sleep 5 + +echo "[2/6] Running Configuration 1: Dynamic Pruning + Skill Filter (FULL)..." +python evaluation/benchmarks/run_bigcodebench_deepseek_dynamic.py --model deepseek-v4-pro --api_base https://api.deepseek.com/v1 --api_key $OPENAI_API_KEY --concurrency 20 + +echo "Shutting down Proxy Server to flush telemetry..." +kill -INT $PROXY_PID +sleep 2 + +echo "=== Dynamic Pruning Proxy Server Telemetry Logs ===" +cat proxy_final_deepseek_dynamic.log + +echo "[3/6] Booting ContextPilot Proxy Server for ALL PLUGINS..." +python -m contextpilot.server.http_server --port 8000 --infer-api-url "$PROXY_TARGET_URL" > proxy_final_deepseek_all.log 2>&1 & +PROXY_PID=$! +sleep 5 + +echo "[4/6] Running Configuration 2: ALL PLUGINS (FULL)..." +python evaluation/benchmarks/run_bigcodebench_deepseek_all.py --model deepseek-v4-pro --api_base https://api.deepseek.com/v1 --api_key $OPENAI_API_KEY --concurrency 20 + +echo "Shutting down Proxy Server to flush telemetry..." +kill -INT $PROXY_PID +sleep 2 + +echo "=== ALL PLUGINS Proxy Server Telemetry Logs ===" +cat proxy_final_deepseek_all.log + +echo "[5/6] Executing Apptainer Sandbox Evaluations..." +cd evaluation/benchmarks + +echo "--- Evaluating Configuration 1 (Dynamic Pruning) Results ---" +cp results_deepseek_dynamic_with_plugin_deepseek-v4-pro.jsonl elm_samples_full.jsonl +bash run_sandbox_eval_full.sh +mv ../sandbox/results/elm_samples_full_eval_results.json ../sandbox/results/final_deepseek_dynamic_eval_results.json + +echo "--- Evaluating Configuration 2 (ALL PLUGINS) Results ---" +cp results_deepseek_all_with_plugin_deepseek-v4-pro.jsonl elm_samples_full.jsonl +bash run_sandbox_eval_full.sh +mv ../sandbox/results/elm_samples_full_eval_results.json ../sandbox/results/final_deepseek_all_eval_results.json + +rm elm_samples_full.jsonl +cd ../.. + +echo "[6/6] Pipeline Complete!" diff --git a/evaluation/slurm_launchers/submit_final_dynamic_skill_elm.slurm b/evaluation/slurm_launchers/submit_final_dynamic_skill_elm.slurm new file mode 100644 index 0000000..6f4135a --- /dev/null +++ b/evaluation/slurm_launchers/submit_final_dynamic_skill_elm.slurm @@ -0,0 +1,60 @@ +#!/bin/bash +#SBATCH --partition=Teaching +#SBATCH --gres=gpu:1 +#SBATCH --mem=16G +#SBATCH --time=24:00:00 +#SBATCH --output=final_dynamic_pruning_%j.out +#SBATCH --error=final_dynamic_pruning_%j.err + +echo "=============================================" +echo " Dynamic Pruning A/B Test Evaluation (FULL)" +echo "=============================================" + +export PYTHONPATH="$(pwd):$PYTHONPATH" +export NO_PROXY="localhost,127.0.0.1,::1" +export no_proxy="localhost,127.0.0.1,::1" + +# Install missing dependencies on compute node before starting the proxy +echo "[0/4] Installing dependencies..." +pip install -r requirements.txt +pip install fastapi uvicorn pydantic aiohttp sentence-transformers + +# 1. Boot the local ContextPilot Proxy Server in the background +echo "[1/4] Booting ContextPilot Proxy Server on port 8000..." +# Determine ELM API Base URL (stripping trailing /v1 if present for the proxy config) +ELM_BASE_URL="${BASE_URL:-https://api.openai.com/v1}" +PROXY_TARGET_URL="${ELM_BASE_URL%/v1}" +python -m contextpilot.server.http_server --port 8000 --infer-api-url "$PROXY_TARGET_URL" > proxy_final_dynamic_pruning.log 2>&1 & +PROXY_PID=$! + +# Wait briefly for the server to be ready +sleep 5 + +# 2. Run the newly refactored A/B generation pipeline (FULL) +echo "[2/4] Running A/B API Evaluation Pipeline (Generation) FULL SUITE..." +python evaluation/benchmarks/run_bigcodebench_dynamic_skill.py --model gpt-5.5 --concurrency 5 + +echo "[3/4] Shutting down Proxy Server to flush telemetry..." +kill -INT $PROXY_PID +sleep 2 + +# 3. Sandbox Evaluation for Baseline +echo "[4/4] Executing Apptainer Sandbox Evaluations..." +cd evaluation/benchmarks + +echo "--- Evaluating Baseline Results ---" +cp results_dynamic_skill_baseline_gpt-5.5.jsonl elm_samples_full.jsonl +bash run_sandbox_eval_full.sh +mv ../sandbox/results/elm_samples_full_eval_results.json ../sandbox/results/final_dynamic_baseline_eval_results.json + +echo "--- Evaluating With-Plugin Results ---" +cp results_dynamic_skill_with_plugin_gpt-5.5.jsonl elm_samples_full.jsonl +bash run_sandbox_eval_full.sh +mv ../sandbox/results/elm_samples_full_eval_results.json ../sandbox/results/final_dynamic_with_plugin_eval_results.json + +rm elm_samples_full.jsonl + +cd ../.. +echo "=============================================" +echo " Test Pipeline Complete!" +echo "=============================================" diff --git a/evaluation/slurm_launchers/submit_final_elm_eval.slurm b/evaluation/slurm_launchers/submit_final_elm_eval.slurm new file mode 100644 index 0000000..b1517aa --- /dev/null +++ b/evaluation/slurm_launchers/submit_final_elm_eval.slurm @@ -0,0 +1,67 @@ +#!/bin/bash +#SBATCH --partition=Teaching +#SBATCH --gres=gpu:1 +#SBATCH --mem=16G +#SBATCH --time=24:00:00 +#SBATCH --output=final_eval_%j.out +#SBATCH --error=final_eval_%j.err + +echo "=============================================" +echo " ContextPilot Final A/B Evaluation (ELM GPT-5.5)" +echo "=============================================" + +export PYTHONPATH="$(pwd):$PYTHONPATH" +export NO_PROXY="localhost,127.0.0.1,::1" +export no_proxy="localhost,127.0.0.1,::1" + +# Install missing dependencies on compute node before starting the proxy +echo "[0/4] Installing dependencies..." +pip install -r requirements.txt +pip install fastapi uvicorn pydantic aiohttp + +# 1. Boot the local ContextPilot Proxy Server in the background +echo "[1/4] Booting ContextPilot Proxy Server on port 8000..." +# Determine ELM API Base URL (stripping trailing /v1 if present for the proxy config) +ELM_BASE_URL="${BASE_URL:-https://api.openai.com/v1}" +PROXY_TARGET_URL="${ELM_BASE_URL%/v1}" +python -m contextpilot.server.http_server --port 8000 --infer-api-url "$PROXY_TARGET_URL" > proxy_server.log 2>&1 & +PROXY_PID=$! + +# Wait briefly for the server to be ready +sleep 5 + +# 2. Run the newly refactored A/B generation pipeline +echo "[2/4] Running A/B API Evaluation Pipeline (Generation)..." +# Concurrency strictly set to 1 to respect ELM API Rate Limit policy +python evaluation/benchmarks/run_bigcodebench_elm_full.py --model gpt-5.5 --concurrency 1 + +echo "[3/4] Shutting down Proxy Server to flush telemetry..." +kill -INT $PROXY_PID +sleep 2 + +# 3. Sandbox Evaluation for Baseline +echo "[4/4] Executing Apptainer Sandbox Evaluations..." + +# The sandbox script is hardcoded to look for "elm_samples_full.jsonl". +# We will temporarily copy each result set to that filename and run the sandbox. +cd evaluation/benchmarks + +echo "--- Evaluating Baseline Results ---" +cp results_baseline_gpt-5.5.jsonl elm_samples_full.jsonl +bash run_sandbox_eval_full.sh +# Rename results so they aren't overwritten +mv ../sandbox/results/elm_samples_full_eval_results.json ../sandbox/results/baseline_eval_results.json + +echo "--- Evaluating With-Plugin Results ---" +cp results_with_plugin_gpt-5.5.jsonl elm_samples_full.jsonl +bash run_sandbox_eval_full.sh +# Rename results +mv ../sandbox/results/elm_samples_full_eval_results.json ../sandbox/results/with_plugin_eval_results.json + +# Cleanup the temporary copy +rm elm_samples_full.jsonl + +cd ../.. +echo "=============================================" +echo " Final Evaluation Pipeline Complete!" +echo "=============================================" diff --git a/evaluation/slurm_launchers/submit_final_mcpatlas_deepseek.slurm b/evaluation/slurm_launchers/submit_final_mcpatlas_deepseek.slurm new file mode 100644 index 0000000..e0d6fb9 --- /dev/null +++ b/evaluation/slurm_launchers/submit_final_mcpatlas_deepseek.slurm @@ -0,0 +1,89 @@ +#!/bin/bash +#SBATCH --partition=Teaching +#SBATCH --gres=gpu:1 +#SBATCH --mem=16G +#SBATCH --time=24:00:00 +#SBATCH --output=final_mcpatlas_deepseek_%j.out +#SBATCH --error=final_mcpatlas_deepseek_%j.err + +echo "=============================================" +echo " ContextPilot MCP-Atlas Final DeepSeek (FULL)" +echo "=============================================" + +export PYTHONPATH="$(pwd):$PYTHONPATH" +export NO_PROXY="localhost,127.0.0.1,::1" +export no_proxy="localhost,127.0.0.1,::1" + +export BASE_URL="https://api.deepseek.com/v1" +export OPENAI_API_KEY=$DEEPSEEK_API_KEY +PROXY_TARGET_URL="${BASE_URL%/v1}" + +echo "[0/3] Installing dependencies..." +pip install -r requirements.txt +pip install fastapi uvicorn pydantic aiohttp datasets + +echo "[1/3] Booting ContextPilot Proxy Server for Baseline..." +python -m contextpilot.server.http_server --port 8000 --infer-api-url "$PROXY_TARGET_URL" > proxy_final_mcpatlas_deepseek_baseline.log 2>&1 & +PROXY_PID=$! + +echo "Waiting for Proxy Server to initialize (may take ~45s for PyTorch to load)..." +MAX_RETRIES=300 +RETRY_COUNT=0 +while ! curl -s http://localhost:8000/health > /dev/null; do + if ! kill -0 $PROXY_PID 2>/dev/null; then + echo "Proxy server crashed before starting!" + exit 1 + fi + sleep 2 + RETRY_COUNT=$((RETRY_COUNT+1)) + if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then + echo "Timeout waiting for proxy server." + exit 1 + fi +done +echo "Proxy Server is ready!" + +echo "[2/3] Running Baseline Evaluation (FULL)..." +python evaluation/benchmarks/run_mcpatlas_eval.py --model deepseek-v4-pro --api_base "$BASE_URL" --api_key "$OPENAI_API_KEY" --concurrency 20 --eval_mode baseline + +echo "Shutting down Proxy Server to flush Baseline telemetry..." +kill -INT $PROXY_PID +sleep 2 + +echo "=== Baseline Proxy Server Telemetry Logs ===" +cat proxy_final_mcpatlas_deepseek_baseline.log + +echo "[3/3] Booting ContextPilot Proxy Server for With-Plugin..." +python -m contextpilot.server.http_server --port 8000 --infer-api-url "$PROXY_TARGET_URL" > proxy_final_mcpatlas_deepseek_plugin.log 2>&1 & +PROXY_PID=$! + +echo "Waiting for Proxy Server to initialize (may take ~45s for PyTorch to load)..." +MAX_RETRIES=300 +RETRY_COUNT=0 +while ! curl -s http://localhost:8000/health > /dev/null; do + if ! kill -0 $PROXY_PID 2>/dev/null; then + echo "Proxy server crashed before starting!" + exit 1 + fi + sleep 2 + RETRY_COUNT=$((RETRY_COUNT+1)) + if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then + echo "Timeout waiting for proxy server." + exit 1 + fi +done +echo "Proxy Server is ready!" + +echo "Running With-Plugin Evaluation (FULL)..." +python evaluation/benchmarks/run_mcpatlas_eval.py --model deepseek-v4-pro --api_base "$BASE_URL" --api_key "$OPENAI_API_KEY" --concurrency 20 --eval_mode with_plugin + +echo "Shutting down Proxy Server to flush With-Plugin telemetry..." +kill -INT $PROXY_PID +sleep 2 + +echo "=== With-Plugin Proxy Server Telemetry Logs ===" +cat proxy_final_mcpatlas_deepseek_plugin.log + +echo "=============================================" +echo " Final Pipeline Complete!" +echo "=============================================" diff --git a/evaluation/slurm_launchers/submit_final_mcpatlas_elm.slurm b/evaluation/slurm_launchers/submit_final_mcpatlas_elm.slurm new file mode 100644 index 0000000..6c5de7b --- /dev/null +++ b/evaluation/slurm_launchers/submit_final_mcpatlas_elm.slurm @@ -0,0 +1,88 @@ +#!/bin/bash +#SBATCH --partition=Teaching +#SBATCH --gres=gpu:1 +#SBATCH --mem=16G +#SBATCH --time=24:00:00 +#SBATCH --output=final_mcpatlas_elm_%j.out +#SBATCH --error=final_mcpatlas_elm_%j.err + +echo "=============================================" +echo " ContextPilot MCP-Atlas Final ELM (FULL)" +echo "=============================================" + +export PYTHONPATH="$(pwd):$PYTHONPATH" +export NO_PROXY="localhost,127.0.0.1,::1" +export no_proxy="localhost,127.0.0.1,::1" + +ELM_BASE_URL="${BASE_URL:-https://api.openai.com/v1}" +PROXY_TARGET_URL="${ELM_BASE_URL%/v1}" + +echo "[0/3] Installing dependencies..." +pip install -r requirements.txt +pip install fastapi uvicorn pydantic aiohttp datasets + +echo "[1/3] Booting ContextPilot Proxy Server for Baseline..." +python -m contextpilot.server.http_server --port 8000 --infer-api-url "$PROXY_TARGET_URL" > proxy_final_mcpatlas_elm_baseline.log 2>&1 & +PROXY_PID=$! + +echo "Waiting for Proxy Server to initialize (may take ~45s for PyTorch to load)..." +MAX_RETRIES=300 +RETRY_COUNT=0 +while ! curl -s http://localhost:8000/health > /dev/null; do + if ! kill -0 $PROXY_PID 2>/dev/null; then + echo "Proxy server crashed before starting!" + exit 1 + fi + sleep 2 + RETRY_COUNT=$((RETRY_COUNT+1)) + if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then + echo "Timeout waiting for proxy server." + exit 1 + fi +done +echo "Proxy Server is ready!" + +echo "[2/3] Running Baseline Evaluation (FULL)..." +python evaluation/benchmarks/run_mcpatlas_eval.py --model gpt-5.5 --api_base "$ELM_BASE_URL" --api_key "$OPENAI_API_KEY" --concurrency 1 --eval_mode baseline + +echo "Shutting down Proxy Server to flush Baseline telemetry..." +kill -INT $PROXY_PID +sleep 2 + +echo "=== Baseline Proxy Server Telemetry Logs ===" +cat proxy_final_mcpatlas_elm_baseline.log + +echo "[3/3] Booting ContextPilot Proxy Server for With-Plugin..." +python -m contextpilot.server.http_server --port 8000 --infer-api-url "$PROXY_TARGET_URL" > proxy_final_mcpatlas_elm_plugin.log 2>&1 & +PROXY_PID=$! + +echo "Waiting for Proxy Server to initialize (may take ~45s for PyTorch to load)..." +MAX_RETRIES=300 +RETRY_COUNT=0 +while ! curl -s http://localhost:8000/health > /dev/null; do + if ! kill -0 $PROXY_PID 2>/dev/null; then + echo "Proxy server crashed before starting!" + exit 1 + fi + sleep 2 + RETRY_COUNT=$((RETRY_COUNT+1)) + if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then + echo "Timeout waiting for proxy server." + exit 1 + fi +done +echo "Proxy Server is ready!" + +echo "Running With-Plugin Evaluation (FULL)..." +python evaluation/benchmarks/run_mcpatlas_eval.py --model gpt-5.5 --api_base "$ELM_BASE_URL" --api_key "$OPENAI_API_KEY" --concurrency 1 --eval_mode with_plugin + +echo "Shutting down Proxy Server to flush With-Plugin telemetry..." +kill -INT $PROXY_PID +sleep 2 + +echo "=== With-Plugin Proxy Server Telemetry Logs ===" +cat proxy_final_mcpatlas_elm_plugin.log + +echo "=============================================" +echo " Final Pipeline Complete!" +echo "=============================================" diff --git a/evaluation/slurm_launchers/submit_full_evaluation.slurm b/evaluation/slurm_launchers/submit_full_evaluation.slurm new file mode 100644 index 0000000..53c4012 --- /dev/null +++ b/evaluation/slurm_launchers/submit_full_evaluation.slurm @@ -0,0 +1,17 @@ +#!/bin/bash +#SBATCH --partition=Teaching +#SBATCH --gres=gpu:1 +#SBATCH --mem=16G +#SBATCH --time=24:00:00 +#SBATCH --output=evaluation_full_%j.out +#SBATCH --error=evaluation_full_%j.err + +export PYTHONPATH="$(pwd):$PYTHONPATH" + +echo "Starting ELM dataset generation..." +python3 evaluation/benchmarks/run_bigcodebench_elm_full.py + +echo "Starting Sandbox evaluation..." +bash evaluation/benchmarks/run_sandbox_eval_full.sh + +echo "Full evaluation pipeline finished!" diff --git a/evaluation/slurm_launchers/submit_full_precision_eval.slurm b/evaluation/slurm_launchers/submit_full_precision_eval.slurm new file mode 100644 index 0000000..be0dc15 --- /dev/null +++ b/evaluation/slurm_launchers/submit_full_precision_eval.slurm @@ -0,0 +1,135 @@ +#!/bin/bash +#SBATCH --partition=Teaching +#SBATCH --nodelist=saxa +#SBATCH --gres=gpu:1 +#SBATCH --mem=32G +#SBATCH --time=24:00:00 +#SBATCH --output=fp16_eval_%j.out +#SBATCH --error=fp16_eval_%j.err + +echo "==========================================================" +echo " ContextPilot Full Precision Evaluation (vLLM + MCP-Atlas)" +echo "==========================================================" + +export PYTHONPATH="$(pwd):$PYTHONPATH" +export NO_PROXY="localhost,127.0.0.1,::1" +export no_proxy="localhost,127.0.0.1,::1" + +# Force JIT compilation to use our local tmp dir instead of system /tmp +export TMPDIR="$(pwd)/tmp" +export TMP="$(pwd)/tmp" + +# Generate a dynamic high port to avoid collisions +PORT=$(shuf -i 15000-20000 -n 1) +echo "Generated vLLM Port: $PORT" + +MODEL_NAME="Qwen/Qwen2.5-7B-Instruct" + +echo "[0/3] Installing dependencies..." +pip install --no-cache-dir -r requirements.txt 2>/dev/null || true +pip install --no-cache-dir vllm datasets 2>/dev/null || true +pip install --no-cache-dir --upgrade scipy "numpy<2.0" 2>/dev/null || true + +# Function to manage vLLM lifecycle +start_vllm() { + local log_file=$1 + echo "Booting vLLM on port $PORT..." + # Boot vLLM with 0.8 gpu utilization for tiny footprint + python -m vllm.entrypoints.openai.api_server \ + --model "$MODEL_NAME" \ + --port "$PORT" \ + --gpu-memory-utilization 0.8 \ + --max-model-len 4096 \ + --enforce-eager \ + --enable-auto-tool-choice \ + --tool-call-parser hermes \ + > "$log_file" 2>&1 & + + VLLM_PID=$! + + echo "Waiting for vLLM to initialize..." + MAX_RETRIES=300 + RETRY_COUNT=0 + while ! curl -s http://localhost:$PORT/health > /dev/null; do + if ! kill -0 $VLLM_PID 2>/dev/null; then + echo "vLLM crashed before starting! Check $log_file" + exit 1 + fi + sleep 2 + RETRY_COUNT=$((RETRY_COUNT+1)) + if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then + echo "Timeout waiting for vLLM." + kill -9 $VLLM_PID + exit 1 + fi + done + echo "vLLM is ready on port $PORT!" +} + +# ========================================== +# PHASE 1: Baseline Evaluation +# ========================================== +echo "[1/3] Phase 1: Baseline Evaluation" +start_vllm "fp16_vllm_baseline.log" + +echo "Starting Metrics Logger for Baseline..." +python scripts/vllm_metrics_logger.py --port $PORT --output fp16_baseline_metrics.json & +LOGGER_PID=$! + +echo "Running Baseline Evaluation (100 Tasks)..." +# Using concurrency=5 to simulate edge constraints +python evaluation/benchmarks/run_mcpatlas_eval.py \ + --model "$MODEL_NAME" \ + --api_base "http://localhost:$PORT/v1" \ + --api_key "dummy" \ + --concurrency 5 \ + --limit 100 \ + --eval_mode baseline + +echo "Stopping Metrics Logger..." +kill -INT $LOGGER_PID +sleep 2 + +echo "Shutting down vLLM..." +kill -TERM $VLLM_PID +wait $VLLM_PID 2>/dev/null || true +sleep 10 +PORT=$(shuf -i 15000-20000 -n 1) + +# ========================================== +# PHASE 2: With-Plugin Evaluation +# ========================================== +echo "[2/3] Phase 2: With-Plugin Evaluation" +start_vllm "fp16_vllm_plugin.log" + +echo "Starting Metrics Logger for With-Plugin..." +python scripts/vllm_metrics_logger.py --port $PORT --output fp16_plugin_metrics.json & +LOGGER_PID=$! + +echo "Running With-Plugin Evaluation (100 Tasks)..." +python evaluation/benchmarks/run_mcpatlas_eval.py \ + --model "$MODEL_NAME" \ + --api_base "http://localhost:$PORT/v1" \ + --api_key "dummy" \ + --concurrency 5 \ + --limit 100 \ + --eval_mode with_plugin + +echo "Stopping Metrics Logger..." +kill -INT $LOGGER_PID +sleep 2 + +echo "Shutting down vLLM..." +kill -9 $VLLM_PID + +echo "[3/3] Displaying Metrics Summaries:" +echo "=== Baseline Peak VRAM ===" +cat fp16_baseline_metrics.json +echo "" +echo "=== With-Plugin Peak VRAM ===" +cat fp16_plugin_metrics.json +echo "" + +echo "==========================================================" +echo " Evaluation Pipeline Complete!" +echo "==========================================================" diff --git a/evaluation/slurm_launchers/submit_quantized_eval.slurm b/evaluation/slurm_launchers/submit_quantized_eval.slurm new file mode 100644 index 0000000..f0d5fe6 --- /dev/null +++ b/evaluation/slurm_launchers/submit_quantized_eval.slurm @@ -0,0 +1,136 @@ +#!/bin/bash +#SBATCH --partition=Teaching +#SBATCH --nodelist=saxa +#SBATCH --gres=gpu:1 +#SBATCH --mem=32G +#SBATCH --time=24:00:00 +#SBATCH --output=quantized_eval_%j.out +#SBATCH --error=quantized_eval_%j.err + +echo "==========================================================" +echo " ContextPilot Quantized AWQ Evaluation (vLLM + MCP-Atlas)" +echo "==========================================================" + +export PYTHONPATH="$(pwd):$PYTHONPATH" +export NO_PROXY="localhost,127.0.0.1,::1" +export no_proxy="localhost,127.0.0.1,::1" + +# Force JIT compilation to use our local tmp dir instead of system /tmp +export TMPDIR="$(pwd)/tmp" +export TMP="$(pwd)/tmp" + +# Generate a dynamic high port to avoid collisions +PORT=$(shuf -i 15000-20000 -n 1) +echo "Generated vLLM Port: $PORT" + +MODEL_NAME="Qwen/Qwen2.5-7B-Instruct-AWQ" + +echo "[0/3] Installing dependencies..." +pip install --no-cache-dir -r requirements.txt 2>/dev/null || true +pip install --no-cache-dir vllm datasets 2>/dev/null || true +pip install --no-cache-dir --upgrade scipy "numpy<2.0" 2>/dev/null || true + +# Function to manage vLLM lifecycle +start_vllm() { + local log_file=$1 + echo "Booting vLLM on port $PORT..." + # Boot vLLM with 0.4 gpu utilization for tiny footprint + python -m vllm.entrypoints.openai.api_server \ + --model "$MODEL_NAME" \ + --port "$PORT" \ + --gpu-memory-utilization 0.8 \ + --quantization awq \ + --max-model-len 4096 \ + --enforce-eager \ + --enable-auto-tool-choice \ + --tool-call-parser hermes \ + > "$log_file" 2>&1 & + + VLLM_PID=$! + + echo "Waiting for vLLM to initialize..." + MAX_RETRIES=300 + RETRY_COUNT=0 + while ! curl -s http://localhost:$PORT/health > /dev/null; do + if ! kill -0 $VLLM_PID 2>/dev/null; then + echo "vLLM crashed before starting! Check $log_file" + exit 1 + fi + sleep 2 + RETRY_COUNT=$((RETRY_COUNT+1)) + if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then + echo "Timeout waiting for vLLM." + kill -9 $VLLM_PID + exit 1 + fi + done + echo "vLLM is ready on port $PORT!" +} + +# ========================================== +# PHASE 1: Baseline Evaluation +# ========================================== +echo "[1/3] Phase 1: Baseline Evaluation" +start_vllm "vllm_baseline.log" + +echo "Starting Metrics Logger for Baseline..." +python scripts/vllm_metrics_logger.py --port $PORT --output baseline_metrics.json & +LOGGER_PID=$! + +echo "Running Baseline Evaluation (100 Tasks)..." +# Using concurrency=5 to simulate edge constraints +python evaluation/benchmarks/run_mcpatlas_eval.py \ + --model "$MODEL_NAME" \ + --api_base "http://localhost:$PORT/v1" \ + --api_key "dummy" \ + --concurrency 5 \ + --limit 100 \ + --eval_mode baseline + +echo "Stopping Metrics Logger..." +kill -INT $LOGGER_PID +sleep 2 + +echo "Shutting down vLLM..." +kill -TERM $VLLM_PID +wait $VLLM_PID 2>/dev/null || true +sleep 10 +PORT=$(shuf -i 15000-20000 -n 1) + +# ========================================== +# PHASE 2: With-Plugin Evaluation +# ========================================== +echo "[2/3] Phase 2: With-Plugin Evaluation" +start_vllm "vllm_plugin.log" + +echo "Starting Metrics Logger for With-Plugin..." +python scripts/vllm_metrics_logger.py --port $PORT --output plugin_metrics.json & +LOGGER_PID=$! + +echo "Running With-Plugin Evaluation (100 Tasks)..." +python evaluation/benchmarks/run_mcpatlas_eval.py \ + --model "$MODEL_NAME" \ + --api_base "http://localhost:$PORT/v1" \ + --api_key "dummy" \ + --concurrency 5 \ + --limit 100 \ + --eval_mode with_plugin + +echo "Stopping Metrics Logger..." +kill -INT $LOGGER_PID +sleep 2 + +echo "Shutting down vLLM..." +kill -9 $VLLM_PID + +echo "[3/3] Displaying Metrics Summaries:" +echo "=== Baseline Peak VRAM ===" +cat baseline_metrics.json +echo "" +echo "=== With-Plugin Peak VRAM ===" +cat plugin_metrics.json +echo "" + +echo "==========================================================" +echo " Evaluation Pipeline Complete!" +echo "==========================================================" diff --git a/evaluation/slurm_launchers/submit_test_ablation_elm.slurm b/evaluation/slurm_launchers/submit_test_ablation_elm.slurm new file mode 100644 index 0000000..ab59885 --- /dev/null +++ b/evaluation/slurm_launchers/submit_test_ablation_elm.slurm @@ -0,0 +1,57 @@ +#!/bin/bash +#SBATCH --partition=Teaching +#SBATCH --gres=gpu:1 +#SBATCH --mem=16G +#SBATCH --time=24:00:00 +#SBATCH --output=test_ablation_elm_%j.out +#SBATCH --error=test_ablation_elm_%j.err + +echo "=============================================" +echo " Dynamic Pruning Ablation Test (LIMIT 5)" +echo "=============================================" + +export PYTHONPATH="$(pwd):$PYTHONPATH" +export NO_PROXY="localhost,127.0.0.1,::1" +export no_proxy="localhost,127.0.0.1,::1" + +# Install missing dependencies on compute node before starting the proxy +echo "[0/3] Installing dependencies..." +pip install -r requirements.txt +pip install fastapi uvicorn pydantic aiohttp sentence-transformers + +# 1. Boot the local ContextPilot Proxy Server in the background +echo "[1/3] Booting ContextPilot Proxy Server on port 8000..." +ELM_BASE_URL="${BASE_URL:-https://api.openai.com/v1}" +PROXY_TARGET_URL="${ELM_BASE_URL%/v1}" +python -m contextpilot.server.http_server --port 8000 --infer-api-url "$PROXY_TARGET_URL" > proxy_test_ablation.log 2>&1 & +PROXY_PID=$! +sleep 5 + +THRESHOLDS=(0.1 0.2 0.4 0.5) + +echo "[2/3] Running Ablation Scripts (LIMIT 5)..." +for THRESHOLD in "${THRESHOLDS[@]}"; do + echo "--- Running Threshold: $THRESHOLD ---" + python evaluation/benchmarks/run_ablation_elm.py --model gpt-5.5 --concurrency 1 --limit 5 --eval_mode with_plugin --threshold $THRESHOLD +done + +echo "Shutting down Proxy Server to flush telemetry..." +kill -INT $PROXY_PID +sleep 2 + +echo "[3/3] Executing Apptainer Sandbox Evaluations..." +cd evaluation/benchmarks + +for THRESHOLD in "${THRESHOLDS[@]}"; do + echo "--- Evaluating Threshold: $THRESHOLD ---" + cp results_ablation_${THRESHOLD}_with_plugin_gpt-5.5.jsonl elm_samples_full.jsonl + bash run_sandbox_eval_full.sh + mv ../sandbox/results/elm_samples_full_eval_results.json ../sandbox/results/test_ablation_${THRESHOLD}_eval_results.json +done + +rm elm_samples_full.jsonl +cd ../.. + +echo "=============================================" +echo " Ablation Test Complete!" +echo "=============================================" diff --git a/evaluation/slurm_launchers/submit_test_all_plugins_elm.slurm b/evaluation/slurm_launchers/submit_test_all_plugins_elm.slurm new file mode 100644 index 0000000..efc8e1b --- /dev/null +++ b/evaluation/slurm_launchers/submit_test_all_plugins_elm.slurm @@ -0,0 +1,60 @@ +#!/bin/bash +#SBATCH --partition=Teaching +#SBATCH --gres=gpu:1 +#SBATCH --mem=16G +#SBATCH --time=24:00:00 +#SBATCH --output=test_all_plugins_%j.out +#SBATCH --error=test_all_plugins_%j.err + +echo "=============================================" +echo " All Plugins A/B Test Evaluation (LIMIT 5)" +echo "=============================================" + +export PYTHONPATH="$(pwd):$PYTHONPATH" +export NO_PROXY="localhost,127.0.0.1,::1" +export no_proxy="localhost,127.0.0.1,::1" + +# Install missing dependencies on compute node before starting the proxy +echo "[0/4] Installing dependencies..." +pip install -r requirements.txt +pip install fastapi uvicorn pydantic aiohttp sentence-transformers + +# 1. Boot the local ContextPilot Proxy Server in the background +echo "[1/4] Booting ContextPilot Proxy Server on port 8000..." +# Determine ELM API Base URL (stripping trailing /v1 if present for the proxy config) +ELM_BASE_URL="${BASE_URL:-https://api.openai.com/v1}" +PROXY_TARGET_URL="${ELM_BASE_URL%/v1}" +python -m contextpilot.server.http_server --port 8000 --infer-api-url "$PROXY_TARGET_URL" > proxy_test_all_plugins.log 2>&1 & +PROXY_PID=$! + +# Wait briefly for the server to be ready +sleep 5 + +# 2. Run the newly refactored A/B generation pipeline (LIMIT 5) +echo "[2/4] Running A/B API Evaluation Pipeline (Generation) with LIMIT 5..." +python evaluation/benchmarks/run_bigcodebench_all_plugins.py --model gpt-5.5 --concurrency 1 --limit 5 + +echo "[3/4] Shutting down Proxy Server to flush telemetry..." +kill -INT $PROXY_PID +sleep 2 + +# 3. Sandbox Evaluation for Baseline +echo "[4/4] Executing Apptainer Sandbox Evaluations..." +cd evaluation/benchmarks + +echo "--- Evaluating Baseline Results ---" +cp results_all_plugins_baseline_gpt-5.5.jsonl elm_samples_full.jsonl +bash run_sandbox_eval_full.sh +mv ../sandbox/results/elm_samples_full_eval_results.json ../sandbox/results/test_all_plugins_baseline_eval_results.json + +echo "--- Evaluating With-Plugin Results ---" +cp results_all_plugins_with_plugin_gpt-5.5.jsonl elm_samples_full.jsonl +bash run_sandbox_eval_full.sh +mv ../sandbox/results/elm_samples_full_eval_results.json ../sandbox/results/test_all_plugins_with_plugin_eval_results.json + +rm elm_samples_full.jsonl + +cd ../.. +echo "=============================================" +echo " Test Pipeline Complete!" +echo "=============================================" diff --git a/evaluation/slurm_launchers/submit_test_deepseek_eval.slurm b/evaluation/slurm_launchers/submit_test_deepseek_eval.slurm new file mode 100644 index 0000000..457c4db --- /dev/null +++ b/evaluation/slurm_launchers/submit_test_deepseek_eval.slurm @@ -0,0 +1,74 @@ +#!/bin/bash +#SBATCH --partition=Teaching +#SBATCH --gres=gpu:1 +#SBATCH --mem=16G +#SBATCH --time=24:00:00 +#SBATCH --output=test_deepseek_eval_%j.out +#SBATCH --error=test_deepseek_eval_%j.err + +echo "=============================================" +echo " ContextPilot A/B Test DeepSeek (LIMIT 5)" +echo "=============================================" + +export PYTHONPATH="$(pwd):$PYTHONPATH" +export NO_PROXY="localhost,127.0.0.1,::1" +export no_proxy="localhost,127.0.0.1,::1" + +export BASE_URL="https://api.deepseek.com/v1" +export OPENAI_API_KEY=$DEEPSEEK_API_KEY + +echo "[0/4] Installing dependencies..." +pip install -r requirements.txt +pip install fastapi uvicorn pydantic aiohttp + +PROXY_TARGET_URL="${BASE_URL%/v1}" + +echo "[1/4] Booting ContextPilot Proxy Server for Baseline..." +python -m contextpilot.server.http_server --port 8000 --infer-api-url "$PROXY_TARGET_URL" > proxy_test_deepseek_server_baseline.log 2>&1 & +PROXY_PID=$! +sleep 5 + +echo "[2/4] Running Baseline Evaluation (LIMIT 5)..." +python evaluation/benchmarks/run_bigcodebench_elm_full.py --model deepseek-v4-pro --api_base https://api.deepseek.com/v1 --api_key $OPENAI_API_KEY --concurrency 20 --limit 5 --eval_mode baseline + +echo "Shutting down Proxy Server to flush Baseline telemetry..." +kill -INT $PROXY_PID +sleep 2 + +echo "=== Baseline Proxy Server Telemetry Logs ===" +cat proxy_test_deepseek_server_baseline.log + +echo "[3/4] Booting ContextPilot Proxy Server for With-Plugin..." +python -m contextpilot.server.http_server --port 8000 --infer-api-url "$PROXY_TARGET_URL" > proxy_test_deepseek_server_plugin.log 2>&1 & +PROXY_PID=$! +sleep 5 + +echo "[4/4] Running With-Plugin Evaluation (LIMIT 5)..." +python evaluation/benchmarks/run_bigcodebench_elm_full.py --model deepseek-v4-pro --api_base https://api.deepseek.com/v1 --api_key $OPENAI_API_KEY --concurrency 20 --limit 5 --eval_mode with_plugin + +echo "Shutting down Proxy Server to flush With-Plugin telemetry..." +kill -INT $PROXY_PID +sleep 2 + +echo "=== With-Plugin Proxy Server Telemetry Logs ===" +cat proxy_test_deepseek_server_plugin.log + +echo "Executing Apptainer Sandbox Evaluations..." +cd evaluation/benchmarks + +echo "--- Evaluating Baseline Results ---" +cp results_baseline_deepseek-v4-pro.jsonl elm_samples_full.jsonl +bash run_sandbox_eval_full.sh +mv ../sandbox/results/elm_samples_full_eval_results.json ../sandbox/results/test_baseline_deepseek_eval_results.json + +echo "--- Evaluating With-Plugin Results ---" +cp results_with_plugin_deepseek-v4-pro.jsonl elm_samples_full.jsonl +bash run_sandbox_eval_full.sh +mv ../sandbox/results/elm_samples_full_eval_results.json ../sandbox/results/test_with_plugin_deepseek_eval_results.json + +rm elm_samples_full.jsonl +cd ../.. + +echo "=============================================" +echo " Test Pipeline Complete!" +echo "=============================================" diff --git a/evaluation/slurm_launchers/submit_test_deepseek_pruning.slurm b/evaluation/slurm_launchers/submit_test_deepseek_pruning.slurm new file mode 100644 index 0000000..7b6960a --- /dev/null +++ b/evaluation/slurm_launchers/submit_test_deepseek_pruning.slurm @@ -0,0 +1,72 @@ +#!/bin/bash +#SBATCH --partition=Teaching +#SBATCH --gres=gpu:1 +#SBATCH --mem=16G +#SBATCH --time=24:00:00 +#SBATCH --output=test_deepseek_pruning_%j.out +#SBATCH --error=test_deepseek_pruning_%j.err + +echo "=============================================" +echo " DeepSeek Dynamic Pruning Test (LIMIT 5)" +echo "=============================================" + +export PYTHONPATH="$(pwd):$PYTHONPATH" +export NO_PROXY="localhost,127.0.0.1,::1" +export no_proxy="localhost,127.0.0.1,::1" + +export BASE_URL="https://api.deepseek.com/v1" +export OPENAI_API_KEY=$DEEPSEEK_API_KEY + +echo "[0/6] Installing dependencies..." +pip install -r requirements.txt +pip install fastapi uvicorn pydantic aiohttp sentence-transformers + +PROXY_TARGET_URL="${BASE_URL%/v1}" + +echo "[1/6] Booting ContextPilot Proxy Server for Dynamic Pruning..." +python -m contextpilot.server.http_server --port 8000 --infer-api-url "$PROXY_TARGET_URL" > proxy_test_deepseek_dynamic.log 2>&1 & +PROXY_PID=$! +sleep 5 + +echo "[2/6] Running Configuration 1: Dynamic Pruning + Skill Filter (LIMIT 5)..." +python evaluation/benchmarks/run_bigcodebench_deepseek_dynamic.py --model deepseek-v4-pro --api_base https://api.deepseek.com/v1 --api_key $OPENAI_API_KEY --concurrency 2 --limit 5 + +echo "Shutting down Proxy Server to flush telemetry..." +kill -INT $PROXY_PID +sleep 2 + +echo "=== Dynamic Pruning Proxy Server Telemetry Logs ===" +cat proxy_test_deepseek_dynamic.log + +echo "[3/6] Booting ContextPilot Proxy Server for ALL PLUGINS..." +python -m contextpilot.server.http_server --port 8000 --infer-api-url "$PROXY_TARGET_URL" > proxy_test_deepseek_all.log 2>&1 & +PROXY_PID=$! +sleep 5 + +echo "[4/6] Running Configuration 2: ALL PLUGINS (LIMIT 5)..." +python evaluation/benchmarks/run_bigcodebench_deepseek_all.py --model deepseek-v4-pro --api_base https://api.deepseek.com/v1 --api_key $OPENAI_API_KEY --concurrency 2 --limit 5 + +echo "Shutting down Proxy Server to flush telemetry..." +kill -INT $PROXY_PID +sleep 2 + +echo "=== ALL PLUGINS Proxy Server Telemetry Logs ===" +cat proxy_test_deepseek_all.log + +echo "[5/6] Executing Apptainer Sandbox Evaluations..." +cd evaluation/benchmarks + +echo "--- Evaluating Configuration 1 (Dynamic Pruning) Results ---" +cp results_deepseek_dynamic_with_plugin_deepseek-v4-pro.jsonl elm_samples_full.jsonl +bash run_sandbox_eval_full.sh +mv ../sandbox/results/elm_samples_full_eval_results.json ../sandbox/results/test_deepseek_dynamic_eval_results.json + +echo "--- Evaluating Configuration 2 (ALL PLUGINS) Results ---" +cp results_deepseek_all_with_plugin_deepseek-v4-pro.jsonl elm_samples_full.jsonl +bash run_sandbox_eval_full.sh +mv ../sandbox/results/elm_samples_full_eval_results.json ../sandbox/results/test_deepseek_all_eval_results.json + +rm elm_samples_full.jsonl +cd ../.. + +echo "[6/6] Pipeline Complete!" diff --git a/evaluation/slurm_launchers/submit_test_dynamic_skill_elm.slurm b/evaluation/slurm_launchers/submit_test_dynamic_skill_elm.slurm new file mode 100644 index 0000000..b14f573 --- /dev/null +++ b/evaluation/slurm_launchers/submit_test_dynamic_skill_elm.slurm @@ -0,0 +1,60 @@ +#!/bin/bash +#SBATCH --partition=Teaching +#SBATCH --gres=gpu:1 +#SBATCH --mem=16G +#SBATCH --time=24:00:00 +#SBATCH --output=test_dynamic_pruning_%j.out +#SBATCH --error=test_dynamic_pruning_%j.err + +echo "=============================================" +echo " Dynamic Pruning A/B Test Evaluation (LIMIT 5)" +echo "=============================================" + +export PYTHONPATH="$(pwd):$PYTHONPATH" +export NO_PROXY="localhost,127.0.0.1,::1" +export no_proxy="localhost,127.0.0.1,::1" + +# Install missing dependencies on compute node before starting the proxy +echo "[0/4] Installing dependencies..." +pip install -r requirements.txt +pip install fastapi uvicorn pydantic aiohttp sentence-transformers + +# 1. Boot the local ContextPilot Proxy Server in the background +echo "[1/4] Booting ContextPilot Proxy Server on port 8000..." +# Determine ELM API Base URL (stripping trailing /v1 if present for the proxy config) +ELM_BASE_URL="${BASE_URL:-https://api.openai.com/v1}" +PROXY_TARGET_URL="${ELM_BASE_URL%/v1}" +python -m contextpilot.server.http_server --port 8000 --infer-api-url "$PROXY_TARGET_URL" > proxy_test_dynamic_pruning.log 2>&1 & +PROXY_PID=$! + +# Wait briefly for the server to be ready +sleep 5 + +# 2. Run the newly refactored A/B generation pipeline (LIMIT 5) +echo "[2/4] Running A/B API Evaluation Pipeline (Generation) with LIMIT 5..." +python evaluation/benchmarks/run_bigcodebench_dynamic_skill.py --model gpt-5.5 --concurrency 1 --limit 5 + +echo "[3/4] Shutting down Proxy Server to flush telemetry..." +kill -INT $PROXY_PID +sleep 2 + +# 3. Sandbox Evaluation for Baseline +echo "[4/4] Executing Apptainer Sandbox Evaluations..." +cd evaluation/benchmarks + +echo "--- Evaluating Baseline Results ---" +cp results_dynamic_skill_baseline_gpt-5.5.jsonl elm_samples_full.jsonl +bash run_sandbox_eval_full.sh +mv ../sandbox/results/elm_samples_full_eval_results.json ../sandbox/results/test_dynamic_baseline_eval_results.json + +echo "--- Evaluating With-Plugin Results ---" +cp results_dynamic_skill_with_plugin_gpt-5.5.jsonl elm_samples_full.jsonl +bash run_sandbox_eval_full.sh +mv ../sandbox/results/elm_samples_full_eval_results.json ../sandbox/results/test_dynamic_with_plugin_eval_results.json + +rm elm_samples_full.jsonl + +cd ../.. +echo "=============================================" +echo " Test Pipeline Complete!" +echo "=============================================" diff --git a/evaluation/slurm_launchers/submit_test_elm_eval.slurm b/evaluation/slurm_launchers/submit_test_elm_eval.slurm new file mode 100644 index 0000000..333a584 --- /dev/null +++ b/evaluation/slurm_launchers/submit_test_elm_eval.slurm @@ -0,0 +1,60 @@ +#!/bin/bash +#SBATCH --partition=Teaching +#SBATCH --gres=gpu:1 +#SBATCH --mem=16G +#SBATCH --time=24:00:00 +#SBATCH --output=test_eval_%j.out +#SBATCH --error=test_eval_%j.err + +echo "=============================================" +echo " ContextPilot A/B Test Evaluation (LIMIT 5)" +echo "=============================================" + +export PYTHONPATH="$(pwd):$PYTHONPATH" +export NO_PROXY="localhost,127.0.0.1,::1" +export no_proxy="localhost,127.0.0.1,::1" + +# Install missing dependencies on compute node before starting the proxy +echo "[0/4] Installing dependencies..." +pip install -r requirements.txt +pip install fastapi uvicorn pydantic aiohttp + +# 1. Boot the local ContextPilot Proxy Server in the background +echo "[1/4] Booting ContextPilot Proxy Server on port 8000..." +# Determine ELM API Base URL (stripping trailing /v1 if present for the proxy config) +ELM_BASE_URL="${BASE_URL:-https://api.openai.com/v1}" +PROXY_TARGET_URL="${ELM_BASE_URL%/v1}" +python -m contextpilot.server.http_server --port 8000 --infer-api-url "$PROXY_TARGET_URL" > proxy_test_server.log 2>&1 & +PROXY_PID=$! + +# Wait briefly for the server to be ready +sleep 5 + +# 2. Run the newly refactored A/B generation pipeline (LIMIT 5) +echo "[2/4] Running A/B API Evaluation Pipeline (Generation) with LIMIT 5..." +python evaluation/benchmarks/run_bigcodebench_elm_full.py --model gpt-5.5 --concurrency 1 --limit 5 + +echo "[3/4] Shutting down Proxy Server to flush telemetry..." +kill -INT $PROXY_PID +sleep 2 + +# 3. Sandbox Evaluation for Baseline +echo "[4/4] Executing Apptainer Sandbox Evaluations..." +cd evaluation/benchmarks + +echo "--- Evaluating Baseline Results ---" +cp results_baseline_gpt-5.5.jsonl elm_samples_full.jsonl +bash run_sandbox_eval_full.sh +mv ../sandbox/results/elm_samples_full_eval_results.json ../sandbox/results/test_baseline_eval_results.json + +echo "--- Evaluating With-Plugin Results ---" +cp results_with_plugin_gpt-5.5.jsonl elm_samples_full.jsonl +bash run_sandbox_eval_full.sh +mv ../sandbox/results/elm_samples_full_eval_results.json ../sandbox/results/test_with_plugin_eval_results.json + +rm elm_samples_full.jsonl + +cd ../.. +echo "=============================================" +echo " Test Pipeline Complete!" +echo "=============================================" diff --git a/evaluation/slurm_launchers/submit_test_mcpatlas_deepseek.slurm b/evaluation/slurm_launchers/submit_test_mcpatlas_deepseek.slurm new file mode 100644 index 0000000..9e8ba77 --- /dev/null +++ b/evaluation/slurm_launchers/submit_test_mcpatlas_deepseek.slurm @@ -0,0 +1,89 @@ +#!/bin/bash +#SBATCH --partition=Teaching +#SBATCH --gres=gpu:1 +#SBATCH --mem=16G +#SBATCH --time=24:00:00 +#SBATCH --output=test_mcpatlas_deepseek_%j.out +#SBATCH --error=test_mcpatlas_deepseek_%j.err + +echo "=============================================" +echo " ContextPilot MCP-Atlas Test DeepSeek (LIMIT 5)" +echo "=============================================" + +export PYTHONPATH="$(pwd):$PYTHONPATH" +export NO_PROXY="localhost,127.0.0.1,::1" +export no_proxy="localhost,127.0.0.1,::1" + +export BASE_URL="https://api.deepseek.com/v1" +export OPENAI_API_KEY=$DEEPSEEK_API_KEY +PROXY_TARGET_URL="${BASE_URL%/v1}" + +echo "[0/3] Installing dependencies..." +pip install -r requirements.txt +pip install fastapi uvicorn pydantic aiohttp datasets + +echo "[1/3] Booting ContextPilot Proxy Server for Baseline..." +python -m contextpilot.server.http_server --port 8000 --infer-api-url "$PROXY_TARGET_URL" > proxy_test_mcpatlas_deepseek_baseline.log 2>&1 & +PROXY_PID=$! + +echo "Waiting for Proxy Server to initialize (may take ~45s for PyTorch to load)..." +MAX_RETRIES=300 +RETRY_COUNT=0 +while ! curl -s http://localhost:8000/health > /dev/null; do + if ! kill -0 $PROXY_PID 2>/dev/null; then + echo "Proxy server crashed before starting!" + exit 1 + fi + sleep 2 + RETRY_COUNT=$((RETRY_COUNT+1)) + if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then + echo "Timeout waiting for proxy server." + exit 1 + fi +done +echo "Proxy Server is ready!" + +echo "[2/3] Running Baseline Evaluation (LIMIT 5)..." +python evaluation/benchmarks/run_mcpatlas_eval.py --model deepseek-v4-pro --api_base "$BASE_URL" --api_key "$OPENAI_API_KEY" --concurrency 20 --limit 5 --eval_mode baseline + +echo "Shutting down Proxy Server to flush Baseline telemetry..." +kill -INT $PROXY_PID +sleep 2 + +echo "=== Baseline Proxy Server Telemetry Logs ===" +cat proxy_test_mcpatlas_deepseek_baseline.log + +echo "[3/3] Booting ContextPilot Proxy Server for With-Plugin..." +python -m contextpilot.server.http_server --port 8000 --infer-api-url "$PROXY_TARGET_URL" > proxy_test_mcpatlas_deepseek_plugin.log 2>&1 & +PROXY_PID=$! + +echo "Waiting for Proxy Server to initialize (may take ~45s for PyTorch to load)..." +MAX_RETRIES=300 +RETRY_COUNT=0 +while ! curl -s http://localhost:8000/health > /dev/null; do + if ! kill -0 $PROXY_PID 2>/dev/null; then + echo "Proxy server crashed before starting!" + exit 1 + fi + sleep 2 + RETRY_COUNT=$((RETRY_COUNT+1)) + if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then + echo "Timeout waiting for proxy server." + exit 1 + fi +done +echo "Proxy Server is ready!" + +echo "Running With-Plugin Evaluation (LIMIT 5)..." +python evaluation/benchmarks/run_mcpatlas_eval.py --model deepseek-v4-pro --api_base "$BASE_URL" --api_key "$OPENAI_API_KEY" --concurrency 20 --limit 5 --eval_mode with_plugin + +echo "Shutting down Proxy Server to flush With-Plugin telemetry..." +kill -INT $PROXY_PID +sleep 2 + +echo "=== With-Plugin Proxy Server Telemetry Logs ===" +cat proxy_test_mcpatlas_deepseek_plugin.log + +echo "=============================================" +echo " Test Pipeline Complete!" +echo "=============================================" diff --git a/evaluation/slurm_launchers/submit_test_mcpatlas_elm.slurm b/evaluation/slurm_launchers/submit_test_mcpatlas_elm.slurm new file mode 100644 index 0000000..d6a96cd --- /dev/null +++ b/evaluation/slurm_launchers/submit_test_mcpatlas_elm.slurm @@ -0,0 +1,88 @@ +#!/bin/bash +#SBATCH --partition=Teaching +#SBATCH --gres=gpu:1 +#SBATCH --mem=16G +#SBATCH --time=24:00:00 +#SBATCH --output=test_mcpatlas_elm_%j.out +#SBATCH --error=test_mcpatlas_elm_%j.err + +echo "=============================================" +echo " ContextPilot MCP-Atlas Test ELM (LIMIT 5)" +echo "=============================================" + +export PYTHONPATH="$(pwd):$PYTHONPATH" +export NO_PROXY="localhost,127.0.0.1,::1" +export no_proxy="localhost,127.0.0.1,::1" + +ELM_BASE_URL="${BASE_URL:-https://api.openai.com/v1}" +PROXY_TARGET_URL="${ELM_BASE_URL%/v1}" + +echo "[0/3] Installing dependencies..." +pip install -r requirements.txt +pip install fastapi uvicorn pydantic aiohttp datasets + +echo "[1/3] Booting ContextPilot Proxy Server for Baseline..." +python -m contextpilot.server.http_server --port 8000 --infer-api-url "$PROXY_TARGET_URL" > proxy_test_mcpatlas_elm_baseline.log 2>&1 & +PROXY_PID=$! + +echo "Waiting for Proxy Server to initialize (may take ~45s for PyTorch to load)..." +MAX_RETRIES=300 +RETRY_COUNT=0 +while ! curl -s http://localhost:8000/health > /dev/null; do + if ! kill -0 $PROXY_PID 2>/dev/null; then + echo "Proxy server crashed before starting!" + exit 1 + fi + sleep 2 + RETRY_COUNT=$((RETRY_COUNT+1)) + if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then + echo "Timeout waiting for proxy server." + exit 1 + fi +done +echo "Proxy Server is ready!" + +echo "[2/3] Running Baseline Evaluation (LIMIT 5)..." +python evaluation/benchmarks/run_mcpatlas_eval.py --model gpt-5.5 --api_base "$ELM_BASE_URL" --api_key "$OPENAI_API_KEY" --concurrency 1 --limit 5 --eval_mode baseline + +echo "Shutting down Proxy Server to flush Baseline telemetry..." +kill -INT $PROXY_PID +sleep 2 + +echo "=== Baseline Proxy Server Telemetry Logs ===" +cat proxy_test_mcpatlas_elm_baseline.log + +echo "[3/3] Booting ContextPilot Proxy Server for With-Plugin..." +python -m contextpilot.server.http_server --port 8000 --infer-api-url "$PROXY_TARGET_URL" > proxy_test_mcpatlas_elm_plugin.log 2>&1 & +PROXY_PID=$! + +echo "Waiting for Proxy Server to initialize (may take ~45s for PyTorch to load)..." +MAX_RETRIES=300 +RETRY_COUNT=0 +while ! curl -s http://localhost:8000/health > /dev/null; do + if ! kill -0 $PROXY_PID 2>/dev/null; then + echo "Proxy server crashed before starting!" + exit 1 + fi + sleep 2 + RETRY_COUNT=$((RETRY_COUNT+1)) + if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then + echo "Timeout waiting for proxy server." + exit 1 + fi +done +echo "Proxy Server is ready!" + +echo "Running With-Plugin Evaluation (LIMIT 5)..." +python evaluation/benchmarks/run_mcpatlas_eval.py --model gpt-5.5 --api_base "$ELM_BASE_URL" --api_key "$OPENAI_API_KEY" --concurrency 1 --limit 5 --eval_mode with_plugin + +echo "Shutting down Proxy Server to flush With-Plugin telemetry..." +kill -INT $PROXY_PID +sleep 2 + +echo "=== With-Plugin Proxy Server Telemetry Logs ===" +cat proxy_test_mcpatlas_elm_plugin.log + +echo "=============================================" +echo " Test Pipeline Complete!" +echo "=============================================" diff --git a/evaluation/slurm_logs/debug_import.slurm b/evaluation/slurm_logs/debug_import.slurm new file mode 100644 index 0000000..cf69362 --- /dev/null +++ b/evaluation/slurm_logs/debug_import.slurm @@ -0,0 +1,15 @@ +#!/bin/bash +#SBATCH --partition=Teaching +#SBATCH --gres=gpu:1 +#SBATCH --mem=16G +#SBATCH --time=00:05:00 +#SBATCH --output=debug_import_%j.out +#SBATCH --error=debug_import_%j.err + +echo "Debugging http_server import on compute node..." +export PYTHONPATH="$(pwd):$PYTHONPATH" +export NO_PROXY="localhost,127.0.0.1,::1" +export no_proxy="localhost,127.0.0.1,::1" + +python -v -c "import contextpilot.server.http_server" 2>&1 | tail -n 100 +echo "Done." diff --git a/evaluation/slurm_logs/fetch_models.slurm b/evaluation/slurm_logs/fetch_models.slurm new file mode 100644 index 0000000..bb2642b --- /dev/null +++ b/evaluation/slurm_logs/fetch_models.slurm @@ -0,0 +1,27 @@ +#!/bin/bash +#SBATCH --partition=Teaching +#SBATCH --gres=gpu:1 +#SBATCH --mem=16G +#SBATCH --time=10:00 +#SBATCH --output=fetch_models_full_%j.out +#SBATCH --error=fetch_models_full_%j.err + +export PYTHONPATH="$(pwd):$PYTHONPATH" +export NO_PROXY="localhost,127.0.0.1,::1" +export no_proxy="localhost,127.0.0.1,::1" + +python -c ' +import os +import requests + +url = os.environ.get("BASE_URL", "https://api.openai.com/v1") + "/models" +api_key = os.environ.get("OPENAI_API_KEY", "dummy") + +try: + response = requests.get(url, headers={"Authorization": f"Bearer {api_key}"}) + data = response.json() + for item in data.get("data", []): + print(item.get("id")) +except Exception as e: + print("Error:", e) +' diff --git a/evaluation/zmq_prototype/benchmark_routing_latency.py b/evaluation/zmq_prototype/benchmark_routing_latency.py new file mode 100644 index 0000000..4022ccb --- /dev/null +++ b/evaluation/zmq_prototype/benchmark_routing_latency.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +""" +benchmark_routing_latency.py + +Micro-benchmark script comparing HTTP polling vs ZMQ (local memory shadow cache) +for KVCacheLookup latency. +""" + +import time +import urllib.request +import json +import sys + +def main(): + print("=== Micro-Benchmark: Routing Latency ===") + + # Simulate 1,000 lookups + num_requests = 1000 + + print(f"\n[Strategy A] HTTP Polling ({num_requests} requests)") + print("Sending synchronous GET requests to http://localhost:8000/cache_status...") + + start_time_a = time.time() + success_count = 0 + + # Disable proxies to avoid 503 errors on cluster environments + proxy_handler = urllib.request.ProxyHandler({}) + opener = urllib.request.build_opener(proxy_handler) + urllib.request.install_opener(opener) + + try: + for _ in range(num_requests): + with urllib.request.urlopen("http://127.0.0.1:8000/cache_status") as response: + data = response.read() + cache = json.loads(data) + success_count += 1 + except Exception as e: + print(f"Error fetching HTTP: {e}") + print("Ensure dummy_sglang_pub.py is running in the background!") + sys.exit(1) + + end_time_a = time.time() + elapsed_a = end_time_a - start_time_a + avg_latency_a = (elapsed_a / num_requests) * 1000 # in ms + + print(f"Total Elapsed Time: {elapsed_a:.4f} seconds") + print(f"Average Latency per Request (ms): {avg_latency_a:.4f} ms") + + + print(f"\n[Strategy B] ZMQ / Local Memory ({num_requests} lookups)") + print("Simulating local dictionary lookups for a shadow cache...") + + # We simulate an in-memory dictionary lookup, since ZMQ maintains a zero-latency shadow cache locally + local_cache = {"12345678": {"parent_block_hash": None, "token_ids": [1,2,3]}} + + start_time_b = time.time() + for i in range(num_requests): + # 1000 dict lookups + _ = local_cache.get("12345678") + end_time_b = time.time() + + elapsed_b = end_time_b - start_time_b + avg_latency_b = (elapsed_b / num_requests) * 1000 + + print(f"Total Elapsed Time: {elapsed_b:.8f} seconds") + print(f"Average Latency per Request (ms): {avg_latency_b:.8f} ms") + + print("\n=== Summary ===") + if avg_latency_b > 0: + factor = avg_latency_a / avg_latency_b + print(f"ZMQ / Local Memory is roughly {factor:,.0f}x faster than HTTP Polling!") + +if __name__ == "__main__": + main() diff --git a/evaluation/zmq_prototype/dummy_sglang_pub.py b/evaluation/zmq_prototype/dummy_sglang_pub.py new file mode 100644 index 0000000..ee93601 --- /dev/null +++ b/evaluation/zmq_prototype/dummy_sglang_pub.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +""" +dummy_sglang_pub.py + +A ZeroMQ Publisher that simulates SGLang's internal Radix Tree KV Cache event stream. +It binds to tcp://*:5557 and randomly publishes BlockStored and BlockRemoved events +in a logically consistent manner (maintaining a simulated tree structure) every second. + +Author: Senior AI Infrastructure Software Engineer +Project: Middleware Token Proxy Middleware - WP2 ZMQ Prototype +""" + +import zmq +import json +import time +import random +import sys +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer + +active_blocks = {} + +class CacheStatusHandler(BaseHTTPRequestHandler): + def do_GET(self): + if self.path == '/cache_status': + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(json.dumps(active_blocks).encode('utf-8')) + else: + self.send_response(404) + self.end_headers() + + def log_message(self, format, *args): + pass # Suppress logging to keep terminal clean + +def run_http_server(): + server = HTTPServer(('localhost', 8000), CacheStatusHandler) + server.serve_forever() + +# ANSI Escape Sequences for beautiful terminal output +class Colors: + HEADER = '\033[95m' + BLUE = '\033[94m' + GREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + CYAN = '\033[96m' + +def log_info(msg): + print(f"{Colors.BLUE}[INFO]{Colors.ENDC} {msg}") + +def log_success(msg): + print(f"{Colors.GREEN}[STORED]{Colors.ENDC} {msg}") + +def log_warning(msg): + print(f"{Colors.WARNING}[REMOVED]{Colors.ENDC} {msg}") + +def main(): + # Setup ZMQ Context and PUB Socket + context = zmq.Context() + publisher = context.socket(zmq.PUB) + + # Configure high-water mark to prevent memory bloat + publisher.set_hwm(1000) + + bind_address = "tcp://*:5557" + try: + publisher.bind(bind_address) + except Exception as e: + print(f"{Colors.FAIL}{Colors.BOLD}Failed to bind to {bind_address}: {e}{Colors.ENDC}", file=sys.stderr) + sys.exit(1) + + print(f"{Colors.HEADER}{Colors.BOLD}" + "="*60 + f"{Colors.ENDC}") + print(f"{Colors.HEADER}{Colors.BOLD}SGLang Dummy KV Cache Event Publisher Started{Colors.ENDC}") + print(f"{Colors.CYAN}Binding Address:{Colors.ENDC} {bind_address}") + print(f"{Colors.CYAN}Simulation Rate:{Colors.ENDC} 1 event / sec") + print(f"{Colors.HEADER}{Colors.BOLD}" + "="*60 + f"{Colors.ENDC}") + print("Press Ctrl+C to terminate the publisher gracefully.\n") + + # Local state to ensure logically consistent events + # Maps block_hash -> parent_block_hash + global active_blocks + active_blocks = {} + + # Start HTTP server in a background thread + http_thread = threading.Thread(target=run_http_server, daemon=True) + http_thread.start() + log_info("HTTP server started on port 8000") + + # Track sequence number to simulate SGLang's monotonic event sequence + sequence_number = 0 + + try: + while True: + # We want to maintain a reasonable number of blocks in the cache (e.g., 3 to 15) + num_blocks = len(active_blocks) + + # Decide whether to store a new block or remove an existing one + # If cache is empty, we must store. If cache is full (> 15), we prefer to remove. + if num_blocks == 0: + action = "store" + elif num_blocks > 12: + action = "remove" if random.random() < 0.7 else "store" + else: + # 65% chance to store (grow), 35% chance to remove (evict) + action = "store" if random.random() < 0.65 else "remove" + + sequence_number += 1 + + if action == "store": + # Generate a unique block hash (simulating a 64-bit integer hash) + block_hash = random.randint(10000000, 99999999) + while block_hash in active_blocks: + block_hash = random.randint(10000000, 99999999) + + # Determine parent block hash (simulate tree branching) + parent_block_hash = None + if active_blocks and random.random() < 0.7: + # Pick an existing block as parent to create a hierarchy + parent_block_hash = random.choice(list(active_blocks.keys())) + + # Generate a list of random token IDs (representing the prefix contents of this block) + # SGLang usually has a block size / page size (e.g. 16 tokens) + block_size = 16 + token_ids = [random.randint(1, 50000) for _ in range(block_size)] + + # Construct BlockStored event payload + event = { + "type": "BlockStored", + "sequence": sequence_number, + "block_hash": block_hash, + "parent_block_hash": parent_block_hash, + "token_ids": token_ids, + "block_size": block_size, + "medium": "GPU" + } + + # Update local tracking + active_blocks[block_hash] = parent_block_hash + + # Publish event + event_str = json.dumps(event) + publisher.send_string(event_str) + + parent_str = f"0x{parent_block_hash:08x}" if parent_block_hash else "None" + log_success(f"Block: 0x{block_hash:08x} | Parent: {parent_str} | Tokens: {token_ids[:3]}... ({block_size} tokens)") + + else: # remove + # Pick a block to remove + # To simulate realistic tree eviction, we should ideally evict leaf blocks. + # Let's find leaf blocks (blocks that are not parents of any other active blocks) + all_parents = set(active_blocks.values()) + leaves = [b for b in active_blocks if b not in all_parents] + + # Fallback to any active block if no leaves are easily found (should always find at least one) + block_to_remove = random.choice(leaves) if leaves else random.choice(list(active_blocks.keys())) + + # Construct BlockRemoved event payload + event = { + "type": "BlockRemoved", + "sequence": sequence_number, + "block_hash": block_to_remove, + "medium": "GPU" + } + + # Update local tracking + del active_blocks[block_to_remove] + + # Publish event + event_str = json.dumps(event) + publisher.send_string(event_str) + + log_warning(f"Block: 0x{block_to_remove:08x}") + + time.sleep(1.0) + + except KeyboardInterrupt: + print(f"\n{Colors.WARNING}Shutting down publisher...{Colors.ENDC}") + finally: + publisher.close() + context.term() + print(f"{Colors.GREEN}Publisher terminated cleanly.{Colors.ENDC}") + +if __name__ == "__main__": + main() diff --git a/evaluation/zmq_prototype/shadow_tree_sub.py b/evaluation/zmq_prototype/shadow_tree_sub.py new file mode 100644 index 0000000..ffa7f39 --- /dev/null +++ b/evaluation/zmq_prototype/shadow_tree_sub.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +""" +shadow_tree_sub.py + +A ZeroMQ Subscriber that connects to tcp://localhost:5557, subscribes to all events, +and maintains a local "shadow_cache" dictionary reflecting SGLang's Radix Tree cache. +When events are received, it updates the shadow cache and prints a beautiful ASCII +visualization of the live tree hierarchy. + +Author: Senior AI Infrastructure Software Engineer +Project: Middleware Token Proxy Middleware - WP2 ZMQ Prototype +""" + +import zmq +import json +import sys +import argparse +import time +import urllib.request +from collections import defaultdict + +# ANSI Escape Sequences for beautiful terminal output +class Colors: + HEADER = '\033[95m' + BLUE = '\033[94m' + GREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + CYAN = '\033[96m' + DARK_GRAY = '\033[90m' + +def print_radix_tree(shadow_cache): + """ + Reconstructs and prints the Radix Tree hierarchy from the flat shadow_cache. + """ + if not shadow_cache: + print(f"{Colors.DARK_GRAY} [Cache is currently empty]{Colors.ENDC}") + return + + # Build adjacency list: parent -> list of child block hashes + children = defaultdict(list) + roots = [] + + for block_hash, block_data in shadow_cache.items(): + parent = block_data.get("parent_block_hash") + # A block is a root if its parent is None OR if its parent is not in the shadow cache + if parent is None or parent not in shadow_cache: + roots.append(block_hash) + else: + children[parent].append(block_hash) + + # Sort roots and children by block hash for deterministic display + roots.sort() + for parent in children: + children[parent].sort() + + total_tokens = sum(len(b.get("token_ids", [])) for b in shadow_cache.values()) + print(f"\n{Colors.BOLD}Shadow Radix Tree Cache State:{Colors.ENDC}") + print(f" ├─ {Colors.CYAN}Total Blocks:{Colors.ENDC} {len(shadow_cache)}") + print(f" └─ {Colors.CYAN}Total Tokens:{Colors.ENDC} {total_tokens}") + print(f"{Colors.DARK_GRAY}Tree Visualization:{Colors.ENDC}") + + def dfs(node_hash, prefix="", is_last=True): + node_data = shadow_cache[node_hash] + token_ids = node_data.get("token_ids", []) + + # Display shortened preview of token IDs + if len(token_ids) > 6: + token_preview = f"[{', '.join(map(str, token_ids[:3]))}, ..., {', '.join(map(str, token_ids[-3:]))}]" + else: + token_preview = str(token_ids) + + connector = "└── " if is_last else "├── " + node_label = f"{Colors.GREEN}Block 0x{node_hash:08x}{Colors.ENDC}" + details = f"{Colors.DARK_GRAY}(tokens: {len(token_ids)}, val: {token_preview}){Colors.ENDC}" + + print(f"{prefix}{connector}{node_label} {details}") + + # Recurse children + node_children = children[node_hash] + child_count = len(node_children) + for i, child_hash in enumerate(node_children): + new_prefix = prefix + (" " if is_last else "│ ") + dfs(child_hash, new_prefix, is_last=(i == child_count - 1)) + + # Print the tree starting from each root node + for idx, root_hash in enumerate(roots): + dfs(root_hash, prefix=" ", is_last=(idx == len(roots) - 1)) + print() + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--mode", choices=["zmq", "http"], default="zmq") + args = parser.parse_args() + + if args.mode == "http": + print(f"{Colors.HEADER}{Colors.BOLD}" + "="*60 + f"{Colors.ENDC}") + print(f"{Colors.HEADER}{Colors.BOLD}SGLang Shadow KV Cache Tree Subscriber (HTTP Polling){Colors.ENDC}") + print(f"{Colors.CYAN}Polling from:{Colors.ENDC} http://localhost:8000/cache_status") + print(f"{Colors.HEADER}{Colors.BOLD}" + "="*60 + f"{Colors.ENDC}") + print("Waiting for HTTP server... (Press Ctrl+C to terminate)\n") + + # Disable proxies to avoid 503 errors on cluster environments + proxy_handler = urllib.request.ProxyHandler({}) + opener = urllib.request.build_opener(proxy_handler) + urllib.request.install_opener(opener) + + try: + while True: + try: + with urllib.request.urlopen("http://127.0.0.1:8000/cache_status") as response: + data = response.read() + cache_data = json.loads(data) + + # Convert string keys back to int for print_radix_tree sorting + shadow_cache = {} + for k, v in cache_data.items(): + shadow_cache[int(k)] = v + + print_radix_tree(shadow_cache) + print("-" * 60) + except Exception as e: + print(f"{Colors.FAIL}[ERROR] Failed to fetch HTTP cache status: {e}{Colors.ENDC}") + + time.sleep(1.0) + except KeyboardInterrupt: + print(f"\n{Colors.WARNING}Shutting down subscriber...{Colors.ENDC}") + print(f"{Colors.GREEN}Subscriber terminated cleanly.{Colors.ENDC}") + sys.exit(0) + + # Setup ZMQ Context and SUB Socket + context = zmq.Context() + subscriber = context.socket(zmq.SUB) + + connect_address = "tcp://localhost:5557" + try: + subscriber.connect(connect_address) + except Exception as e: + print(f"{Colors.FAIL}{Colors.BOLD}Failed to connect to {connect_address}: {e}{Colors.ENDC}", file=sys.stderr) + sys.exit(1) + + # Subscribe to all events (empty prefix string) + subscriber.setsockopt_string(zmq.SUBSCRIBE, "") + + print(f"{Colors.HEADER}{Colors.BOLD}" + "="*60 + f"{Colors.ENDC}") + print(f"{Colors.HEADER}{Colors.BOLD}SGLang Shadow KV Cache Tree Subscriber Started{Colors.ENDC}") + print(f"{Colors.CYAN}Connecting to:{Colors.ENDC} {connect_address}") + print(f"{Colors.CYAN}Subscription topic:{Colors.ENDC} [ALL EVENTS]") + print(f"{Colors.HEADER}{Colors.BOLD}" + "="*60 + f"{Colors.ENDC}") + print("Waiting for SGLang publisher events... (Press Ctrl+C to terminate)\n") + + # In-memory dictionary tracking cache state: block_hash -> metadata dict + shadow_cache = {} + + try: + while True: + # Receive event string + event_str = subscriber.recv_string() + + try: + event = json.loads(event_str) + except json.JSONDecodeError as je: + print(f"{Colors.FAIL}[ERROR] Failed to parse JSON event: {je}{Colors.ENDC}", file=sys.stderr) + continue + + event_type = event.get("type") + seq = event.get("sequence", 0) + block_hash = event.get("block_hash") + + if not event_type or block_hash is None: + print(f"{Colors.WARNING}[WARN] Received invalid event structure: {event}{Colors.ENDC}") + continue + + print(f"{Colors.BLUE}[Seq: {seq:03d}]{Colors.ENDC} Received {Colors.BOLD}{event_type}{Colors.ENDC} for Block {Colors.BOLD}0x{block_hash:08x}{Colors.ENDC}") + + if event_type == "BlockStored": + # Store the block details in the local shadow tree cache + shadow_cache[block_hash] = { + "parent_block_hash": event.get("parent_block_hash"), + "token_ids": event.get("token_ids", []), + "block_size": event.get("block_size", 0), + "medium": event.get("medium", "GPU"), + "seq": seq + } + elif event_type == "BlockRemoved": + # Remove the block from the local shadow tree cache + if block_hash in shadow_cache: + del shadow_cache[block_hash] + else: + print(f" {Colors.WARNING}* Block 0x{block_hash:08x} was not found in local shadow cache, skipping deletion *{Colors.ENDC}") + else: + print(f" {Colors.WARNING}* Unknown event type: {event_type} *{Colors.ENDC}") + + # Reconstruct and display the current cache tree structure + print_radix_tree(shadow_cache) + print("-" * 60) + + except KeyboardInterrupt: + print(f"\n{Colors.WARNING}Shutting down subscriber...{Colors.ENDC}") + finally: + subscriber.close() + context.term() + print(f"{Colors.GREEN}Subscriber terminated cleanly.{Colors.ENDC}") + +if __name__ == "__main__": + main() diff --git a/examples/http_server_example.py b/examples/http_server_example.py index 6959429..c5c2fee 100644 --- a/examples/http_server_example.py +++ b/examples/http_server_example.py @@ -32,7 +32,7 @@ def check_server(): try: response = requests.get(f"{BASE_URL}/health", timeout=2.0) health = response.json() - print(f"✓ Server status: {health['status']}") + print(f"+ Server status: {health['status']}") return health except Exception as e: print(f"✗ Server not running: {e}") @@ -81,7 +81,7 @@ def build_index(): ) result = response.json() - print(f"✓ Index built: {len(result['request_ids'])} request IDs") + print(f"+ Index built: {len(result['request_ids'])} request IDs") print(f" Reordered contexts for optimal cache sharing") return result @@ -140,7 +140,7 @@ def stateless_schedule(): ) result = response.json() - print(f"✓ Reordered into {len(result['groups'])} groups") + print(f"+ Reordered into {len(result['groups'])} groups") return result @@ -219,7 +219,7 @@ def main(): print() print("=" * 70) - print("✓ Example complete!") + print("+ Example complete!") print("=" * 70) diff --git a/examples/stateless_batch_example.py b/examples/stateless_batch_example.py index 3d62f07..2cf8652 100644 --- a/examples/stateless_batch_example.py +++ b/examples/stateless_batch_example.py @@ -44,7 +44,7 @@ def example_with_client(): result = client.reorder_raw(contexts) if result: - print(f"\n✓ Batch reordered successfully!") + print(f"\n+ Batch reordered successfully!") print(f" Mode: {result.get('mode', 'stateless')}") print(f" Number of contexts: {result['num_contexts']}") print(f" Number of execution groups: {result['num_groups']}") @@ -87,7 +87,7 @@ def example_with_function(): ) if result: - print(f"✓ Reordered {result['num_contexts']} contexts into {result['num_groups']} groups") + print(f"+ Reordered {result['num_contexts']} contexts into {result['num_groups']} groups") print(f"Original indices order: {result['original_indices']}") else: print("Failed to reorder batch") @@ -122,7 +122,7 @@ def example_direct_http(): if response.status_code == 200: result = response.json() - print(f"✓ Reordered successfully!") + print(f"+ Reordered successfully!") print(f" Groups: {result['num_groups']}") print(f" Order: {result['original_indices']}") else: @@ -173,8 +173,8 @@ def batch_processing_workflow(): return execution_order = result['original_indices'] - print(f" ✓ Optimal order: {execution_order}") - print(f" ✓ {result['num_groups']} execution groups") + print(f" + Optimal order: {execution_order}") + print(f" + {result['num_groups']} execution groups") # Step 3: Reorder your data according to the execution order print("\n3. Reordering data for inference...") @@ -219,7 +219,7 @@ def batch_processing_workflow(): response = requests.get("http://localhost:8765/health", timeout=2) if response.status_code == 200: health = response.json() - print(f"\n✓ Server is running (mode: {health.get('mode', 'unknown')})") + print(f"\n+ Server is running (mode: {health.get('mode', 'unknown')})") else: print(f"\n✗ Server returned status {response.status_code}") sys.exit(1) diff --git a/examples/stateless_sglang_e2e.py b/examples/stateless_sglang_e2e.py index b59beb2..f4d2eb5 100644 --- a/examples/stateless_sglang_e2e.py +++ b/examples/stateless_sglang_e2e.py @@ -177,9 +177,9 @@ def run_rag_with_contextpilot( # 2. IDs within each context reordered (shared IDs as prefix) reordered_contexts = schedule_result['reordered_contexts'] num_groups = schedule_result['num_groups'] - print(f" ✓ Optimal order: {scheduled_order}") - print(f" ✓ Grouped into {num_groups} execution groups") - print(f" ✓ Document IDs reordered within each context for prefix sharing") + print(f" + Optimal order: {scheduled_order}") + print(f" + Grouped into {num_groups} execution groups") + print(f" + Document IDs reordered within each context for prefix sharing") else: print(" ⚠ ContextPilot unavailable, using original order") scheduled_order = list(range(n)) @@ -214,7 +214,7 @@ def run_rag_with_contextpilot( print(f" Generating response {i+1}/{len(prompts)}...", end=" ") response = llm_generate(prompt) responses.append(response) - print("✓") + print("+") # Option B: Batch (uncomment to use) # responses = llm_generate_batch(prompts) @@ -236,7 +236,7 @@ def run_rag_with_contextpilot( 'scheduled_position': scheduled_pos, }) - print(" ✓ Results reordered to match original query order") + print(" + Results reordered to match original query order") return results @@ -255,7 +255,7 @@ def main(): try: r = requests.get(f"{CONTEXTPILOT_URL}/health", timeout=2) - print(f" ContextPilot: ✓ ({r.json().get('mode', 'unknown')} mode)") + print(f" ContextPilot: + ({r.json().get('mode', 'unknown')} mode)") contextpilot_available = True except: print(f" ContextPilot: ✗ Not available at {CONTEXTPILOT_URL}") @@ -263,7 +263,7 @@ def main(): try: r = requests.get(f"{INFERENCE_URL}/health", timeout=2) - print(f" Inference engine: ✓ Ready") + print(f" Inference engine: + Ready") engine_available = True except: print(f" Inference engine: ✗ Not available at {INFERENCE_URL}") diff --git a/pyproject.toml b/pyproject.toml index 9757c7f..21bd07a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ contextpilot-llama-server = "contextpilot._llamacpp_hook:main" [project.optional-dependencies] gpu = ["cupy-cuda12x"] -dev = ["black", "bumpver", "isort", "pip-tools", "pytest", "pytest-cov", "ipython"] +dev = ["black", "bumpver", "isort", "pip-tools", "pytest", "pytest-cov", "ipython", "pytest-asyncio"] sglang = ["sglang>=0.5"] [tool.pytest.ini_options] diff --git a/refactored_plugins/__init__.py b/refactored_plugins/__init__.py new file mode 100644 index 0000000..92c7b0b --- /dev/null +++ b/refactored_plugins/__init__.py @@ -0,0 +1 @@ +# Token Proxy Plugins package diff --git a/refactored_plugins/base.py b/refactored_plugins/base.py new file mode 100644 index 0000000..2755def --- /dev/null +++ b/refactored_plugins/base.py @@ -0,0 +1,48 @@ +from abc import ABC, abstractmethod +from typing import Any, Dict, List, Optional + + +class BasePlugin(ABC): + """ + Abstract base class for all Token Proxy Plugins. + Each plugin intercepts a request and performs a specific optimization. + """ + + def __init__(self, name: str): + self.name = name + + @abstractmethod + async def process(self, request_data: Any) -> Any: + """ + Process the incoming request data and return the optimized version. + """ + pass + + @abstractmethod + def get_plugin_metrics(self) -> Dict[str, float]: + """ + Return a dictionary of performance and optimization metrics. + """ + pass + + +class ContextReorderPlugin(BasePlugin): + """ + Plugin for reordering prompts to maximize KV Cache prefix sharing. + Uses ContextPilot clustering and scheduling logic. + """ + + def __init__(self, alpha: float = 0.001, use_gpu: bool = False): + super().__init__("context_reorder") + from contextpilot.server.live_index import ContextPilot + + self.pilot = ContextPilot(alpha=alpha, use_gpu=use_gpu) + + async def process(self, request_batch: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """ + Specialized process for batches. + Note: The Proxy framework will need to handle batching. + """ + # Extract prompts/messages from batch + # This is where we'll bridge the OpenAI format to ContextPilot's token lists + pass diff --git a/refactored_plugins/dedup.py b/refactored_plugins/dedup.py new file mode 100644 index 0000000..1205c4e --- /dev/null +++ b/refactored_plugins/dedup.py @@ -0,0 +1,105 @@ +import logging +import time +import uuid +from typing import Any, Dict, List, Optional +from .base import BasePlugin + +logger = logging.getLogger(__name__) + + +class ContextDedupPlugin(BasePlugin): + """ + Plugin for deduplicating redundant conversational history in multi-turn requests. + Uses ContextPilot's ConversationTracker to replace repeated messages with reference hints. + """ + + def __init__(self, hint_template: str = "[Reference to Turn {turn_number}]", shadow_mode: bool = False): + super().__init__("context_dedup") + from contextpilot.server.conversation_tracker import ConversationTracker + + self.tracker = ConversationTracker(hint_template=hint_template) + self.shadow_mode = shadow_mode + self._content_to_id = {} + self._next_id = 0 + + # Telemetry + self.total_original_chars = 0 + self.total_chars_saved = 0 + self.total_requests_processed = 0 + self.last_execution_time_ms = 0.0 + + def _get_id(self, content: str) -> int: + """Map message content to a unique integer ID.""" + if content not in self._content_to_id: + self._content_to_id[content] = self._next_id + self._next_id += 1 + return self._content_to_id[content] + + async def process(self, request_data: Dict[str, Any]) -> Dict[str, Any]: + """ + Deduplicate a single OpenAI request. + """ + messages = request_data.get("messages", []) + if not messages: + return request_data + + start_time = time.perf_counter() + + # Calculate original char length for telemetry + original_len = sum(len(m.get("content", "")) for m in messages) + + conv_id = request_data.get("user_id", "default_session") + parent_id = request_data.get("parent_id") + + # 1. Convert messages to IDs + message_ids = [self._get_id(m.get("content", "")) for m in messages] + + # 2. Run Deduplication + current_req_id = str(uuid.uuid4()) + result = self.tracker.deduplicate(request_id=current_req_id, docs=message_ids, parent_request_id=parent_id) + + # 3. Reconstruct messages with hints (Shadow Mode for metrics only) + new_messages = [] + for i, m in enumerate(messages): + msg_id = message_ids[i] + if msg_id in result.overlapping_docs: + hint_idx = result.overlapping_docs.index(msg_id) + hint_text = result.reference_hints[hint_idx] + new_messages.append({"role": m.get("role"), "content": hint_text}) + else: + new_messages.append(m) + + # Update Request + optimized_request = dict(request_data) + if self.shadow_mode: + # SHADOW MODE: We do not actually send the hints to the LLM + # because black-box APIs like ELM/DeepSeek cannot resolve them without our modified engine. + # We restore the original messages to ensure accuracy is preserved. + optimized_request["messages"] = messages + else: + optimized_request["messages"] = new_messages + + optimized_request["current_id"] = current_req_id + + # Update Telemetry + dedup_len = sum(len(m.get("content", "")) for m in new_messages) + self.total_original_chars += original_len + self.total_chars_saved += original_len - dedup_len + self.total_requests_processed += 1 + self.last_execution_time_ms = (time.perf_counter() - start_time) * 1000 + + logger.info( + f"Deduplicated request in {self.last_execution_time_ms:.2f}ms. Saved {original_len - dedup_len} chars." + ) + return optimized_request + + def get_plugin_metrics(self) -> Dict[str, float]: + """Return deduplication metrics.""" + saving_percentage = (self.total_chars_saved / self.total_original_chars * 100) if self.total_original_chars > 0 else 0.0 + return { + "total_original_chars": float(self.total_original_chars), + "total_chars_saved": float(self.total_chars_saved), + "chars_saved_percentage": saving_percentage, + "total_requests_processed": float(self.total_requests_processed), + "last_execution_time_ms": self.last_execution_time_ms, + } diff --git a/refactored_plugins/dynamic_pruning.py b/refactored_plugins/dynamic_pruning.py new file mode 100644 index 0000000..7b7d43a --- /dev/null +++ b/refactored_plugins/dynamic_pruning.py @@ -0,0 +1,96 @@ +import logging +import time +from typing import Any, Dict, List +import torch +import torch.nn.functional as F + +from .base import BasePlugin + +logger = logging.getLogger(__name__) + +class DynamicPruningPlugin(BasePlugin): + """ + Plugin for aggressively pruning redundant conversational history using Semantic Similarity. + """ + + def __init__(self, similarity_threshold: float = 0.3): + super().__init__("dynamic_pruning") + from sentence_transformers import SentenceTransformer + self.model = SentenceTransformer('all-MiniLM-L6-v2') + self.similarity_threshold = similarity_threshold + + # Telemetry + self.total_original_chars = 0 + self.total_chars_saved = 0 + self.total_requests_processed = 0 + self.last_execution_time_ms = 0.0 + + async def process(self, request_data: Dict[str, Any]) -> Dict[str, Any]: + messages = request_data.get("messages", []) + if not messages or len(messages) <= 2: + # Nothing to prune if only system + user + return request_data + + start_time = time.perf_counter() + + original_len = sum(len(m.get("content", "")) for m in messages) + + # 1. Identify system prompt and current query + system_idx = 0 if messages[0].get("role") == "system" else -1 + current_query_idx = len(messages) - 1 + current_query_text = messages[current_query_idx].get("content", "") + + # 2. Extract intermediate historical messages + start_idx = 1 if system_idx == 0 else 0 + history_indices = list(range(start_idx, current_query_idx)) + + if not history_indices: + return request_data + + history_texts = [messages[i].get("content", "") for i in history_indices] + + # 3. Calculate Cosine Similarity + # Encode current query and history texts + query_embedding = self.model.encode(current_query_text, convert_to_tensor=True) + history_embeddings = self.model.encode(history_texts, convert_to_tensor=True) + + # Compute cosine similarities + # query_embedding is (dim,), so unsqueeze to (1, dim) for broadcasting + cosine_scores = F.cosine_similarity(query_embedding.unsqueeze(0), history_embeddings) + + # 4. Filter messages + new_messages = [] + if system_idx == 0: + new_messages.append(messages[0]) + + for i, score in enumerate(cosine_scores): + if score.item() >= self.similarity_threshold: + new_messages.append(messages[history_indices[i]]) + + new_messages.append(messages[current_query_idx]) + + # Update Request + optimized_request = dict(request_data) + optimized_request["messages"] = new_messages + + # Update Telemetry + pruned_len = sum(len(m.get("content", "")) for m in new_messages) + self.total_original_chars += original_len + self.total_chars_saved += (original_len - pruned_len) + self.total_requests_processed += 1 + self.last_execution_time_ms = (time.perf_counter() - start_time) * 1000 + + logger.info( + f"Pruned request in {self.last_execution_time_ms:.2f}ms. Saved {original_len - pruned_len} chars." + ) + return optimized_request + + def get_plugin_metrics(self) -> Dict[str, float]: + saving_percentage = (self.total_chars_saved / self.total_original_chars * 100) if self.total_original_chars > 0 else 0.0 + return { + "total_original_chars": float(self.total_original_chars), + "total_chars_saved": float(self.total_chars_saved), + "chars_saved_percentage": saving_percentage, + "total_requests_processed": float(self.total_requests_processed), + "last_execution_time_ms": self.last_execution_time_ms, + } diff --git a/refactored_plugins/kv_lookup.py b/refactored_plugins/kv_lookup.py new file mode 100644 index 0000000..e0e65d9 --- /dev/null +++ b/refactored_plugins/kv_lookup.py @@ -0,0 +1,165 @@ +import asyncio +import logging +import zmq +import zmq.asyncio +import msgspec +from typing import Any, Dict, List, Optional +from .base import BasePlugin + +logger = logging.getLogger(__name__) + + +class ShadowRadixTree: + """ + Maintains a shadow copy of the worker's KV cache Radix Tree state + by mapping block hashes to their parents and token contents. + """ + + def __init__(self): + # block_hash -> {"parent": parent_hash, "tokens": token_ids} + self.state: Dict[int, Dict[str, Any]] = {} + # Cache to speed up prefix matching queries + self._full_tokens_cache: Dict[int, List[int]] = {} + + def add_block(self, block_hash: int, parent_hash: Optional[int], token_ids: List[int]): + self.state[block_hash] = {"parent": parent_hash, "tokens": token_ids} + self._full_tokens_cache.clear() + + def remove_block(self, block_hash: int): + if block_hash in self.state: + del self.state[block_hash] + self._full_tokens_cache.clear() + + def _get_tokens(self, block_hash: int) -> List[int]: + if block_hash in self._full_tokens_cache: + return self._full_tokens_cache[block_hash] + + path = [] + curr = block_hash + while curr is not None and curr in self.state: + path.append(curr) + curr = self.state[curr]["parent"] + + tokens = [] + for b_hash in reversed(path): + tokens.extend(self.state[b_hash]["tokens"]) + + self._full_tokens_cache[block_hash] = tokens + return tokens + + def longest_prefix_match(self, target_token_ids: List[int]) -> int: + """ + Traverses from root blocks down to find how many tokens match the target. + Returns the maximum matched tokens. + """ + best_match = 0 + for block_hash in self.state: + node_tokens = self._get_tokens(block_hash) + match_len = 0 + for t1, t2 in zip(target_token_ids, node_tokens): + if t1 == t2: + match_len += 1 + else: + break + if match_len > best_match: + best_match = match_len + return best_match + + +class KVCacheLookupPlugin(BasePlugin): + """ + Plugin for routing requests to the worker with the highest KV cache prefix match. + Subscribes to worker ZMQ streams to maintain shadow Radix trees. + """ + + def __init__(self, endpoints: List[str], model_name: str = "Qwen/Qwen2.5-7B-Instruct"): + super().__init__("kv_cache_lookup") + from contextpilot.utils.prompt_generator import get_tokenizer + + self.tokenizer = get_tokenizer(model_name) + if self.tokenizer is None: + logger.warning(f"Could not load tokenizer for {model_name}. Using fallback char-split.") + + self.endpoints = endpoints + self.trees: Dict[str, ShadowRadixTree] = {endpoint: ShadowRadixTree() for endpoint in endpoints} + + self.ctx = zmq.asyncio.Context() + self.listener_tasks = [] + + # Spawn ZMQ listener tasks for each endpoint + for endpoint in endpoints: + task = asyncio.create_task(self._listen(endpoint)) + self.listener_tasks.append(task) + + async def _listen(self, endpoint: str): + sub = self.ctx.socket(zmq.SUB) + sub.connect(endpoint) + sub.setsockopt_string(zmq.SUBSCRIBE, "") + + tree = self.trees[endpoint] + + while True: + try: + parts = await sub.recv_multipart() + # SGLang format: topic, seq, msgpack_payload + if len(parts) >= 3: + payload = parts[2] + elif len(parts) == 1: + payload = parts[0] + else: + continue + + event = msgspec.msgpack.decode(payload) + event_type = event.get("type") or event.get("event_type") + + if event_type == "BlockStored": + block_hash = event.get("block_hash") + parent_hash = event.get("parent_block_hash") + token_ids = event.get("token_ids", []) + if block_hash is not None: + tree.add_block(block_hash, parent_hash, token_ids) + + elif event_type == "BlockRemoved": + block_hash = event.get("block_hash") + if block_hash is not None: + tree.remove_block(block_hash) + + except asyncio.CancelledError: + break + except Exception as e: + logger.error(f"Error in ZMQ listener for {endpoint}: {e}") + + def _tokenize(self, text: str) -> List[int]: + if self.tokenizer: + return self.tokenizer.encode(text, add_special_tokens=False) + return [ord(c) for c in text] + + async def process(self, request_data: Dict[str, Any]) -> Dict[str, Any]: + """ + Tokenize the request messages and query all worker Shadow Radix Trees + for the longest prefix match. Inject '_route_to' into the request. + """ + messages = request_data.get("messages", []) + if not messages: + return request_data + + full_text = "\n".join([m.get("content", "") for m in messages]) + target_tokens = self._tokenize(full_text) + + best_endpoint = None + max_match = -1 + + for endpoint, tree in self.trees.items(): + match_len = tree.longest_prefix_match(target_tokens) + if match_len > max_match: + max_match = match_len + best_endpoint = endpoint + + optimized_request = dict(request_data) + if best_endpoint: + optimized_request["_route_to"] = best_endpoint + + return optimized_request + + def get_plugin_metrics(self) -> Dict[str, float]: + return {} diff --git a/refactored_plugins/reorder.py b/refactored_plugins/reorder.py new file mode 100644 index 0000000..1d1ccd1 --- /dev/null +++ b/refactored_plugins/reorder.py @@ -0,0 +1,70 @@ +import logging +import time +from typing import Any, Dict, List, Optional +from .base import BasePlugin + +logger = logging.getLogger(__name__) + + +class ContextReorderPlugin(BasePlugin): + """ + Plugin for reordering prompts to maximize KV Cache prefix sharing. + Optimized for OpenAI-formatted request batches. + """ + + def __init__(self, model_name: str = "Qwen/Qwen2.5-7B-Instruct", alpha: float = 0.001, use_gpu: bool = False): + super().__init__("context_reorder") + from contextpilot.server.live_index import ContextPilot + from contextpilot.utils.prompt_generator import get_tokenizer + + self.pilot = ContextPilot(alpha=alpha, use_gpu=use_gpu, linkage_method="single") + self.pilot.num_workers = 1 + self.tokenizer = get_tokenizer(model_name) + if self.tokenizer is None: + logger.warning(f"Could not load tokenizer for {model_name}. Using fallback char-split.") + + # Telemetry + self.total_processed_batches = 0 + self.last_execution_time_ms = 0.0 + + def _tokenize(self, text: str) -> List[int]: + """Convert text to token IDs using the configured tokenizer.""" + if self.tokenizer: + return self.tokenizer.encode(text, add_special_tokens=False) + return [ord(c) for c in text] + + async def process(self, request_batch: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """ + Process a batch of OpenAI requests. + """ + if not request_batch: + return [] + + start_time = time.perf_counter() + + # 1. Extract and Tokenize + tokenized_contexts = [] + for req in request_batch: + full_text = "\n".join([m.get("content", "") for m in req.get("messages", [])]) + tokenized_contexts.append(self._tokenize(full_text)) + + # 2. Run ContextPilot Scheduling + result = self.pilot.build_and_schedule(tokenized_contexts) + + # 3. Reorder the original JSON objects + new_order_indices = result["original_indices"] + reordered_batch = [request_batch[i] for i in new_order_indices] + + # Update Telemetry + self.last_execution_time_ms = (time.perf_counter() - start_time) * 1000 + self.total_processed_batches += 1 + + logger.info(f"Reordered batch of {len(request_batch)} requests in {self.last_execution_time_ms:.2f}ms") + return reordered_batch + + def get_plugin_metrics(self) -> Dict[str, float]: + """Return reordering metrics.""" + return { + "total_processed_batches": float(self.total_processed_batches), + "last_execution_time_ms": self.last_execution_time_ms, + } diff --git a/refactored_plugins/skill_index.py b/refactored_plugins/skill_index.py new file mode 100644 index 0000000..9cd0f26 --- /dev/null +++ b/refactored_plugins/skill_index.py @@ -0,0 +1,64 @@ +import logging +import time +from typing import Any, Dict +from .base import BasePlugin + +logger = logging.getLogger(__name__) + + +class SkillAwareContextPlugin(BasePlugin): + """ + Plugin for dynamically injecting OpenAI tool schemas based on the required + skills specified by the framework's router. + """ + + def __init__(self, tool_registry: Dict[str, Dict]): + super().__init__("skill_aware_context") + self.tool_registry = tool_registry + + # Telemetry variables + self.total_original_tools = 0 + self.total_tools_filtered = 0 + self.last_execution_time_ms = 0.0 + + async def process(self, request_data: Dict[str, Any]) -> Dict[str, Any]: + """ + Process the incoming request and inject the required tool schemas. + """ + start_time = time.perf_counter() + optimized_request = dict(request_data) + + required_skills = optimized_request.get("_required_skills") + if required_skills is not None: + injected_tools = [] + for skill in required_skills: + if skill in self.tool_registry: + injected_tools.append(self.tool_registry[skill]) + else: + logger.warning(f"Skill '{skill}' requested but not found in tool registry.") + + # OpenAI requires a 'tools' array + optimized_request["tools"] = injected_tools + + # Update telemetry + original_tools_count = len(self.tool_registry) + self.total_original_tools += original_tools_count + + # Number of tools filtered out = total available - total injected + filtered_count = original_tools_count - len(injected_tools) + self.total_tools_filtered += filtered_count + + self.last_execution_time_ms = (time.perf_counter() - start_time) * 1000 + return optimized_request + + def get_plugin_metrics(self) -> Dict[str, float]: + """ + Return the telemetry data. + """ + saving_percentage = (self.total_tools_filtered / self.total_original_tools * 100) if self.total_original_tools > 0 else 0.0 + return { + "total_original_tools": float(self.total_original_tools), + "total_tools_filtered": float(self.total_tools_filtered), + "tools_filtered_percentage": saving_percentage, + "last_execution_time_ms": self.last_execution_time_ms, + } diff --git a/requirements.txt b/requirements.txt index c9b66cf..b34b6b0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,6 @@ fastapi[all]>=0.115.0 uvicorn[standard]>=0.32.0 httpx>=0.28.0 pytest +pyzmq +msgspec +sentence-transformers diff --git a/scripts/benchmark_pageindex.py b/scripts/benchmark_pageindex.py index 9a571d5..fad5a83 100644 --- a/scripts/benchmark_pageindex.py +++ b/scripts/benchmark_pageindex.py @@ -545,7 +545,7 @@ async def run_query_benchmark( 'num_contexts': len(contexts), }) - self._log(f" ✓ {sr['qid']}: {sr['search_time']:.2f}s search, {gen_time:.2f}s gen") + self._log(f" + {sr['qid']}: {sr['search_time']:.2f}s search, {gen_time:.2f}s gen") return { 'results': results, diff --git a/scripts/vllm_metrics_logger.py b/scripts/vllm_metrics_logger.py new file mode 100644 index 0000000..eb5b4a1 --- /dev/null +++ b/scripts/vllm_metrics_logger.py @@ -0,0 +1,74 @@ +import argparse +import time +import requests +import json +import signal +import sys +import re + +running = True + +def signal_handler(sig, frame): + global running + running = False + +signal.signal(signal.SIGINT, signal_handler) +signal.signal(signal.SIGTERM, signal_handler) + +def parse_metrics(metrics_text): + # Search for vllm:gpu_cache_usage_perc{...} + # Format typically: vllm:gpu_cache_usage_perc{model_name="Qwen/Qwen2.5-7B-Instruct-AWQ"} 0.05 + peak_val = 0.0 + for line in metrics_text.split('\n'): + if line.startswith('vllm:gpu_cache_usage_perc') or line.startswith('vllm_gpu_cache_usage_perc') or line.startswith('vllm:kv_cache_usage_perc') or line.startswith('vllm_kv_cache_usage_perc'): + try: + # Extract the float value at the end of the line + parts = line.rsplit(' ', 1) + if len(parts) == 2: + val = float(parts[1]) + if val > peak_val: + peak_val = val + except ValueError: + pass + return peak_val + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--port", type=int, required=True) + parser.add_argument("--output", type=str, required=True) + parser.add_argument("--interval", type=float, default=0.05) + args = parser.parse_args() + + url = f"http://localhost:{args.port}/metrics" + peak_cache_usage = 0.0 + + print(f"Metrics logger started, polling {url} every {args.interval}s") + + while running: + try: + response = requests.get(url, timeout=2.0) + if response.status_code == 200: + with open("metrics_dump.txt", "w") as dump_f: + dump_f.write(response.text) + current_usage = parse_metrics(response.text) + if current_usage > peak_cache_usage: + peak_cache_usage = current_usage + except Exception: + # vLLM might be starting up or overloaded + pass + + time.sleep(args.interval) + + # Save final peak usage + result = { + "peak_gpu_cache_usage_perc": peak_cache_usage, + "peak_gpu_cache_usage_human": f"{peak_cache_usage * 100:.2f}%" + } + + with open(args.output, "w") as f: + json.dump(result, f, indent=4) + + print(f"Metrics logger stopped. Peak GPU cache usage: {result['peak_gpu_cache_usage_human']}") + +if __name__ == "__main__": + main() diff --git a/tests/test_dedup_plugin.py b/tests/test_dedup_plugin.py new file mode 100644 index 0000000..7eb2303 --- /dev/null +++ b/tests/test_dedup_plugin.py @@ -0,0 +1,75 @@ + +import pytest +import asyncio +from refactored_plugins.dedup import ContextDedupPlugin + +@pytest.mark.asyncio +async def test_context_dedup_plugin_multi_turn(): + """ + Test: Deduplicate a two-turn conversation. + Turn 1: System + Question 1 + Turn 2: System + Question 1 + Answer 1 + Question 2 + """ + plugin = ContextDedupPlugin() + + # --- TURN 1 --- + system_msg = {"role": "system", "content": "You are a helpful assistant."} + q1 = {"role": "user", "content": "What is the capital of France?"} + + request_t1 = { + "user_id": "user123", + "messages": [system_msg, q1] + } + + # Process Turn 1 (should be no change, but registers the history) + resp_t1 = await plugin.process(request_t1) + assert len(resp_t1["messages"]) == 2 + assert resp_t1["messages"][0]["content"] == system_msg["content"] + + t1_id = resp_t1["current_id"] + + # --- TURN 2 (The Agent framework sends the whole history again) --- + a1 = {"role": "assistant", "content": "The capital of France is Paris."} + q2 = {"role": "user", "content": "And Germany?"} + + request_t2 = { + "user_id": "user123", + "parent_id": t1_id, # Link to Turn 1 + "messages": [ + system_msg, # Duplicate + q1, # Duplicate + a1, # New + q2 # New + ] + } + + # Calculate original length + original_total_len = sum(len(m["content"]) for m in request_t2["messages"]) + + # Process Turn 2 + resp_t2 = await plugin.process(request_t2) + + # Calculate deduplicated length + dedup_total_len = sum(len(m["content"]) for m in resp_t2["messages"]) + + # --- ASSERTIONS --- + assert len(resp_t2["messages"]) == 4 + + # The first two messages should now be hints + assert "[Reference to Turn" in resp_t2["messages"][0]["content"] + assert "[Reference to Turn" in resp_t2["messages"][1]["content"] + + # The new messages should remain intact + assert resp_t2["messages"][2]["content"] == a1["content"] + assert resp_t2["messages"][3]["content"] == q2["content"] + + # Compression check + assert dedup_total_len < original_total_len + + print(f"\nOriginal Length: {original_total_len} chars") + print(f"Dedup Length: {dedup_total_len} chars") + print(f"Compression: {(1 - dedup_total_len/original_total_len)*100:.2f}%") + print("\nSUCCESS: ContextDedupPlugin compressed multi-turn history using reference hints.") + +if __name__ == "__main__": + asyncio.run(test_context_dedup_plugin_multi_turn()) diff --git a/tests/test_kv_lookup.py b/tests/test_kv_lookup.py new file mode 100644 index 0000000..53ae1fe --- /dev/null +++ b/tests/test_kv_lookup.py @@ -0,0 +1,35 @@ +import pytest +from refactored_plugins.kv_lookup import ShadowRadixTree + + +@pytest.mark.asyncio +async def test_shadow_radix_tree(): + """ + Test the manual adding of overlapping blocks and longest_prefix_match functionality + of the ShadowRadixTree. + """ + tree = ShadowRadixTree() + + # 1. Manually add 3 overlapping blocks + # Block 1: parent None + tree.add_block(1, None, [100, 200, 300]) + + # Block 2: parent is Block 1 + tree.add_block(2, 1, [400, 500, 600]) + + # Block 3: parent is Block 2 + tree.add_block(3, 2, [700, 800, 900]) + + # Test 1: Completely new sequence (no match) + assert tree.longest_prefix_match([999, 888]) == 0 + + # Test 2: Partially matching sequence + # Matches tokens from Block 1 and part of Block 2 + assert tree.longest_prefix_match([100, 200, 300, 400, 500, 999]) == 5 + + # Test 3: Fully matching sequence + # Matches all tokens across all blocks + assert tree.longest_prefix_match([100, 200, 300, 400, 500, 600, 700, 800, 900]) == 9 + + # Test 4: Another partial match targeting only Block 1 + assert tree.longest_prefix_match([100, 200, 300, 999]) == 3 diff --git a/tests/test_reorder_plugin.py b/tests/test_reorder_plugin.py new file mode 100644 index 0000000..dfa81f5 --- /dev/null +++ b/tests/test_reorder_plugin.py @@ -0,0 +1,50 @@ + +import pytest +import asyncio +from refactored_plugins.reorder import ContextReorderPlugin + +@pytest.mark.asyncio +async def test_context_reorder_plugin_end_to_end(): + """ + E2E Test: Reorder a batch of OpenAI requests with overlapping system prompts. + """ + # 1. Setup Mock OpenAI Requests + # Group A: Common system prompt + Tool Set A + # Group B: Common system prompt + Tool Set B + system_prompt = "You are an assistant." + tools_a = "Tools for math: add, subtract." + tools_b = "Tools for coding: python, bash." + + requests = [ + {"id": "req1", "messages": [{"role": "system", "content": system_prompt}, {"role": "user", "content": "How's the weather?"}]}, # No tools + {"id": "req2", "messages": [{"role": "system", "content": system_prompt}, {"role": "system", "content": tools_a}, {"role": "user", "content": "Add 5+5"}]}, # Tools A + {"id": "req3", "messages": [{"role": "system", "content": system_prompt}, {"role": "system", "content": tools_b}, {"role": "user", "content": "Write python script"}]}, # Tools B + {"id": "req4", "messages": [{"role": "system", "content": system_prompt}, {"role": "system", "content": tools_a}, {"role": "user", "content": "Subtract 10-2"}]}, # Tools A + ] + + # 2. Initialize Plugin (using small alpha to detect overlap) + # We use a dummy model name to avoid downloading 7B tokenizer during test (uses fallback) + plugin = ContextReorderPlugin(model_name="test-model", alpha=0.1) + + # 3. Process Batch + reordered_requests = await plugin.process(requests) + + # 4. Assertions + assert len(reordered_requests) == len(requests) + + # In a perfect world, req2 and req4 (Tools A) should be adjacent + # and follow the same prefix logic we saw in the verification script. + + # Let's find the positions of Tools A requests + tools_a_indices = [i for i, r in enumerate(reordered_requests) if "Add 5+5" in str(r) or "Subtract 10-2" in str(r)] + + # Check if they are adjacent + is_adjacent = abs(tools_a_indices[0] - tools_a_indices[1]) == 1 + + print("\nReordered Sequence IDs:", [r["id"] for r in reordered_requests]) + + assert is_adjacent, f"Tools A requests were not grouped together! Indices: {tools_a_indices}" + print("\nSUCCESS: ContextReorderPlugin grouped requests with shared tool definitions.") + +if __name__ == "__main__": + asyncio.run(test_context_reorder_plugin_end_to_end()) diff --git a/tests/test_skill_index.py b/tests/test_skill_index.py new file mode 100644 index 0000000..d01a2ee --- /dev/null +++ b/tests/test_skill_index.py @@ -0,0 +1,40 @@ +import pytest +from refactored_plugins.skill_index import SkillAwareContextPlugin + + +@pytest.fixture +def mock_tool_registry(): + return { + "math": { + "type": "function", + "function": {"name": "math_tool", "description": "Performs mathematical calculations"}, + }, + "weather": { + "type": "function", + "function": {"name": "weather_tool", "description": "Gets the current weather"}, + }, + "database": {"type": "function", "function": {"name": "db_tool", "description": "Queries the database"}}, + } + + +@pytest.mark.asyncio +async def test_skill_aware_context_plugin(mock_tool_registry): + # Initialize the plugin + plugin = SkillAwareContextPlugin(tool_registry=mock_tool_registry) + + # Pass a mock OpenAI request with "_required_skills": ["math"] + mock_request = {"messages": [{"role": "user", "content": "What is 2 + 2?"}], "_required_skills": ["math"]} + + # Process the request + modified_request = await plugin.process(mock_request) + + # Asserts that the returned request contains exactly 1 tool in its "tools" array + assert "tools" in modified_request + assert len(modified_request["tools"]) == 1 + + # Asserts that it is the correct "math" schema + assert modified_request["tools"][0]["function"]["name"] == "math_tool" + + # Asserts that the telemetry correctly tracks that 2 tools were filtered out. + metrics = plugin.get_plugin_metrics() + assert metrics["total_tools_filtered"] == 2.0 diff --git a/tests/test_utils.py b/tests/test_utils.py index 29d4a8a..73e656f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -20,6 +20,6 @@ def generate_contexts(num_contexts: int, chunk_ids = list(set(chunk_ids)) contexts.append(chunk_ids) - print(f"✓ Generated {num_contexts:,} contexts") + print(f"+ Generated {num_contexts:,} contexts") print(f" Avg chunks per context: {np.mean([len(c) for c in contexts]):.1f}") return contexts \ No newline at end of file diff --git a/verify_prefix_sharing.py b/verify_prefix_sharing.py new file mode 100644 index 0000000..dc4c63c --- /dev/null +++ b/verify_prefix_sharing.py @@ -0,0 +1,73 @@ + +import time +from typing import List +from contextpilot.server.live_index import ContextPilot, compute_prefix_length + +def calculate_total_prefix_sharing(contexts: List[List[int]]) -> int: + if not contexts: + return 0 + + total_sharing = 0 + for i in range(1, len(contexts)): + shared = compute_prefix_length(contexts[i-1], contexts[i]) + total_sharing += shared + return total_sharing + +def run_verification(): + print("=" * 60) + print("VERIFYING PREFIX SHARING BENEFIT") + print("=" * 60) + + # 1. Generate overlapping synthetic contexts + system_prompt = list(range(1, 101)) # 100 + tool_defs = list(range(101, 201)) # 100 + + contexts = [] + for i in range(20): + if i < 10: + ctx = system_prompt + tool_defs + [1000 + i] + else: + ctx = system_prompt + [2000 + i] + contexts.append(ctx) + + # 2. Calculate sharing BEFORE reordering + import random + random.seed(42) + shuffled_contexts = list(contexts) + random.shuffle(shuffled_contexts) + + sharing_before = calculate_total_prefix_sharing(shuffled_contexts) + + # 3. Apply ContextPilot reordering (using the full Pilot) + pilot = ContextPilot(use_gpu=False, alpha=0.1) + # build_and_schedule returns reordered_contexts in the scheduled order + result = pilot.build_and_schedule(shuffled_contexts) + reordered_contexts = result["reordered_contexts"] + + sharing_after = calculate_total_prefix_sharing(reordered_contexts) + + # 4. Debug: Check the first few reordered contexts + print("\nDEBUG: Reordered Context Structure") + for i in range(min(10, len(reordered_contexts))): + ctx = reordered_contexts[i] + has_tools = all(t in ctx for t in tool_defs[:10]) + print(f" Context {i}: len={len(ctx)}, has_tool_defs={has_tools}") + + # 5. Results + print(f"\nBatch Size: {len(contexts)} contexts") + print(f"Sharing BEFORE: {sharing_before} tokens") + print(f"Sharing AFTER: {sharing_after} tokens") + + improvement = (sharing_after - sharing_before) / (sharing_before + 1e-9) * 100 + print(f"Improvement: {improvement:.2f}%") + + if sharing_after > sharing_before: + print("\nSUCCESS: ContextPilot increased prefix sharing!") + elif sharing_after == sharing_before: + print("\nNEUTRAL: No change. Is the baseline already optimal?") + else: + print("\nFAILURE: Reordering decreased sharing!") + print("=" * 60) + +if __name__ == "__main__": + run_verification()