Added --exit-on-first-fail and --no-results-on-fail flags

Author: eugr

README.md

@@ -176,6 +176,8 @@ Generally you don't need to disable prompt caching on the server, as a probabili
 -   `--format`: Output format: 'md', 'json', 'csv' (Default: 'md').
 -   `--save-total-throughput-timeseries`: Save calculated TOTAL throughput for each 1 second window inside peak throughput calculation during the run (default: off).
 -   `--save-all-throughput-timeseries`: Save calculated throughput timeseries for EACH individual request (default: off).
+-   `--exit-on-first-fail`: Stop execution on first failed test and exit with non-zero status.
+-   `--no-results-on-fail`: Prevent saving/printing any results when error is experienced, turns on --exit-on-first-fail as well.
 
 ### Metrics
 

src/llama_benchy/config.py

@@ -27,6 +27,8 @@ class BenchmarkConfig(BaseModel):
     result_format: str = Field("md", description="Output format (md, json, csv)")
     save_total_throughput_timeseries: bool = Field(False, description="Save calculated TOTAL throughput for each 1 second window inside peak throughput calculation during the run.")
     save_all_throughput_timeseries: bool = Field(False, description="Save calculated throughput timeseries for EACH individual request.")
+    exit_on_first_fail: bool = Field(False, description="Stop execution on first failed test and exit with non-zero status")
+    no_results_on_fail: bool = Field(False, description="Prevent saving/printing results when error is experienced, turns on --exit-on-first-fail as well")
 
     @classmethod
     def from_args(cls):
@@ -55,8 +57,13 @@ def from_args(cls):
         parser.add_argument("--format", type=str, default="md", choices=["md", "json", "csv"], help="Output format")
         parser.add_argument("--save-total-throughput-timeseries", action="store_true", help="Save calculated TOTAL throughput for each 1 second window inside peak throughput calculation during the run.")
         parser.add_argument("--save-all-throughput-timeseries", action="store_true", help="Save calculated throughput timeseries for EACH individual request.")
+        parser.add_argument("--exit-on-first-fail", action="store_true", help="Stop execution on first failed test and exit with non-zero status")
+        parser.add_argument("--no-results-on-fail", action="store_true", help="Prevent saving/printing results when error is experienced, turns on --exit-on-first-fail as well")
 
         args = parser.parse_args()
+
+        if args.no_results_on_fail:
+            args.exit_on_first_fail = True
 
         return cls(
             base_url=args.base_url,
@@ -80,5 +87,7 @@ def from_args(cls):
             save_result=args.save_result,
             result_format=args.format,
             save_total_throughput_timeseries=args.save_total_throughput_timeseries,
-            save_all_throughput_timeseries=args.save_all_throughput_timeseries
+            save_all_throughput_timeseries=args.save_all_throughput_timeseries,
+            exit_on_first_fail=args.exit_on_first_fail,
+            no_results_on_fail=args.no_results_on_fail
         )

src/llama_benchy/runner.py

@@ -1,6 +1,7 @@
 import asyncio
 import subprocess
 import time
+import sys
 from datetime import datetime, timezone
 from typing import List
 import aiohttp
@@ -11,6 +12,9 @@
 from .prompts import PromptGenerator
 from .results import BenchmarkResults, BenchmarkMetadata
 
+class BenchmarkFailure(Exception):
+    pass
+
 class BenchmarkRunner:
     def __init__(self, config: BenchmarkConfig, client: LLMClient, prompt_generator: PromptGenerator):
         self.config = config
@@ -101,6 +105,11 @@ async def run_suite(self):
                                         load_results = await asyncio.gather(*load_tasks)
                                         run_ctx_results.append(load_results)
 
+                                        if self.config.exit_on_first_fail and any(r.error for r in load_results):
+                                            first_error = next(r.error for r in load_results if r.error)
+                                            print(f"\n[Error] Stopping due to error in context load: {first_error}")
+                                            raise BenchmarkFailure()
+
                                         # Phase 2: Inference
                                         print(f"  Run {run+1}/{self.config.num_runs} (Inference, batch size {concurrency})...")
                                         inf_tasks = []
@@ -117,6 +126,11 @@ async def run_suite(self):
                                         batch_results = await asyncio.gather(*inf_tasks)
                                         run_std_results.append(batch_results)
 
+                                        if self.config.exit_on_first_fail and any(r.error for r in batch_results):
+                                            first_error = next(r.error for r in batch_results if r.error)
+                                            print(f"\n[Error] Stopping due to error in inference: {first_error}")
+                                            raise BenchmarkFailure()
+
                                     else:
                                         # Standard Run
                                         print(f"  Run {run+1}/{self.config.num_runs} (batch size {concurrency})...")
@@ -135,6 +149,11 @@ async def run_suite(self):
                                         batch_results = await asyncio.gather(*batch_tasks)
                                         run_std_results.append(batch_results)
 
+                                        if self.config.exit_on_first_fail and any(r.error for r in batch_results):
+                                            first_error = next(r.error for r in batch_results if r.error)
+                                            print(f"\n[Error] Stopping due to error in standard run: {first_error}")
+                                            raise BenchmarkFailure()
+
 
                                     # Post Run Command
                                     if self.config.post_run_cmd:
@@ -164,18 +183,27 @@ async def run_suite(self):
 
             self.results.save_report(self.config.save_result, self.config.result_format, max(self.config.concurrency_levels) if self.config.concurrency_levels else 1)
 
-        except (asyncio.CancelledError, KeyboardInterrupt):
+        except (asyncio.CancelledError, KeyboardInterrupt, BenchmarkFailure) as e:
             if self.results.runs:
-                print("\n[Interrupted] Saving partial results...")
-                if self.results.metadata is None:
-                    self.results.metadata = BenchmarkMetadata(
-                        version=__version__,
-                        timestamp=datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%SZ"),
-                        latency_mode=self.config.latency_mode,
-                        latency_ms=latency * 1000,
-                        model=self.config.model,
-                        prefix_caching_enabled=self.config.enable_prefix_caching,
-                        max_concurrency=max_concurrency
-                    )
-                self.results.save_report(self.config.save_result, self.config.result_format, max_concurrency)
+                should_save = True
+                if isinstance(e, BenchmarkFailure) and self.config.no_results_on_fail:
+                    should_save = False
+                    print("\n[Failed] Results discarded per --no-results-on-fail.")
+
+                if should_save:
+                    print("\n[Interrupted/Failed] Saving partial results...")
+                    if self.results.metadata is None:
+                        self.results.metadata = BenchmarkMetadata(
+                            version=__version__,
+                            timestamp=datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%SZ"),
+                            latency_mode=self.config.latency_mode,
+                            latency_ms=latency * 1000,
+                            model=self.config.model,
+                            prefix_caching_enabled=self.config.enable_prefix_caching,
+                            max_concurrency=max_concurrency
+                        )
+                    self.results.save_report(self.config.save_result, self.config.result_format, max_concurrency)
+            
+            if isinstance(e, BenchmarkFailure):
+                sys.exit(1)
             raise