feat(simulate): retry still-failing scenarios in CI with --retries (default 3)

After a run finishes with failures, CI mode re-runs only the failing
scenarios against the same already-registered agent, up to --retries
times; a scenario passes when any attempt does. The scenarios come from
the finished run itself, so generated-from-source runs retry without
re-uploading or regenerating. Systemic conditions (broken agent, quota
exhaustion) are never retried.

The verdict (counts, --baseline comparison, exit error) reads each
scenario's outcome from the last attempt that ran it to a terminal
state, so a cancelled retry cannot launder an earlier failure. Every
attempt's transcript still prints; a failure that later passed keeps
its transcript but loses its ::error:: annotation, and the final counts
name the scenarios that passed on retry so flakes stay visible.

Author: u9g

cmd/lk/simulate.go

@@ -105,6 +105,11 @@ var simulateCommand = &cli.Command{
 			Name:  "baseline",
 			Usage: "Compare failures against the finished run with run `ID`: only scenarios that pass there and fail here fail the exit code. Non-interactive (CI) runs only",
 		},
+		&cli.IntFlag{
+			Name:  "retries",
+			Value: 3,
+			Usage: "Times to re-run scenarios that still fail before giving up (0 disables). A scenario passes when any attempt passes. Non-interactive (CI) runs only",
+		},
 		&cli.StringFlag{
 			Name:  "agent-name",
 			Usage: "Run against an already-running agent instead of spawning one locally. Pass the registered `NAME`, or \"\" to target the project's default agent (the one that auto-joins every room). Requires --scenarios.",
@@ -219,6 +224,7 @@ type simulateConfig struct {
 	scenariosPath  string   // path to the --scenarios file (empty when generating from source)
 	viewModeRunID  string   // non-empty when --view opens a pre-existing run
 	baselineRunID  string   // --baseline: failures this run also has don't fail CI
+	retries        int      // --retries: times to re-run still-failing scenarios (CI only)
 	liveAgent      bool     // --agent-name: run against an already-running agent, don't spawn one
 	warnings       []string // config-level warnings surfaced at setup (e.g. ignored flags)
 
@@ -422,6 +428,7 @@ func runSimulate(ctx context.Context, cmd *cli.Command, simulationMode livekit.S
 		scenariosPath:  scenariosPath,
 		viewModeRunID:  runID,
 		baselineRunID:  cmd.String("baseline"),
+		retries:        int(cmd.Int("retries")),
 		liveAgent:      liveAgent,
 		warnings:       simulateConfigWarnings(mode, numSimulations),
 	}

cmd/lk/simulate_ci.go

@@ -145,33 +145,76 @@ func runSimulateCI(ctx context.Context, config *simulateConfig) error {
 		return err
 	}
 	runFinished = true
+	firstRunID := runID
+
+	// --- Retry still-failing scenarios ---
+	//
+	// Each retry re-runs only the scenarios still failing, against the same
+	// already-registered agent. Systemic conditions (broken agent, quota
+	// exhaustion) fail the same way again, so they are not retried.
+	attempts := []*livekit.SimulationRun{run}
+	for len(attempts)-1 < config.retries && !poller.brokenAgent && !poller.quotaWarned {
+		failed := failedScenarioKeys(mergedFinalRun(attempts))
+		if len(failed) == 0 {
+			break
+		}
+		group := retryScenarioGroup(run, config.scenarioGroup, failed)
+		if group == nil {
+			break
+		}
+
+		report.RetryingFailed(len(attempts), config.retries, failed)
+
+		retryCfg := *config
+		retryCfg.mode = modeScenarios
+		retryCfg.scenarioGroup = group
+		retryID, _, err := createSimulationRun(ctx, &retryCfg)
+		if err != nil {
+			out.Warnf("Warning: could not create the retry run: %v", err)
+			break
+		}
+		runID, runFinished = retryID, false
+		report.RunCreated(runID, simulationDashboardURL(config.pc.ProjectId, runID))
+
+		retryRun, err := poller.poll(ctx, runID)
+		if err != nil {
+			return err
+		}
+		runFinished = true
+		attempts = append(attempts, retryRun)
+	}
 	brokenAgent := poller.brokenAgent
+	finalRun := mergedFinalRun(attempts)
 
 	// --- Results ---
 
 	if !out.Interactive() {
-		report.Results(run, agent)
+		report.ResultsAll(attempts, agent)
 	} else {
 		// A terminal is watching; we just couldn't open the TUI (e.g. stdin
 		// isn't a TTY). Keep it to counts and pointers, the per-scenario
 		// transcripts go to a report file like the TUI's.
-		dashboardURL := simulationDashboardURL(config.pc.ProjectId, runID)
-		if path := newRunReporter().Finish(run, agent, brokenAgent, dashboardURL); path != "" {
+		dashboardURL := simulationDashboardURL(config.pc.ProjectId, firstRunID)
+		if path := newRunReporter().FinishAll(attempts, agent, brokenAgent, dashboardURL); path != "" {
 			out.Statusf("Run report: %s", path)
 		}
-		total, _, passed, failedN := simulationJobCounts(run)
-		fmt.Fprintf(out.ResultWriter(), "%d total, %d passed, %d failed\n", total, passed, failedN)
+		total, _, passed, failedN := simulationJobCounts(finalRun)
+		line := fmt.Sprintf("%d total, %d passed, %d failed", total, passed, failedN)
+		if flaky := passedOnRetry(attempts); len(flaky) > 0 {
+			line += fmt.Sprintf(" (%d passed on retry)", len(flaky))
+		}
+		fmt.Fprintln(out.ResultWriter(), line)
 	}
 
 	if brokenAgent && agent != nil {
 		writeBrokenAgentNote(out.WarnWriter(), agent)
 	}
 
-	if url := simulationDashboardURL(config.pc.ProjectId, runID); url != "" {
+	if url := simulationDashboardURL(config.pc.ProjectId, firstRunID); url != "" {
 		out.Statusf("Dashboard:  %s", url)
 	}
 
-	return baselineFailureError(ctx, config, run)
+	return baselineFailureError(ctx, config, finalRun)
 }
 
 // ciRunPoller polls a run until it reaches a terminal state. Detection state

cmd/lk/simulate_report.go

@@ -127,8 +127,48 @@ func (l *simLog) BrokenAgent() {
 	fmt.Fprintln(l.info, "The agent is failing to run jobs; cancelling the run.")
 }
 
+// RetryingFailed announces the next retry run and resets per-run progress
+// tracking so the retry's counts print from zero.
+func (l *simLog) RetryingFailed(retryNum, maxRetries int, keys []string) {
+	l.prevStatus = livekit.SimulationRun_Status(-1)
+	l.prevDone = 0
+	fmt.Fprintln(l.out)
+	fmt.Fprintf(l.out, "%d scenario(s) still failing, retrying (%d of %d): %s\n",
+		len(keys), retryNum, maxRetries, strings.Join(keys, ", "))
+}
+
 func (l *simLog) Results(run *livekit.SimulationRun, ap *AgentProcess) {
-	writeRunResults(l.out, run, ap)
+	l.ResultsAll([]*livekit.SimulationRun{run}, ap)
+}
+
+// ResultsAll writes each attempt's results in order. A job that failed but
+// passed on a later attempt keeps its transcript and loses only the ::error::
+// annotation; the merged counts at the end are the run's verdict.
+func (l *simLog) ResultsAll(attempts []*livekit.SimulationRun, ap *AgentProcess) {
+	if len(attempts) == 0 {
+		return
+	}
+	flaky := passedOnRetry(attempts)
+	resolved := make(map[string]bool, len(flaky))
+	for _, key := range flaky {
+		resolved[key] = true
+	}
+	for i, run := range attempts {
+		if i > 0 {
+			fmt.Fprintln(l.out)
+			fmt.Fprintf(l.out, "--- Retry %d ---\n", i)
+		}
+		writeRunResults(l.out, run, ap, resolved)
+	}
+	if len(attempts) > 1 {
+		total, _, passed, failed := simulationJobCounts(mergedFinalRun(attempts))
+		fmt.Fprintln(l.out)
+		fmt.Fprintf(l.out, "After retries: %d total, %d passed, %d failed", total, passed, failed)
+		if len(flaky) > 0 {
+			fmt.Fprintf(l.out, " (passed on retry: %s)", strings.Join(flaky, ", "))
+		}
+		fmt.Fprintln(l.out)
+	}
 	if l.quotaNote != "" {
 		fmt.Fprintf(l.out, "\n⚠ %s\n", l.quotaNote)
 	}
@@ -161,8 +201,10 @@ func (a asciiWriter) Write(p []byte) (int, error) {
 }
 
 // writeRunResults writes the per-job results and the run summary, with GitHub
-// group markers (a useful delimiter outside GitHub too).
-func writeRunResults(w io.Writer, run *livekit.SimulationRun, ap *AgentProcess) {
+// group markers (a useful delimiter outside GitHub too). Failed jobs whose
+// scenario is in passedOnRetry get no ::error:: annotation — a later attempt
+// passed them.
+func writeRunResults(w io.Writer, run *livekit.SimulationRun, ap *AgentProcess, passedOnRetry map[string]bool) {
 	if run == nil {
 		return
 	}
@@ -227,7 +269,7 @@ func writeRunResults(w io.Writer, run *livekit.SimulationRun, ap *AgentProcess)
 
 		fmt.Fprintln(w, "::endgroup::")
 
-		if job.Status == livekit.SimulationRun_Job_STATUS_FAILED {
+		if job.Status == livekit.SimulationRun_Job_STATUS_FAILED && !passedOnRetry[scenarioKey(job)] {
 			firstLine, _, _ := strings.Cut(job.Error, "\n")
 			fmt.Fprintf(w, "::error::Job %d failed: %s\n", i+1, firstLine)
 		}
@@ -340,12 +382,18 @@ func newRunReporter() *runReporter {
 }
 
 func (r *runReporter) Finish(run *livekit.SimulationRun, ap *AgentProcess, brokenAgent bool, dashboardURL string) string {
+	var attempts []*livekit.SimulationRun
+	if run != nil {
+		attempts = []*livekit.SimulationRun{run}
+	}
+	return r.FinishAll(attempts, ap, brokenAgent, dashboardURL)
+}
+
+func (r *runReporter) FinishAll(attempts []*livekit.SimulationRun, ap *AgentProcess, brokenAgent bool, dashboardURL string) string {
 	if r.f == nil {
 		return ""
 	}
-	if run != nil {
-		r.Results(run, ap)
-	}
+	r.ResultsAll(attempts, ap)
 	if brokenAgent && ap != nil {
 		writeBrokenAgentNote(r.info, ap)
 	}

cmd/lk/simulate_retry.go

@@ -0,0 +1,105 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"github.com/livekit/protocol/livekit"
+	"google.golang.org/protobuf/proto"
+)
+
+// scenarioKey identifies a scenario across runs: the label, or the
+// instructions for generated jobs that carry no label.
+func scenarioKey(job *livekit.SimulationRun_Job) string {
+	if l := job.GetLabel(); l != "" {
+		return l
+	}
+	return job.GetInstructions()
+}
+
+// retryScenarioGroup returns the scenarios matching keys, taken from the
+// finished run (which carries its scenarios in both --scenarios and generated
+// modes) or from fallback when the run carries none. Nil when nothing matches:
+// there is nothing to re-run.
+func retryScenarioGroup(run *livekit.SimulationRun, fallback *livekit.ScenarioGroup, keys []string) *livekit.ScenarioGroup {
+	group := run.GetScenarioGroup()
+	if len(group.GetScenarios()) == 0 {
+		group = fallback
+	}
+	want := make(map[string]bool, len(keys))
+	for _, k := range keys {
+		want[k] = true
+	}
+	out := &livekit.ScenarioGroup{Name: group.GetName()}
+	for _, s := range group.GetScenarios() {
+		key := s.GetLabel()
+		if key == "" {
+			key = s.GetInstructions()
+		}
+		if want[key] {
+			out.Scenarios = append(out.Scenarios, s)
+		}
+	}
+	if len(out.Scenarios) == 0 {
+		return nil
+	}
+	return out
+}
+
+// mergedFinalRun folds retry attempts into the first run: each job's outcome
+// comes from the last attempt that ran its scenario to a terminal state, so a
+// cancelled retry cannot launder an earlier failure. The CI verdict (counts,
+// baseline comparison, exit error) reads final outcomes; the printed
+// per-attempt results stay verbatim.
+func mergedFinalRun(attempts []*livekit.SimulationRun) *livekit.SimulationRun {
+	if len(attempts) == 1 {
+		return attempts[0]
+	}
+	final := make(map[string]*livekit.SimulationRun_Job)
+	for _, run := range attempts[1:] {
+		for _, job := range run.GetJobs() {
+			if isTerminalJobStatus(job.GetStatus()) {
+				final[scenarioKey(job)] = job
+			}
+		}
+	}
+	merged := proto.Clone(attempts[0]).(*livekit.SimulationRun)
+	for i, job := range merged.Jobs {
+		if f, ok := final[scenarioKey(job)]; ok {
+			merged.Jobs[i] = f
+		}
+	}
+	return merged
+}
+
+// passedOnRetry returns the scenarios that failed on some attempt but passed
+// on a later one, in first-run job order.
+func passedOnRetry(attempts []*livekit.SimulationRun) []string {
+	failedEver := make(map[string]bool)
+	for _, run := range attempts {
+		for _, key := range failedScenarioKeys(run) {
+			failedEver[key] = true
+		}
+	}
+	seen := make(map[string]bool)
+	var flaky []string
+	for _, job := range mergedFinalRun(attempts).GetJobs() {
+		key := scenarioKey(job)
+		if failedEver[key] && !seen[key] && job.GetStatus() == livekit.SimulationRun_Job_STATUS_COMPLETED {
+			seen[key] = true
+			flaky = append(flaky, key)
+		}
+	}
+	return flaky
+}