release: To Prod

.dockerignore

@@ -45,6 +45,10 @@ scripts/dev-mint-session.ts
 scripts/dev-login.ts
 scripts/dev-login-browser.ts
 
+# Coverage yardstick (pnpm coverage:report). CI/local-only and imports the
+# fixture planner from tests/, which is excluded above.
+scripts/protocol-coverage-report.ts
+
 # Dev-only config files
 playwright.config.ts
 vitest.config.mts

.github/workflows/e2e-tests-ephemeral.yml

@@ -86,7 +86,7 @@ jobs:
             fi
           fi
 
-          if [[ "$EFFECTIVE_EVENT" == "push" || "$EFFECTIVE_EVENT" == "workflow_dispatch" ]]; then
+          if [[ "$EFFECTIVE_EVENT" == "push" || "$EFFECTIVE_EVENT" == "workflow_dispatch" || "$EFFECTIVE_EVENT" == "schedule" ]]; then
             echo "run-e2e=true" >> "$GITHUB_OUTPUT"
           elif [[ "$EFFECTIVE_EVENT" == "pull_request" ]]; then
             # Run if the e2e label is present (covers both labeled and synchronize events)
@@ -95,6 +95,58 @@ jobs:
             fi
           fi
 
+  # Tier 1 fork simulations: every registry action executed directly
+  # against an anvil mainnet fork (impersonation instead of signing, no
+  # app build), asserting encoding and on-chain semantics with the same
+  # output oracle as the e2e suites. Runs in parallel with the full e2e
+  # jobs and finishes in a fraction of their time; the executed-test
+  # floor guards against silent self-skips. See
+  # specs/protocol-coverage-methodology.md.
+  tier1-simulations:
+    needs: should-run
+    if: needs.should-run.outputs.run-e2e == 'true'
+    runs-on: ubuntu-latest
+    environment: staging
+    timeout-minutes: 20
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v7
+        with:
+          fetch-depth: 1
+
+      - name: Setup Node.js and pnpm
+        uses: ./.github/actions/setup-node-pnpm
+
+      - name: Start anvil Ethereum mainnet fork
+        run: |
+          docker network create keeperhub-network 2>/dev/null || true
+          docker compose --profile test up -d test-anvil-fork-mainnet
+          retries=0
+          until curl -sf -X POST http://localhost:8548 \
+                -H "Content-Type: application/json" \
+                -d '{"jsonrpc":"2.0","method":"eth_chainId","id":1}' | grep -q '"0x1"'; do
+            retries=$((retries + 1))
+            if [ "$retries" -ge 30 ]; then
+              echo "anvil mainnet fork did not become ready in 60s"
+              docker logs keeperhub-test-anvil-fork-mainnet || true
+              exit 1
+            fi
+            sleep 2
+          done
+        env:
+          ANVIL_FORK_MAINNET_URL: ${{ secrets.ANVIL_FORK_MAINNET_URL }}
+
+      - name: Run Tier 1 simulations (chain 1)
+        run: |
+          pnpm vitest run tests/e2e/vitest/protocol-simulation \
+            --reporter=default --reporter=json --outputFile=sim-results.json
+          # Floor 150: the chain-1 sweep currently executes ~200 tests;
+          # below 150 means suites are self-skipping or a protocol lost
+          # its bindings.
+          pnpm tsx scripts/protocol-coverage-floor.ts sim-results.json 150
+        env:
+          PROTOCOL_SIM_RPC_1: http://localhost:8548
+
   # DISABLED: ephemeral E2E tests temporarily disabled while KEEP-1351 stabilises auth/rate-limit infrastructure
   e2e-vitest-ephemeral:
     needs: should-run
@@ -361,7 +413,21 @@ jobs:
           PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d keeperhub_test \
             -c "UPDATE chains SET default_primary_rpc = 'http://localhost:8547', default_fallback_rpc = NULL WHERE chain_id = 11155111" \
             -c "UPDATE chains SET default_primary_rpc = 'http://localhost:8548', default_fallback_rpc = NULL WHERE chain_id = 1"
-          pnpm test:protocol
+          pnpm vitest run tests/e2e/vitest/protocol-coverage \
+            --reporter=default --reporter=json --outputFile=protocol-results.json
+          # Executed-test floor: every suite self-skips when its infra
+          # gates are absent, so a green exit code alone can mean "ran
+          # nothing" (that is how coverage silently collapsed to ~35
+          # executed tests). Fails when executed tests drop below the
+          # floor; publishes run/skip counts to the step summary.
+          #
+          # The floor is coupled to PROTOCOL_E2E_REPRESENTATIVES: full
+          # mode expects ~35 executed today (ajna 25 + superfluid 10;
+          # raise when ANVIL_FORK_MAINNET_URL activates the mainnet
+          # suites), representatives mode expects ~2 per active suite.
+          # Deriving it here keeps the two from drifting apart.
+          if [ "${PROTOCOL_E2E_REPRESENTATIVES:-}" = "1" ]; then FLOOR=4; else FLOOR=30; fi
+          pnpm tsx scripts/protocol-coverage-floor.ts protocol-results.json "$FLOOR"
         env:
           DATABASE_URL: postgresql://postgres:postgres@localhost:5432/keeperhub_test
           BETTER_AUTH_SECRET: ${{ secrets.BETTER_AUTH_SECRET }}

.github/workflows/protocol-nightly.yml

@@ -0,0 +1,38 @@
+name: Protocol Nightly
+
+# Nightly depth for protocol node coverage: the full ephemeral e2e run
+# (including the per-action protocol suites and Tier 1 simulations) plus
+# the Tier 0 mutation check that proves the calldata tests still have
+# teeth. PR runs stay fast; breadth and falsification live here. See
+# specs/protocol-coverage-methodology.md.
+
+on:
+  schedule:
+    - cron: "0 16 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  packages: read
+
+jobs:
+  e2e-full:
+    uses: ./.github/workflows/e2e-tests-ephemeral.yml
+    with:
+      caller_event: schedule
+    secrets: inherit
+
+  mutation-check:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v7
+        with:
+          fetch-depth: 1
+
+      - name: Setup Node.js and pnpm
+        uses: ./.github/actions/setup-node-pnpm
+
+      - name: Run mutation check
+        run: RUN_LOCAL=1 scripts/protocol-mutation-check.sh

.gitignore

@@ -142,6 +142,13 @@ packages/*/dist/
 .claude/test-*-output.txt
 .claude/worktrees/
 
+# Local protocol-coverage rig and tier output (scripts/protocol-local.sh,
+# vitest --outputFile dumps); pattern, not filenames, so new tiers cannot
+# leak results artifacts into the repo again
+.claude/protocol-*-results.json
+.claude/floor-test-*.json
+.claude/tier*.json
+
 # Local dev session cookie minted by scripts/dev-mint-session.ts
 .claude/.dev-session-cookie-LOCAL
 

app/api/internal/reaper/route.ts

@@ -1,35 +1,13 @@
-import { and, eq, gt, inArray, lt, notInArray, or, sql } from "drizzle-orm";
 import { NextResponse } from "next/server";
-import { db } from "@/lib/db";
-import { workflowExecutionLogs, workflowExecutions } from "@/lib/db/schema";
-import { walletLocks } from "@/lib/db/schema-extensions";
-import { classifyExecutionError } from "@/lib/errors/classify";
-import type { ErrorCode } from "@/lib/errors/error-codes";
-import { recordExecutionErrorFinalized } from "@/lib/errors/finalize-error";
 import { authenticateInternalService } from "@/lib/internal-service-auth";
 import { ErrorCategory, logSystemError } from "@/lib/logging";
-
-const DEFAULT_THRESHOLD_MINUTES = 30;
-
-// Pending rows that have produced no step logs are unambiguously stuck:
-// the executor inserted but the runtime never started. A few minutes is
-// generous for image pulls and k8s scheduling; beyond that, it's dead.
-const PENDING_THRESHOLD_MINUTES = 5;
-
-function getThresholdMinutes(): number {
-  const envValue = process.env.STALE_EXECUTION_THRESHOLD_MINUTES;
-  if (!envValue) {
-    return DEFAULT_THRESHOLD_MINUTES;
-  }
-  const parsed = Number.parseInt(envValue, 10);
-  return Number.isNaN(parsed) ? DEFAULT_THRESHOLD_MINUTES : parsed;
-}
+import { reapStaleExecutions } from "@/lib/reaper/reap-stale-executions";
 
 /**
  * GET /api/internal/reaper
  *
- * Finds workflow executions stuck in "running" state with no recent activity
- * and marks them as "error" with a timeout message.
+ * Finds workflow executions stuck past their threshold and marks them as
+ * system_error with a timeout classification.
  *
  * Intended to be called by an external cron job (K8s CronJob).
  */
@@ -43,147 +21,7 @@ export async function GET(request: Request): Promise<NextResponse> {
   }
 
   try {
-    const thresholdMinutes = getThresholdMinutes();
-    const runningCutoff = new Date(Date.now() - thresholdMinutes * 60 * 1000);
-    const pendingCutoff = new Date(
-      Date.now() - PENDING_THRESHOLD_MINUTES * 60 * 1000
-    );
-
-    // Find execution IDs that have recent step activity (should NOT be reaped)
-    const activeExecutionIds = await db
-      .select({
-        executionId: workflowExecutionLogs.executionId,
-      })
-      .from(workflowExecutionLogs)
-      .where(gt(workflowExecutionLogs.completedAt, runningCutoff))
-      .groupBy(workflowExecutionLogs.executionId);
-
-    const excludeIds = activeExecutionIds.map((row) => row.executionId);
-
-    // Bulk update all stale executions in a single query.
-    //
-    // Three independent stuck-state predicates combined with OR. The per-row
-    // error_code below is a CASE on the pre-update status:
-    //   - status='running' rows that haven't recorded recent step activity
-    //     within the running threshold (default 30 min). The runtime claimed
-    //     the execution but stopped progressing -> E-0001.
-    //   - status='pending' rows older than the pending threshold (5 min) with
-    //     no step logs at all. The SQS executor inserted the row but the runtime
-    //     never started (dispatch failed, pod crash before first step) -> P-0001.
-    //     Guarded by NOT EXISTS rather than a NOT IN list because every
-    //     successful execution has step logs and a NOT IN would load that entire
-    //     set into the predicate.
-    //   - status='phantom' rows older than the pending threshold. KEEP-693: the
-    //     scheduler/event-tracker pre-created the row but the executor never
-    //     dequeued it (dispatcher down, SQS lost) -> P-0005.
-    const staleConditions = or(
-      and(
-        eq(workflowExecutions.status, "running"),
-        lt(workflowExecutions.startedAt, runningCutoff),
-        excludeIds.length > 0
-          ? notInArray(workflowExecutions.id, excludeIds)
-          : undefined
-      ),
-      and(
-        eq(workflowExecutions.status, "pending"),
-        lt(workflowExecutions.startedAt, pendingCutoff),
-        sql`NOT EXISTS (
-          SELECT 1 FROM ${workflowExecutionLogs}
-          WHERE ${workflowExecutionLogs.executionId} = ${workflowExecutions.id}
-        )`
-      ),
-      and(
-        eq(workflowExecutions.status, "phantom"),
-        lt(workflowExecutions.startedAt, pendingCutoff)
-      )
-    );
-
-    // KEEP-545: reaped executions are by definition stuck/timed-out, which
-    // classifies as system-side. Write the classification columns and
-    // increment the per-org counter per reaped row.
-    const reaperErrorMessage = `Execution timed out: no progress for ${thresholdMinutes} minutes`;
-    const reaperClassification = classifyExecutionError(reaperErrorMessage);
-
-    const reaped = await db
-      .update(workflowExecutions)
-      .set({
-        // KEEP-853: reaped rows are always system/infra failures (timeouts,
-        // never-started, SQS-lost), so they carry the system_error status.
-        status: "system_error",
-        error: reaperErrorMessage,
-        // KEEP-651: errorCategory is per pre-update status. running -> the
-        // runtime claimed the execution but stopped progressing, which is a
-        // workflow engine failure (workflow_engine). pending/phantom -> the
-        // pod never started or the dispatcher never dequeued -- these are
-        // infrastructure failures (infrastructure), not runtime engine faults,
-        // and must be a distinct metric series so a FailedCreatePodSandBox
-        // burst surfaces separately from ordinary execution timeouts.
-        errorCategory: sql<string>`CASE
-          WHEN ${workflowExecutions.status} = 'running' THEN 'workflow_engine'
-          ELSE 'infrastructure' END`,
-        errorType: reaperClassification.errorType,
-        // KEEP-693: attribute the code to the pre-update status -- running ->
-        // timed out (E-0001), pending -> never started (P-0001), phantom ->
-        // never picked up (P-0005).
-        errorCode: sql<ErrorCode>`CASE
-          WHEN ${workflowExecutions.status} = 'pending' THEN 'P-0001'
-          WHEN ${workflowExecutions.status} = 'phantom' THEN 'P-0005'
-          ELSE 'E-0001' END`,
-        completedAt: new Date(),
-        duration: sql`ROUND(EXTRACT(EPOCH FROM (NOW() - ${workflowExecutions.startedAt})) * 1000)`,
-      })
-      .where(staleConditions)
-      .returning({
-        id: workflowExecutions.id,
-        workflowId: workflowExecutions.workflowId,
-        errorCategory: workflowExecutions.errorCategory,
-      });
-
-    const reapedIds = reaped.map((row) => row.id);
-
-    // KEEP-545: emit one counter increment per reaped row so the timeout
-    // family shows up in the new classified counter the SLA alert watches.
-    for (const row of reaped) {
-      await recordExecutionErrorFinalized({
-        workflowId: row.workflowId,
-        errorMessage: reaperErrorMessage,
-        persistedStatus: "system_error",
-        errorCategory: row.errorCategory ?? undefined,
-      });
-    }
-
-    // KEEP-333: Close orphaned 'running' step logs for reaped executions so
-    // the UI doesn't show stuck spinners after the workflow has been marked
-    // as timed out.
-    if (reapedIds.length > 0) {
-      await db
-        .update(workflowExecutionLogs)
-        .set({
-          status: "error",
-          error: "Step did not record completion",
-          completedAt: new Date(),
-        })
-        .where(
-          and(
-            inArray(workflowExecutionLogs.executionId, reapedIds),
-            eq(workflowExecutionLogs.status, "running")
-          )
-        );
-
-      // KEEP-344: Release any nonce locks still held by reaped executions.
-      // The wallet_locks TTL would clear them on its own, but releasing them
-      // here unwedges affected wallets immediately when reaper runs, instead
-      // of leaving the user blocked until expires_at.
-      await db
-        .update(walletLocks)
-        .set({
-          lockedBy: null,
-          lockedAt: null,
-          expiresAt: sql`NOW()`,
-        })
-        .where(inArray(walletLocks.lockedBy, reapedIds));
-    }
-
+    const reapedIds = await reapStaleExecutions();
     return NextResponse.json({
       reapedCount: reapedIds.length,
       reapedIds,

app/api/workflows/executions/[executionId]/logs/route.ts

@@ -3,6 +3,7 @@ import { NextResponse } from "next/server";
 import { db } from "@/lib/db";
 import { ErrorCategory, logSystemError } from "@/lib/logging";
 import { workflowExecutionLogs } from "@/lib/db/schema";
+import { redactAllUrls, redactSecretUrls } from "@/lib/rpc/scrub-rpc-urls";
 import { redactSensitiveData } from "@/lib/utils/redact";
 import { resolveAuthorizedExecution } from "@/lib/workflow/execution-access";
 
@@ -64,6 +65,15 @@ export async function GET(
       );
     }
     const { execution } = resolved;
+    // Redact on read so rows persisted before URL redaction existed do not
+    // re-display provider RPC URLs. web3 step errors only ever contain
+    // provider URLs, so drop every URL there; other steps may legitimately
+    // reference user-owned URLs.
+    const scrubbedExecution = {
+      ...execution,
+      error:
+        execution.error === null ? null : redactSecretUrls(execution.error),
+    };
 
     const logs = await db.query.workflowExecutionLogs.findMany({
       where: eq(workflowExecutionLogs.executionId, executionId),
@@ -74,11 +84,17 @@ export async function GET(
       ...log,
       input: redactSensitiveData(log.input),
       output: redactSensitiveData(log.output),
+      error:
+        log.error === null
+          ? null
+          : log.nodeType.startsWith("web3/")
+            ? redactAllUrls(log.error)
+            : redactSecretUrls(log.error),
     }));
 
     if (!hasAnyPartialParam) {
       return NextResponse.json({
-        execution,
+        execution: scrubbedExecution,
         logs: redactedLogs,
       });
     }
@@ -105,7 +121,7 @@ export async function GET(
     });
 
     return NextResponse.json({
-      execution,
+      execution: scrubbedExecution,
       logs: transformedLogs,
     });
   } catch (error) {