fix(benches): bench sequentially

.github/workflows/benchmarks.yml

@@ -28,8 +28,8 @@ env:
   DEFAULT_REPOS: "ithacaxyz/account:v0.3.2,Vectorized/solady:v0.1.22"
 
 jobs:
-  setup:
-    name: Setup and Build
+  run-benchmarks:
+    name: Run All Benchmarks
     runs-on: foundry-runner
     steps:
       - name: Checkout repository
@@ -59,22 +59,16 @@ jobs:
       - name: Build benchmark binary
         run: cargo build --release --bin foundry-bench
 
-      - name: Upload benchmark binary
-        uses: actions/upload-artifact@v4
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
         with:
-          name: foundry-bench
-          path: target/release/foundry-bench
-
-  forge-test-bench:
-    name: Forge Test Benchmarks
-    needs: setup
-    runs-on: foundry-runner
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
+          node-version: "24"
 
-      - name: Benchmark setup
-        uses: ./.github/actions/benchmark-setup
+      - name: Install hyperfine
+        run: |
+          curl -L https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine-v1.19.0-x86_64-unknown-linux-gnu.tar.gz | tar xz
+          sudo mv hyperfine-v1.19.0-x86_64-unknown-linux-gnu/hyperfine /usr/local/bin/
+          rm -rf hyperfine-v1.19.0-x86_64-unknown-linux-gnu
 
       - name: Run forge test benchmarks
         env:
@@ -83,115 +77,88 @@ jobs:
           VERSIONS="${{ github.event.inputs.versions || 'stable,nightly' }}"
           REPOS="${{ github.event.inputs.repos || env.DEFAULT_REPOS }}"
 
-          ./foundry-bench --output-dir ./benches --force-install \
+          ./target/release/foundry-bench --output-dir ./benches --force-install \
             --versions $VERSIONS \
             --repos $REPOS \
             --benchmarks forge_test,forge_fuzz_test \
             --output-file forge_test_bench.md
 
-      - name: Upload test benchmark results
-        uses: actions/upload-artifact@v4
-        with:
-          name: forge-test-results
-          path: benches/forge_test_bench.md
-
-  forge-build-bench:
-    name: Forge Build Benchmarks
-    needs: setup
-    runs-on: foundry-runner
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Benchmark setup
-        uses: ./.github/actions/benchmark-setup
-
       - name: Run forge build benchmarks
         env:
           FOUNDRY_DIR: ${{ github.workspace }}/.foundry
         run: |
           VERSIONS="${{ github.event.inputs.versions || 'stable,nightly' }}"
           REPOS="${{ github.event.inputs.repos || env.DEFAULT_REPOS }}"
 
-          ./foundry-bench --output-dir ./benches --force-install \
+          ./target/release/foundry-bench --output-dir ./benches --force-install \
             --versions $VERSIONS \
             --repos $REPOS \
             --benchmarks forge_build_no_cache,forge_build_with_cache \
             --output-file forge_build_bench.md
 
-      - name: Upload build benchmark results
-        uses: actions/upload-artifact@v4
-        with:
-          name: forge-build-results
-          path: benches/forge_build_bench.md
-
-  forge-coverage-bench:
-    name: Forge Coverage Benchmarks
-    needs: setup
-    runs-on: foundry-runner
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Benchmark setup
-        uses: ./.github/actions/benchmark-setup
-
       - name: Run forge coverage benchmarks
         env:
           FOUNDRY_DIR: ${{ github.workspace }}/.foundry
         run: |
           VERSIONS="${{ github.event.inputs.versions || 'stable,nightly' }}"
           # Coverage only runs on ithacaxyz/account:v0.3.2
 
-          ./foundry-bench --output-dir ./benches --force-install \
+          ./target/release/foundry-bench --output-dir ./benches --force-install \
             --versions $VERSIONS \
             --repos ${{ env.ITHACAXYZ_ACCOUNT }} \
             --benchmarks forge_coverage \
             --output-file forge_coverage_bench.md
 
-      - name: Upload coverage benchmark results
+      - name: Combine benchmark results
+        run: ./.github/scripts/combine-benchmarks.sh benches
+
+      - name: Commit and read benchmark results
+        id: benchmark_results
+        env:
+          GITHUB_HEAD_REF: ${{ github.head_ref }}
+        run: ./.github/scripts/commit-and-read-benchmarks.sh benches "${{ github.event_name }}" "${{ github.repository }}"
+
+      - name: Upload benchmark results as artifacts
         uses: actions/upload-artifact@v4
         with:
-          name: forge-coverage-results
-          path: benches/forge_coverage_bench.md
-
-  combine-results:
-    name: Combine and Publish Results
-    needs: [forge-test-bench, forge-build-bench, forge-coverage-bench]
+          name: benchmark-results
+          path: |
+            benches/forge_test_bench.md
+            benches/forge_build_bench.md
+            benches/forge_coverage_bench.md
+            benches/LATEST.md
+
+    outputs:
+      branch_name: ${{ steps.benchmark_results.outputs.branch_name }}
+      pr_comment: ${{ steps.benchmark_results.outputs.pr_comment }}
+
+  publish-results:
+    name: Publish Results
+    needs: run-benchmarks
     runs-on: foundry-runner
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Download all benchmark results
+      - name: Download benchmark results
         uses: actions/download-artifact@v4
         with:
-          pattern: forge-*-results
+          name: benchmark-results
           path: benches/
-          merge-multiple: true
-
-      - name: Combine benchmark results
-        run: ./.github/scripts/combine-benchmarks.sh benches
-
-      - name: Commit and read benchmark results
-        id: benchmark_results
-        env:
-          GITHUB_HEAD_REF: ${{ github.head_ref }}
-        run: ./.github/scripts/commit-and-read-benchmarks.sh benches "${{ github.event_name }}" "${{ github.repository }}"
 
       - name: Push branch for manual runs
         if: github.event_name == 'workflow_dispatch'
         run: |
-          git push origin "${{ steps.benchmark_results.outputs.branch_name }}"
-          echo "Pushed branch: ${{ steps.benchmark_results.outputs.branch_name }}"
+          git push origin "${{ needs.run-benchmarks.outputs.branch_name }}"
+          echo "Pushed branch: ${{ needs.run-benchmarks.outputs.branch_name }}"
 
       - name: Create PR for manual runs
         if: github.event_name == 'workflow_dispatch'
         uses: actions/github-script@v7
         with:
           script: |
-            const branchName = '${{ steps.benchmark_results.outputs.branch_name }}';
-            const prComment = `${{ steps.benchmark_results.outputs.pr_comment }}`;
+            const branchName = '${{ needs.run-benchmarks.outputs.branch_name }}';
+            const prComment = `${{ needs.run-benchmarks.outputs.pr_comment }}`;
 
             // Create the pull request
             const { data: pr } = await github.rest.pulls.create({
@@ -219,7 +186,7 @@ jobs:
         with:
           script: |
             const prNumber = ${{ github.event.inputs.pr_number || github.event.pull_request.number }};
-            const prComment = `${{ steps.benchmark_results.outputs.pr_comment }}`;
+            const prComment = `${{ needs.run-benchmarks.outputs.pr_comment }}`;
 
             const comment = `${prComment}
 

benches/LATEST.md

@@ -1,6 +1,6 @@
 # 📊 Foundry Benchmark Results
 
-**Generated at**: 2025-07-11 11:48:21 UTC
+**Generated at**: 2025-07-17 14:25:08 UTC
 
 ## Forge Test
 
@@ -11,19 +11,19 @@
 ### Foundry Versions
 
 - **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08)
-- **nightly**: forge Version: 1.2.3-nightly (d592b3e 2025-07-11)
+- **nightly**: forge Version: 1.3.0-nightly (0af4341 2025-07-17)
 
 | Repository | stable | nightly |
 |------------|----------|----------|
-| ithacaxyz-account | 6.86 s | 6.03 s |
-| solady | 7.70 s | 6.94 s |
+| ithacaxyz-account | 5.12 s | 5.15 s |
+| solady | 2.99 s | 2.87 s |
 
 ## Forge Fuzz Test
 
 | Repository | stable | nightly |
 |------------|----------|----------|
-| ithacaxyz-account | 6.65 s | 6.28 s |
-| solady | 7.87 s | 6.98 s |
+| ithacaxyz-account | 5.45 s | 5.20 s |
+| solady | 3.08 s | 3.00 s |
 
 ## Forge Build
 
@@ -34,21 +34,21 @@
 ### Foundry Versions
 
 - **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08)
-- **nightly**: forge Version: 1.2.3-nightly (d592b3e 2025-07-11)
+- **nightly**: forge Version: 1.3.0-nightly (0af4341 2025-07-17)
 
 ### No Cache
 
 | Repository | stable | nightly |
 |------------|----------|----------|
-| ithacaxyz-account | 2.57 s | 2.61 s |
-| solady | 3.95 s | 4.08 s |
+| ithacaxyz-account | 2.58 s | 2.83 s |
+| solady | 3.88 s | 3.98 s |
 
 ### With Cache
 
 | Repository | stable | nightly |
 |------------|----------|----------|
-| ithacaxyz-account | 2.58 s | 2.60 s |
-| solady | 4.04 s | 4.07 s |
+| ithacaxyz-account | 0.210 s | 0.443 s |
+| solady | 0.093 s | 0.195 s |
 
 ## Forge Coverage
 
@@ -58,15 +58,15 @@
 ### Foundry Versions
 
 - **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08)
-- **nightly**: forge Version: 1.2.3-nightly (d592b3e 2025-07-11)
+- **nightly**: forge Version: 1.3.0-nightly (0af4341 2025-07-17)
 
 | Repository | stable | nightly |
 |------------|----------|----------|
-| ithacaxyz-account | 21.49 s | 20.85 s |
+| ithacaxyz-account | 22.03 s | 21.98 s |
 
 ## System Information
 
 
 - **OS**: linux
 - **CPU**: 8
-- **Rustc**: unknown
+- **Rustc**: rustc 1.88.0 (6b00bc388 2025-06-23)

benches/src/main.rs

@@ -1,7 +1,7 @@
 use clap::Parser;
 use eyre::{Result, WrapErr};
 use foundry_bench::{
-    BENCHMARK_REPOS, FOUNDRY_VERSIONS, RUNS, RepoConfig, get_forge_version,
+    BENCHMARK_REPOS, BenchmarkProject, FOUNDRY_VERSIONS, RUNS, RepoConfig, get_forge_version,
     get_forge_version_details,
     results::{BenchmarkResults, HyperfineResult},
     switch_foundry_version,
@@ -118,6 +118,34 @@ fn main() -> Result<()> {
         results.set_baseline_version(first_version.clone());
     }
 
+    // Setup all projects upfront before version loop
+    sh_println!("📦 Setting up projects to benchmark");
+    let projects: Vec<(RepoConfig, BenchmarkProject)> = repos
+        .par_iter()
+        .map(|repo_config| -> Result<(RepoConfig, BenchmarkProject)> {
+            sh_println!("Setting up {}/{}", repo_config.org, repo_config.repo);
+            let project = BenchmarkProject::setup(repo_config).wrap_err(format!(
+                "Failed to setup project for {}/{}",
+                repo_config.org, repo_config.repo
+            ))?;
+            Ok((repo_config.clone(), project))
+        })
+        .collect::<Result<Vec<_>>>()?;
+
+    sh_println!("✅ All projects setup complete");
+
+    // Create a list of all benchmark tasks (same for all versions)
+    let benchmark_tasks: Vec<_> = projects
+        .iter()
+        .flat_map(|(repo_config, project)| {
+            benchmarks
+                .iter()
+                .map(move |benchmark| (repo_config.clone(), project, benchmark.clone()))
+        })
+        .collect();
+
+    sh_println!("Will run {} benchmark tasks per version", benchmark_tasks.len());
+
     // Run benchmarks for each version
     for version in &versions {
         sh_println!("🔧 Switching to Foundry version: {version}");
@@ -131,32 +159,12 @@ fn main() -> Result<()> {
         let version_details = get_forge_version_details()?;
         results.add_version_details(version, version_details);
 
-        // Create a list of all benchmark tasks
-        let benchmark_tasks: Vec<_> = repos
-            .iter()
-            .flat_map(|repo| {
-                benchmarks.iter().map(move |benchmark| (repo.clone(), benchmark.clone()))
-            })
-            .collect();
-
-        sh_println!("Running {} benchmark tasks in parallel...", benchmark_tasks.len());
+        sh_println!("Running benchmark tasks for version {version}...");
 
-        // Run all benchmarks in parallel
+        // Run all benchmarks sequentially
         let version_results = benchmark_tasks
-            .par_iter()
-            .map(|(repo_config, benchmark)| -> Result<(String, String, HyperfineResult)> {
-                sh_println!(
-                    "Setting up {}/{} for {}",
-                    repo_config.org,
-                    repo_config.repo,
-                    benchmark
-                );
-
-                // Setup a fresh project for this specific benchmark
-                let project = foundry_bench::BenchmarkProject::setup(repo_config).wrap_err(
-                    format!("Failed to setup project for {}/{}", repo_config.org, repo_config.repo),
-                )?;
-
+            .iter()
+            .map(|(repo_config, project, benchmark)| -> Result<(String, String, HyperfineResult)> {
                 sh_println!("Running {} on {}/{}", benchmark, repo_config.org, repo_config.repo);
 
                 // Determine runs based on benchmark type