diff --git a/.github/actions/benchmark-setup/action.yml b/.github/actions/benchmark-setup/action.yml deleted file mode 100644 index 8278e12adaacd..0000000000000 --- a/.github/actions/benchmark-setup/action.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: "Benchmark Setup" -description: "Common setup steps for benchmark jobs" - -runs: - using: "composite" - steps: - - name: Setup Foundry - shell: bash - env: - FOUNDRY_DIR: ${{ github.workspace }}/.foundry - run: | - ./.github/scripts/setup-foundryup.sh - echo "${{ github.workspace }}/.foundry/bin" >> $GITHUB_PATH - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: "24" - - - name: Install hyperfine - shell: bash - run: | - curl -L https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine-v1.19.0-x86_64-unknown-linux-gnu.tar.gz | tar xz - sudo mv hyperfine-v1.19.0-x86_64-unknown-linux-gnu/hyperfine /usr/local/bin/ - rm -rf hyperfine-v1.19.0-x86_64-unknown-linux-gnu - - - name: Download benchmark binary - uses: actions/download-artifact@v4 - with: - name: foundry-bench - - - name: Make binary executable - shell: bash - run: chmod +x foundry-bench diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index c664680b31daa..699d6287fb476 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -28,8 +28,8 @@ env: DEFAULT_REPOS: "ithacaxyz/account:v0.3.2,Vectorized/solady:v0.1.22" jobs: - setup: - name: Setup and Build + run-benchmarks: + name: Run All Benchmarks runs-on: foundry-runner steps: - name: Checkout repository @@ -59,22 +59,16 @@ jobs: - name: Build benchmark binary run: cargo build --release --bin foundry-bench - - name: Upload benchmark binary - uses: actions/upload-artifact@v4 + - name: Setup Node.js + uses: actions/setup-node@v4 with: - name: foundry-bench - path: target/release/foundry-bench - - forge-test-bench: - name: Forge Test Benchmarks - needs: setup - runs-on: foundry-runner - steps: - - name: Checkout repository - uses: actions/checkout@v4 + node-version: "24" - - name: Benchmark setup - uses: ./.github/actions/benchmark-setup + - name: Install hyperfine + run: | + curl -L https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine-v1.19.0-x86_64-unknown-linux-gnu.tar.gz | tar xz + sudo mv hyperfine-v1.19.0-x86_64-unknown-linux-gnu/hyperfine /usr/local/bin/ + rm -rf hyperfine-v1.19.0-x86_64-unknown-linux-gnu - name: Run forge test benchmarks env: @@ -83,29 +77,12 @@ jobs: VERSIONS="${{ github.event.inputs.versions || 'stable,nightly' }}" REPOS="${{ github.event.inputs.repos || env.DEFAULT_REPOS }}" - ./foundry-bench --output-dir ./benches --force-install \ + ./target/release/foundry-bench --output-dir ./benches --force-install \ --versions $VERSIONS \ --repos $REPOS \ --benchmarks forge_test,forge_fuzz_test \ --output-file forge_test_bench.md - - name: Upload test benchmark results - uses: actions/upload-artifact@v4 - with: - name: forge-test-results - path: benches/forge_test_bench.md - - forge-build-bench: - name: Forge Build Benchmarks - needs: setup - runs-on: foundry-runner - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Benchmark setup - uses: ./.github/actions/benchmark-setup - - name: Run forge build benchmarks env: FOUNDRY_DIR: ${{ github.workspace }}/.foundry @@ -113,29 +90,12 @@ jobs: VERSIONS="${{ github.event.inputs.versions || 'stable,nightly' }}" REPOS="${{ github.event.inputs.repos || env.DEFAULT_REPOS }}" - ./foundry-bench --output-dir ./benches --force-install \ + ./target/release/foundry-bench --output-dir ./benches --force-install \ --versions $VERSIONS \ --repos $REPOS \ --benchmarks forge_build_no_cache,forge_build_with_cache \ --output-file forge_build_bench.md - - name: Upload build benchmark results - uses: actions/upload-artifact@v4 - with: - name: forge-build-results - path: benches/forge_build_bench.md - - forge-coverage-bench: - name: Forge Coverage Benchmarks - needs: setup - runs-on: foundry-runner - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Benchmark setup - uses: ./.github/actions/benchmark-setup - - name: Run forge coverage benchmarks env: FOUNDRY_DIR: ${{ github.workspace }}/.foundry @@ -143,55 +103,62 @@ jobs: VERSIONS="${{ github.event.inputs.versions || 'stable,nightly' }}" # Coverage only runs on ithacaxyz/account:v0.3.2 - ./foundry-bench --output-dir ./benches --force-install \ + ./target/release/foundry-bench --output-dir ./benches --force-install \ --versions $VERSIONS \ --repos ${{ env.ITHACAXYZ_ACCOUNT }} \ --benchmarks forge_coverage \ --output-file forge_coverage_bench.md - - name: Upload coverage benchmark results + - name: Combine benchmark results + run: ./.github/scripts/combine-benchmarks.sh benches + + - name: Commit and read benchmark results + id: benchmark_results + env: + GITHUB_HEAD_REF: ${{ github.head_ref }} + run: ./.github/scripts/commit-and-read-benchmarks.sh benches "${{ github.event_name }}" "${{ github.repository }}" + + - name: Upload benchmark results as artifacts uses: actions/upload-artifact@v4 with: - name: forge-coverage-results - path: benches/forge_coverage_bench.md - - combine-results: - name: Combine and Publish Results - needs: [forge-test-bench, forge-build-bench, forge-coverage-bench] + name: benchmark-results + path: | + benches/forge_test_bench.md + benches/forge_build_bench.md + benches/forge_coverage_bench.md + benches/LATEST.md + + outputs: + branch_name: ${{ steps.benchmark_results.outputs.branch_name }} + pr_comment: ${{ steps.benchmark_results.outputs.pr_comment }} + + publish-results: + name: Publish Results + needs: run-benchmarks runs-on: foundry-runner steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Download all benchmark results + - name: Download benchmark results uses: actions/download-artifact@v4 with: - pattern: forge-*-results + name: benchmark-results path: benches/ - merge-multiple: true - - - name: Combine benchmark results - run: ./.github/scripts/combine-benchmarks.sh benches - - - name: Commit and read benchmark results - id: benchmark_results - env: - GITHUB_HEAD_REF: ${{ github.head_ref }} - run: ./.github/scripts/commit-and-read-benchmarks.sh benches "${{ github.event_name }}" "${{ github.repository }}" - name: Push branch for manual runs if: github.event_name == 'workflow_dispatch' run: | - git push origin "${{ steps.benchmark_results.outputs.branch_name }}" - echo "Pushed branch: ${{ steps.benchmark_results.outputs.branch_name }}" + git push origin "${{ needs.run-benchmarks.outputs.branch_name }}" + echo "Pushed branch: ${{ needs.run-benchmarks.outputs.branch_name }}" - name: Create PR for manual runs if: github.event_name == 'workflow_dispatch' uses: actions/github-script@v7 with: script: | - const branchName = '${{ steps.benchmark_results.outputs.branch_name }}'; - const prComment = `${{ steps.benchmark_results.outputs.pr_comment }}`; + const branchName = '${{ needs.run-benchmarks.outputs.branch_name }}'; + const prComment = `${{ needs.run-benchmarks.outputs.pr_comment }}`; // Create the pull request const { data: pr } = await github.rest.pulls.create({ @@ -219,7 +186,7 @@ jobs: with: script: | const prNumber = ${{ github.event.inputs.pr_number || github.event.pull_request.number }}; - const prComment = `${{ steps.benchmark_results.outputs.pr_comment }}`; + const prComment = `${{ needs.run-benchmarks.outputs.pr_comment }}`; const comment = `${prComment} diff --git a/benches/LATEST.md b/benches/LATEST.md index e4f600ad96626..9a140dda70573 100644 --- a/benches/LATEST.md +++ b/benches/LATEST.md @@ -1,6 +1,6 @@ # 📊 Foundry Benchmark Results -**Generated at**: 2025-07-11 11:48:21 UTC +**Generated at**: 2025-07-17 14:25:08 UTC ## Forge Test @@ -11,19 +11,19 @@ ### Foundry Versions - **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08) -- **nightly**: forge Version: 1.2.3-nightly (d592b3e 2025-07-11) +- **nightly**: forge Version: 1.3.0-nightly (0af4341 2025-07-17) | Repository | stable | nightly | |------------|----------|----------| -| ithacaxyz-account | 6.86 s | 6.03 s | -| solady | 7.70 s | 6.94 s | +| ithacaxyz-account | 5.12 s | 5.15 s | +| solady | 2.99 s | 2.87 s | ## Forge Fuzz Test | Repository | stable | nightly | |------------|----------|----------| -| ithacaxyz-account | 6.65 s | 6.28 s | -| solady | 7.87 s | 6.98 s | +| ithacaxyz-account | 5.45 s | 5.20 s | +| solady | 3.08 s | 3.00 s | ## Forge Build @@ -34,21 +34,21 @@ ### Foundry Versions - **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08) -- **nightly**: forge Version: 1.2.3-nightly (d592b3e 2025-07-11) +- **nightly**: forge Version: 1.3.0-nightly (0af4341 2025-07-17) ### No Cache | Repository | stable | nightly | |------------|----------|----------| -| ithacaxyz-account | 2.57 s | 2.61 s | -| solady | 3.95 s | 4.08 s | +| ithacaxyz-account | 2.58 s | 2.83 s | +| solady | 3.88 s | 3.98 s | ### With Cache | Repository | stable | nightly | |------------|----------|----------| -| ithacaxyz-account | 2.58 s | 2.60 s | -| solady | 4.04 s | 4.07 s | +| ithacaxyz-account | 0.210 s | 0.443 s | +| solady | 0.093 s | 0.195 s | ## Forge Coverage @@ -58,15 +58,15 @@ ### Foundry Versions - **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08) -- **nightly**: forge Version: 1.2.3-nightly (d592b3e 2025-07-11) +- **nightly**: forge Version: 1.3.0-nightly (0af4341 2025-07-17) | Repository | stable | nightly | |------------|----------|----------| -| ithacaxyz-account | 21.49 s | 20.85 s | +| ithacaxyz-account | 22.03 s | 21.98 s | ## System Information - **OS**: linux - **CPU**: 8 -- **Rustc**: unknown +- **Rustc**: rustc 1.88.0 (6b00bc388 2025-06-23) diff --git a/benches/src/main.rs b/benches/src/main.rs index f9fb27d5ac04a..aca0e069d582e 100644 --- a/benches/src/main.rs +++ b/benches/src/main.rs @@ -1,7 +1,7 @@ use clap::Parser; use eyre::{Result, WrapErr}; use foundry_bench::{ - BENCHMARK_REPOS, FOUNDRY_VERSIONS, RUNS, RepoConfig, get_forge_version, + BENCHMARK_REPOS, BenchmarkProject, FOUNDRY_VERSIONS, RUNS, RepoConfig, get_forge_version, get_forge_version_details, results::{BenchmarkResults, HyperfineResult}, switch_foundry_version, @@ -118,6 +118,34 @@ fn main() -> Result<()> { results.set_baseline_version(first_version.clone()); } + // Setup all projects upfront before version loop + sh_println!("📦 Setting up projects to benchmark"); + let projects: Vec<(RepoConfig, BenchmarkProject)> = repos + .par_iter() + .map(|repo_config| -> Result<(RepoConfig, BenchmarkProject)> { + sh_println!("Setting up {}/{}", repo_config.org, repo_config.repo); + let project = BenchmarkProject::setup(repo_config).wrap_err(format!( + "Failed to setup project for {}/{}", + repo_config.org, repo_config.repo + ))?; + Ok((repo_config.clone(), project)) + }) + .collect::>>()?; + + sh_println!("✅ All projects setup complete"); + + // Create a list of all benchmark tasks (same for all versions) + let benchmark_tasks: Vec<_> = projects + .iter() + .flat_map(|(repo_config, project)| { + benchmarks + .iter() + .map(move |benchmark| (repo_config.clone(), project, benchmark.clone())) + }) + .collect(); + + sh_println!("Will run {} benchmark tasks per version", benchmark_tasks.len()); + // Run benchmarks for each version for version in &versions { sh_println!("🔧 Switching to Foundry version: {version}"); @@ -131,32 +159,12 @@ fn main() -> Result<()> { let version_details = get_forge_version_details()?; results.add_version_details(version, version_details); - // Create a list of all benchmark tasks - let benchmark_tasks: Vec<_> = repos - .iter() - .flat_map(|repo| { - benchmarks.iter().map(move |benchmark| (repo.clone(), benchmark.clone())) - }) - .collect(); - - sh_println!("Running {} benchmark tasks in parallel...", benchmark_tasks.len()); + sh_println!("Running benchmark tasks for version {version}..."); - // Run all benchmarks in parallel + // Run all benchmarks sequentially let version_results = benchmark_tasks - .par_iter() - .map(|(repo_config, benchmark)| -> Result<(String, String, HyperfineResult)> { - sh_println!( - "Setting up {}/{} for {}", - repo_config.org, - repo_config.repo, - benchmark - ); - - // Setup a fresh project for this specific benchmark - let project = foundry_bench::BenchmarkProject::setup(repo_config).wrap_err( - format!("Failed to setup project for {}/{}", repo_config.org, repo_config.repo), - )?; - + .iter() + .map(|(repo_config, project, benchmark)| -> Result<(String, String, HyperfineResult)> { sh_println!("Running {} on {}/{}", benchmark, repo_config.org, repo_config.repo); // Determine runs based on benchmark type