Skip to content

feat(benches): forge test --isolate #11037

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Jul 23, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .github/scripts/combine-benchmarks.sh
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ EOF
FIRST_FILE=1
SYSTEM_INFO=""

for bench_file in "forge_test_bench.md" "forge_build_bench.md" "forge_coverage_bench.md"; do
for bench_file in "forge_test_bench.md" "forge_isolate_test_bench.md" "forge_build_bench.md" "forge_coverage_bench.md"; do
if [ -f "$OUTPUT_DIR/$bench_file" ]; then
echo "Processing $bench_file..."

Expand All @@ -135,6 +135,9 @@ for bench_file in "forge_test_bench.md" "forge_build_bench.md" "forge_coverage_b
"forge_test_bench.md")
SECTION_NAME="Forge Test"
;;
"forge_isolate_test_bench.md")
SECTION_NAME="Forge Test (Isolated)"
;;
"forge_build_bench.md")
SECTION_NAME="Forge Build"
;;
Expand Down
15 changes: 15 additions & 0 deletions .github/workflows/benchmarks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,20 @@ jobs:
--benchmarks forge_test,forge_fuzz_test \
--output-file forge_test_bench.md

- name: Run forge isolate test benchmarks
env:
FOUNDRY_DIR: ${{ github.workspace }}/.foundry
run: |
VERSIONS="${{ github.event.inputs.versions || 'stable,nightly' }}"
# Isolate tests default to Vectorized/solady but can be overridden
REPOS="${{ github.event.inputs.repos || env.VECTORIZED_SOLADY }}"

./target/release/foundry-bench --output-dir ./benches --force-install \
--versions $VERSIONS \
--repos $REPOS \
--benchmarks forge_isolate_test \
--output-file forge_isolate_test_bench.md

- name: Run forge build benchmarks
env:
FOUNDRY_DIR: ${{ github.workspace }}/.foundry
Expand Down Expand Up @@ -124,6 +138,7 @@ jobs:
name: benchmark-results
path: |
benches/forge_test_bench.md
benches/forge_isolate_test_bench.md
benches/forge_build_bench.md
benches/forge_coverage_bench.md
benches/LATEST.md
Expand Down
40 changes: 27 additions & 13 deletions benches/LATEST.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# 📊 Foundry Benchmark Results

**Generated at**: 2025-07-17 14:25:08 UTC
**Generated at**: 2025-07-23 09:01:16 UTC

## Forge Test

Expand All @@ -11,19 +11,33 @@
### Foundry Versions

- **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08)
- **nightly**: forge Version: 1.3.0-nightly (0af4341 2025-07-17)
- **nightly**: forge Version: 1.3.0-nightly (cb8f3bf 2025-07-23)

| Repository | stable | nightly |
|------------|----------|----------|
| ithacaxyz-account | 5.12 s | 5.15 s |
| solady | 2.99 s | 2.87 s |
| ithacaxyz-account | 5.44 s | 4.91 s |
| solady | 3.02 s | 2.76 s |

## Forge Fuzz Test

| Repository | stable | nightly |
|------------|----------|----------|
| ithacaxyz-account | 5.45 s | 5.20 s |
| solady | 3.08 s | 3.00 s |
| ithacaxyz-account | 5.27 s | 4.90 s |
| solady | 2.89 s | 2.82 s |

## Forge Test (Isolated)

### Repositories Tested

1. [Vectorized/solady](https://github.com/Vectorized/solady)
### Foundry Versions

- **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08)
- **nightly**: forge Version: 1.3.0-nightly (cb8f3bf 2025-07-23)

| Repository | stable | nightly |
|------------|----------|----------|
| solady | 3.21 s | 2.88 s |

## Forge Build

Expand All @@ -34,21 +48,21 @@
### Foundry Versions

- **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08)
- **nightly**: forge Version: 1.3.0-nightly (0af4341 2025-07-17)
- **nightly**: forge Version: 1.3.0-nightly (cb8f3bf 2025-07-23)

### No Cache

| Repository | stable | nightly |
|------------|----------|----------|
| ithacaxyz-account | 2.58 s | 2.83 s |
| solady | 3.88 s | 3.98 s |
| ithacaxyz-account | 2.58 s | 2.79 s |
| solady | 3.79 s | 3.89 s |

### With Cache

| Repository | stable | nightly |
|------------|----------|----------|
| ithacaxyz-account | 0.210 s | 0.443 s |
| solady | 0.093 s | 0.195 s |
| ithacaxyz-account | 0.209 s | 0.458 s |
| solady | 0.096 s | 0.226 s |

## Forge Coverage

Expand All @@ -58,11 +72,11 @@
### Foundry Versions

- **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08)
- **nightly**: forge Version: 1.3.0-nightly (0af4341 2025-07-17)
- **nightly**: forge Version: 1.3.0-nightly (cb8f3bf 2025-07-23)

| Repository | stable | nightly |
|------------|----------|----------|
| ithacaxyz-account | 22.03 s | 21.98 s |
| ithacaxyz-account | 21.88 s | 22.52 s |

## System Information

Expand Down
20 changes: 20 additions & 0 deletions benches/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -355,6 +355,25 @@ impl BenchmarkProject {
)
}

/// Benchmark forge test with --isolate flag
pub fn bench_forge_isolate_test(
&self,
version: &str,
runs: u32,
verbose: bool,
) -> Result<HyperfineResult> {
// Build before running tests
self.hyperfine(
"forge_isolate_test",
version,
"forge test --isolate",
runs,
Some("forge build"),
None,
verbose,
)
}

/// Get the root path of the project
pub fn root(&self) -> &Path {
&self.root_path
Expand All @@ -374,6 +393,7 @@ impl BenchmarkProject {
"forge_build_with_cache" => self.bench_forge_build_with_cache(version, runs, verbose),
"forge_fuzz_test" => self.bench_forge_fuzz_test(version, runs, verbose),
"forge_coverage" => self.bench_forge_coverage(version, runs, verbose),
"forge_isolate_test" => self.bench_forge_isolate_test(version, runs, verbose),
_ => eyre::bail!("Unknown benchmark: {}", benchmark),
}
}
Expand Down
3 changes: 2 additions & 1 deletion benches/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,13 @@ use foundry_common::sh_println;
use rayon::prelude::*;
use std::{fs, path::PathBuf, process::Command, sync::Mutex};

const ALL_BENCHMARKS: [&str; 5] = [
const ALL_BENCHMARKS: [&str; 6] = [
"forge_test",
"forge_build_no_cache",
"forge_build_with_cache",
"forge_fuzz_test",
"forge_coverage",
"forge_isolate_test",
];

/// Foundry Benchmark Runner
Expand Down
1 change: 1 addition & 0 deletions benches/src/results.rs
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,7 @@ pub fn format_benchmark_name(name: &str) -> String {
"forge_build_with_cache" => "Forge Build (With Cache)",
"forge_fuzz_test" => "Forge Fuzz Test",
"forge_coverage" => "Forge Coverage",
"forge_isolate_test" => "Forge Test (Isolated)",
_ => name,
}
.to_string()
Expand Down