diff --git a/Cargo.lock b/Cargo.lock index ccf4c0a9..0b02de21 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6854,8 +6854,9 @@ dependencies = [ "ff 0.13.1", "glob", "halo2curves-axiom 0.7.0", + "metrics", "metrics-tracing-context", - "once_cell", + "metrics-util", "openvm-build", "openvm-circuit", "openvm-native-circuit", @@ -6863,6 +6864,7 @@ dependencies = [ "openvm-native-recursion", "openvm-native-transpiler", "openvm-sdk", + "openvm-stark-sdk", "openvm-transpiler", "rayon", "regex", @@ -6876,6 +6878,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "snark-verifier-sdk", + "thousands", "tracing", "tracing-subscriber 0.3.19", "vm-zstd", @@ -6896,7 +6899,6 @@ dependencies = [ "metrics-tracing-context", "metrics-util", "munge", - "once_cell", "openvm-circuit", "openvm-continuations", "openvm-native-circuit", @@ -7708,6 +7710,12 @@ dependencies = [ "syn 2.0.101", ] +[[package]] +name = "thousands" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bf63baf9f5039dadc247375c29eb13706706cfde997d0330d05aa63a77d8820" + [[package]] name = "thread_local" version = "1.1.8" diff --git a/Makefile b/Makefile index c28fb8ba..0e38f74b 100644 --- a/Makefile +++ b/Makefile @@ -56,6 +56,9 @@ test-execute-chunk-multi: test-cycle: @cargo test --release -p scroll-zkvm-integration --test chunk_circuit test_cycle -- --exact --nocapture +test-chunk-cell: + @cargo test --release -p scroll-zkvm-integration --test chunk_circuit test_cell -- --exact --nocapture + test-execute-batch: $(TESTDATA_PATH)/proofs/chunk-$(CHUNK_PROOF).json @cargo test --release -p scroll-zkvm-integration --test batch_circuit test_e2e_execute -- --exact --nocapture diff --git a/crates/integration/Cargo.toml b/crates/integration/Cargo.toml index 3b8cd56e..daa582c8 100644 --- a/crates/integration/Cargo.toml +++ b/crates/integration/Cargo.toml @@ -11,11 +11,14 @@ sbv-primitives = { workspace = true } tracing.workspace = true rkyv.workspace = true tracing-subscriber.workspace = true +metrics.workspace = true +metrics-util.workspace = true metrics-tracing-context.workspace = true openvm-build = { workspace = true, default-features = false } openvm-circuit.workspace = true openvm-sdk = { workspace = true, default-features = false } +openvm-stark-sdk = { workspace = true, default-features = false } openvm-native-circuit = { workspace = true, default-features = false } openvm-native-compiler = { workspace = true, default-features = false } openvm-native-recursion = { workspace = true, default-features = false } @@ -33,7 +36,6 @@ vm-zstd = { workspace = true, features = ["zstd"] } chrono = "0.4" ff = "0.13" glob = "0.3" -once_cell = "1.20" revm = { workspace = true } serde_json = "1.0" sha2 = "0.10" @@ -47,6 +49,7 @@ regex = "1.11.1" [dev-dependencies] halo2curves-axiom = "0.7.0" glob = "0.3" +thousands = "0.2" [features] default = [] diff --git a/crates/integration/src/lib.rs b/crates/integration/src/lib.rs index a9d4d248..41141f86 100644 --- a/crates/integration/src/lib.rs +++ b/crates/integration/src/lib.rs @@ -1,9 +1,17 @@ +#![feature(exit_status_error)] + use cargo_metadata::MetadataCommand; -use once_cell::sync::OnceCell; +use eyre::Context; +use metrics_tracing_context::TracingContextLayer; +use metrics_util::{ + debugging::{DebuggingRecorder, Snapshotter}, + layers::Layer, +}; use openvm_sdk::{ F, Sdk, config::{AppConfig, SdkVmConfig}, }; +use openvm_stark_sdk::bench::serialize_metric_snapshot; use scroll_zkvm_prover::{ ProverType, WrappedProof, setup::{read_app_config, read_app_exe}, @@ -12,7 +20,8 @@ use scroll_zkvm_prover::{ use std::{ path::{Path, PathBuf}, process, - sync::LazyLock, + process::Command, + sync::{LazyLock, OnceLock}, }; use tracing::instrument; use tracing_subscriber::{fmt::format::FmtSpan, layer::SubscriberExt, util::SubscriberInitExt}; @@ -40,9 +49,20 @@ static DIR_OUTPUT: LazyLock<&Path> = LazyLock::new(|| { Box::leak(path.into_boxed_path()) }); +pub static METRIC_SNAPSHOTTER: LazyLock = LazyLock::new(|| { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let recorder = TracingContextLayer::all().layer(recorder); + metrics::set_global_recorder(recorder).unwrap(); + snapshotter +}); + /// Directory to store proofs on disc. const DIR_PROOFS: &str = "proofs"; +/// Directory to store metrics on disc. +const DIR_METRICS: &str = "metrics"; + /// File descriptor for app openvm config. const FD_APP_CONFIG: &str = "openvm.toml"; @@ -58,7 +78,7 @@ const ENV_OUTPUT_DIR: &str = "OUTPUT_DIR"; /// - /chunk-tests-{timestamp} /// - /batch-tests-{timestamp} /// - /bundle-tests-{timestamp} -static DIR_TESTRUN: OnceCell = OnceCell::new(); +static DIR_TESTRUN: OnceLock = OnceLock::new(); /// Circuit that implements functionality required to run e2e tests. pub trait ProverTester { @@ -102,6 +122,10 @@ pub trait ProverTester { Ok(()) } + fn setup_metrics() { + LazyLock::force(&METRIC_SNAPSHOTTER); + } + /// Load the app config. fn load_with_exe_fd( app_exe_fd: &str, @@ -150,6 +174,36 @@ pub trait ProverTester { ) -> eyre::Result> { Self::execute(app_config, &Self::gen_proving_task()?, exe_path) } + + fn export_metrics() -> eyre::Result<()> { + Self::export_metrics_with_name("default") + } + + fn export_metrics_with_name(name: &str) -> eyre::Result<()> { + let snapshot = METRIC_SNAPSHOTTER.snapshot(); + + let dir = DIR_TESTRUN + .get() + .ok_or(eyre::eyre!("missing assets dir"))? + .join(Self::DIR_ASSETS) + .join(DIR_METRICS); + std::fs::create_dir_all(&dir)?; + + let path = dir.join(format!("metrics-{name}")).with_extension("json"); + tracing::info!("exporting metrics to {}", path.display()); + let f = std::fs::File::create(&path).context("failed to create metrics file")?; + + serde_json::to_writer_pretty(f, &serialize_metric_snapshot(snapshot)) + .context("failed to serialize metrics snapshot")?; + + Command::new("openvm-prof") + .arg("--json-paths") + .arg(&path) + .status() + .context("failed to run openvm-prof")? + .exit_ok() + .context("openvm-prof failed") + } } /// The outcome of a successful prove-verify run. @@ -203,6 +257,8 @@ fn setup_logger() -> eyre::Result<()> { .try_init()?; } + LazyLock::force(&METRIC_SNAPSHOTTER); + Ok(()) } diff --git a/crates/integration/tests/chunk_circuit.rs b/crates/integration/tests/chunk_circuit.rs index 72719461..09810590 100644 --- a/crates/integration/tests/chunk_circuit.rs +++ b/crates/integration/tests/chunk_circuit.rs @@ -1,6 +1,7 @@ use eyre::Ok; +use metrics_util::{MetricKind, debugging::DebugValue}; use scroll_zkvm_integration::{ - ProverTester, prove_verify_multi, prove_verify_single, + METRIC_SNAPSHOTTER, ProverTester, prove_verify_multi, prove_verify_single, testers::chunk::{ChunkProverTester, MultiChunkProverTester, read_block_witness_from_testdata}, utils::testing_hardfork, }; @@ -10,6 +11,7 @@ use scroll_zkvm_prover::{ task::{ProvingTask, chunk::ChunkProvingTask}, utils::{self, vm::ExecutionResult}, }; +use thousands::Separable; fn exec_chunk(task: &ChunkProvingTask) -> eyre::Result<(ExecutionResult, u64)> { let (_path_app_config, app_config, path_exe) = @@ -61,6 +63,64 @@ fn test_cycle() -> eyre::Result<()> { Ok(()) } +#[test] +fn test_cell() -> eyre::Result<()> { + let task = ChunkProverTester::gen_proving_task()?; + let (exec_result, total_gas_used) = exec_chunk(&task)?; + + ChunkProverTester::setup()?; + prove_verify_single::(None)?; + + let snapshot = METRIC_SNAPSHOTTER.snapshot().into_hashmap(); + let (total_cells, total_cycles) = snapshot + .iter() + .filter(|(ck, _)| { + matches!(ck.kind(), MetricKind::Counter) + && (ck.key().name() == "total_cells" || ck.key().name() == "total_cycles") + && ck + .key() + .labels() + .find(|label| label.key() == "segment") + .is_some() // filter only segment labels + }) + .fold( + (0u64, 0u64), + |(cells_acc, cycles_acc), (ck, (_, _, value))| { + let DebugValue::Counter(value) = value else { + panic!("Expected a counter value for total_cells or total_cycles"); + }; + if ck.key().name() == "total_cells" { + (cells_acc + value, cycles_acc) + } else if ck.key().name() == "total_cycles" { + (cells_acc, cycles_acc + value) + } else { + unreachable!() + } + }, + ); + + println!("Total cells: {}", total_cells.separate_with_commas()); + println!("Total cycles: {}", total_cycles.separate_with_commas()); + let cycles_per_gas = total_cycles as f64 / total_gas_used as f64; + let cells_per_gas = total_cells as f64 / total_gas_used as f64; + let cells_per_cycle = total_cells as f64 / total_cycles as f64; + println!( + "Cycles per gas: {}", + format!("{cycles_per_gas:.2}").separate_with_commas() + ); + println!( + "Cells per gas: {}", + format!("{cells_per_gas:.2}").separate_with_commas() + ); + println!( + "Cells per cycle: {}", + format!("{cells_per_cycle:.2}").separate_with_commas() + ); + assert_eq!(exec_result.total_cycle, total_cycles); + + Ok(()) +} + #[test] fn test_execute() -> eyre::Result<()> { ChunkProverTester::setup()?; @@ -69,7 +129,9 @@ fn test_execute() -> eyre::Result<()> { let (exec_result, total_gas_used) = exec_chunk(&task)?; let cycle_per_gas = exec_result.total_cycle / total_gas_used; assert_ne!(cycle_per_gas, 0); - assert!(cycle_per_gas <= 35); + // assert!(cycle_per_gas <= 35); + ChunkProverTester::export_metrics()?; + Ok(()) } @@ -200,6 +262,7 @@ fn setup_prove_verify_single() -> eyre::Result<()> { prove_verify_single::(None)?; + ChunkProverTester::export_metrics()?; Ok(()) } diff --git a/crates/prover/Cargo.toml b/crates/prover/Cargo.toml index af89630f..9f05c6ec 100644 --- a/crates/prover/Cargo.toml +++ b/crates/prover/Cargo.toml @@ -33,7 +33,6 @@ base64 = "0.22" git-version = "0.3.5" hex = "0.4" munge = "=0.4.1" -once_cell = "1.20" serde = "1.0" serde_json = "1.0" serde_stacker = "0.1" diff --git a/crates/prover/src/prover/mod.rs b/crates/prover/src/prover/mod.rs index b0702de3..42cbfef6 100644 --- a/crates/prover/src/prover/mod.rs +++ b/crates/prover/src/prover/mod.rs @@ -1,10 +1,3 @@ -use std::{ - marker::PhantomData, - path::{Path, PathBuf}, - sync::Arc, -}; - -use once_cell::sync::Lazy; use openvm_circuit::{arch::SingleSegmentVmExecutor, system::program::trace::VmCommittedExe}; use openvm_native_recursion::{ halo2::{ @@ -22,17 +15,22 @@ use openvm_sdk::{ prover::{AggStarkProver, AppProver, EvmHalo2Prover}, }; use openvm_stark_sdk::config::baby_bear_poseidon2::BabyBearPoseidon2Engine; -use tracing::{debug, instrument}; +use std::{ + marker::PhantomData, + path::{Path, PathBuf}, + sync::{Arc, LazyLock}, +}; +use tracing::{debug, info_span, instrument}; // Re-export from openvm_sdk. -pub use openvm_sdk::{self, SC}; - use crate::{ Error, proof::{PersistableProof, ProofMetadata, WrappedProof}, setup::{read_app_config, read_app_exe}, task::ProvingTask, }; +pub use openvm_sdk::{self, SC}; +use openvm_stark_sdk::openvm_stark_backend::utils::metrics_span; use scroll_zkvm_types::{ proof::{EvmProof, ProofEnum, RootProof}, @@ -50,8 +48,8 @@ pub use chunk::{ChunkProver, ChunkProverType, GenericChunkProverType}; /// Proving key for STARK aggregation. Primarily used to aggregate /// [continuation proofs][openvm_sdk::prover::vm::ContinuationVmProof]. -static AGG_STARK_PROVING_KEY: Lazy = - Lazy::new(|| AggStarkProvingKey::keygen(AggStarkConfig::default())); +static AGG_STARK_PROVING_KEY: LazyLock = + LazyLock::new(|| AggStarkProvingKey::keygen(AggStarkConfig::default())); /// The default directory to locate openvm's halo2 SRS parameters. const DEFAULT_PARAMS_DIR: &str = concat!(env!("HOME"), "/.openvm/params/"); @@ -139,6 +137,14 @@ impl Prover { /// Read app exe, proving key and return committed data. #[instrument("Prover::init")] pub fn init(config: &ProverConfig) -> Result { + let circuit_name = config + .path_app_exe + .parent() + .and_then(|p| p.parent()) + .and_then(|p| p.file_name()) + .and_then(|p| p.to_str()) + .unwrap_or("unknown"); + let app_exe = read_app_exe(&config.path_app_exe)?; let mut app_config = read_app_config(&config.path_app_config)?; let segment_len = config.segment_len.unwrap_or(Type::SEGMENT_SIZE); @@ -149,12 +155,18 @@ impl Prover { .with_max_segment_len(segment_len); let sdk = Sdk::new(); - let app_pk = sdk - .app_keygen(app_config) - .map_err(|e| Error::Keygen(e.to_string()))?; - let app_committed_exe = sdk - .commit_app_exe(app_pk.app_fri_params(), app_exe) - .map_err(|e| Error::Commit(e.to_string()))?; + let app_pk = info_span!("keygen", group = circuit_name).in_scope(|| { + metrics_span("keygen_time_ms", || { + sdk.app_keygen(app_config) + .map_err(|e| Error::Keygen(e.to_string())) + }) + })?; + let app_committed_exe = info_span!("commit_exe", group = circuit_name).in_scope(|| { + metrics_span("commit_exe_time_ms", || { + sdk.commit_app_exe(app_pk.app_fri_params(), app_exe) + .map_err(|e| Error::Commit(e.to_string())) + }) + })?; Ok((app_committed_exe, Arc::new(app_pk))) }