[Feat] For prover 4.6.1

Cargo.toml

@@ -17,9 +17,10 @@ repository = "https://github.com/scroll-tech/scroll"
 version = "4.5.47"
 
 [workspace.dependencies]
-scroll-zkvm-prover = { git = "https://github.com/scroll-tech/zkvm-prover", rev = "060be4c" }
-scroll-zkvm-verifier = { git = "https://github.com/scroll-tech/zkvm-prover", rev = "060be4c" }
-scroll-zkvm-types = { git = "https://github.com/scroll-tech/zkvm-prover", rev = "060be4c" }
+# include compatiblity fixing from "fix/coordinator"
+scroll-zkvm-prover = { git = "https://github.com/scroll-tech/zkvm-prover", rev = "2c7fd3c" }
+scroll-zkvm-verifier = { git = "https://github.com/scroll-tech/zkvm-prover", rev = "2c7fd3c" }
+scroll-zkvm-types = { git = "https://github.com/scroll-tech/zkvm-prover", rev = "2c7fd3c" }
 
 sbv-primitives = { git = "https://github.com/scroll-tech/stateless-block-verifier", branch = "master", features = ["scroll", "rkyv"] }
 sbv-utils = { git = "https://github.com/scroll-tech/stateless-block-verifier", branch = "master" }

common/version/version.go

@@ -5,7 +5,7 @@ import (
 	"runtime/debug"
 )
 
-var tag = "v4.5.47"
+var tag = "v4.6.1"
 
 var commit = func() string {
 	if info, ok := debug.ReadBuildInfo(); ok {

coordinator/Makefile

@@ -35,8 +35,9 @@ coordinator_tool:
 	go build -ldflags "-X scroll-tech/common/version.ZkVersion=${ZK_VERSION}" -o $(PWD)/build/bin/coordinator_tool ./cmd/tool
 
 localsetup: coordinator_api ## Local setup: build coordinator_api, copy config, and setup releases
+	mkdir -p build/bin/conf
 	@echo "Copying configuration files..."
-	cp -r $(PWD)/conf $(PWD)/build/bin/
+	cp -r $(PWD)/conf/config.json $(PWD)/build/bin/conf/config.template.json
 	@echo "Setting up releases..."
 	cd $(PWD)/build && bash setup_releases.sh
 

coordinator/build/setup_releases.sh

@@ -7,7 +7,7 @@ if [ -z "${SCROLL_ZKVM_VERSION}" ]; then
 fi
 
 # set ASSET_DIR by reading from config.json
-CONFIG_FILE="bin/conf/config.json"
+CONFIG_FILE="bin/conf/config.template.json"
 if [ ! -f "$CONFIG_FILE" ]; then
   echo "Config file $CONFIG_FILE not found"
   exit 1
@@ -53,6 +53,7 @@ for ((i=0; i<$VERIFIER_COUNT; i++)); do
   # assets for verifier-only mode
   echo "Downloading assets for $FORK_NAME to $ASSET_DIR..."
   wget https://circuit-release.s3.us-west-2.amazonaws.com/scroll-zkvm/releases/$SCROLL_ZKVM_VERSION/verifier/verifier.bin -O ${ASSET_DIR}/verifier.bin
+  wget https://circuit-release.s3.us-west-2.amazonaws.com/scroll-zkvm/releases/$SCROLL_ZKVM_VERSION/verifier/root_verifier_vk -O ${ASSET_DIR}/root_verifier_vk
   wget https://circuit-release.s3.us-west-2.amazonaws.com/scroll-zkvm/releases/$SCROLL_ZKVM_VERSION/verifier/openVmVk.json -O ${ASSET_DIR}/openVmVk.json
 
   echo "Completed downloading assets for $FORK_NAME"

coordinator/conf/config.json

@@ -22,7 +22,7 @@
   },
   "db": {
     "driver_name": "postgres",
-    "dsn": "postgres://localhost/scroll?sslmode=disable",
+    "dsn": "postgres://dev:dev@localhost/scroll?sslmode=disable",
     "maxOpenNum": 200,
     "maxIdleNum": 20
   },

coordinator/internal/logic/libzkp/lib.go

@@ -147,3 +147,20 @@ func SetDynamicFeature(feats string) {
 	defer freeCString(cFeats)
 	C.set_dynamic_feature(cFeats)
 }
+
+// UnivTaskCompatibilityFix calls the universal task compatibility fix function
+func UnivTaskCompatibilityFix(taskJSON string) (string, error) {
+	cTaskJSON := goToCString(taskJSON)
+	defer freeCString(cTaskJSON)
+
+	resultPtr := C.univ_task_compatibility_fix(cTaskJSON)
+	if resultPtr == nil {
+		return "", fmt.Errorf("univ_task_compatibility_fix failed")
+	}
+
+	// Convert result to Go string and free C memory
+	result := C.GoString(resultPtr)
+	C.release_string(resultPtr)
+
+	return result, nil
+}

coordinator/internal/logic/libzkp/libzkp.h

@@ -56,5 +56,7 @@ void release_string(char* string_ptr);
 
 void set_dynamic_feature(const char* feats);
 
+// Universal task compatibility fix function
+char* univ_task_compatibility_fix(char* task_json);
 
 #endif /* LIBZKP_H */

coordinator/internal/logic/provertask/batch_prover_task.go

@@ -213,6 +213,14 @@ func (bp *BatchProverTask) Assign(ctx *gin.Context, getTaskParameter *coordinato
 			return nil, ErrCoordinatorInternalFailure
 		}
 		proverTask.Metadata = metadata
+
+		if isCompatibilityFixingVersion(taskCtx.ProverVersion) {
+			log.Info("Apply compatibility fixing for prover", "version", taskCtx.ProverVersion)
+			if err := fixCompatibility(taskMsg); err != nil {
+				log.Error("apply compatibility failure", "err", err)
+				return nil, ErrCoordinatorInternalFailure
+			}
+		}
 	}
 
 	// Store session info.

coordinator/internal/logic/provertask/bundle_prover_task.go

@@ -211,6 +211,14 @@ func (bp *BundleProverTask) Assign(ctx *gin.Context, getTaskParameter *coordinat
 		// bundle proof require snark
 		taskMsg.UseSnark = true
 		proverTask.Metadata = metadata
+
+		if isCompatibilityFixingVersion(taskCtx.ProverVersion) {
+			log.Info("Apply compatibility fixing for prover", "version", taskCtx.ProverVersion)
+			if err := fixCompatibility(taskMsg); err != nil {
+				log.Error("apply compatibility failure", "err", err)
+				return nil, ErrCoordinatorInternalFailure
+			}
+		}
 	}
 
 	// Store session info.

coordinator/internal/logic/provertask/prover_task.go

@@ -14,6 +14,7 @@ import (
 	"gorm.io/gorm"
 
 	"scroll-tech/common/types/message"
+	"scroll-tech/common/version"
 
 	"scroll-tech/coordinator/internal/config"
 	"scroll-tech/coordinator/internal/logic/libzkp"
@@ -201,6 +202,23 @@ func (b *BaseProverTask) applyUniversal(schema *coordinatorType.GetTaskSchema) (
 	return schema, []byte(metadata), nil
 }
 
+const CompatibilityVersion = "4.5.43"
+
+func isCompatibilityFixingVersion(ver string) bool {
+	return !version.CheckScrollRepoVersion(ver, CompatibilityVersion)
+}
+
+func fixCompatibility(schema *coordinatorType.GetTaskSchema) error {
+
+	fixedTask, err := libzkp.UnivTaskCompatibilityFix(schema.TaskData)
+	if err != nil {
+		return err
+	}
+	schema.TaskData = fixedTask
+
+	return nil
+}
+
 func newGetTaskCounterVec(factory promauto.Factory, taskType string) *prometheus.CounterVec {
 	getTaskCounterInitOnce.Do(func() {
 		getTaskCounterVec = factory.NewCounterVec(prometheus.CounterOpts{

coordinator/test/api_test.go

@@ -132,7 +132,7 @@ func setupCoordinator(t *testing.T, proversPerSession uint8, coordinatorURL stri
 func setEnv(t *testing.T) {
 	var err error
 
-	version.Version = "v4.4.89"
+	version.Version = "v4.5.45"
 
 	glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.LogfmtFormat()))
 	glogger.Verbosity(log.LvlInfo)

crates/libzkp/src/lib.rs

@@ -46,6 +46,47 @@ pub fn checkout_chunk_task(
     Ok(ret)
 }
 
+/// Convert the universal task json into compatible form for old prover
+pub fn univ_task_compatibility_fix(task_json: &str) -> eyre::Result<String> {
+    use scroll_zkvm_types::proof::VmInternalStarkProof;
+
+    let u_task: tasks::ProvingTask = serde_json::from_str(task_json)?;
+    let aggregated_proofs: Vec<VmInternalStarkProof> = u_task
+        .aggregated_proofs
+        .into_iter()
+        .map(|proof| VmInternalStarkProof {
+            proofs: proof.proofs,
+            public_values: proof.public_values,
+        })
+        .collect();
+
+    #[derive(Serialize)]
+    struct CompatibleProvingTask {
+        /// seralized witness which should be written into stdin first
+        pub serialized_witness: Vec<Vec<u8>>,
+        /// aggregated proof carried by babybear fields, should be written into stdin
+        /// followed `serialized_witness`
+        pub aggregated_proofs: Vec<VmInternalStarkProof>,
+        /// Fork name specify
+        pub fork_name: String,
+        /// The vk of app which is expcted to prove this task
+        pub vk: Vec<u8>,
+        /// An identifier assigned by coordinator, it should be kept identify for the
+        /// same task (for example, using chunk, batch and bundle hashes)
+        pub identifier: String,
+    }
+
+    let compatible_u_task = CompatibleProvingTask {
+        serialized_witness: u_task.serialized_witness,
+        aggregated_proofs,
+        fork_name: u_task.fork_name,
+        vk: u_task.vk,
+        identifier: u_task.identifier,
+    };
+
+    Ok(serde_json::to_string(&compatible_u_task)?)
+}
+
 /// Generate required staff for proving tasks
 /// return (pi_hash, metadata, task)
 pub fn gen_universal_task(

crates/libzkp_c/src/lib.rs

@@ -238,6 +238,19 @@ pub unsafe extern "C" fn gen_wrapped_proof(
     }
 }
 
+/// # Safety
+#[no_mangle]
+pub unsafe extern "C" fn univ_task_compatibility_fix(task_json: *const c_char) -> *mut c_char {
+    let task_json_str = c_char_to_str(task_json);
+    match libzkp::univ_task_compatibility_fix(task_json_str) {
+        Ok(result) => CString::new(result).unwrap().into_raw(),
+        Err(e) => {
+            tracing::error!("univ_task_compability_fix failed, error: {:#}", e);
+            std::ptr::null_mut()
+        }
+    }
+}
+
 /// # Safety
 #[no_mangle]
 pub unsafe extern "C" fn release_string(ptr: *mut c_char) {

tests/prover-e2e/Makefile

@@ -1,7 +1,8 @@
 .PHONY: clean setup_db test_tool all check_vars
 
 GOOSE_CMD?=goose
-
+BEGIN_BLOCK?=10973711
+END_BLOCK?=10973721
 
 all: setup_db test_tool import_data
 

tests/prover-e2e/prepare/dump_block_records.sql

@@ -1,7 +1,12 @@
 -- Create a file with INSERT statements for the specific records
-\o block_export.sql
+\o 00100_import_blocks.sql
 \t on
 \a
+-- Write header comment
+SELECT '-- +goose Up';
+SELECT '-- +goose StatementBegin';
+SELECT '';
+
 SELECT 'INSERT INTO l2_block (number, hash, parent_hash, header, withdraw_root,
        state_root, tx_num, gas_used, block_timestamp, row_consumption,
        chunk_hash, transactions
@@ -22,6 +27,15 @@ SELECT 'INSERT INTO l2_block (number, hash, parent_hash, header, withdraw_root,
 FROM l2_block 
 WHERE number >= 10973700 and number <= 10973730
 ORDER BY number ASC;
+
+-- Write footer
+SELECT '';
+SELECT '-- +goose StatementEnd';
+SELECT '-- +goose Down';
+SELECT '-- +goose StatementBegin';
+SELECT 'DELETE FROM l2_block;';
+SELECT '-- +goose StatementEnd';
+
 \t off
 \a
 \o