wip: debugging failed proofs (#106)

* debug failed proofs

* got compilation, working, still issues

* wip

* wip

* add

* push

---------

Co-authored-by: Ubuntu <[email protected]>

Author: ratankaliani

Cargo.lock

@@ -40,17 +40,27 @@ kzg-rs = { version = "0.2.2" }
 # TODO: Change to stable tag when released.
 # Note: Explicitly needed to downgrade superchain-primitives to 0.3.1.
 # cargo update -p [email protected] --precise 0.3.1
-kona-common = { git = "https://github.com/ethereum-optimism/kona", branch = "main" }
-kona-common-proc = { git = "https://github.com/ethereum-optimism/kona", branch = "main" }
-kona-preimage = { git = "https://github.com/ethereum-optimism/kona", branch = "main", features = [
+kona-common = { git = "https://github.com/moongate-forks/kona", branch = "ratan/add-converter" }
+kona-common-proc = { git = "https://github.com/moongate-forks/kona", branch = "ratan/add-converter" }
+kona-preimage = { git = "https://github.com/moongate-forks/kona", branch = "ratan/add-converter", features = [
     "rkyv",
 ] }
-kona-primitives = { git = "https://github.com/ethereum-optimism/kona", branch = "main" }
-kona-mpt = { git = "https://github.com/ethereum-optimism/kona", branch = "main" }
-kona-derive = { git = "https://github.com/ethereum-optimism/kona", branch = "main", default-features = false }
-kona-executor = { git = "https://github.com/ethereum-optimism/kona", branch = "main" }
-kona-client = { git = "https://github.com/ethereum-optimism/kona", branch = "main" }
-kona-host = { git = "https://github.com/ethereum-optimism/kona", branch = "main" }
+kona-primitives = { git = "https://github.com/moongate-forks/kona", branch = "ratan/add-converter" }
+kona-mpt = { git = "https://github.com/moongate-forks/kona", branch = "ratan/add-converter" }
+kona-derive = { git = "https://github.com/moongate-forks/kona", branch = "ratan/add-converter", default-features = false }
+kona-executor = { git = "https://github.com/moongate-forks/kona", branch = "ratan/add-converter" }
+kona-client = { git = "https://github.com/moongate-forks/kona", branch = "ratan/add-converter" }
+kona-host = { git = "https://github.com/moongate-forks/kona", branch = "ratan/add-converter" }
+# kona-common = { path = "../kona/crates/common" }
+# kona-common-proc = { path = "../kona/crates/common-proc" }
+# kona-preimage = { path = "../kona/crates/preimage", features = ["rkyv"] }
+# kona-primitives = { path = "../kona/crates/primitives" }
+# kona-mpt = { path = "../kona/crates/mpt" }
+# kona-derive = { path = "../kona/crates/derive", default-features = false }
+# kona-executor = { path = "../kona/crates/executor" }
+# kona-client = { path = "../kona/bin/client" }
+# kona-host = { path = "../kona/bin/host" }
+
 
 # op-succinct
 op-succinct-prove = { path = "scripts/prove" }
@@ -97,3 +107,7 @@ bn = { git = "https://github.com/0xWOLAND/bn.git", package = "substrate-bn" }
 # Note: Patch alloy-primitives to use sha3 instead of tiny-keccak. Reduces cycle count for Keccak by 50%.
 sha3 = { git = "https://github.com/sp1-patches/RustCrypto-hashes", package = "sha3", branch = "patch-sha3-v0.10.8" }
 alloy-primitives = { git = "https://github.com/sp1-patches/alloy-core", branch = "patch-v0.7.7" }
+
+# Patch op-alloy-protocol for recent exports for compatibility with kona.
+# TODO: Can remove later.
+op-alloy-protocol = { git = "https://github.com/alloy-rs/op-alloy" }

Cargo.toml

@@ -366,6 +366,7 @@ func (db *ProofDB) TryCreateAggProofFromSpanProofs(from, minTo uint64) (bool, ui
 	start := from
 	var end uint64
 	for {
+		fmt.Println("querying for span proof that starts at", start)
 		spanProof, err := tx.ProofRequest.Query().
 			Where(
 				proofrequest.TypeEQ(proofrequest.TypeSPAN),
@@ -379,6 +380,7 @@ func (db *ProofDB) TryCreateAggProofFromSpanProofs(from, minTo uint64) (bool, ui
 			}
 			return false, 0, fmt.Errorf("failed to query SPAN proof: %w", err)
 		}
+		fmt.Println("found span proof", start, spanProof.EndBlock)
 		end = spanProof.EndBlock
 		start = end + 1
 	}

elf/range-elf

@@ -559,22 +559,22 @@ func (l *L2OutputSubmitter) loopL2OO(ctx context.Context) {
 	for {
 		select {
 		case <-ticker.C:
-			// 1) Queue up any span batches that are ready to prove.
-			// This is done by checking the chain for completed channels and pulling span batches out.
-			// We break apart span batches if they exceed the max size, and gracefully handle bugs in span batch decoding.
-			// We add ranges to be proven to the DB as "UNREQ" so they are queued up to request later.
-			l.Log.Info("Stage 1: Deriving Span Batches...")
-			err := l.DeriveNewSpanBatches(ctx)
-			if err != nil {
-				l.Log.Error("failed to add next span batches to db", "err", err)
-				continue
-			}
+			// // 1) Queue up any span batches that are ready to prove.
+			// // This is done by checking the chain for completed channels and pulling span batches out.
+			// // We break apart span batches if they exceed the max size, and gracefully handle bugs in span batch decoding.
+			// // We add ranges to be proven to the DB as "UNREQ" so they are queued up to request later.
+			// l.Log.Info("Stage 1: Deriving Span Batches...")
+			// err := l.DeriveNewSpanBatches(ctx)
+			// if err != nil {
+			// 	l.Log.Error("failed to add next span batches to db", "err", err)
+			// 	continue
+			// }
 
 			// 2) Check the statuses of all requested proofs.
 			// If it's successfully returned, we validate that we have it on disk and set status = "COMPLETE".
 			// If it fails or times out, we set status = "FAILED" (and, if it's a span proof, split the request in half to try again).
 			l.Log.Info("Stage 2: Processing Pending Proofs...")
-			err = l.ProcessPendingProofs()
+			err := l.ProcessPendingProofs()
 			if err != nil {
 				l.Log.Error("failed to update requested proofs", "err", err)
 				continue
@@ -590,24 +590,24 @@ func (l *L2OutputSubmitter) loopL2OO(ctx context.Context) {
 				continue
 			}
 
-			// 4) Request all unrequested proofs from the prover network.
-			// Any DB entry with status = "UNREQ" means it's queued up and ready.
-			// We request all of these (both span and agg) from the prover network.
-			// For agg proofs, we also checkpoint the blockhash in advance.
-			l.Log.Info("Stage 4: Requesting Queued Proofs...")
-			err = l.RequestQueuedProofs(ctx)
-			if err != nil {
-				l.Log.Error("failed to request unrequested proofs", "err", err)
-				continue
-			}
-
-			// 5) Submit agg proofs on chain.
-			// If we have a completed agg proof waiting in the DB, we submit them on chain.
-			l.Log.Info("Stage 5: Submitting Agg Proofs...")
-			err = l.SubmitAggProofs(ctx)
-			if err != nil {
-				l.Log.Error("failed to submit agg proofs", "err", err)
-			}
+			// // 4) Request all unrequested proofs from the prover network.
+			// // Any DB entry with status = "UNREQ" means it's queued up and ready.
+			// // We request all of these (both span and agg) from the prover network.
+			// // For agg proofs, we also checkpoint the blockhash in advance.
+			// l.Log.Info("Stage 4: Requesting Queued Proofs...")
+			// err = l.RequestQueuedProofs(ctx)
+			// if err != nil {
+			// 	l.Log.Error("failed to request unrequested proofs", "err", err)
+			// 	continue
+			// }
+
+			// // 5) Submit agg proofs on chain.
+			// // If we have a completed agg proof waiting in the DB, we submit them on chain.
+			// l.Log.Info("Stage 5: Submitting Agg Proofs...")
+			// err = l.SubmitAggProofs(ctx)
+			// if err != nil {
+			// 	l.Log.Error("failed to submit agg proofs", "err", err)
+			// }
 		case <-l.done:
 			return
 		}

proposer/op/proposer/db/db.go

@@ -61,7 +61,7 @@ fn main() {
         // Note: Don't comment this out, because the Docker program depends on the native program
         // for range being built.
         build_native_program(program);
-        // build_zkvm_program(program);
+        build_zkvm_program(program);
     }
 
     // build_zkvm_program("aggregation");

proposer/op/proposer/driver.go

@@ -0,0 +1 @@
+3.12

proposer/succinct/build.rs

@@ -0,0 +1,6 @@
+def main():
+    print("Hello from python-db-explorations!")
+
+
+if __name__ == "__main__":
+    main()

python-db-explorations/.python-version

@@ -0,0 +1,10 @@
+[project]
+name = "python-db-explorations"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = []
+
+[project.scripts]
+query-proofs = "query_proofs.py"

python-db-explorations/README.md

@@ -0,0 +1,61 @@
+import sqlite3
+from enum import Enum
+
+class ProofType(Enum):
+    SPAN = 1
+    AGG = 2
+
+class ProofStatus(Enum):
+    UNREQ = 1
+    REQ = 2
+    COMPLETE = 3
+    FAILED = 4
+
+def query_span_proofs(db_path, start_block):
+    conn = sqlite3.connect(db_path)
+    cursor = conn.cursor()
+
+    # query = """
+    # SELECT * FROM proof_requests
+    # WHERE type = ? AND status = ? AND start_block = ?
+    # """
+    # cursor.execute(query, (ProofType.SPAN.value, ProofStatus.COMPLETE.value, start_block))
+
+    query = """
+    SELECT * FROM proof_requests
+    WHERE type = 'SPAN' AND start_block = ?
+    """
+    cursor.execute(query, (start_block,))
+    
+    results = cursor.fetchall()
+    if results:
+        first_result = results[0]
+        # print(f"ID: {first_result[0]}")
+        # print(f"Type: {first_result[1]}")
+        # print(f"Start Block: {first_result[2]}")
+        # print(f"End Block: {first_result[3]}")
+        # print(f"Status: {first_result[4]}")
+        # print(f"Request Added Time: {first_result[5]}")
+        # print(f"Prover Request ID: {first_result[6]}")
+        # print(f"Proof Request Time: {first_result[7]}")
+        # print(f"L1 Block Number: {first_result[8]}")
+        # print(f"L1 Block Hash: {first_result[9]}")
+        # print(f"Proof: {first_result[10]}")
+
+    # print("Results:", results)
+    conn.close()
+    
+    return results
+
+if __name__ == "__main__":
+    print("Querying span proofs")
+    db_path = "../db/proofs.db"
+
+    start_block = 16843141  # Replace with the desired start block
+    for i in range(1000):
+    
+        proofs = query_span_proofs(db_path, start_block)
+    
+        for proof in proofs:
+            print(f"Proof ID: {proof[0]}, Start Block: {proof[2]}, End Block: {proof[3]}, Status: {proof[4]}, Prover Request ID: {proof[6]}")
+        start_block += 1

python-db-explorations/hello.py

@@ -1,3 +1,3 @@
 [toolchain]
-channel = "nightly-2024-08-01"
+channel = "1.79"
 components = ["llvm-tools", "rustc-dev"]

python-db-explorations/pyproject.toml

@@ -62,3 +62,4 @@ rayon = "1.10.0"
 [build-dependencies]
 sp1-build = { workspace = true }
 op-succinct-host-utils = { workspace = true }
+cargo_metadata.workspace = true

python-db-explorations/query_proofs.py

@@ -1,39 +1,71 @@
-// use std::process::Command;
+use std::process::Command;
 
-// use sp1_build::{build_program_with_args, BuildArgs};
+use sp1_build::{build_program_with_args, BuildArgs};
 
 /// Build a native program.
-// fn build_native_program(program: &str) {
-//     let status = Command::new("cargo")
-//         .args(["build", "--workspace", "--bin", program, "--profile", "release-client-lto"])
-//         .status()
-//         .expect("Failed to execute cargo build command");
-
-//     if !status.success() {
-//         panic!("Failed to build {}", program);
-//     }
-
-//     println!("cargo:warning={} built with release-client-lto profile", program);
-// }
-
-// /// Build a program for the zkVM.
-// fn build_zkvm_program(program: &str) {
-//     build_program_with_args(
-//         &format!("../../programs/{}", program),
-//         BuildArgs {
-//             elf_name: format!("{}-elf", program),
-//             // docker: true,
-//             ..Default::default()
-//         },
-//     );
-// }
+fn build_native_program(program: &str) {
+    let status = Command::new("cargo")
+        .args(["build", "--workspace", "--bin", program, "--profile", "release-client-lto"])
+        .status()
+        .expect("Failed to execute cargo build command");
+
+    if !status.success() {
+        panic!("Failed to build {}", program);
+    }
+
+    println!("cargo:warning={} built with release-client-lto profile", program);
+}
+
+/// Build the native host runner to a separate target directory to avoid build lockups.
+fn build_native_host_runner() {
+    let metadata =
+        cargo_metadata::MetadataCommand::new().exec().expect("Failed to get cargo metadata");
+    let target_dir = metadata.target_directory.join("native_host_runner");
+
+    let status = Command::new("cargo")
+        .args([
+            "build",
+            "--workspace",
+            "--bin",
+            "native_host_runner",
+            "--release",
+            "--target-dir",
+            target_dir.as_ref(),
+        ])
+        .status()
+        .expect("Failed to execute cargo build command");
+    if !status.success() {
+        panic!("Failed to build native_host_runner");
+    }
+
+    println!("cargo:warning=native_host_runner built with release profile",);
+}
+
+/// Build a program for the zkVM.
+#[allow(dead_code)]
+fn build_zkvm_program(program: &str) {
+    build_program_with_args(
+        &format!("../../programs/{}", program),
+        BuildArgs {
+            elf_name: format!("{}-elf", program),
+            // docker: true,
+            ..Default::default()
+        },
+    );
+}
 
 fn main() {
-    // let programs = vec!["range"];
-    // for program in programs {
-    // build_native_program(program);
-    // build_zkvm_program(program);
-    // }
+    let programs = vec!["range"];
+
+    for program in programs {
+        // Note: Don't comment this out, because the Docker program depends on the native program
+        // for range being built.
+        build_native_program(program);
+        build_zkvm_program(program);
+    }
 
     // build_zkvm_program("aggregation");
+    // Note: Don't comment this out, because the Docker program depends on the native host runner
+    // being built.
+    build_native_host_runner();
 }

python-db-explorations/uv.lock

@@ -12,6 +12,7 @@ alloy-sol-types.workspace = true
 op-succinct-client-utils.workspace = true
 rkyv.workspace = true
 kona-host.workspace = true
+kona-preimage.workspace = true
 sp1-sdk.workspace = true
 anyhow.workspace = true
 cargo_metadata.workspace = true

rust-toolchain

@@ -13,7 +13,7 @@ pub fn load_kv_store(data_dir: &PathBuf) -> HashMap<[u8; 32], Vec<u8>, BytesHash
         if path.is_file() {
             // Extract the file name
             let file_name = path.file_stem().unwrap().to_str().unwrap();
-
+            println!("File name: {:?}", file_name);
             // Convert the file name to PreimageKey
             if let Ok(key) = hex::decode(file_name) {
                 // Read the file contents