feat: Load OP Stack Rollup Config (#121)

* load op stack rollup cfg fixes

* Parse the rust rollup config and default return the entire range

* fix: import paths for rollup cfg

* add

* update

* docs

* lint

---------

Co-authored-by: Ubuntu <[email protected]>

programs/range/src/main.rs

@@ -171,8 +171,10 @@ fn main() {
                 Sealed::new_unchecked(new_block_header.clone(), new_block_header.hash_slow()),
             );
 
-            // Produce the next payload.
+            println!("cycle-tracker-report-start: payload-derivation");
+            // Produce the next payload. If a span batch boundary is passed, the driver will step until the next batch.
             payload = driver.produce_payloads().await.unwrap();
+            println!("cycle-tracker-report-end: payload-derivation");
         }
 
         println!("cycle-tracker-start: output-root");

proposer/op/Dockerfile.op_proposer

@@ -25,6 +25,9 @@ COPY --from=optimism-builder /optimism/op-proposer/proposer/bin/op-proposer /usr
 # Set the entrypoint to run op-proposer with environment variables
 COPY ./proposer/op/op_proposer.sh /usr/local/bin/op_proposer.sh
 
+# Copy the rollup configs
+COPY ../rollup-configs /rollup-configs
+
 # Make the binary and entrypoint executable.
 RUN ls -l /usr/local/bin/
 RUN chmod +x /usr/local/bin/op-proposer

proposer/op/Dockerfile.span_batch_server

@@ -25,6 +25,9 @@ WORKDIR /app
 # Copy the local proposer/op directory
 COPY ./proposer/op /app/op-proposer-go
 
+# Copy the rollup configs
+COPY ../rollup-configs /rollup-configs
+
 # Change to the server directory and build the application
 WORKDIR /app/op-proposer-go/server
 

proposer/op/cmd/main.go

@@ -1,12 +1,13 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"log"
 	"os"
 
-	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum-optimism/optimism/op-service/dial"
+	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/joho/godotenv"
 	"github.com/succinctlabs/op-succinct-go/proposer/utils"
 	"github.com/urfave/cli/v2"
@@ -60,7 +61,18 @@ func main() {
 			},
 		},
 		Action: func(cliCtx *cli.Context) error {
-			rollupCfg, err := utils.GetRollupConfigFromL2Rpc(cliCtx.String("l2"))
+			// Get the chain ID from the L2 RPC.
+			l2Client, err := ethclient.Dial(cliCtx.String("l2"))
+			if err != nil {
+				log.Fatal(err)
+			}
+			chainID, err := l2Client.ChainID(context.Background())
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			// Load the rollup config for the given L2 chain ID.
+			rollupCfg, err := utils.LoadOPStackRollupConfigFromChainID(chainID.Uint64())
 			if err != nil {
 				log.Fatal(err)
 			}

proposer/op/proposer/span_batches.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"math/big"
 
-	"github.com/ethereum-optimism/optimism/op-node/rollup"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/succinctlabs/op-succinct-go/proposer/db/ent"
 	"github.com/succinctlabs/op-succinct-go/proposer/utils"
@@ -51,7 +50,7 @@ func (l *L2OutputSubmitter) DeriveNewSpanBatches(ctx context.Context) error {
 	}
 
 	// Get the rollup config for the chain to fetch the batcher address.
-	rollupCfg, err := rollup.LoadOPStackRollupConfig(l.Cfg.L2ChainID)
+	rollupCfg, err := utils.LoadOPStackRollupConfigFromChainID(l.Cfg.L2ChainID)
 	if err != nil {
 		return fmt.Errorf("failed to load rollup config: %w", err)
 	}

proposer/op/proposer/utils/utils.go

@@ -2,12 +2,17 @@ package utils
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"log"
 	"math/big"
 	"os"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
 	"time"
 
 	"github.com/ethereum-optimism/optimism/op-node/cmd/batch_decoder/fetch"
@@ -47,24 +52,135 @@ type BatchDecoderConfig struct {
 	DataDir           string
 }
 
-// Get the rollup config from the given L2 RPC.
-func GetRollupConfigFromL2Rpc(l2Rpc string) (*rollup.Config, error) {
-	l2Client, err := ethclient.Dial(l2Rpc)
+// CustomBytes32 is a wrapper around eth.Bytes32 that can unmarshal from both
+// full-length and minimal hex strings.
+type CustomBytes32 eth.Bytes32
+
+// Unmarshal some data into a CustomBytes32.
+func (b *CustomBytes32) UnmarshalJSON(data []byte) error {
+	var s string
+	if err := json.Unmarshal(data, &s); err != nil {
+		return err
+	}
+
+	// Remove "0x" prefix if present.
+	s = strings.TrimPrefix(s, "0x")
+
+	// Pad the string to 64 characters (32 bytes) with leading zeros.
+	s = fmt.Sprintf("%064s", s)
+
+	// Add back the "0x" prefix.
+	s = "0x" + s
+
+	bytes, err := common.ParseHexOrString(s)
 	if err != nil {
-		return nil, fmt.Errorf("failed to dial L2 client: %w", err)
+		return err
+	}
+
+	if len(bytes) != 32 {
+		return fmt.Errorf("invalid length for Bytes32: got %d, want 32", len(bytes))
 	}
 
-	chainID, err := l2Client.ChainID(context.Background())
+	copy((*b)[:], bytes)
+	return nil
+}
+
+// LoadOPStackRollupConfigFromChainID loads and parses the rollup config for the given L2 chain ID.
+func LoadOPStackRollupConfigFromChainID(l2ChainId uint64) (*rollup.Config, error) {
+	// Determine the path to the rollup config file.
+	_, currentFile, _, _ := runtime.Caller(0)
+	currentDir := filepath.Dir(currentFile)
+	path := filepath.Join(currentDir, "..", "..", "..", "..", "rollup-configs", fmt.Sprintf("%d.json", l2ChainId))
+
+	// Read the rollup config file.
+	rollupCfg, err := os.ReadFile(path)
 	if err != nil {
-		return nil, fmt.Errorf("failed to get chain ID: %w", err)
+		return nil, fmt.Errorf("failed to read rollup config: %w", err)
 	}
 
-	rollupCfg, err := rollup.LoadOPStackRollupConfig(chainID.Uint64())
+	// Parse the JSON config.
+	var rawConfig map[string]interface{}
+	if err := json.Unmarshal(rollupCfg, &rawConfig); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal rollup config: %w", err)
+	}
+
+	// Convert the Rust SuperchainConfig types to Go types, as they differ in a few places.
+	convertedConfig, err := convertConfigTypes(rawConfig)
 	if err != nil {
-		return nil, fmt.Errorf("failed to load rollup config: %w", err)
+		return nil, fmt.Errorf("failed to convert config types: %w", err)
 	}
 
-	return rollupCfg, nil
+	// Marshal the converted config back to JSON.
+	modifiedConfig, err := json.Marshal(convertedConfig)
+	if err != nil {
+		return nil, fmt.Errorf("failed to re-marshal modified config: %w", err)
+	}
+
+	// Unmarshal into the actual rollup.Config struct.
+	var config rollup.Config
+	if err := json.Unmarshal(modifiedConfig, &config); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal modified rollup config: %w", err)
+	}
+
+	return &config, nil
+}
+
+// The JSON serialization of the Rust superchain-primitives types differ from the Go types (ex. U256 instead of Bytes32, U64 instead of uint64, etc.)
+// This function converts the Rust types in the rollup config JSON to the Go types.
+func convertConfigTypes(rawConfig map[string]interface{}) (map[string]interface{}, error) {
+	// Convert genesis block numbers.
+	if genesis, ok := rawConfig["genesis"].(map[string]interface{}); ok {
+		convertBlockNumber(genesis, "l1")
+		convertBlockNumber(genesis, "l2")
+		convertSystemConfig(genesis)
+	}
+
+	// Convert base fee parameters.
+	convertBaseFeeParams(rawConfig, "base_fee_params")
+	convertBaseFeeParams(rawConfig, "canyon_base_fee_params")
+
+	return rawConfig, nil
+}
+
+// convertBlockNumber converts the block number from hex string to integer.
+func convertBlockNumber(data map[string]interface{}, key string) {
+	if block, ok := data[key].(map[string]interface{}); ok {
+		if number, ok := block["number"].(string); ok {
+			if intNumber, err := strconv.ParseInt(strings.TrimPrefix(number, "0x"), 16, 64); err == nil {
+				block["number"] = intNumber
+			}
+		}
+	}
+}
+
+// convertSystemConfig converts the overhead and scalar fields in the system config.
+func convertSystemConfig(genesis map[string]interface{}) {
+	if systemConfig, ok := genesis["system_config"].(map[string]interface{}); ok {
+		convertBytes32Field(systemConfig, "overhead")
+		convertBytes32Field(systemConfig, "scalar")
+	}
+}
+
+// convertBytes32Field converts a hex string to CustomBytes32 which can unmarshal from both
+// full-length and minimal hex strings.
+func convertBytes32Field(data map[string]interface{}, key string) {
+	if value, ok := data[key].(string); ok {
+		var customValue CustomBytes32
+		if err := customValue.UnmarshalJSON([]byte(`"` + value + `"`)); err == nil {
+			data[key] = eth.Bytes32(customValue)
+		}
+	}
+}
+
+// convertBaseFeeParams converts the max_change_denominator from hex string to integer.
+func convertBaseFeeParams(rawConfig map[string]interface{}, key string) {
+	if params, ok := rawConfig[key].(map[string]interface{}); ok {
+		if maxChangeDenominator, ok := params["max_change_denominator"].(string); ok {
+			if intValue, err := strconv.ParseInt(strings.TrimPrefix(maxChangeDenominator, "0x"), 16, 64); err == nil {
+				params["max_change_denominator"] = intValue
+			}
+		}
+	}
 }
 
 // GetAllSpanBatchesInBlockRange fetches span batches within a range of L2 blocks.
@@ -104,6 +220,7 @@ func GetAllSpanBatchesInL2BlockRange(config BatchDecoderConfig) ([]SpanBatchRang
 	return ranges, nil
 }
 
+// / Get the L2 block number for the given L2 timestamp.
 func TimestampToBlock(rollupCfg *rollup.Config, l2Timestamp uint64) uint64 {
 	return ((l2Timestamp - rollupCfg.Genesis.L2Time) / rollupCfg.BlockTime) + rollupCfg.Genesis.L2.Number
 }
@@ -128,7 +245,10 @@ func GetSpanBatchRanges(config reassemble.Config, rollupCfg *rollup.Config, star
 			batchStartBlock := TimestampToBlock(rollupCfg, b.GetTimestamp())
 			spanBatch, success := b.AsSpanBatch()
 			if !success {
-				log.Fatalf("couldn't convert batch %v to span batch\n", idx)
+				// If AsSpanBatch fails, return the entire range.
+				log.Printf("couldn't convert batch %v to span batch\n", idx)
+				ranges = append(ranges, SpanBatchRange{Start: startBlock, End: endBlock})
+				return ranges, nil
 			}
 			blockCount := spanBatch.GetBlockCount()
 			batchEndBlock := batchStartBlock + uint64(blockCount) - 1
@@ -144,42 +264,9 @@ func GetSpanBatchRanges(config reassemble.Config, rollupCfg *rollup.Config, star
 	return ranges, nil
 }
 
-// Find the span batch that contains the given L2 block. If no span batch contains the given block, return the start block of the span batch that is closest to the given block.
-func GetSpanBatchRange(config reassemble.Config, rollupCfg *rollup.Config, l2Block, maxSpanBatchDeviation uint64) (uint64, uint64, error) {
-	frames := reassemble.LoadFrames(config.InDirectory, config.BatchInbox)
-	framesByChannel := make(map[derive.ChannelID][]reassemble.FrameWithMetadata)
-	for _, frame := range frames {
-		framesByChannel[frame.Frame.ID] = append(framesByChannel[frame.Frame.ID], frame)
-	}
-	for id, frames := range framesByChannel {
-		ch := processFrames(config, rollupCfg, id, frames)
-		if len(ch.Batches) == 0 {
-			log.Fatalf("no span batches in channel")
-			return 0, 0, errors.New("no span batches in channel")
-		}
-
-		for idx, b := range ch.Batches {
-			startBlock := TimestampToBlock(rollupCfg, b.GetTimestamp())
-			spanBatch, success := b.AsSpanBatch()
-			if !success {
-				log.Fatalf("couldn't convert batch %v to span batch\n", idx)
-				return 0, 0, errors.New("couldn't convert batch to span batch")
-			}
-			blockCount := spanBatch.GetBlockCount()
-			endBlock := startBlock + uint64(blockCount) - 1
-			if l2Block >= startBlock && l2Block <= endBlock {
-				return startBlock, endBlock, nil
-			} else if l2Block+maxSpanBatchDeviation < startBlock {
-				return l2Block, startBlock - 1, ErrMaxDeviationExceeded
-			}
-		}
-	}
-	return 0, 0, ErrNoSpanBatchFound
-}
-
 // Set up the batch decoder config.
 func setupBatchDecoderConfig(config *BatchDecoderConfig) (*rollup.Config, error) {
-	rollupCfg, err := rollup.LoadOPStackRollupConfig(config.L2ChainID.Uint64())
+	rollupCfg, err := LoadOPStackRollupConfigFromChainID(config.L2ChainID.Uint64())
 	if err != nil {
 		return nil, err
 	}

proposer/op/server/main_test.go

@@ -28,8 +28,17 @@ func TestHandleSpanBatchRanges(t *testing.T) {
 		t.Fatalf("Required environment variables are not set")
 	}
 
-	// Rollup config
-	rollupCfg, err := utils.GetRollupConfigFromL2Rpc(l2Rpc)
+	// Get the L2 chain ID from the L2 RPC.
+	l2Client, err := ethclient.Dial(l2Rpc)
+	if err != nil {
+		t.Fatalf("Failed to connect to L2 RPC: %v", err)
+	}
+	chainID, err := l2Client.ChainID(context.Background())
+	if err != nil {
+		t.Fatalf("Failed to get chain ID: %v", err)
+	}
+	// Load the rollup config for the given L2 chain ID.
+	rollupCfg, err := utils.LoadOPStackRollupConfigFromChainID(chainID.Uint64())
 	if err != nil {
 		t.Fatalf("Failed to get rollup config: %v", err)
 	}

scripts/prove/bin/cost_estimator.rs

@@ -97,6 +97,7 @@ async fn get_span_batch_ranges_from_server(
         env::var("SPAN_BATCH_SERVER_URL").unwrap_or("http://localhost:8089".to_string());
     let query_url = format!("{}/span-batch-ranges", span_batch_server_url);
 
+    // Send the request to the span batch server. If the request fails, return the corresponding error.
     let response: SpanBatchResponse =
         client.post(&query_url).json(&request).send().await?.json().await?;
 
@@ -343,8 +344,6 @@ fn manage_span_batch_server_container() -> Result<()> {
     // Start the Docker container.
     let run_status = Command::new("docker")
         .args(["run", "-p", "8089:8089", "-d", "span_batch_server"])
-        .stdout(Stdio::null())
-        .stderr(Stdio::null())
         .status()?;
     if !run_status.success() {
         return Err(anyhow::anyhow!("Failed to start Docker container"));

utils/host/src/rollup_config.rs

@@ -131,8 +131,7 @@ pub fn save_rollup_config(rollup_config: &RollupConfig) -> Result<()> {
 /// Read rollup config from rollup-configs/{l2_chain_id}.json in the workspace root.
 pub fn read_rollup_config(l2_chain_id: u64) -> Result<RollupConfig> {
     let workspace_root = cargo_metadata::MetadataCommand::new().exec()?.workspace_root;
-    let rollup_config_path =
-        workspace_root.join(format!("rollup-configs/{}.json", l2_chain_id));
+    let rollup_config_path = workspace_root.join(format!("rollup-configs/{}.json", l2_chain_id));
     let rollup_config_str = fs::read_to_string(rollup_config_path)?;
     let rollup_config: RollupConfig = serde_json::from_str(&rollup_config_str)?;
     Ok(rollup_config)

utils/host/src/witnessgen.rs

@@ -14,6 +14,11 @@ pub fn convert_host_cli_to_args(host_cli: &HostCli) -> Vec<String> {
         format!("--l2-block-number={}", host_cli.l2_block_number),
         format!("--l2-chain-id={}", host_cli.l2_chain_id),
     ];
+    // The verbosity should be passed as -v, -vv, -vvv, etc.
+    if host_cli.v > 0 {
+        args.push(format!("-{}", "v".repeat(host_cli.v as usize)));
+    }
+
     if let Some(addr) = &host_cli.l2_node_address {
         args.push("--l2-node-address".to_string());
         args.push(addr.to_string());
@@ -45,7 +50,7 @@ pub fn convert_host_cli_to_args(host_cli: &HostCli) -> Vec<String> {
 }
 
 /// Default timeout for witness generation.
-pub const WITNESSGEN_TIMEOUT: Duration = Duration::from_secs(120);
+pub const WITNESSGEN_TIMEOUT: Duration = Duration::from_secs(1000);
 
 struct WitnessGenProcess {
     child: tokio::process::Child,