From 2b7f5729283edf471cdd52fb4ce35a2df66b37cd Mon Sep 17 00:00:00 2001 From: Ratan Kaliani Date: Wed, 15 Jan 2025 17:19:43 -0800 Subject: [PATCH] feat(proposer): Environment variables for configuring request strategy (#327) * feat: remove reserved from server * wip * clean * add * add * feat: docs * add * docs --- README.md | 4 +- book/advanced/proposer.md | 69 ++++++++++++++++++----------- book/quick-start/full.md | 6 +-- book/quick-start/mock.md | 2 +- proposer/op/proposer/config.go | 9 ---- proposer/op/proposer/flags/flags.go | 19 -------- proposer/op/proposer/service.go | 6 --- proposer/succinct/bin/server.rs | 30 +++++++++---- proposer/succinct/src/lib.rs | 6 ++- 9 files changed, 75 insertions(+), 76 deletions(-) diff --git a/README.md b/README.md index e283abd2..d874e708 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ OP Succinct turns any OP stack rollup into a full type-1 zkEVM Rollup in 1 hour ## Getting Started -Today, you can already use OP Succinct to upgrade any existing OP Stack rollup to a type-1 zkEVM rollup. To get started, make sure you have [Rust](https://rustup.rs/), [Foundry](https://book.getfoundry.sh/), and [Docker](https://docs.docker.com/engine/install/) installed. Then, follow the steps in the [book](https://succinctlabs.github.io/op-succinct/) to deploy the `OPSuccinctL2OutputOracle` contract and start the `op-succinct-proposer` service. +Today, you can already use OP Succinct to upgrade any existing OP Stack rollup to a type-1 zkEVM rollup. To get started, make sure you have [Rust](https://rustup.rs/), [Foundry](https://book.getfoundry.sh/), and [Docker](https://docs.docker.com/engine/install/) installed. Then, follow the steps in the [book](https://succinctlabs.github.io/op-succinct/) to deploy the `OPSuccinctL2OutputOracle` contract and start the OP Succinct service. ## Repository Overview @@ -19,7 +19,7 @@ The repository is organized into the following directories: - `book`: The documentation for OP Succinct users and developers. - `contracts`: The solidity contracts for posting state roots to L1. - `programs`: The programs for proving the execution and derivation of the L2 state transitions and proof aggregation. -- `proposer`: The implementation of the `op-succinct-proposer` service. +- `proposer`: The implementation of the `op-succinct/op-proposer` and `op-succinct/succinct-proposer` services. - `scripts`: The scripts for proving and generating witnesses. ## Acknowledgments diff --git a/book/advanced/proposer.md b/book/advanced/proposer.md index 67763e22..b3e53e7c 100644 --- a/book/advanced/proposer.md +++ b/book/advanced/proposer.md @@ -1,18 +1,18 @@ # Proposer The `op-succinct` service consists of two containers: -- `op-succinct-server`: Receives proof requests from the `op-succinct-proposer`, generates the witness for the proof, and submits the proof to the Succinct Prover Network. Handles the communication with the [Succinct's Prover Network](https://docs.succinct.xyz/docs/generating-proofs/prover-network) to fetch the proof status and completed proof data. -- `op-succinct-proposer`: Monitors L1 state to determine when to request a proof. Sends proof requests to the `op-succinct-server`. Once proofs have been generated for a sufficiently large range, aggregates range proofs into an aggregation proof. Submits the aggregation proof to the `OPSuccinctL2OutputOracle` contract which includes the L2 state outputs. +- [`op-succinct/succinct-proposer`](https://ghcr.io/succinctlabs/op-succinct/succinct-proposer): Receives proof requests from the `op-succinct/op-proposer`, generates the witness for the proof, and submits the proof to the Succinct Prover Network. Handles the communication with the [Succinct's Prover Network](https://docs.succinct.xyz/docs/generating-proofs/prover-network) to fetch the proof status and completed proof data. +- [`op-succinct/op-proposer`](https://ghcr.io/succinctlabs/op-succinct/op-proposer): Monitors L1 state to determine when to request a proof. Sends proof requests to the `op-succinct/succinct-proposer`. Once proofs have been generated for a sufficiently large range, aggregates range proofs into an aggregation proof. Submits the aggregation proof to the `OPSuccinctL2OutputOracle` contract which includes the L2 state outputs. We've packaged the `op-succinct` service in a docker compose file to make it easier to run. -## Prerequisites +# Prerequisites -### RPC Requirements +## RPC Requirements Confirm that your RPC's have all of the required endpoints. More details can be found in the [prerequisites](../quick-start/prerequisites.md#requirements) section. -### Hardware Requirements +## Hardware Requirements We recommend the following hardware configuration for the `op-succinct` service containers: @@ -23,18 +23,22 @@ Using the docker compose file: Running as separate containers: -- `op-succinct-server` +- [`op-succinct/succinct-proposer`](https://ghcr.io/succinctlabs/op-succinct/succinct-proposer): - Full `op-succinct` service: 16 vCPUs, 64GB RAM. - Mock `op-succinct` service: 32 vCPUs, 128GB RAM. Increased memory because the machine is executing the proofs locally. -- `op-succinct-proposer`: 1 vCPU, 4GB RAM +- [`op-succinct/op-proposer`](https://ghcr.io/succinctlabs/op-succinct/op-proposer): 1 vCPU, 4GB RAM For advanced configurations, depending on the number of concurrent requests you expect, you may need to increase the number of vCPUs and memory allocated to the `op-succinct-server` container. -## Environment Setup +# Environment Setup -### Required Environment Variables +If you are running the `op-succinct` service as a Docker Compose service, you should include *all* of the required environment variables in the `.env` file for both the `op-succinct/succinct-proposer` and `op-succinct/op-proposer` services. -Before starting the proposer, the following environment variables should be in your `.env` file. You should have already set up your environment when you deployed the L2 Output Oracle. If you have not done so, follow the steps in the [Contract Configuration](../contracts/configuration.md) section. +Before starting the proposer, ensure you have deployed the L2 Output Oracle and have the address of the proxy contract ready. Follow the steps in the [Contract Configuration](../contracts/configuration.md) section. + +## Required Environment Variables + +### `op-succinct/succinct-proposer` | Parameter | Description | |-----------|-------------| @@ -42,15 +46,30 @@ Before starting the proposer, the following environment variables should be in y | `L1_BEACON_RPC` | L1 Consensus (Beacon) Node. | | `L2_RPC` | L2 Execution Node (`op-geth`). | | `L2_NODE_RPC` | L2 Rollup Node (`op-node`). | -| `NETWORK_RPC_URL` | RPC URL for the Succinct Prover Network. | | `NETWORK_PRIVATE_KEY` | Key for the Succinct Prover Network. Get access [here](https://docs.succinct.xyz/docs/generating-proofs/prover-network). | -| `SP1_PROVER` | Default: `network`. Set to `network` to use the Succinct Prover Network. | -| `PRIVATE_KEY` | Private key for the account that will be deploying the contract and posting output roots to L1. | + +### `op-succinct/op-proposer` + +| Parameter | Description | +|-----------|-------------| +| `L1_RPC` | L1 Archive Node. | +| `L1_BEACON_RPC` | L1 Consensus (Beacon) Node. | +| `L2_RPC` | L2 Execution Node (`op-geth`). | +| `L2_NODE_RPC` | L2 Rollup Node (`op-node`). | | `L2OO_ADDRESS` | Address of the `OPSuccinctL2OutputOracle` contract. | +| `PRIVATE_KEY` | Private key for the account that will be posting output roots to L1. | + +## Advanced Environment Variables + +### `op-succinct/succinct-proposer` -### Advanced Environment Variables +| Parameter | Description | +|-----------|-------------| +| `NETWORK_RPC_URL` | Default: `https://rpc.production.succinct.xyz`. RPC URL for the Succinct Prover Network. | +| `RANGE_PROOF_STRATEGY` | Default: `reserved`. Set to `hosted` to use hosted proof strategy. | +| `AGG_PROOF_STRATEGY` | Default: `reserved`. Set to `hosted` to use hosted proof strategy. | -The following environment variables are optional. +### `op-succinct/op-proposer` | Parameter | Description | |-----------|-------------| @@ -59,38 +78,38 @@ The following environment variables are optional. | `WITNESS_GEN_TIMEOUT` | Default: `1200`. The maximum time in seconds to spend generating a witness for `op-succinct-server`. | | `MAX_BLOCK_RANGE_PER_SPAN_PROOF` | Default: `300`. The maximum number of blocks to include in each span proof. For chains with high throughput, you need to decrease this value. | | `OP_SUCCINCT_MOCK` | Default: `false`. Set to `true` to run in mock proof mode. The `OPSuccinctL2OutputOracle` contract must be configured to use an `SP1MockVerifier`. | -| `OP_SUCCINCT_SERVER_URL` | Default: `http://op-succinct-server:3000`. The URL of the `op-succinct-server` service which the `op-succinct-proposer` will send proof requests to. | +| `OP_SUCCINCT_SERVER_URL` | Default: `http://op-succinct-server:3000`. The URL of the `op-succinct-server` service which the `op-succinct/op-proposer` will send proof requests to. | | `METRICS_ENABLED` | Default: `true`. Set to `false` to disable metrics collection. | | `METRICS_PORT` | Default: `7300`. The port to run the metrics server on. | | `DB_PATH` | Default: `/usr/local/bin/dbdata`. The path to the database directory within the container. | -| `POLL_INTERVAL` | Default: `20s`. The interval at which the `op-succinct-proposer` service runs. | +| `POLL_INTERVAL` | Default: `20s`. The interval at which the `op-succinct/op-proposer` service runs. | | `USE_CACHED_DB` | Default: `false`. Set to `true` to use cached proofs from previous runs when restarting the service, avoiding regeneration of unused proofs. | -## Build the Proposer Service +# Build the Proposer Service -Build the docker images for the `op-succinct-proposer` service. +Build the docker images for the OP Succinct services. ```bash docker compose build ``` -## Run the Proposer +# Run the Proposer -This command launches the `op-succinct-proposer` service in the background. It launches two containers: one container that manages proof generation and another container that is a small fork of the original `op-proposer` service. - -After a few minutes, you should see the `op-succinct-proposer` service start to generate range proofs. Once enough range proofs have been generated, they will be verified in an aggregate proof and submitted to the L1. +Now, launch both services in the background. ```bash docker compose up ``` -To see the logs of the `op-succinct-proposer` service, run: +To see the logs of the OP Succinct services, run: ```bash docker compose logs -f ``` -and to stop the `op-succinct-proposer` service, run: +After a few minutes, you should see the OP Succinct services start to generate range proofs. Once enough range proofs have been generated, they will be verified in an aggregate proof and submitted to the L1. + +To stop the OP Succinct services, run: ```bash docker compose stop diff --git a/book/quick-start/full.md b/book/quick-start/full.md index 4418b962..d6be2e0b 100644 --- a/book/quick-start/full.md +++ b/book/quick-start/full.md @@ -104,11 +104,11 @@ We provide a Docker Compose file for running the `op-succinct` service. docker compose build ``` -#### Run the Proposer +#### Run OP Succinct -This command launches the [op-succinct service](../advanced/proposer.md) in the background. It launches two containers: one container that manages proof generation and another container that is a small fork of the original op-proposer service. +This command launches [OP Succinct](../advanced/proposer.md) in the background. It launches two containers: one container that manages proof generation and another container that is a small fork of the original op-proposer service. -After a few minutes, you should see the op-succinct-proposer service start to request proofs from the Succinct Prover Network. Once enough proofs have been generated, an aggregate proof will be requested and submitted to the L1. +After a few minutes, you should see the OP Succinct service start to request proofs from the Succinct Prover Network. Once enough proofs have been generated, an aggregate proof will be requested and submitted to the L1. ```shell docker compose up diff --git a/book/quick-start/mock.md b/book/quick-start/mock.md index d3dd8113..9f088116 100644 --- a/book/quick-start/mock.md +++ b/book/quick-start/mock.md @@ -152,7 +152,7 @@ docker compose build This command launches the [op-succinct service](../advanced/proposer.md) in the background. It launches two containers: one container that manages proof generation and another container that is a small fork of the original op-proposer service. -After a few minutes, you should see the op-succinct-proposer service start to generate mock range proofs. Once enough range proofs have been generated, a mock aggregate proof will be created and submitted to the L1. +After a few minutes, you should see the OP Succinct service start to generate mock range proofs. Once enough range proofs have been generated, a mock aggregate proof will be created and submitted to the L1. ```shell docker compose up diff --git a/proposer/op/proposer/config.go b/proposer/op/proposer/config.go index ebe0abac..a8a01889 100644 --- a/proposer/op/proposer/config.go +++ b/proposer/op/proposer/config.go @@ -79,8 +79,6 @@ type CLIConfig struct { // L1 Beacon RPC URL used to determine span batch boundaries. BeaconRpc string - // Directory to store the transaction cache when determining span batch boundaries. - TxCacheOutDir string // The max size (in blocks) of a proof we will attempt to generate. If span batches are larger, we break them up. MaxBlockRangePerSpanProof uint64 // The max number of concurrent witness generation processes. @@ -95,10 +93,6 @@ type CLIConfig struct { OPSuccinctServerUrl string // The maximum proofs that can be requested from the server concurrently. MaxConcurrentProofRequests uint64 - // The batch inbox on L1 to read batches from. Note that this is ignored if L2 Chain ID is in rollup config. - BatchInbox string - // The batcher address to include transactions from. Note that this is ignored if L2 Chain ID is in rollup config. - BatcherAddress string // Mock is a flag to use the mock OP Succinct server. Mock bool } @@ -166,11 +160,8 @@ func NewConfig(ctx *cli.Context) *CLIConfig { MaxConcurrentWitnessGen: ctx.Uint64(flags.MaxConcurrentWitnessGenFlag.Name), WitnessGenTimeout: ctx.Uint64(flags.WitnessGenTimeoutFlag.Name), ProofTimeout: ctx.Uint64(flags.ProofTimeoutFlag.Name), - TxCacheOutDir: ctx.String(flags.TxCacheOutDirFlag.Name), OPSuccinctServerUrl: ctx.String(flags.OPSuccinctServerUrlFlag.Name), MaxConcurrentProofRequests: ctx.Uint64(flags.MaxConcurrentProofRequestsFlag.Name), - BatchInbox: ctx.String(flags.BatchInboxFlag.Name), - BatcherAddress: ctx.String(flags.BatcherAddressFlag.Name), Mock: ctx.Bool(flags.MockFlag.Name), } } diff --git a/proposer/op/proposer/flags/flags.go b/proposer/op/proposer/flags/flags.go index ea591acd..fe73b11e 100644 --- a/proposer/op/proposer/flags/flags.go +++ b/proposer/op/proposer/flags/flags.go @@ -129,22 +129,6 @@ var ( Value: 20, EnvVars: prefixEnvVars("MAX_CONCURRENT_PROOF_REQUESTS"), } - TxCacheOutDirFlag = &cli.StringFlag{ - Name: "tx-cache-out-dir", - Usage: "Cache directory for the found transactions to determine span batch boundaries", - Value: "/tmp/batch_decoder/transactions_cache", - EnvVars: prefixEnvVars("TX_CACHE_OUT_DIR"), - } - BatchInboxFlag = &cli.StringFlag{ - Name: "batch-inbox", - Usage: "Batch Inbox Address", - EnvVars: prefixEnvVars("BATCH_INBOX"), - } - BatcherAddressFlag = &cli.StringFlag{ - Name: "batcher-address", - Usage: "Batch Sender Address", - EnvVars: prefixEnvVars("BATCHER_ADDRESS"), - } MockFlag = &cli.BoolFlag{ Name: "mock", Usage: "Use the mock OP Succinct server", @@ -175,12 +159,9 @@ var optionalFlags = []cli.Flag{ SlackTokenFlag, MaxBlockRangePerSpanProofFlag, MaxConcurrentWitnessGenFlag, - TxCacheOutDirFlag, OPSuccinctServerUrlFlag, ProofTimeoutFlag, MaxConcurrentProofRequestsFlag, - BatchInboxFlag, - BatcherAddressFlag, MockFlag, WitnessGenTimeoutFlag, } diff --git a/proposer/op/proposer/service.go b/proposer/op/proposer/service.go index 1aea3ea8..ff6840ab 100644 --- a/proposer/op/proposer/service.go +++ b/proposer/op/proposer/service.go @@ -57,7 +57,6 @@ type ProposerConfig struct { SlackToken string BeaconRpc string RollupRpc string - TxCacheOutDir string MaxBlockRangePerSpanProof uint64 MaxConcurrentWitnessGen uint64 WitnessGenTimeout uint64 @@ -65,8 +64,6 @@ type ProposerConfig struct { ProofTimeout uint64 OPSuccinctServerUrl string MaxConcurrentProofRequests uint64 - BatchInbox common.Address - BatcherAddress common.Address Mock bool } @@ -122,7 +119,6 @@ func (ps *ProposerService) initFromCLIConfig(ctx context.Context, version string ps.SlackToken = cfg.SlackToken ps.BeaconRpc = cfg.BeaconRpc ps.RollupRpc = cfg.RollupRpc - ps.TxCacheOutDir = cfg.TxCacheOutDir ps.MaxBlockRangePerSpanProof = cfg.MaxBlockRangePerSpanProof ps.MaxConcurrentWitnessGen = cfg.MaxConcurrentWitnessGen ps.WitnessGenTimeout = cfg.WitnessGenTimeout @@ -130,8 +126,6 @@ func (ps *ProposerService) initFromCLIConfig(ctx context.Context, version string ps.ProofTimeout = cfg.ProofTimeout ps.L2ChainID = cfg.L2ChainID ps.MaxConcurrentProofRequests = cfg.MaxConcurrentProofRequests - ps.BatchInbox = common.HexToAddress(cfg.BatchInbox) - ps.BatcherAddress = common.HexToAddress(cfg.BatcherAddress) ps.Mock = cfg.Mock ps.initL2ooAddress(cfg) diff --git a/proposer/succinct/bin/server.rs b/proposer/succinct/bin/server.rs index a4d6691b..7685c815 100644 --- a/proposer/succinct/bin/server.rs +++ b/proposer/succinct/bin/server.rs @@ -20,7 +20,7 @@ use op_succinct_host_utils::{ L2OutputOracle, ProgramType, }; use op_succinct_proposer::{ - AggProofRequest, ContractConfig, ProofResponse, ProofStatus, SpanProofRequest, + AggProofRequest, ProofResponse, ProofStatus, SpanProofRequest, SuccinctProposerConfig, ValidateConfigRequest, ValidateConfigResponse, }; use sp1_sdk::{ @@ -64,8 +64,18 @@ async fn main() -> Result<()> { // [`RollupConfig`] is released from `op-alloy`. let rollup_config_hash = hash_rollup_config(fetcher.rollup_config.as_ref().unwrap()); + // Set the proof strategies based on environment variables. Default to reserved to keep existing behavior. + let range_proof_strategy = match env::var("RANGE_PROOF_STRATEGY") { + Ok(strategy) if strategy.to_lowercase() == "hosted" => FulfillmentStrategy::Hosted, + _ => FulfillmentStrategy::Reserved, + }; + let agg_proof_strategy = match env::var("AGG_PROOF_STRATEGY") { + Ok(strategy) if strategy.to_lowercase() == "hosted" => FulfillmentStrategy::Hosted, + _ => FulfillmentStrategy::Reserved, + }; + // Initialize global hashes. - let global_hashes = ContractConfig { + let global_hashes = SuccinctProposerConfig { agg_vkey_hash, range_vkey_commitment, rollup_config_hash, @@ -73,6 +83,8 @@ async fn main() -> Result<()> { range_pk, agg_vk, agg_pk, + range_proof_strategy, + agg_proof_strategy, }; let app = Router::new() @@ -98,7 +110,7 @@ async fn main() -> Result<()> { /// Validate the configuration of the L2 Output Oracle. async fn validate_config( - State(state): State, + State(state): State, Json(payload): Json, ) -> Result<(StatusCode, Json), AppError> { info!("Received validate config request: {:?}", payload); @@ -127,7 +139,7 @@ async fn validate_config( /// Request a proof for a span of blocks. async fn request_span_proof( - State(state): State, + State(state): State, Json(payload): Json, ) -> Result<(StatusCode, Json), AppError> { info!("Received span proof request: {:?}", payload); @@ -193,7 +205,7 @@ async fn request_span_proof( let proof_id = client .prove(&state.range_pk, &sp1_stdin) .compressed() - .strategy(FulfillmentStrategy::Reserved) + .strategy(state.range_proof_strategy) .skip_simulation(true) .cycle_limit(1_000_000_000_000) .request_async() @@ -213,7 +225,7 @@ async fn request_span_proof( /// Request an aggregation proof for a set of subproofs. async fn request_agg_proof( - State(state): State, + State(state): State, Json(payload): Json, ) -> Result<(StatusCode, Json), AppError> { info!("Received agg proof request"); @@ -302,7 +314,7 @@ async fn request_agg_proof( let proof_id = match prover .prove(&state.agg_pk, &stdin) .groth16() - .strategy(FulfillmentStrategy::Reserved) + .strategy(state.agg_proof_strategy) .request_async() .await { @@ -323,7 +335,7 @@ async fn request_agg_proof( /// Request a mock proof for a span of blocks. async fn request_mock_span_proof( - State(state): State, + State(state): State, Json(payload): Json, ) -> Result<(StatusCode, Json), AppError> { info!("Received mock span proof request: {:?}", payload); @@ -428,7 +440,7 @@ async fn request_mock_span_proof( /// Request mock aggregation proof. async fn request_mock_agg_proof( - State(state): State, + State(state): State, Json(payload): Json, ) -> Result<(StatusCode, Json), AppError> { info!("Received mock agg proof request!"); diff --git a/proposer/succinct/src/lib.rs b/proposer/succinct/src/lib.rs index 1dd8970b..5599715e 100644 --- a/proposer/succinct/src/lib.rs +++ b/proposer/succinct/src/lib.rs @@ -2,7 +2,7 @@ use alloy_primitives::B256; use base64::{engine::general_purpose, Engine as _}; use serde::{Deserialize, Deserializer, Serialize}; use serde_repr::{Deserialize_repr, Serialize_repr}; -use sp1_sdk::{SP1ProvingKey, SP1VerifyingKey}; +use sp1_sdk::{network::FulfillmentStrategy, SP1ProvingKey, SP1VerifyingKey}; #[derive(Serialize, Deserialize, Debug)] pub struct ValidateConfigRequest { @@ -75,7 +75,7 @@ pub struct ProofStatus { /// Configuration of the L2 Output Oracle contract. Created once at server start-up, monitors if there are any changes /// to the contract's configuration. #[derive(Clone)] -pub struct ContractConfig { +pub struct SuccinctProposerConfig { pub range_vk: SP1VerifyingKey, pub range_pk: SP1ProvingKey, pub agg_pk: SP1ProvingKey, @@ -83,6 +83,8 @@ pub struct ContractConfig { pub agg_vkey_hash: B256, pub range_vkey_commitment: B256, pub rollup_config_hash: B256, + pub range_proof_strategy: FulfillmentStrategy, + pub agg_proof_strategy: FulfillmentStrategy, } /// Deserialize a vector of base64 strings into a vector of vectors of bytes. Go serializes