Metrics and load testing (#121)

* skeleton test setup

* add basic tests and rpc client

* add client

* full e2e tests, with local node

* Ignore test that requires node running

* Mark full node tests with env flag

* Finish merge

* readme + fmt

* yarn update

* Update tests for clean fail

* fmt

* Add mock provider to avoid need to run node for e2e tests

* Cargo

* lint + clippy

* Update env var flag and readme

* run tests with full services

* update integration tests filename

* create runnable bin for load testing

* Restructure for runnable binary

* Separate load testing from integration testing

* refine load testing

* Fix toml

* Cargo.toml update

* Load test cleanup

* fix readme

* METRICS.md fix + fmt

Author: kmchicoine

Cargo.lock

@@ -7,12 +7,13 @@ homepage = "https://github.com/base/tips"
 repository = "https://github.com/base/tips"
 
 [workspace]
-members = ["crates/audit", "crates/ingress-rpc", "crates/core", "crates/account-abstraction-core"]
+members = ["crates/audit", "crates/ingress-rpc", "crates/core", "crates/account-abstraction-core", "crates/system-tests"]
 resolver = "2"
 
 [workspace.dependencies]
 tips-audit = { path = "crates/audit" }
 tips-core = { path = "crates/core" }
+tips-system-tests = { path = "crates/system-tests" }
 account-abstraction-core = { path = "crates/account-abstraction-core" }
 
 # Reth
@@ -27,8 +28,10 @@ alloy-primitives = { version = "1.4.1", default-features = false, features = [
 ] }
 alloy-consensus = { version = "1.0.41" }
 alloy-provider = { version = "1.0.41" }
-alloy-serde = "1.0.41"
 alloy-rpc-types = "1.1.2"
+alloy-signer = { version = "1.0.41" }
+alloy-network = { version = "1.0.41" }
+alloy-serde = "1.0.41"
 alloy-sol-types = { version = "1.4.1", default-features = false }
 
 # op-alloy

Cargo.toml

@@ -47,6 +47,13 @@ hex = "0.4.3"
 jsonrpsee = { workspace = true }
 op-revm = { workspace = true }
 
+# Load test dependencies
+clap = { version = "4.5", features = ["derive", "env"] }
+indicatif = "0.17"
+rand = "0.8"
+rand_chacha = "0.3"
+dashmap = "6.0"
+
 [dev-dependencies]
 tokio = { workspace = true, features = ["test-util"] }
 testcontainers = { workspace = true }

crates/system-tests/Cargo.toml

@@ -0,0 +1,133 @@
+# TIPS Load Testing
+
+Multi-wallet concurrent load testing tool for measuring TIPS performance.
+
+## Quick Start
+
+```bash
+# 1. Build
+cargo build --release --bin load-test
+
+# 2. Setup wallets
+./target/release/load-test setup \
+  --master-key 0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d \
+  --output wallets.json
+
+# 3. Run load test
+./target/release/load-test load --wallets wallets.json 
+```
+
+---
+
+## Configuration Options
+
+### Setup Command
+
+Create and fund test wallets from a master wallet. Test wallets are saved to allow test reproducibility and avoid the need to create new wallets for every test run.
+
+**Usage:**
+```bash
+./target/release/load-test setup --master-key <KEY> --output <FILE> [OPTIONS]
+```
+
+**Options:**
+
+| Flag | Description | Default | Example |
+|------|-------------|---------|---------|
+| `--master-key` | Private key of funded wallet (required) | - | `0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d` |
+| `--output` | Save wallets to JSON file (required) | - | `wallets.json` |
+| `--sequencer` | L2 sequencer RPC URL | `http://localhost:8547` | `http://localhost:8547` |
+| `--num-wallets` | Number of wallets to create | `10` | `100` |
+| `--fund-amount` | ETH to fund each wallet | `0.1` | `0.5` |
+
+**Environment Variables:**
+- `MASTER_KEY` - Alternative to `--master-key` flag
+- `SEQUENCER_URL` - Alternative to `--sequencer` flag
+
+### Load Command
+
+Run load test with funded wallets. Use the `--seed` flag to set the RNG seed for test reproducibility.
+
+**Usage:**
+```bash
+./target/release/load-test load --wallets <FILE> [OPTIONS]
+```
+
+**Options:**
+
+| Flag | Description | Default | Example |
+|------|-------------|---------|---------|
+| `--wallets` | Path to wallets JSON file (required) | - | `wallets.json` |
+| `--target` | TIPS ingress RPC URL | `http://localhost:8080` | `http://localhost:8080` |
+| `--sequencer` | L2 sequencer RPC URL | `http://localhost:8547` | `http://localhost:8547` |
+| `--rate` | Target transaction rate (tx/s) | `100` | `500` |
+| `--duration` | Test duration in seconds | `60` | `100` |
+| `--tx-timeout` | Timeout for tx inclusion (seconds) | `60` | `120` |
+| `--seed` | Random seed for reproducibility | (none) | `42` |
+| `--output` | Save metrics to JSON file | (none) | `metrics.json` |
+
+**Environment Variables:**
+- `INGRESS_URL` - Alternative to `--target` flag
+- `SEQUENCER_URL` - Alternative to `--sequencer` flag
+
+---
+---
+
+## Metrics Explained
+
+### Output Example
+
+```
+Load Test Results
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+Configuration:
+  Target:              http://localhost:8080
+  Sequencer:           http://localhost:8547
+  Wallets:             100
+  Target Rate:         100 tx/s
+  Duration:            60s
+  TX Timeout:          60s
+
+Throughput:
+  Sent:                100.0 tx/s (6000 total)
+  Included:            98.5 tx/s (5910 total)
+  Success Rate:        98.5%
+
+Transaction Results:
+  Included:            5910 (98.5%)
+  Reverted:            10 (0.2%)
+  Timed Out:           70 (1.2%)
+  Send Errors:         10 (0.1%)
+```
+
+### Metrics Definitions
+
+**Throughput:**
+- `Sent Rate` - Transactions sent to TIPS per second
+- `Included Rate` - Transactions included in blocks per second
+- `Success Rate` - Percentage of sent transactions that were included
+
+**Transaction Results:**
+- `Included` - Successfully included in a block with status == true
+- `Reverted` - Included in a block but transaction reverted (status == false)
+- `Timed Out` - Not included within timeout period
+- `Send Errors` - Failed to send to TIPS RPC
+
+---
+
+## Architecture
+
+```
+Sender Tasks (1 per wallet)          Receipt Poller
+        │                                  │
+        ▼                                  ▼
+   Send to TIPS ──► Tracker ◄── Poll sequencer every 2s
+   (retry 3x)       (pending)     │
+        │               │         ├─ status=true  → included
+        │               │         ├─ status=false → reverted
+        │               │         └─ timeout      → timed_out
+        ▼               ▼
+   rate/N tx/s    Calculate Results → Print Summary
+```
+
+---

crates/system-tests/METRICS.md

@@ -0,0 +1,13 @@
+use anyhow::Result;
+use clap::Parser;
+use tips_system_tests::load_test::{config, load, setup};
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let cli = config::Cli::parse();
+
+    match cli.command {
+        config::Commands::Setup(args) => setup::run(args).await,
+        config::Commands::Load(args) => load::run(args).await,
+    }
+}

crates/system-tests/src/bin/load-test.rs

@@ -1,2 +1,3 @@
 pub mod client;
 pub mod fixtures;
+pub mod load_test;

crates/system-tests/src/lib.rs

@@ -0,0 +1,76 @@
+use clap::{Parser, Subcommand};
+use std::path::PathBuf;
+
+#[derive(Parser)]
+#[command(name = "load-test")]
+#[command(about = "Load testing tool for TIPS ingress service", long_about = None)]
+pub struct Cli {
+    #[command(subcommand)]
+    pub command: Commands,
+}
+
+#[derive(Subcommand)]
+pub enum Commands {
+    /// Setup: Fund N wallets from a master wallet
+    Setup(SetupArgs),
+    /// Load: Run load test with funded wallets
+    Load(LoadArgs),
+}
+
+#[derive(Parser)]
+pub struct SetupArgs {
+    /// Master wallet private key (must have funds)
+    #[arg(long, env = "MASTER_KEY")]
+    pub master_key: String,
+
+    /// Sequencer RPC URL
+    #[arg(long, env = "SEQUENCER_URL", default_value = "http://localhost:8547")]
+    pub sequencer: String,
+
+    /// Number of wallets to create and fund
+    #[arg(long, default_value = "10")]
+    pub num_wallets: usize,
+
+    /// Amount of ETH to fund each wallet
+    #[arg(long, default_value = "0.1")]
+    pub fund_amount: f64,
+
+    /// Output file for wallet data (required)
+    #[arg(long)]
+    pub output: PathBuf,
+}
+
+#[derive(Parser)]
+pub struct LoadArgs {
+    /// TIPS ingress RPC URL
+    #[arg(long, env = "INGRESS_URL", default_value = "http://localhost:8080")]
+    pub target: String,
+
+    /// Sequencer RPC URL (for nonce fetching and receipt polling)
+    #[arg(long, env = "SEQUENCER_URL", default_value = "http://localhost:8547")]
+    pub sequencer: String,
+
+    /// Path to wallets JSON file (required)
+    #[arg(long)]
+    pub wallets: PathBuf,
+
+    /// Target transaction rate (transactions per second)
+    #[arg(long, default_value = "100")]
+    pub rate: u64,
+
+    /// Test duration in seconds
+    #[arg(long, default_value = "60")]
+    pub duration: u64,
+
+    /// Timeout for transaction inclusion (seconds)
+    #[arg(long, default_value = "60")]
+    pub tx_timeout: u64,
+
+    /// Random seed for reproducibility
+    #[arg(long)]
+    pub seed: Option<u64>,
+
+    /// Output file for metrics (JSON)
+    #[arg(long)]
+    pub output: Option<PathBuf>,
+}