Add benchmarking (#254)

* add test files

* fix path

* fix fibonacci.cairo

* change test to use first_contract.cairo

* add run_bench script and increase amount of iteratiions

* save work

* try with fibonacci

* update names

* save work

* replace bench

* use bench/internals for flamegraph

* use mimalloc, fix bench

* bench intenrals

* use directly state

* add bin for test

* save work

* add test bins

* add bench scripts

* use makefile to run scripts for benching

* update paths

* remove unnecessary operations

* increase amount of iterations

* increase amount of iterations

* add new bin

* re-add file

* remove pedersen

* add bench for fibonacci

* fix build and bench

* remove executionInfo and apply_to_copy

* Add doc, update makefile, add fib

* rollback bench/internals.rs:

* add -N flag to hyperfine

* restore Makefile

* address clippy errors

* Update src/testing/starknet_state.rs

* use new method

* add EOF in .gitignore

* remove pub from fields

* replace vec reference with slice

---------

Co-authored-by: juanbono <[email protected]>

Author: SantiagoPittella

.gitignore

@@ -42,6 +42,7 @@ starknet-venv/
 default.profraw
 **.DS_Store
 /scripts/reports
-
+__pycache__/
+.pytest_cache/
 # codecov report
 lcov.info

Cargo.lock

@@ -3,6 +3,10 @@ name = "starknet-rs"
 version = "0.1.0"
 edition = "2021"
 
+[features]
+default = ["with_mimalloc"]
+with_mimalloc = ["mimalloc"]
+
 [dependencies]
 cairo-rs = { git = "https://github.com/lambdaclass/cairo-rs", package = "cairo-vm", rev = "eebfd612c5f969c759cb982e8fb777b2a1a1dab0" }
 felt = { git = "https://github.com/lambdaclass/cairo-rs", package = "cairo-felt", rev = "eebfd612c5f969c759cb982e8fb777b2a1a1dab0" }
@@ -22,6 +26,7 @@ thiserror = "1.0.32"
 clap = { version = "4.1.8", features = ["derive"] }
 actix-web = "4.3.1"
 awc = "3.1.1"
+mimalloc = { version = "0.1.29", default-features = false, optional = true }
 hex = "0.4.3"
 cargo-llvm-cov = "0.5.14"
 

Cargo.toml

@@ -89,3 +89,10 @@ heaptrack:
 
 flamegraph: compile-cairo compile-starknet
 	CARGO_PROFILE_RELEASE_DEBUG=true cargo flamegraph --root --bench internals
+
+benchmark: compile-cairo compile-starknet
+	cargo build --release --all-targets
+	./scripts/bench-invoke.sh
+	./scripts/bench-deploy-invoke.sh
+	./scripts/bench-fibonacci.sh
+	./scripts/bench-deploy.sh

Makefile

@@ -19,19 +19,24 @@ StarkNet library in Rust, featuring [⚡cairo-rs⚡](https://github.com/lambdacl
 </div>
 
 ## Table of Contents
-- [Disclaimer](#%EF%B8%8F-disclaimer)
-- [About](#-about)
-- [Getting Started](#-getting-started)
-  * [Dependencies](#dependencies)
-  * [Installation](#installation)
-- [Usage](#-usage)
-  * [Running simple contracts](#running-simple-contracts)
-  * [Testing](#testing)
-- [Contributing](#-contributing)
-- [Related Projects](#-related-projects)
-- [Documentation](#-documentation)
-  * [StarkNet](#starknet)
-- [License](#%EF%B8%8F-license)
+- [Table of Contents](#table-of-contents)
+- [⚠️ Disclaimer](#️-disclaimer)
+- [📖 About](#-about)
+- [🌅 Getting Started](#-getting-started)
+  - [Dependencies](#dependencies)
+  - [Installation](#installation)
+    - [How to manually install the script dependencies](#how-to-manually-install-the-script-dependencies)
+- [🚀 Usage](#-usage)
+  - [Running simple contracts](#running-simple-contracts)
+  - [Using the Cli](#using-the-cli)
+  - [Testing](#testing)
+  - [Profiling](#profiling)
+  - [Benchmarking](#benchmarking)
+- [🛠 Contributing](#-contributing)
+- [🌞 Related Projects](#-related-projects)
+- [📚 Documentation](#-documentation)
+  - [StarkNet](#starknet)
+- [⚖️ License](#️-license)
 
 ## ⚠️ Disclaimer
 
@@ -104,6 +109,14 @@ $ make flamegraph
 
 to generate a flamegraph with info of the execution of the main operations.
 
+### Benchmarking
+
+Read the 'bench_integration.py' file to identify which lines need to be commented out for accurate results. Comment out those lines and then run the following command:
+
+```bash
+$ make benchmark
+```
+
 ## 🛠 Contributing
 
 The open source community is a fantastic place for learning, inspiration, and creation, and this is all thanks to contributions from people like you. Your contributions are **greatly appreciated**. 

README.md

@@ -45,7 +45,6 @@ lazy_static! {
 fn scope<T>(f: impl FnOnce() -> T) -> T {
     f()
 }
-
 // We don't use the cargo test harness because it uses
 // FnOnce calls for each test, that are merged in the flamegraph.
 fn main() {
@@ -170,9 +169,7 @@ fn invoke() {
     state
         .set_contract_class(&CLASS_HASH, &CONTRACT_CLASS)
         .unwrap();
-
     let config = &Default::default();
-
     let salt = Address(felt_str!(
         "2669425616857739096022668060305620640217901643963991674344872184515580705509"
     ));

bench/internals.rs

@@ -0,0 +1,46 @@
+import os
+
+import pytest
+import pytest_asyncio
+from starkware.starknet.compiler.compile import compile_starknet_files
+from starkware.starknet.testing.starknet import StarknetState
+from starkware.starknet.services.api.contract_class import ContractClass
+CONTRACT_FILE = os.path.join(os.path.dirname(__file__), "starknet_programs/first_contract.json")
+FIBONACCI_FILE = os.path.join(os.path.dirname(__file__), "starknet_programs/fibonacci.json")
+
+
+@pytest.mark.asyncio
+async def test_invoke():
+    runs = 10000
+    starknet = await StarknetState.empty()
+    json_program = open(CONTRACT_FILE).read()
+    contract_class = ContractClass.loads(json_program)
+    contract_address, _ = await starknet.deploy(contract_class=contract_class, constructor_calldata=[])
+    for i in range(runs):
+        # take into account that you need to comment verify_version()
+        # from cairo-lang in order to be able to run the selectors below
+        # because invoke_raw inserts a default version=0 that throws an
+        # error.
+        res_1 = await starknet.invoke_raw(contract_address=contract_address, selector="increase_balance", calldata=[1000], max_fee=0)
+        res_2 = await starknet.invoke_raw(contract_address=contract_address, selector="get_balance", calldata=[], max_fee=0)
+        assert(res_2.call_info.retdata == [i*1000 + 1000])
+
+@pytest.mark.asyncio
+async def test_deploy():
+    runs = 100
+    starknet = await StarknetState.empty()
+    json_program = open(CONTRACT_FILE).read()
+    contract_class = ContractClass.loads(json_program)
+    for i in range(runs):
+        contract_address, _ = await starknet.deploy(contract_class=contract_class, constructor_calldata=[], contract_address_salt=i)
+
+@pytest.mark.asyncio
+async def test_fibonacci():
+    runs = 1000
+    starknet = await StarknetState.empty()
+    json_program = open(FIBONACCI_FILE).read()
+    contract_class = ContractClass.loads(json_program)
+    contract_address, _ = await starknet.deploy(contract_class=contract_class, constructor_calldata=[])
+    for i in range(runs):
+        call = await starknet.invoke_raw(contract_address=contract_address, selector="fib", calldata=[1, 1, 1000], max_fee=0)
+        assert(call.call_info.retdata == [222450955505511890955301767713383614666194461405743219770606958667979327682])

bench_integration.py

@@ -0,0 +1,4 @@
+#!/usr/bin/env sh
+hyperfine -N -w 3 -r 5 \
+	    -n "cairo-lang (CPython) read/write storage 10k with deploy" "pytest bench_integration.py::test_invoke" \
+	    -n "starknet_in_rust read/write storage 10k with deploy" "./target/release/deploy_invoke"

scripts/bench-deploy-invoke.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env sh
+hyperfine -N -w 3 -r 5 \
+	    -n "cairo-lang (CPython) deploy 10k" "pytest bench_integration.py::test_deploy" \
+	    -n "starknet_in_rust deploy 10k" "./target/release/deploy"

scripts/bench-deploy.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env sh
+hyperfine -N -w 3 -r 5 \
+	    -n "cairo-lang (CPython) fib 15k" "pytest bench_integration.py::test_fibonacci" \
+	    -n "starknet_in_rust fib 15k" "./target/release/fibonacci"

scripts/bench-fibonacci.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env sh
+hyperfine -N -w 3 -r 5 \
+	    -n "cairo-lang (CPython) read/write storage 10k" "pytest bench_integration.py::test_invoke" \
+	    -n "starknet_in_rust read/write storage 10k without deploy" "./target/release/invoke"

scripts/bench-invoke.sh

@@ -0,0 +1,34 @@
+use std::path::PathBuf;
+
+use lazy_static::lazy_static;
+use starknet_rs::{
+    services::api::contract_class::ContractClass, testing::starknet_state::StarknetState,
+    utils::Address,
+};
+
+#[cfg(feature = "with_mimalloc")]
+use mimalloc::MiMalloc;
+
+#[cfg(feature = "with_mimalloc")]
+#[global_allocator]
+static ALLOC: MiMalloc = MiMalloc;
+
+lazy_static! {
+    // include_str! doesn't seem to work in CI
+    static ref CONTRACT_CLASS: ContractClass = ContractClass::try_from(PathBuf::from(
+        "starknet_programs/first_contract.json",
+    )).unwrap();
+}
+
+fn main() {
+    const RUNS: usize = 100;
+    let mut starknet_state = StarknetState::new(None);
+
+    for n in 0..RUNS {
+        let contract_address_salt = Address(n.into());
+
+        starknet_state
+            .deploy(CONTRACT_CLASS.clone(), vec![], contract_address_salt)
+            .unwrap();
+    }
+}

src/bin/deploy.rs

@@ -0,0 +1,81 @@
+use std::path::PathBuf;
+
+use felt::{felt_str, Felt252};
+use num_traits::Zero;
+
+use starknet_rs::{
+    services::api::contract_class::ContractClass, testing::starknet_state::StarknetState,
+    utils::Address,
+};
+
+use lazy_static::lazy_static;
+
+#[cfg(feature = "with_mimalloc")]
+use mimalloc::MiMalloc;
+
+#[cfg(feature = "with_mimalloc")]
+#[global_allocator]
+static ALLOC: MiMalloc = MiMalloc;
+
+lazy_static! {
+    // include_str! doesn't seem to work in CI
+    static ref CONTRACT_CLASS: ContractClass = ContractClass::try_from(PathBuf::from(
+        "starknet_programs/first_contract.json",
+    )).unwrap();
+
+    static ref CONTRACT_PATH: PathBuf = PathBuf::from("starknet_programs/first_contract.json");
+
+    static ref CONTRACT_CLASS_HASH: [u8; 32] = [5, 133, 114, 83, 104, 231, 159, 23, 87, 255, 235, 75, 170, 4, 84, 140, 49, 77, 101, 41, 147, 198, 201, 231, 38, 189, 215, 84, 231, 141, 140, 122];
+
+    static ref CONTRACT_ADDRESS: Address = Address(1.into());
+
+    static ref INCREASE_BALANCE_SELECTOR: Felt252 = felt_str!("1530486729947006463063166157847785599120665941190480211966374137237989315360");
+
+    static ref GET_BALANCE_SELECTOR: Felt252 = felt_str!("1636223440827086009537493065587328807418413867743950350615962740049133672085");
+}
+
+fn main() {
+    const RUNS: usize = 10000;
+    let mut starknet_state = StarknetState::new(None);
+    let contract_address_salt = Address(1.into());
+
+    let (contract_address, _exec_info) = starknet_state
+        .deploy(CONTRACT_CLASS.to_owned(), vec![], contract_address_salt)
+        .unwrap();
+
+    // Statement **not** in blockifier.
+    starknet_state
+        .state
+        .cache_mut()
+        .nonce_initial_values_mut()
+        .insert(contract_address.clone(), Felt252::zero());
+
+    for i in 0..RUNS {
+        starknet_state
+            .invoke_raw(
+                contract_address.clone(),
+                INCREASE_BALANCE_SELECTOR.clone(),
+                vec![1000.into()],
+                0,
+                Some(Vec::new()),
+                Some(Felt252::from(i * 2)),
+            )
+            .unwrap();
+
+        let tx_exec_info = starknet_state
+            .invoke_raw(
+                contract_address.clone(),
+                GET_BALANCE_SELECTOR.clone(),
+                vec![],
+                0,
+                Some(Vec::new()),
+                Some(Felt252::from((i * 2) + 1)),
+            )
+            .unwrap();
+
+        assert_eq!(
+            tx_exec_info.call_info.unwrap().retdata,
+            vec![((1000 * i) + 1000).into()]
+        );
+    }
+}

src/bin/deploy_invoke.rs

@@ -0,0 +1,95 @@
+use std::{collections::HashMap, path::PathBuf};
+
+use felt::{felt_str, Felt252};
+use num_traits::Zero;
+
+use lazy_static::lazy_static;
+use starknet_rs::{
+    business_logic::{
+        fact_state::in_memory_state_reader::InMemoryStateReader, state::cached_state::CachedState,
+    },
+    services::api::contract_class::ContractClass,
+    testing::starknet_state::StarknetState,
+    utils::Address,
+};
+
+#[cfg(feature = "with_mimalloc")]
+use mimalloc::MiMalloc;
+
+#[cfg(feature = "with_mimalloc")]
+#[global_allocator]
+static ALLOC: MiMalloc = MiMalloc;
+
+lazy_static! {
+    // include_str! doesn't seem to work in CI
+    static ref CONTRACT_CLASS: ContractClass = ContractClass::try_from(PathBuf::from(
+        "starknet_programs/fibonacci.json",
+    )).unwrap();
+
+    static ref CONTRACT_PATH: PathBuf = PathBuf::from("starknet_programs/fibonacci.json");
+
+    static ref CONTRACT_CLASS_HASH: [u8; 32] = [1; 32];
+
+    static ref CONTRACT_ADDRESS: Address = Address(1.into());
+
+    static ref FIB_SELECTOR: Felt252 = felt_str!("485685360977693822178494178685050472186234432883326654755380582597179924681");
+
+    static ref EXPECTED_RES: Felt252 = felt_str!("222450955505511890955301767713383614666194461405743219770606958667979327682");
+}
+
+fn main() {
+    const RUNS: usize = 1000;
+    let cached_state = create_initial_state();
+
+    let mut starknet_state = StarknetState::new_with_states(Default::default(), cached_state);
+
+    starknet_state
+        .state
+        .cache_mut()
+        .nonce_initial_values_mut()
+        .insert(CONTRACT_ADDRESS.clone(), Felt252::zero());
+
+    for i in 0..RUNS {
+        let tx_exec_info = starknet_state
+            .invoke_raw(
+                CONTRACT_ADDRESS.clone(),
+                FIB_SELECTOR.clone(),
+                [1.into(), 1.into(), 1000.into()].into(),
+                0,
+                Some(Vec::new()),
+                Some(Felt252::from(i)),
+            )
+            .unwrap();
+
+        assert_eq!(
+            tx_exec_info.call_info.unwrap().retdata,
+            vec![EXPECTED_RES.clone()]
+        )
+    }
+}
+
+fn create_initial_state() -> CachedState<InMemoryStateReader> {
+    let cached_state = CachedState::new(
+        {
+            let mut state_reader = InMemoryStateReader::default();
+            state_reader
+                .address_to_class_hash_mut()
+                .insert(CONTRACT_ADDRESS.clone(), *CONTRACT_CLASS_HASH);
+
+            state_reader
+                .address_to_nonce_mut()
+                .insert(CONTRACT_ADDRESS.clone(), Felt252::zero());
+            state_reader
+                .class_hash_to_contract_class_mut()
+                .insert(*CONTRACT_CLASS_HASH, CONTRACT_CLASS.clone());
+
+            state_reader
+                .address_to_storage_mut()
+                .insert((CONTRACT_ADDRESS.clone(), [0; 32]), Felt252::zero());
+            state_reader
+        },
+        Some(HashMap::new()),
+    );
+
+    cached_state
+}