refactor: improve mod and test layout

Author: prestwich

crates/blobber/src/cache.rs

@@ -2,7 +2,7 @@ use crate::{BlobFetcherError, Blobs, FetchResult};
 use alloy::consensus::{SidecarCoder, SimpleCoder, Transaction as _};
 use alloy::eips::eip7691::MAX_BLOBS_PER_BLOCK_ELECTRA;
 use alloy::eips::merge::EPOCH_SLOTS;
-use alloy::primitives::{keccak256, Bytes, B256};
+use alloy::primitives::{B256, Bytes, keccak256};
 use reth::transaction_pool::TransactionPool;
 use reth::{network::cache::LruMap, primitives::Receipt};
 use signet_extract::ExtractedEvent;
@@ -206,3 +206,92 @@ impl<Pool: TransactionPool + 'static> BlobCacher<Pool> {
         CacheHandle { sender }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use crate::BlobFetcher;
+
+    use super::*;
+    use alloy::{
+        consensus::{SidecarBuilder, SignableTransaction as _, TxEip2930},
+        eips::Encodable2718,
+        primitives::{TxKind, U256, bytes},
+        rlp::encode,
+        signers::{SignerSync, local::PrivateKeySigner},
+    };
+    use init4_bin_base::utils::calc::SlotCalculator;
+    use reth::primitives::Transaction;
+    use reth_transaction_pool::{
+        PoolTransaction, TransactionOrigin,
+        test_utils::{MockTransaction, testing_pool},
+    };
+    use signet_types::{constants::SignetSystemConstants, primitives::TransactionSigned};
+
+    #[tokio::test]
+    async fn test_fetch_from_pool() -> eyre::Result<()> {
+        let wallet = PrivateKeySigner::random();
+        let pool = testing_pool();
+
+        let test = signet_constants::KnownChains::Test;
+
+        let constants: SignetSystemConstants = test.try_into().unwrap();
+        let calc = SlotCalculator::new(0, 0, 12);
+
+        let explorer_url = "https://api.holesky.blobscan.com/";
+        let client = reqwest::Client::builder().use_rustls_tls();
+
+        let tx = Transaction::Eip2930(TxEip2930 {
+            chain_id: 17001,
+            nonce: 2,
+            gas_limit: 50000,
+            gas_price: 1_500_000_000,
+            to: TxKind::Call(constants.host_zenith()),
+            value: U256::from(1_f64),
+            input: bytes!(""),
+            ..Default::default()
+        });
+
+        let encoded_transactions =
+            encode(vec![sign_tx_with_key_pair(wallet.clone(), tx).encoded_2718()]);
+
+        let result = SidecarBuilder::<SimpleCoder>::from_slice(&encoded_transactions).build();
+        assert!(result.is_ok());
+
+        let mut mock_transaction = MockTransaction::eip4844_with_sidecar(result.unwrap().into());
+        let transaction =
+            sign_tx_with_key_pair(wallet, Transaction::from(mock_transaction.clone()));
+
+        mock_transaction.set_hash(*transaction.hash());
+
+        pool.add_transaction(TransactionOrigin::Local, mock_transaction.clone()).await?;
+
+        // Spawn the cache
+        let cache = BlobFetcher::builder()
+            .with_pool(pool.clone())
+            .with_explorer_url(explorer_url)
+            .with_client_builder(client)
+            .unwrap()
+            .with_slot_calculator(calc)
+            .build_cache()?;
+        let handle = cache.spawn();
+
+        let got = handle
+            .fetch_blobs(
+                0, // this is ignored by the pool
+                *mock_transaction.hash(),
+                mock_transaction.blob_versioned_hashes().unwrap().to_owned(),
+            )
+            .await;
+        assert!(got.is_ok());
+
+        let got_blobs = got.unwrap();
+        assert!(got_blobs.len() == 1);
+
+        Ok(())
+    }
+
+    fn sign_tx_with_key_pair(wallet: PrivateKeySigner, tx: Transaction) -> TransactionSigned {
+        let signature = wallet.sign_hash_sync(&tx.signature_hash()).unwrap();
+        TransactionSigned::new_unhashed(tx, signature)
+    }
+}

crates/blobber/src/fetch.rs

@@ -1,6 +1,6 @@
 use crate::{
     BlobFetcherBuilder, BlobFetcherError, FetchResult, error::UnrecoverableBlobError,
-    shim::ExtractableChainShim,
+    shim::ExtractableChainShim, utils::extract_blobs_from_bundle,
 };
 use alloy::{
     consensus::{Blob, SidecarCoder, SimpleCoder, Transaction as _},
@@ -14,7 +14,6 @@ use reth::{
 };
 use signet_extract::{ExtractedEvent, Extracts};
 use signet_zenith::{Zenith::BlockSubmitted, ZenithBlock};
-use smallvec::SmallVec;
 use std::{ops::Deref, sync::Arc};
 use tokio::select;
 use tracing::{error, instrument, trace};
@@ -305,98 +304,23 @@ where
     }
 }
 
-/// Extracts the blobs from the [`BeaconBlobBundle`], and returns the blobs that match the versioned hashes in the transaction.
-/// This also dedups any duplicate blobs if a builder lands the same blob multiple times in a block.
-fn extract_blobs_from_bundle(
-    bundle: BeaconBlobBundle,
-    versioned_hashes: &[B256],
-) -> FetchResult<Blobs> {
-    let mut blobs = vec![];
-    // NB: There can be, at most, 9 blobs per block from Pectra forwards. We'll never need more space than this, unless blob capacity is increased again or made dynamic.
-    let mut seen_versioned_hashes: SmallVec<[B256; 9]> = SmallVec::new();
-
-    for item in bundle.data.iter() {
-        let versioned_hash =
-            alloy::eips::eip4844::kzg_to_versioned_hash(item.kzg_commitment.as_ref());
-
-        if versioned_hashes.contains(&versioned_hash)
-            && !seen_versioned_hashes.contains(&versioned_hash)
-        {
-            blobs.push(*item.blob);
-            seen_versioned_hashes.push(versioned_hash);
-        }
-    }
-
-    Ok(blobs.into())
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
     use alloy::{
-        consensus::{
-            BlobTransactionSidecar, SidecarBuilder, SignableTransaction as _, TxEip2930, TxEnvelope,
-        },
+        consensus::{SidecarBuilder, SignableTransaction as _, TxEip2930},
         eips::Encodable2718,
         primitives::{TxKind, U256, bytes},
         rlp::encode,
         signers::{SignerSync, local::PrivateKeySigner},
     };
-    use foundry_blob_explorers::TransactionDetails;
     use reth::primitives::Transaction;
     use reth_transaction_pool::{
         PoolTransaction, TransactionOrigin,
         test_utils::{MockTransaction, testing_pool},
     };
     use signet_types::{constants::SignetSystemConstants, primitives::TransactionSigned};
 
-    const BLOBSCAN_BLOB_RESPONSE: &str = include_str!("../../../tests/artifacts/blob.json");
-    /// Blob from Slot 2277733, corresponding to block 277722 on Pecorino host.
-    const CL_BLOB_RESPONSE: &str = include_str!("../../../tests/artifacts/cl_blob.json");
-    /// EIP4844 blob tx with hash 0x73d1c682fae85c761528a0a7ec22fac613b25ede87b80f0ac052107f3444324f,
-    /// corresponding to blob sent to block 277722 on Pecorino host.
-    const CL_BLOB_TX: &str = include_str!("../../../tests/artifacts/cl_blob_tx.json");
-    /// Blob sidecar from Pylon, corresponding to block 277722 on Pecorino host.
-    const PYLON_BLOB_RESPONSE: &str = include_str!("../../../tests/artifacts/pylon_blob.json");
-
-    #[test]
-    fn test_process_blob_extraction() {
-        let bundle: BeaconBlobBundle = serde_json::from_str(CL_BLOB_RESPONSE).unwrap();
-        let tx: TxEnvelope = serde_json::from_str::<TxEnvelope>(CL_BLOB_TX).unwrap();
-        let tx: TransactionSigned = tx.into();
-
-        let versioned_hashes =
-            tx.blob_versioned_hashes().map(ToOwned::to_owned).unwrap_or_default();
-
-        // Extract the blobs from the CL beacon blob bundle.
-        let cl_blobs = extract_blobs_from_bundle(bundle, &versioned_hashes).unwrap();
-        assert_eq!(cl_blobs.len(), 1);
-
-        // Now, process the pylon blobs which come in a [`BlobTransactionSidecar`].
-        // NB: this should be changes to `BlobTransactionSidecarVariant` in the
-        // future. After https://github.com/alloy-rs/alloy/pull/2713
-        // The json is definitely a `BlobTransactionSidecar`, so we can
-        // deserialize it directly and it doesn't really matter much.
-        let sidecar: BlobTransactionSidecar =
-            serde_json::from_str::<BlobTransactionSidecar>(PYLON_BLOB_RESPONSE).unwrap();
-        let pylon_blobs: Blobs = Arc::<BlobTransactionSidecarVariant>::new(sidecar.into()).into();
-
-        // Make sure that both blob sources have the same blobs after being processed.
-        assert_eq!(cl_blobs.len(), pylon_blobs.len());
-        assert_eq!(cl_blobs.as_slice(), pylon_blobs.as_slice());
-
-        // Make sure both can be decoded
-        let cl_decoded = SimpleCoder::default().decode_all(cl_blobs.as_ref()).unwrap();
-        let pylon_decoded = SimpleCoder::default().decode_all(pylon_blobs.as_ref()).unwrap();
-        assert_eq!(cl_decoded.len(), pylon_decoded.len());
-        assert_eq!(cl_decoded, pylon_decoded);
-    }
-
-    #[test]
-    fn test_deser_blob() {
-        let _: TransactionDetails = serde_json::from_str(BLOBSCAN_BLOB_RESPONSE).unwrap();
-    }
-
     #[tokio::test]
     async fn test_fetch_from_pool() -> eyre::Result<()> {
         let wallet = PrivateKeySigner::random();

crates/blobber/src/lib.rs

@@ -28,3 +28,17 @@ pub use fetch::{BlobFetcher, Blobs};
 
 mod shim;
 pub use shim::ExtractableChainShim;
+
+pub(crate) mod utils;
+
+#[cfg(test)]
+mod test {
+    use crate::utils::tests::BLOBSCAN_BLOB_RESPONSE;
+    use foundry_blob_explorers::TransactionDetails;
+
+    // Sanity check on dependency compatibility.
+    #[test]
+    fn test_deser_blob() {
+        let _: TransactionDetails = serde_json::from_str(BLOBSCAN_BLOB_RESPONSE).unwrap();
+    }
+}

crates/blobber/src/utils.rs

@@ -0,0 +1,87 @@
+use crate::{Blobs, FetchResult};
+use alloy::{
+    eips::{eip4844::kzg_to_versioned_hash, eip7691::MAX_BLOBS_PER_BLOCK_ELECTRA},
+    primitives::B256,
+};
+use reth::rpc::types::beacon::sidecar::BeaconBlobBundle;
+use smallvec::SmallVec;
+
+/// Extracts the blobs from the [`BeaconBlobBundle`], and returns the blobs that match the versioned hashes in the transaction.
+/// This also dedups any duplicate blobs if a builder lands the same blob multiple times in a block.
+pub(crate) fn extract_blobs_from_bundle(
+    bundle: BeaconBlobBundle,
+    versioned_hashes: &[B256],
+) -> FetchResult<Blobs> {
+    let mut blobs = vec![];
+    // NB: There can be, at most, 9 blobs per block from Pectra forwards. We'll never need more space than this, unless blob capacity is increased again or made dynamic.
+    let mut seen_versioned_hashes: SmallVec<[B256; MAX_BLOBS_PER_BLOCK_ELECTRA as usize]> =
+        SmallVec::new();
+
+    for item in bundle.data.iter() {
+        let versioned_hash = kzg_to_versioned_hash(item.kzg_commitment.as_ref());
+
+        if versioned_hashes.contains(&versioned_hash)
+            && !seen_versioned_hashes.contains(&versioned_hash)
+        {
+            blobs.push(*item.blob);
+            seen_versioned_hashes.push(versioned_hash);
+        }
+    }
+
+    Ok(blobs.into())
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+
+    use super::*;
+    use alloy::{
+        consensus::{BlobTransactionSidecar, SidecarCoder, SimpleCoder, Transaction, TxEnvelope},
+        eips::eip7594::BlobTransactionSidecarVariant,
+    };
+    use signet_types::primitives::TransactionSigned;
+    use std::sync::Arc;
+
+    pub(crate) const BLOBSCAN_BLOB_RESPONSE: &str =
+        include_str!("../../../tests/artifacts/blob.json");
+    /// Blob from Slot 2277733, corresponding to block 277722 on Pecorino host.
+    pub(crate) const CL_BLOB_RESPONSE: &str = include_str!("../../../tests/artifacts/cl_blob.json");
+    /// EIP4844 blob tx with hash 0x73d1c682fae85c761528a0a7ec22fac613b25ede87b80f0ac052107f3444324f,
+    /// corresponding to blob sent to block 277722 on Pecorino host.
+    pub(crate) const CL_BLOB_TX: &str = include_str!("../../../tests/artifacts/cl_blob_tx.json");
+    /// Blob sidecar from Pylon, corresponding to block 277722 on Pecorino host.
+    pub(crate) const PYLON_BLOB_RESPONSE: &str =
+        include_str!("../../../tests/artifacts/pylon_blob.json");
+
+    #[test]
+    fn test_process_blob_extraction() {
+        let bundle: BeaconBlobBundle = serde_json::from_str(CL_BLOB_RESPONSE).unwrap();
+        let tx: TxEnvelope = serde_json::from_str::<TxEnvelope>(CL_BLOB_TX).unwrap();
+        let tx: TransactionSigned = tx.into();
+
+        let versioned_hashes = tx.blob_versioned_hashes().unwrap().to_owned();
+
+        // Extract the blobs from the CL beacon blob bundle.
+        let cl_blobs = extract_blobs_from_bundle(bundle, &versioned_hashes).unwrap();
+        assert_eq!(cl_blobs.len(), 1);
+
+        // Now, process the pylon blobs which come in a [`BlobTransactionSidecar`].
+        // NB: this should be changes to `BlobTransactionSidecarVariant` in the
+        // future. After https://github.com/alloy-rs/alloy/pull/2713
+        // The json is definitely a `BlobTransactionSidecar`, so we can
+        // deserialize it directly and it doesn't really matter much.
+        let sidecar: BlobTransactionSidecar =
+            serde_json::from_str::<BlobTransactionSidecar>(PYLON_BLOB_RESPONSE).unwrap();
+        let pylon_blobs: Blobs = Arc::<BlobTransactionSidecarVariant>::new(sidecar.into()).into();
+
+        // Make sure that both blob sources have the same blobs after being processed.
+        assert_eq!(cl_blobs.len(), pylon_blobs.len());
+        assert_eq!(cl_blobs.as_slice(), pylon_blobs.as_slice());
+
+        // Make sure both can be decoded
+        let cl_decoded = SimpleCoder::default().decode_all(cl_blobs.as_ref()).unwrap();
+        let pylon_decoded = SimpleCoder::default().decode_all(pylon_blobs.as_ref()).unwrap();
+        assert_eq!(cl_decoded.len(), pylon_decoded.len());
+        assert_eq!(cl_decoded, pylon_decoded);
+    }
+}