Move instruction profiling to a dedicated thread

crates/core/src/runloops/mod.rs

@@ -40,7 +40,7 @@ use surfpool_subgraph::SurfpoolSubgraphPlugin;
 use surfpool_types::{
     BlockProductionMode, ClockCommand, ClockEvent, DEFAULT_MAINNET_RPC_URL, DataIndexingCommand,
     SimnetCommand, SimnetConfig, SimnetEvent, SubgraphCommand, SubgraphPluginConfig,
-    SurfpoolConfig,
+    SurfpoolConfig, TransactionStatusEvent,
 };
 type PluginConstructor = unsafe fn() -> *mut dyn GeyserPlugin;
 use txtx_addon_kit::helpers::fs::FileLocation;
@@ -53,7 +53,7 @@ use crate::{
         admin::AdminRpc, bank_data::BankData, full::Full, minimal::Minimal,
         surfnet_cheatcodes::SurfnetCheatcodes, ws::Rpc,
     },
-    surfnet::{GeyserEvent, locker::SurfnetSvmLocker, remote::SurfnetRemoteClient},
+    surfnet::{GeyserEvent, ProfilingJob, locker::SurfnetSvmLocker, remote::SurfnetRemoteClient},
 };
 
 const BLOCKHASH_SLOT_TTL: u64 = 75;
@@ -167,6 +167,9 @@ pub async fn start_local_surfnet_runloop(
                 simnet_events_tx_cc.send(SimnetEvent::error(format!("Geyser plugin failed: {e}")));
         }
     };
+
+    setup_profiling(&svm_locker);
+
 
     let (clock_event_rx, clock_command_tx) =
         start_clock_runloop(simnet_config.slot_time, Some(simnet_events_tx_cc.clone()));
@@ -1169,3 +1172,57 @@ async fn start_ws_rpc_server_runloop(
         .map_err(|e| format!("Failed to spawn WebSocket RPC Handler thread: {:?}", e))?;
     Ok(_ws_handle)
 }
+
+
+pub fn setup_profiling(svm_locker: &SurfnetSvmLocker) {
+    let simnet_events_tx = svm_locker.simnet_events_tx();
+    let (profiling_job_tx, profiling_job_rx) = crossbeam_channel::bounded(128);
+    start_profiling_runloop(profiling_job_rx, simnet_events_tx);
+    svm_locker.with_svm_writer(|svm| {
+        svm.profiling_job_tx = Some(profiling_job_tx);
+    });
+}
+
+pub fn start_profiling_runloop(
+    profiling_job_rx: Receiver<ProfilingJob>,
+    simnet_events_tx: Sender<SimnetEvent>,
+) { 
+    // no need of this channel in profiling 
+    let (temp_status_tx, _) = crossbeam_channel::unbounded();
+    hiro_system_kit::thread_named("Instruction Profiler").spawn(move || {
+        let rt = tokio::runtime::Builder::new_multi_thread()
+            .worker_threads(1)
+            .enable_all()
+            .build()
+            .expect("Failed to build profiling runtime");
+
+        while let Ok(job) = profiling_job_rx.recv() {
+            let result = rt.block_on(async {
+                let profiling_locker = SurfnetSvmLocker::new(job.profiling_svm);
+                profiling_locker
+                    .generate_instruction_profiles(
+                        &job.transaction,
+                        &job.transaction_accounts,
+                        &job.loaded_addresses,
+                        &job.accounts_before,
+                        &job.token_accounts_before,
+                        &job.token_programs,
+                        job.pre_execution_capture,
+                        &temp_status_tx
+                    )
+                    .await
+            });
+
+            let profiles = match result {
+                Ok(profiles) => profiles,
+                Err(e) => {
+                    let _ = simnet_events_tx.try_send(SimnetEvent::error(format!(
+                        "Instruction profiling failed: {}", e
+                    )));
+                    None
+                }
+            };
+            let _ = job.result_tx.send(profiles);
+        }
+    }).expect("Failed to spawn Instruction Profiler thread");
+}

crates/core/src/surfnet/locker.rs

@@ -69,7 +69,7 @@ use crate::{
     error::{SurfpoolError, SurfpoolResult},
     helpers::time_travel::calculate_time_travel_clock,
     rpc::utils::{convert_transaction_metadata_from_canonical, verify_pubkey},
-    surfnet::{FINALIZATION_SLOT_THRESHOLD, SLOTS_PER_EPOCH},
+    surfnet::{FINALIZATION_SLOT_THRESHOLD, ProfilingJob, SLOTS_PER_EPOCH},
     types::{
         GeyserAccountUpdate, RemoteRpcResult, SurfnetTransactionStatus, TimeTravelConfig,
         TokenAccount, TransactionLoadedAddresses, TransactionWithStatusMeta,
@@ -1084,6 +1084,11 @@ impl SurfnetSvmLocker {
         Ok(self.with_contextualized_svm_reader(|_| uuid))
     }
 
+
+    pub fn profiling_job_tx(&self) -> Option<Sender<ProfilingJob>> {
+        self.with_svm_reader(|svm| svm.profiling_job_tx.clone())
+    }
+
     async fn fetch_all_tx_accounts_then_process_tx_returning_profile_res(
         &self,
         remote_ctx: &Option<(SurfnetRemoteClient, CommitmentConfig)>,
@@ -1230,29 +1235,28 @@ impl SurfnetSvmLocker {
 
         let loaded_addresses = tx_loaded_addresses.as_ref().map(|l| l.loaded_addresses());
 
-        let ix_profiles = if self.is_instruction_profiling_enabled() {
-            match self
-                .generate_instruction_profiles(
-                    &transaction,
-                    &transaction_accounts,
-                    &tx_loaded_addresses,
-                    &accounts_before,
-                    &token_accounts_before,
-                    &token_programs,
-                    pre_execution_capture.clone(),
-                    &status_tx,
-                )
-                .await
-            {
-                Ok(profiles) => profiles,
-                Err(e) => {
-                    let _ = self.simnet_events_tx().try_send(SimnetEvent::error(format!(
-                        "Failed to generate instruction profiles: {}",
-                        e
-                    )));
-                    None
-                }
+        let ix_profile_rx = if self.is_instruction_profiling_enabled() {
+            if let Some(profiling_job_tx) = self.profiling_job_tx() {
+                let profiling_svm = self.with_svm_reader(|r| r.clone_for_profiling());
+                let (result_tx, result_rx) = crossbeam_channel::bounded(1);
+                let job = ProfilingJob {
+                    profiling_svm,
+                    transaction: transaction.clone(),
+                    transaction_accounts: transaction_accounts.to_vec(),
+                    loaded_addresses: tx_loaded_addresses.clone(),
+                    accounts_before: accounts_before.to_vec(),
+                    token_accounts_before: token_accounts_before.to_vec(),
+                    token_programs: token_programs.to_vec(),
+                    pre_execution_capture: pre_execution_capture.clone(),
+                    result_tx,
+                };
+                let _ = profiling_job_tx.send(job);
+                Some(result_rx)
             }
+            else {
+                None
+            }
+
         } else {
             None
         };
@@ -1272,6 +1276,12 @@ impl SurfnetSvmLocker {
                 do_propagate,
             )
             .await?;
+
+        let ix_profiles = match ix_profile_rx {
+            Some(rx) => tokio::task::block_in_place(|| rx.recv().ok().flatten()),
+            None => None,
+        };
+
 
         Ok(KeyedProfileResult::new(
             latest_absolute_slot,
@@ -1283,7 +1293,7 @@ impl SurfnetSvmLocker {
     }
 
     #[allow(clippy::too_many_arguments)]
-    async fn generate_instruction_profiles(
+    pub async fn generate_instruction_profiles(
         &self,
         transaction: &VersionedTransaction,
         transaction_accounts: &[Pubkey],

crates/core/src/surfnet/mod.rs

@@ -14,11 +14,12 @@ use solana_signature::Signature;
 use solana_transaction::versioned::VersionedTransaction;
 use solana_transaction_error::TransactionError;
 use solana_transaction_status::{EncodedConfirmedTransactionWithStatusMeta, TransactionStatus};
+use surfpool_types::{ExecutionCapture, ProfileResult};
 use svm::SurfnetSvm;
 
 use crate::{
     error::{SurfpoolError, SurfpoolResult},
-    types::{GeyserAccountUpdate, TransactionWithStatusMeta},
+    types::{GeyserAccountUpdate, TokenAccount, TransactionLoadedAddresses, TransactionWithStatusMeta},
 };
 
 pub mod locker;
@@ -160,6 +161,18 @@ pub enum SnapshotImportStatus {
     Failed,
 }
 
+pub struct ProfilingJob {
+    pub profiling_svm: SurfnetSvm,
+    pub transaction: VersionedTransaction,
+    pub transaction_accounts: Vec<Pubkey>,
+    pub loaded_addresses: Option<TransactionLoadedAddresses>,
+    pub accounts_before: Vec<Option<Account>>,
+    pub token_accounts_before: Vec<(usize, TokenAccount)>,
+    pub token_programs: Vec<Pubkey>,
+    pub pre_execution_capture: ExecutionCapture,
+    pub result_tx: Sender<Option<Vec<ProfileResult>>>,
+}
+
 #[derive(Debug, Clone, PartialEq)]
 pub enum SignatureSubscriptionType {
     Received,

crates/core/src/surfnet/svm.rs

@@ -98,7 +98,7 @@ use crate::{
     scenarios::TemplateRegistry,
     storage::{OverlayStorage, Storage, new_kv_store, new_kv_store_with_default},
     surfnet::{
-        LogsSubscriptionData, locker::is_supported_token_program, surfnet_lite_svm::SurfnetLiteSvm,
+        LogsSubscriptionData, ProfilingJob, locker::is_supported_token_program, surfnet_lite_svm::SurfnetLiteSvm
     },
     types::{
         GeyserAccountUpdate, MintAccount, SerializableAccountAdditionalData,
@@ -251,6 +251,7 @@ pub struct SurfnetSvm {
         Sender<TransactionStatusEvent>,
         Option<TransactionError>,
     )>,
+    pub profiling_job_tx: Option<Sender<ProfilingJob>>,
     pub perf_samples: VecDeque<RpcPerfSample>,
     pub transactions_processed: u64,
     pub latest_epoch_info: EpochInfo,
@@ -357,6 +358,7 @@ impl SurfnetSvm {
             inner: self.inner.clone_for_profiling(),
             remote_rpc_url: self.remote_rpc_url.clone(),
             chain_tip: self.chain_tip.clone(),
+            profiling_job_tx: self.profiling_job_tx.clone(),
 
             // Wrap all storage fields with OverlayStorage
             blocks: OverlayStorage::wrap(self.blocks.clone_box()),
@@ -567,6 +569,7 @@ impl SurfnetSvm {
             chain_tip,
             blocks: blocks_db,
             transactions: transactions_db,
+            profiling_job_tx: None,
             perf_samples: VecDeque::new(),
             transactions_processed,
             simnet_events_tx,

crates/core/src/tests/integration.rs

@@ -53,7 +53,7 @@ use crate::{
         minimal::MinimalClient,
         surfnet_cheatcodes::{SurfnetCheatcodes, SurfnetCheatcodesRpc},
     },
-    runloops::start_local_surfnet_runloop,
+    runloops::{setup_profiling, start_local_surfnet_runloop},
     storage::tests::TestType,
     surfnet::{SignatureSubscriptionType, locker::SurfnetSvmLocker, svm::SurfnetSvm},
     tests::helpers::get_free_port,
@@ -1552,6 +1552,7 @@ async fn test_profile_transaction_basic(test_type: TestType) {
     // Set up test environment
     let (svm_instance, _simnet_events_rx, _geyser_events_rx) = test_type.initialize_svm();
     let svm_locker = SurfnetSvmLocker::new(svm_instance);
+    setup_profiling(&svm_locker);
 
     // Set up test accounts
     let payer = Keypair::new();
@@ -1639,6 +1640,7 @@ async fn test_profile_transaction_basic(test_type: TestType) {
 async fn test_profile_transaction_multi_instruction_basic(test_type: TestType) {
     let (svm_instance, _simnet_events_rx, _geyser_events_rx) = test_type.initialize_svm();
     let svm_locker = SurfnetSvmLocker::new(svm_instance);
+    setup_profiling(&svm_locker);
 
     let payer = Keypair::new();
     let lamports_to_send = 1_000_000;
@@ -2037,6 +2039,7 @@ async fn test_profile_transaction_with_tag(test_type: TestType) {
     // Set up test environment
     let (svm_instance, _simnet_events_rx, _geyser_events_rx) = test_type.initialize_svm();
     let svm_locker = SurfnetSvmLocker::new(svm_instance);
+    setup_profiling(&svm_locker);
 
     // Set up test accounts
     let payer = Keypair::new();
@@ -2197,6 +2200,7 @@ async fn test_profile_transaction_with_tag(test_type: TestType) {
 async fn test_profile_transaction_token_transfer(test_type: TestType) {
     let (svm_instance, _simnet_events_rx, _geyser_events_rx) = test_type.initialize_svm();
     let svm_locker = SurfnetSvmLocker::new(svm_instance);
+    setup_profiling(&svm_locker);
 
     // Set up test accounts
     let payer = Keypair::new();
@@ -2637,6 +2641,7 @@ async fn test_profile_transaction_token_transfer(test_type: TestType) {
 async fn test_profile_transaction_insufficient_funds(test_type: TestType) {
     let (svm_instance, _simnet_events_rx, _geyser_events_rx) = test_type.initialize_svm();
     let svm_locker = SurfnetSvmLocker::new(svm_instance);
+    setup_profiling(&svm_locker);
 
     // Set up test accounts with insufficient funds
     let payer = Keypair::new();
@@ -2707,6 +2712,7 @@ async fn test_profile_transaction_insufficient_funds(test_type: TestType) {
 async fn test_profile_transaction_multi_instruction_failure(test_type: TestType) {
     let (svm_instance, _simnet_events_rx, _geyser_events_rx) = test_type.initialize_svm();
     let svm_locker = SurfnetSvmLocker::new(svm_instance);
+    setup_profiling(&svm_locker);
 
     // Set up test accounts
     let payer = Keypair::new();
@@ -2791,6 +2797,7 @@ async fn test_profile_transaction_multi_instruction_failure(test_type: TestType)
 async fn test_profile_transaction_with_encoding(test_type: TestType) {
     let (svm_instance, _simnet_events_rx, _geyser_events_rx) = test_type.initialize_svm();
     let svm_locker = SurfnetSvmLocker::new(svm_instance);
+    setup_profiling(&svm_locker);
 
     // Set up test accounts
     let payer = Keypair::new();
@@ -2863,6 +2870,7 @@ async fn test_profile_transaction_with_encoding(test_type: TestType) {
 async fn test_profile_transaction_with_tag_and_retrieval(test_type: TestType) {
     let (svm_instance, _simnet_events_rx, _geyser_events_rx) = test_type.initialize_svm();
     let svm_locker = SurfnetSvmLocker::new(svm_instance);
+    setup_profiling(&svm_locker);
 
     // Set up test accounts
     let payer = Keypair::new();
@@ -2967,6 +2975,7 @@ async fn test_profile_transaction_with_tag_and_retrieval(test_type: TestType) {
 async fn test_profile_transaction_empty_instruction(test_type: TestType) {
     let (svm_instance, _simnet_events_rx, _geyser_events_rx) = test_type.initialize_svm();
     let svm_locker = SurfnetSvmLocker::new(svm_instance);
+    setup_profiling(&svm_locker);
 
     // Set up test accounts
     let payer = Keypair::new();
@@ -3027,6 +3036,7 @@ async fn test_profile_transaction_empty_instruction(test_type: TestType) {
 async fn test_profile_transaction_versioned_message(test_type: TestType) {
     let (svm_instance, _simnet_events_rx, _geyser_events_rx) = test_type.initialize_svm();
     let svm_locker = SurfnetSvmLocker::new(svm_instance);
+    setup_profiling(&svm_locker);
 
     // Set up test accounts
     let payer = Keypair::new();
@@ -6951,6 +6961,7 @@ async fn test_profile_transaction_does_not_mutate_state(test_type: TestType) {
 
     // Create the locker and runloop context
     let svm_locker = SurfnetSvmLocker::new(svm_instance);
+    setup_profiling(&svm_locker);
     let (simnet_cmd_tx, _simnet_cmd_rx) = crossbeam_unbounded::<SimnetCommand>();
     let (plugin_cmd_tx, _plugin_cmd_rx) = crossbeam_unbounded::<PluginManagerCommand>();
 

crates/core/src/types.rs

@@ -1257,7 +1257,7 @@ impl std::fmt::Display for TimeTravelError {
 
 impl std::error::Error for TimeTravelError {}
 
-#[derive(Debug, Default)]
+#[derive(Debug, Default, Clone)]
 /// Tracks the loaded addresses with its associated index within an Address Lookup Table
 pub struct IndexedLoadedAddresses {
     pub writable: Vec<(u8, Pubkey)>,
@@ -1279,7 +1279,7 @@ impl IndexedLoadedAddresses {
     }
 }
 
-#[derive(Debug, Default)]
+#[derive(Debug, Default, Clone)]
 /// Maps an Address Lookup Table entry to its indexed loaded addresses
 pub struct TransactionLoadedAddresses(IndexMap<Pubkey, IndexedLoadedAddresses>);