chore(ntx): replace in memory with sqlite database

CHANGELOG.md

@@ -15,6 +15,7 @@
 - Refactored NTX Builder actor state into `AccountDeltaTracker` and `NotePool` for clarity, and added tracing instrumentation to event broadcasting ([#1611](https://github.com/0xMiden/miden-node/pull/1611)).
 - Add #[track_caller] to tracing/logging helpers ([#1651](https://github.com/0xMiden/miden-node/pull/1651)).
 - Improved tracing span fields ([#1650](https://github.com/0xMiden/miden-node/pull/1650))
+- Replaced NTX Builder's in-memory state management with SQLite-backed persistence; account states, notes, and transaction effects are now stored in the database and inflight state is purged on startup ([#1662](https://github.com/0xMiden/miden-node/pull/1662)).
 
 ## v0.13.5 (TBD)
 

bin/node/src/commands/bundled.rs

@@ -315,6 +315,7 @@ impl BundledCommand {
                 store_ntx_builder_url,
                 block_producer_url,
                 validator_url,
+                &data_directory,
             );
 
             let id = join_set

bin/node/src/commands/mod.rs

@@ -1,4 +1,5 @@
 use std::num::NonZeroUsize;
+use std::path::{Path, PathBuf};
 use std::time::Duration;
 
 use miden_node_block_producer::{
@@ -78,19 +79,37 @@ pub struct NtxBuilderConfig {
         default_value_t = DEFAULT_NTX_SCRIPT_CACHE_SIZE
     )]
     pub script_cache_size: NonZeroUsize,
+
+    /// Directory for the ntx-builder's persistent database.
+    ///
+    /// If not set, defaults to the node's data directory.
+    #[arg(long = "ntx-builder.data-directory", value_name = "DIR")]
+    pub data_directory: Option<PathBuf>,
 }
 
 impl NtxBuilderConfig {
     /// Converts this CLI config into the ntx-builder's internal config.
+    ///
+    /// The `node_data_directory` is used as the default location for the ntx-builder's database
+    /// if `--ntx-builder.data-directory` is not explicitly set.
     pub fn into_builder_config(
         self,
         store_url: Url,
         block_producer_url: Url,
         validator_url: Url,
+        node_data_directory: &Path,
     ) -> miden_node_ntx_builder::NtxBuilderConfig {
-        miden_node_ntx_builder::NtxBuilderConfig::new(store_url, block_producer_url, validator_url)
-            .with_tx_prover_url(self.tx_prover_url)
-            .with_script_cache_size(self.script_cache_size)
+        let data_dir = self.data_directory.unwrap_or_else(|| node_data_directory.to_path_buf());
+        let database_filepath = data_dir.join("ntx-builder.sqlite3");
+
+        miden_node_ntx_builder::NtxBuilderConfig::new(
+            store_url,
+            block_producer_url,
+            validator_url,
+            database_filepath,
+        )
+        .with_tx_prover_url(self.tx_prover_url)
+        .with_script_cache_size(self.script_cache_size)
     }
 }
 

crates/ntx-builder/Cargo.toml

@@ -21,12 +21,12 @@ deadpool-sync              = { features = ["tracing"], workspace = true }
 diesel                     = { features = ["numeric", "sqlite"], workspace = true }
 diesel_migrations          = { features = ["sqlite"], workspace = true }
 futures                    = { workspace = true }
-indexmap                   = { workspace = true }
 miden-node-proto           = { workspace = true }
 miden-node-utils           = { workspace = true }
 miden-protocol             = { default-features = true, workspace = true }
 miden-remote-prover-client = { features = ["tx-prover"], workspace = true }
 miden-tx                   = { default-features = true, workspace = true }
+prost                      = { workspace = true }
 thiserror                  = { workspace = true }
 tokio                      = { features = ["rt-multi-thread"], workspace = true }
 tokio-stream               = { workspace = true }
@@ -39,5 +39,6 @@ url                        = { workspace = true }
 miden-node-test-macro = { path = "../test-macro" }
 miden-node-utils      = { features = ["testing"], workspace = true }
 miden-protocol        = { default-features = true, features = ["testing"], workspace = true }
-miden-standards       = { workspace = true }
+miden-standards       = { features = ["testing"], workspace = true }
+rand_chacha           = { workspace = true }
 rstest                = { workspace = true }

crates/ntx-builder/src/actor/account_effect.rs

@@ -0,0 +1,73 @@
+use miden_node_proto::domain::account::NetworkAccountId;
+use miden_protocol::account::delta::AccountUpdateDetails;
+use miden_protocol::account::{Account, AccountDelta, AccountId};
+
+// NETWORK ACCOUNT EFFECT
+// ================================================================================================
+
+/// Represents the effect of a transaction on a network account.
+#[derive(Clone)]
+pub enum NetworkAccountEffect {
+    Created(Account),
+    Updated(AccountDelta),
+}
+
+impl NetworkAccountEffect {
+    pub fn from_protocol(update: &AccountUpdateDetails) -> Option<Self> {
+        let update = match update {
+            AccountUpdateDetails::Private => return None,
+            AccountUpdateDetails::Delta(update) if update.is_full_state() => {
+                NetworkAccountEffect::Created(
+                    Account::try_from(update)
+                        .expect("Account should be derivable by full state AccountDelta"),
+                )
+            },
+            AccountUpdateDetails::Delta(update) => NetworkAccountEffect::Updated(update.clone()),
+        };
+
+        update.protocol_account_id().is_network().then_some(update)
+    }
+
+    pub fn network_account_id(&self) -> NetworkAccountId {
+        // SAFETY: This is a network account by construction.
+        self.protocol_account_id().try_into().unwrap()
+    }
+
+    fn protocol_account_id(&self) -> AccountId {
+        match self {
+            NetworkAccountEffect::Created(acc) => acc.id(),
+            NetworkAccountEffect::Updated(delta) => delta.id(),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use miden_protocol::block::BlockNumber;
+
+    #[rstest::rstest]
+    #[test]
+    #[case::all_zero(Some(BlockNumber::GENESIS), BlockNumber::GENESIS, 0, true)]
+    #[case::no_attempts(None, BlockNumber::GENESIS, 0, true)]
+    #[case::one_attempt(Some(BlockNumber::GENESIS), BlockNumber::from(2), 1, true)]
+    #[case::three_attempts(Some(BlockNumber::GENESIS), BlockNumber::from(3), 3, true)]
+    #[case::ten_attempts(Some(BlockNumber::GENESIS), BlockNumber::from(13), 10, true)]
+    #[case::twenty_attempts(Some(BlockNumber::GENESIS), BlockNumber::from(149), 20, true)]
+    #[case::one_attempt_false(Some(BlockNumber::GENESIS), BlockNumber::from(1), 1, false)]
+    #[case::three_attempts_false(Some(BlockNumber::GENESIS), BlockNumber::from(2), 3, false)]
+    #[case::ten_attempts_false(Some(BlockNumber::GENESIS), BlockNumber::from(12), 10, false)]
+    #[case::twenty_attempts_false(Some(BlockNumber::GENESIS), BlockNumber::from(148), 20, false)]
+    fn backoff_has_passed(
+        #[case] last_attempt_block_num: Option<BlockNumber>,
+        #[case] current_block_num: BlockNumber,
+        #[case] attempt_count: usize,
+        #[case] backoff_should_have_passed: bool,
+    ) {
+        use crate::actor::has_backoff_passed;
+
+        assert_eq!(
+            backoff_should_have_passed,
+            has_backoff_passed(current_block_num, last_attempt_block_num, attempt_count)
+        );
+    }
+}