[storage/archive] Add Lite Index + Remove Journal Padding

consensus/src/marshal/actor.rs

@@ -180,8 +180,9 @@ where
             partition_prefix: format!("{}-cache", config.partition_prefix.clone()),
             prunable_items_per_section: config.prunable_items_per_section,
             replay_buffer: config.replay_buffer,
-            write_buffer: config.write_buffer,
-            freezer_journal_buffer_pool: config.buffer_pool.clone(),
+            key_write_buffer: config.key_write_buffer,
+            value_write_buffer: config.value_write_buffer,
+            key_buffer_pool: config.buffer_pool.clone(),
         };
         let cache = cache::Manager::init(
             context.with_label("cache"),

consensus/src/marshal/cache.rs

@@ -28,8 +28,9 @@ pub(crate) struct Config {
     pub partition_prefix: String,
     pub prunable_items_per_section: NonZero<u64>,
     pub replay_buffer: NonZeroUsize,
-    pub write_buffer: NonZeroUsize,
-    pub freezer_journal_buffer_pool: PoolRef,
+    pub key_write_buffer: NonZeroUsize,
+    pub value_write_buffer: NonZeroUsize,
+    pub key_buffer_pool: PoolRef,
 }
 
 /// Prunable archives for a single epoch.
@@ -189,14 +190,16 @@ impl<R: Rng + Spawner + Metrics + Clock + Storage, B: Block, S: Scheme> Manager<
     ) -> prunable::Archive<TwoCap, R, B::Commitment, T> {
         let start = Instant::now();
         let cfg = prunable::Config {
-            partition: format!("{}-cache-{epoch}-{name}", self.cfg.partition_prefix),
             translator: TwoCap,
+            key_partition: format!("{}-cache-{epoch}-{name}-key", self.cfg.partition_prefix),
+            key_buffer_pool: self.cfg.key_buffer_pool.clone(),
+            value_partition: format!("{}-cache-{epoch}-{name}-value", self.cfg.partition_prefix),
             items_per_section: self.cfg.prunable_items_per_section,
             compression: None,
             codec_config,
-            buffer_pool: self.cfg.freezer_journal_buffer_pool.clone(),
             replay_buffer: self.cfg.replay_buffer,
-            write_buffer: self.cfg.write_buffer,
+            key_write_buffer: self.cfg.key_write_buffer,
+            value_write_buffer: self.cfg.value_write_buffer,
         };
         let archive = prunable::Archive::init(self.context.with_label(name), cfg)
             .await

consensus/src/marshal/config.rs

@@ -39,8 +39,11 @@ where
     /// The size of the replay buffer for storage archives.
     pub replay_buffer: NonZeroUsize,
 
-    /// The size of the write buffer for storage archives.
-    pub write_buffer: NonZeroUsize,
+    /// The size of the write buffer for the key journal of storage archives.
+    pub key_write_buffer: NonZeroUsize,
+
+    /// The size of the write buffer for the value journal of storage archives.
+    pub value_write_buffer: NonZeroUsize,
 
     /// Codec configuration for block type.
     pub block_codec_config: B::Cfg,

consensus/src/marshal/mod.rs

@@ -189,7 +189,8 @@ mod tests {
             partition_prefix: format!("validator-{}", validator.clone()),
             prunable_items_per_section: NZU64!(10),
             replay_buffer: NZUsize!(1024),
-            write_buffer: NZUsize!(1024),
+            key_write_buffer: NZUsize!(1024),
+            value_write_buffer: NZUsize!(1024),
             buffer_pool: PoolRef::new(PAGE_SIZE, PAGE_CACHE_SIZE),
         };
 
@@ -237,21 +238,27 @@ mod tests {
                 freezer_table_initial_size: 64,
                 freezer_table_resize_frequency: 10,
                 freezer_table_resize_chunk_size: 10,
-                freezer_journal_partition: format!(
-                    "{}-finalizations-by-height-freezer-journal",
+                freezer_key_partition: format!(
+                    "{}-finalizations-by-height-freezer-key",
                     config.partition_prefix
                 ),
-                freezer_journal_target_size: 1024,
-                freezer_journal_compression: None,
-                freezer_journal_buffer_pool: config.buffer_pool.clone(),
+                freezer_key_buffer_pool: config.buffer_pool.clone(),
+                freezer_value_partition: format!(
+                    "{}-finalizations-by-height-freezer-value",
+                    config.partition_prefix
+                ),
+                freezer_value_target_size: 1024,
+                freezer_value_compression: None,
                 ordinal_partition: format!(
                     "{}-finalizations-by-height-ordinal",
                     config.partition_prefix
                 ),
                 items_per_section: NZU64!(10),
                 codec_config: S::certificate_codec_config_unbounded(),
                 replay_buffer: config.replay_buffer,
-                write_buffer: config.write_buffer,
+                freezer_key_write_buffer: config.key_write_buffer,
+                freezer_value_write_buffer: config.value_write_buffer,
+                ordinal_write_buffer: config.key_write_buffer,
             },
         )
         .await
@@ -274,18 +281,24 @@ mod tests {
                 freezer_table_initial_size: 64,
                 freezer_table_resize_frequency: 10,
                 freezer_table_resize_chunk_size: 10,
-                freezer_journal_partition: format!(
-                    "{}-finalized_blocks-freezer-journal",
+                freezer_key_partition: format!(
+                    "{}-finalized_blocks-freezer-key",
+                    config.partition_prefix
+                ),
+                freezer_key_buffer_pool: config.buffer_pool.clone(),
+                freezer_value_partition: format!(
+                    "{}-finalized_blocks-freezer-value",
                     config.partition_prefix
                 ),
-                freezer_journal_target_size: 1024,
-                freezer_journal_compression: None,
-                freezer_journal_buffer_pool: config.buffer_pool.clone(),
+                freezer_value_target_size: 1024,
+                freezer_value_compression: None,
                 ordinal_partition: format!("{}-finalized_blocks-ordinal", config.partition_prefix),
                 items_per_section: NZU64!(10),
                 codec_config: config.block_codec_config,
                 replay_buffer: config.replay_buffer,
-                write_buffer: config.write_buffer,
+                freezer_key_write_buffer: config.key_write_buffer,
+                freezer_value_write_buffer: config.value_write_buffer,
+                ordinal_write_buffer: config.key_write_buffer,
             },
         )
         .await

examples/reshare/src/engine.rs

@@ -41,8 +41,8 @@ const PRUNABLE_ITEMS_PER_SECTION: NonZero<u64> = NZU64!(4_096);
 const IMMUTABLE_ITEMS_PER_SECTION: NonZero<u64> = NZU64!(262_144);
 const FREEZER_TABLE_RESIZE_FREQUENCY: u8 = 4;
 const FREEZER_TABLE_RESIZE_CHUNK_SIZE: u32 = 2u32.pow(16); // 3MB
-const FREEZER_JOURNAL_TARGET_SIZE: u64 = 1024 * 1024 * 1024; // 1GB
-const FREEZER_JOURNAL_COMPRESSION: Option<u8> = Some(3);
+const FREEZER_VALUE_TARGET_SIZE: u64 = 1024 * 1024 * 1024; // 1GB
+const FREEZER_VALUE_COMPRESSION: Option<u8> = Some(3);
 const REPLAY_BUFFER: NonZero<usize> = NZUsize!(8 * 1024 * 1024); // 8MB
 const WRITE_BUFFER: NonZero<usize> = NZUsize!(1024 * 1024); // 1MB
 const BUFFER_POOL_PAGE_SIZE: NonZero<usize> = NZUsize!(4_096); // 4KB
@@ -171,21 +171,27 @@ where
                 freezer_table_initial_size: config.freezer_table_initial_size,
                 freezer_table_resize_frequency: FREEZER_TABLE_RESIZE_FREQUENCY,
                 freezer_table_resize_chunk_size: FREEZER_TABLE_RESIZE_CHUNK_SIZE,
-                freezer_journal_partition: format!(
-                    "{}-finalizations-by-height-freezer-journal",
+                freezer_key_partition: format!(
+                    "{}-finalizations-by-height-freezer-key",
                     config.partition_prefix
                 ),
-                freezer_journal_target_size: FREEZER_JOURNAL_TARGET_SIZE,
-                freezer_journal_compression: FREEZER_JOURNAL_COMPRESSION,
-                freezer_journal_buffer_pool: buffer_pool.clone(),
+                freezer_key_buffer_pool: buffer_pool.clone(),
+                freezer_value_partition: format!(
+                    "{}-finalizations-by-height-freezer-value",
+                    config.partition_prefix
+                ),
+                freezer_value_target_size: FREEZER_VALUE_TARGET_SIZE,
+                freezer_value_compression: FREEZER_VALUE_COMPRESSION,
                 ordinal_partition: format!(
                     "{}-finalizations-by-height-ordinal",
                     config.partition_prefix
                 ),
                 items_per_section: IMMUTABLE_ITEMS_PER_SECTION,
                 codec_config: S::certificate_codec_config_unbounded(),
                 replay_buffer: REPLAY_BUFFER,
-                write_buffer: WRITE_BUFFER,
+                freezer_key_write_buffer: WRITE_BUFFER,
+                freezer_value_write_buffer: WRITE_BUFFER,
+                ordinal_write_buffer: WRITE_BUFFER,
             },
         )
         .await
@@ -208,18 +214,24 @@ where
                 freezer_table_initial_size: config.freezer_table_initial_size,
                 freezer_table_resize_frequency: FREEZER_TABLE_RESIZE_FREQUENCY,
                 freezer_table_resize_chunk_size: FREEZER_TABLE_RESIZE_CHUNK_SIZE,
-                freezer_journal_partition: format!(
-                    "{}-finalized_blocks-freezer-journal",
+                freezer_key_partition: format!(
+                    "{}-finalized_blocks-freezer-key",
+                    config.partition_prefix
+                ),
+                freezer_key_buffer_pool: buffer_pool.clone(),
+                freezer_value_partition: format!(
+                    "{}-finalized_blocks-freezer-value",
                     config.partition_prefix
                 ),
-                freezer_journal_target_size: FREEZER_JOURNAL_TARGET_SIZE,
-                freezer_journal_compression: FREEZER_JOURNAL_COMPRESSION,
-                freezer_journal_buffer_pool: buffer_pool.clone(),
+                freezer_value_target_size: FREEZER_VALUE_TARGET_SIZE,
+                freezer_value_compression: FREEZER_VALUE_COMPRESSION,
                 ordinal_partition: format!("{}-finalized_blocks-ordinal", config.partition_prefix),
                 items_per_section: IMMUTABLE_ITEMS_PER_SECTION,
                 codec_config: num_participants,
                 replay_buffer: REPLAY_BUFFER,
-                write_buffer: WRITE_BUFFER,
+                freezer_key_write_buffer: WRITE_BUFFER,
+                freezer_value_write_buffer: WRITE_BUFFER,
+                ordinal_write_buffer: WRITE_BUFFER,
             },
         )
         .await
@@ -259,7 +271,8 @@ where
                 prunable_items_per_section: PRUNABLE_ITEMS_PER_SECTION,
                 buffer_pool: buffer_pool.clone(),
                 replay_buffer: REPLAY_BUFFER,
-                write_buffer: WRITE_BUFFER,
+                key_write_buffer: WRITE_BUFFER,
+                value_write_buffer: WRITE_BUFFER,
                 block_codec_config: num_participants,
                 max_repair: MAX_REPAIR,
             },

runtime/src/deterministic.rs

@@ -941,6 +941,11 @@ impl Context {
         self.executor().auditor.clone()
     }
 
+    /// Compute a [Sha256] digest of all storage contents.
+    pub fn storage_audit(&self) -> Digest {
+        self.storage.inner().inner().audit()
+    }
+
     /// Register a DNS mapping for a hostname.
     ///
     /// If `addrs` is `None`, the mapping is removed.

runtime/src/storage/audited.rs

@@ -13,6 +13,11 @@ impl<S: crate::Storage> Storage<S> {
     pub const fn new(inner: S, auditor: Arc<Auditor>) -> Self {
         Self { inner, auditor }
     }
+
+    /// Get a reference to the inner storage.
+    pub const fn inner(&self) -> &S {
+        &self.inner
+    }
 }
 
 impl<S: crate::Storage> crate::Storage for Storage<S> {

runtime/src/storage/memory.rs

@@ -1,6 +1,7 @@
 use super::Header;
 use commonware_codec::Encode;
 use commonware_utils::{hex, StableBuf};
+use sha2::{Digest, Sha256};
 use std::{
     collections::BTreeMap,
     ops::RangeInclusive,
@@ -21,6 +22,24 @@ impl Default for Storage {
     }
 }
 
+impl Storage {
+    /// Compute a [Sha256] digest of all blob contents.
+    pub fn audit(&self) -> [u8; 32] {
+        let partitions = self.partitions.lock().unwrap();
+        let mut hasher = Sha256::new();
+
+        for (partition_name, blobs) in partitions.iter() {
+            for (blob_name, content) in blobs.iter() {
+                hasher.update(partition_name.as_bytes());
+                hasher.update(blob_name);
+                hasher.update(content);
+            }
+        }
+
+        hasher.finalize().into()
+    }
+}
+
 impl crate::Storage for Storage {
     type Blob = Blob;
 

runtime/src/storage/metered.rs

@@ -72,6 +72,11 @@ impl<S> Storage<S> {
             metrics: Metrics::new(registry).into(),
         }
     }
+
+    /// Get a reference to the inner storage.
+    pub const fn inner(&self) -> &S {
+        &self.inner
+    }
 }
 
 impl<S: crate::Storage> crate::Storage for Storage<S> {

storage/conformance.toml

@@ -1,11 +1,27 @@
+["commonware_storage::archive::conformance::ArchiveImmutable"]
+n_cases = 128
+hash = "1fb629cfd9c3dfacd97cf674147387b4c9c1e36a1d64568fa0f782f96589e2e9"
+
+["commonware_storage::archive::conformance::ArchivePrunable"]
+n_cases = 128
+hash = "674e81c769c06a3965dc691b1f8c0327374f427e8a4bf67895c6ad4e566fed20"
+
 ["commonware_storage::archive::immutable::storage::conformance::CodecConformance<Record>"]
 n_cases = 65536
-hash = "892ba87ed9fa60ee60f694f927de60c3ee7fcdddd5a04d330c7511533cc2ca8d"
+hash = "9a040034af8177874f3c8be9fc5d5148fd40bf4991a62ca72708ff9c74a3f558"
+
+["commonware_storage::archive::prunable::storage::conformance::CodecConformance<IndexEntry<U64>>"]
+n_cases = 65536
+hash = "96805f8ed19133a058b7fab313f35eaf0cd085d08c7600010c217d28b9348960"
 
 ["commonware_storage::archive::prunable::storage::conformance::CodecConformance<Record<U64,Vec<u8>>>"]
 n_cases = 65536
 hash = "e681f76832577cad99b881684c56d820f9059d117ed1fef9fa3c7735c2eb24f9"
 
+["commonware_storage::archive::prunable::storage::conformance::CodecConformance<Record<U64>>"]
+n_cases = 65536
+hash = "3cb6882637c1c1a929a50b3ab425311f3ef342184dc46a80b1eae616ca7b64a4"
+
 ["commonware_storage::bmt::tests::conformance::CodecConformance<Bounds<Sha256Digest>>"]
 n_cases = 65536
 hash = "927da8204f16522162168efcae8ca126b927c96fb08289e805859071647b9333"
@@ -18,33 +34,65 @@ hash = "fcb0baa781f374ca04f2c967abc3c82c7b99d26239b387472de23c85aebd2790"
 n_cases = 65536
 hash = "035babcc7aa34c3d4b263924865c4da8206b226e7c372484b89e5430166068c3"
 
-["commonware_storage::cache::storage::conformance::CodecConformance<Record<u64>>"]
-n_cases = 65536
-hash = "9add8634d4f06c3fb7f6dfd80630f38013ecb20849c18accd0579d7eb2ca11d5"
-
 ["commonware_storage::freezer::storage::conformance::CodecConformance<Checkpoint>"]
 n_cases = 65536
-hash = "2066c89982e914e02272e613a7577bd22d15f4f7ce0a03c6b2fb5d3bdaff78fd"
+hash = "b0979a7998b8bde23262563837f9f42a461a69b83ca095d50da9b0f8f52f2ed0"
 
 ["commonware_storage::freezer::storage::conformance::CodecConformance<Cursor>"]
 n_cases = 65536
-hash = "074b9674e0997e724949866ca8ae5c52a15d270140c1ae82626ea6d917e089f8"
+hash = "fa5e588e9074776440c9d7e46729cc60c8d0cea87e5afc4326640106abcea7ba"
 
 ["commonware_storage::freezer::storage::conformance::CodecConformance<Entry>"]
 n_cases = 65536
-hash = "8c43d6d3a0ffb749db46d02a461cdc131c14b331aed670ee36e76bd716cdbaec"
+hash = "572cd3da262e0bb13d2c6780c5be7bbadb21f93e4dbf2c787febecdba62671df"
+
+["commonware_storage::freezer::storage::conformance::CodecConformance<KeyEntry<U64>>"]
+n_cases = 65536
+hash = "1a6d7f060e48860493f25a91b92d8a3ce7d6d51c6b15b2483f58ac2edb8d6b8a"
 
 ["commonware_storage::freezer::storage::conformance::CodecConformance<Record<U64,U64>>"]
 n_cases = 65536
 hash = "b4f2eb51a158d964317fb1714dbe708ffbe6673453dc648eabdd72409fb30440"
 
+["commonware_storage::freezer::storage::conformance::CodecConformance<Record<U64>>"]
+n_cases = 65536
+hash = "13b3e99a8c74b50dc18150194a92306de670b94e6642758feb6d9b6e9881f827"
+
+["commonware_storage::journal::conformance::ContiguousFixed"]
+n_cases = 512
+hash = "134bb8b838241c2dedf98d96130f014bea19f1bc7580307c9798540466eb81c6"
+
+["commonware_storage::journal::conformance::ContiguousVariable"]
+n_cases = 512
+hash = "29d37f2309943dd27d4344710a900bb3b992c0a1089ff9734cddbfa78c039200"
+
 ["commonware_storage::journal::conformance::FixedJournal"]
 n_cases = 512
 hash = "9cd764e31b5dbc0bd78cd0908851ba1d645f083884beacd2c8a63f66de0fb9db"
 
+["commonware_storage::journal::conformance::SegmentedFixed"]
+n_cases = 512
+hash = "505611ba11d6380254c159eb6234f87cc19a62b0919bc96d59e83de498b458fa"
+
+["commonware_storage::journal::conformance::SegmentedFixedJournal"]
+n_cases = 512
+hash = "cc21c23946ca33bc828d4734e1871cd84a37ac359f8a957d82a636efe1f4f675"
+
+["commonware_storage::journal::conformance::SegmentedGlob"]
+n_cases = 512
+hash = "adb1efeef12c203c05879ce4d1d03ef443c767737a6c6b57433189100eec9197"
+
+["commonware_storage::journal::conformance::SegmentedOversized"]
+n_cases = 512
+hash = "b98d56d2eb039657b3452135666795eeeefdc83e9d6f3cb070e7ca114b4621a1"
+
+["commonware_storage::journal::conformance::SegmentedVariable"]
+n_cases = 512
+hash = "cd79e09ca53917f78c290e67efe08bf17b3ec0d0faf1b5f6507d4665749574b1"
+
 ["commonware_storage::journal::conformance::VariableJournal"]
 n_cases = 512
-hash = "c0af6899248693a3262f31b9a8554cd64c014d9b59f2514840c8828ad74ddf85"
+hash = "955f2d9497e33cc74eb7673000de0e23d01439d07378a092d78178668b8d8b3e"
 
 ["commonware_storage::mmr::proof::tests::conformance::CodecConformance<Proof<Sha256Digest>>"]
 n_cases = 65536