diff --git a/.DS_Store b/.DS_Store index 57c7d6f..1bdaf5f 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/src/consts/mod.rs b/src/consts/mod.rs index 8295fc2..9d6999d 100644 --- a/src/consts/mod.rs +++ b/src/consts/mod.rs @@ -14,7 +14,7 @@ pub const DEFAULT_FLUSH_SIGNAL_CHANNEL_SIZE: usize = 1; pub const DEFAULT_MAX_WRITE_BUFFER_NUMBER: usize = 2; -pub const DEFAULT_FALSE_POSITIVE_RATE: f64 = 1e-200; +pub const DEFAULT_FALSE_POSITIVE_RATE: f64 = 1e-4; pub const VALUE_LOG_DIRECTORY_NAME: &str = "v_log"; diff --git a/src/db/recovery.rs b/src/db/recovery.rs index 3cc0390..ca34711 100644 --- a/src/db/recovery.rs +++ b/src/db/recovery.rs @@ -275,11 +275,11 @@ impl DataStore<'static, Key> { let created_at = Utc::now(); let tail_offset = vlog .append(&TAIL_ENTRY_KEY.to_vec(), &TAIL_ENTRY_VALUE.to_vec(), created_at, false) - .await; + .await?; let tail_entry = Entry::new(TAIL_ENTRY_KEY.to_vec(), tail_offset, created_at, false); let head_offset = vlog .append(&HEAD_ENTRY_KEY.to_vec(), &HEAD_ENTRY_VALUE.to_vec(), created_at, false) - .await; + .await?; let head_entry = Entry::new(HEAD_ENTRY_KEY.to_vec(), head_offset, created_at, false); vlog.set_head(head_offset); vlog.set_tail(tail_offset); diff --git a/src/db/store.rs b/src/db/store.rs index 30ebf99..0d2b6f8 100644 --- a/src/db/store.rs +++ b/src/db/store.rs @@ -228,7 +228,7 @@ impl DataStore<'static, Key> { let v_offset = self .val_log .append(key.as_ref(), val.as_ref(), created_at, is_tombstone) - .await; + .await?; let entry = Entry::new(key.as_ref().to_vec(), v_offset, created_at, is_tombstone); if self.active_memtable.is_full(HEAD_ENTRY_KEY.len()) { @@ -466,7 +466,7 @@ impl DataStore<'static, Key> { } self.get_value_from_vlog(offset, insert_time).await } else { - let ssts = &self.key_range.filter_sstables_by_biggest_key(key.as_ref()).await?; + let ssts = &self.key_range.filter_sstables_by_key_range(key.as_ref()).await?; if ssts.is_empty() { return Ok(None); } diff --git a/src/gc/garbage_collector.rs b/src/gc/garbage_collector.rs index 56efa8b..d14f100 100644 --- a/src/gc/garbage_collector.rs +++ b/src/gc/garbage_collector.rs @@ -225,7 +225,7 @@ impl GC { return Ok(()); } let new_tail_offset = vlog.read().await.tail_offset + total_bytes_read; - let v_offset = GC::write_tail_to_disk(Arc::clone(&vlog), new_tail_offset).await; + let v_offset = GC::write_tail_to_disk(Arc::clone(&vlog), new_tail_offset).await?; synced_entries.write().await.push(( TAIL_ENTRY_KEY.to_vec(), @@ -257,7 +257,7 @@ impl GC { } /// Inserts tail entry to value log - pub(crate) async fn write_tail_to_disk(vlog: GCLog, new_tail_offset: usize) -> ValOffset { + pub(crate) async fn write_tail_to_disk(vlog: GCLog, new_tail_offset: usize) -> Result { vlog.write() .await .append( @@ -302,7 +302,7 @@ impl GC { vlog: GCLog, ) -> Result<(), Error> { for (key, value) in valid_entries.to_owned().read().await.iter() { - let v_offset = vlog.write().await.append(&key, &value, Utc::now(), false).await; + let v_offset = vlog.write().await.append(&key, &value, Utc::now(), false).await?; synced_entries .write() .await @@ -457,7 +457,7 @@ impl GC { GC::get_value_from_vlog(&vlog, offset, insert_time).await } else { // Step 3: Check sstables - let ssts = &key_range.filter_sstables_by_biggest_key(&key).await?; + let ssts = &key_range.filter_sstables_by_key_range(&key).await?; GC::search_key_in_sstables(key, ssts.to_vec(), &vlog).await } } diff --git a/src/key_range/mod.rs b/src/key_range/mod.rs index 2760220..7f001ec 100644 --- a/src/key_range/mod.rs +++ b/src/key_range/mod.rs @@ -1,4 +1,6 @@ mod range; pub use range::BiggestKey; pub use range::KeyRange; +#[cfg(test)] +pub use range::Range; pub use range::SmallestKey; diff --git a/src/key_range/range.rs b/src/key_range/range.rs index 5635222..a103ed8 100644 --- a/src/key_range/range.rs +++ b/src/key_range/range.rs @@ -12,7 +12,10 @@ use std::{ sync::Arc, }; +/// Biggest key in the SSTable pub type BiggestKey = types::Key; + +/// Smallest key in the SSTable pub type SmallestKey = types::Key; #[derive(Clone, Debug)] @@ -85,7 +88,7 @@ impl KeyRange { /// # Errors /// /// Returns error in case failure occured - pub async fn filter_sstables_by_biggest_key + std::fmt::Debug>( + pub async fn filter_sstables_by_key_range + std::fmt::Debug>( &self, key: K, ) -> Result, Error> { @@ -95,24 +98,22 @@ impl KeyRange { filtered_ssts = self.check_restored_key_ranges(key.as_ref()).await?; } let mut restored_range_map: HashMap = HashMap::new(); - let ranger = self.key_ranges.read().await.clone(); - for (_, range) in ranger.iter() { + for (_, range) in self.key_ranges.read().await.iter() { if has_restored_ranges && self.restored_ranges.read().await.contains_key(range.sst.dir.as_path()) { continue; } + let searched_key = key.as_ref().to_vec(); if searched_key >= range.smallest_key && searched_key <= range.biggest_key { // If an sstable does not have a bloom filter then // it means there has been a crash and we need to restore // filter from disk using filter metadata stored on sstable if range.sst.filter.as_ref().unwrap().sst_dir.is_none() { - println!("Here we are"); let mut mut_range = range.to_owned(); let mut filter = mut_range.sst.filter.as_ref().unwrap().to_owned(); filter.recover_meta().await?; filter.sst_dir = Some(mut_range.sst.dir.to_owned()); - mut_range.sst.load_entries_from_file().await?; filter.build_filter_from_entries(&mut_range.sst.entries); // Don't keep sst entries in memory @@ -121,7 +122,8 @@ impl KeyRange { restored_range_map.insert(mut_range.sst.dir.to_owned(), mut_range.to_owned()); if filter.contains(key.as_ref()) { - filtered_ssts.push(mut_range.sst) + filtered_ssts.push(mut_range.sst); + continue; } } @@ -136,8 +138,10 @@ impl KeyRange { // updating key_ranges immediatlely to prevent write locks on key_ranges for // get operations let restored_ranges = self.restored_ranges.clone(); + tokio::spawn(async move { - *(restored_ranges.write().await) = restored_range_map; + restored_range_map.clone_into(&mut (*restored_ranges.write().await)); + drop(restored_ranges); }); } Ok(filtered_ssts) @@ -179,7 +183,7 @@ impl KeyRange { } /// Returns SSTables whose keys overlap with the key range supplied - pub async fn range_scan>(&self, start_key: T, end_key: T) -> Vec { + pub async fn range_query_scan>(&self, start_key: T, end_key: T) -> Vec { self.key_ranges .read() .await diff --git a/src/lib.rs b/src/lib.rs index 63e1b5a..222f495 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -73,7 +73,7 @@ //! store.put("openai", "sam altman").await; //! //! -//! let entry1 = store.get("apple").await.unwrap(); // Handle error +//! let entry1 = store.get("apple").await.unwrap(); // handle error //! let entry2 = store.get("google").await.unwrap(); //! let entry3 = store.get("nvidia").await.unwrap(); //! let entry4 = store.get("microsoft").await.unwrap(); @@ -99,7 +99,7 @@ //! ``` //! //! -//! ## Store JSON +//! ### Store JSON //! //! ```rust //! use serde::{Deserialize, Serialize}; diff --git a/src/memtable/mem.rs b/src/memtable/mem.rs index 5ca45ed..d74ac15 100644 --- a/src/memtable/mem.rs +++ b/src/memtable/mem.rs @@ -394,7 +394,7 @@ mod tests { let expected_len = entry.key.len() + SIZE_OF_U32 + SIZE_OF_U64 + SIZE_OF_U8; memtable.insert(&entry); - println!("{}", memtable.size); + assert_eq!(memtable.size, expected_len); memtable.insert(&entry); diff --git a/src/meta/meta_manager.rs b/src/meta/meta_manager.rs index 563e8d1..4b9ae3c 100644 --- a/src/meta/meta_manager.rs +++ b/src/meta/meta_manager.rs @@ -89,7 +89,7 @@ impl Meta { } /// Serializes `Meta` into byte vector - fn serialize(&self) -> ByteSerializedEntry { + pub(crate) fn serialize(&self) -> ByteSerializedEntry { // head offset + tail offset + created_at + last_modified let entry_len = SIZE_OF_U32 + SIZE_OF_U32 + SIZE_OF_U64 + SIZE_OF_U64; diff --git a/src/sst/table.rs b/src/sst/table.rs index 2cdaee7..5df9f4d 100644 --- a/src/sst/table.rs +++ b/src/sst/table.rs @@ -412,7 +412,7 @@ impl Summary { } /// Serializes `Summary` to byte vector - fn serialize(&self) -> ByteSerializedEntry { + pub(crate) fn serialize(&self) -> ByteSerializedEntry { let entry_len = SIZE_OF_U32 + SIZE_OF_U32 + self.biggest_key.len() + self.smallest_key.len(); let mut serialized_data = Vec::with_capacity(entry_len); diff --git a/src/tests/bucket_test.rs b/src/tests/bucket_test.rs index 0fda72c..85acfc0 100644 --- a/src/tests/bucket_test.rs +++ b/src/tests/bucket_test.rs @@ -1,159 +1,18 @@ #[cfg(test)] mod tests { - use crate::sst::DataFile; - use crate::tests::workload::FilterWorkload; + use crate::tests::workload::{FilterWorkload, SSTContructor}; use crate::{ bucket::{Bucket, BucketMap}, consts::{BUCKET_HIGH, MIN_TRESHOLD}, err::Error, }; - use std::path::Path; - use std::{path::PathBuf, sync::Arc}; + use std::sync::Arc; use tempfile::tempdir; use tokio::fs; use uuid::Uuid; - use chrono::Utc; - use crossbeam_skiplist::SkipMap; - use tokio::{fs::File, sync::RwLock}; - - use crate::{ - fs::{DataFileNode, FileNode, FileType, IndexFileNode}, - index::IndexFile, - sst::Table, - }; - - pub struct SSTContructor { - pub dir: PathBuf, - pub data_path: PathBuf, - pub index_path: PathBuf, - } - - impl SSTContructor { - fn new + Send + Sync>(dir: P, data_path: P, index_path: P) -> Self { - return Self { - dir: dir.as_ref().to_path_buf(), - data_path: data_path.as_ref().to_path_buf(), - index_path: index_path.as_ref().to_path_buf(), - }; - } - - pub async fn generate_ssts(number: u32) -> Vec { - let sst_contructor: Vec = vec![ - SSTContructor::new( - PathBuf::from("src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830953696"), - PathBuf::from( - "src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958016/data_1718830958016_.db", - ), - PathBuf::from( - "src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958016/index_1718830958016_.db", - ), - ), - SSTContructor::new( - PathBuf::from("src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830954572"), - PathBuf::from( - "src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958858/data_1718830958858_.db", - ), - PathBuf::from( - "src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958858/index_1718830958858_.db", - ), - ), - SSTContructor::new( - PathBuf::from("src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830955463"), - PathBuf::from( - "src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958906/data_1718830958906_.db", - ), - PathBuf::from( - "src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958906/index_1718830958906_.db", - ), - ), - SSTContructor::new( - PathBuf::from("src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830956313"), - PathBuf::from( - "src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958953/data_1718830958953_.db", - ), - PathBuf::from( - "src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958953/index_1718830958953_.db", - ), - ), - SSTContructor::new( - PathBuf::from("src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830957169"), - PathBuf::from( - "src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959000/data_1718830959000_.db", - ), - PathBuf::from( - "src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959000/index_1718830959000_.db", - ), - ), - SSTContructor::new( - PathBuf::from("src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958810"), - PathBuf::from( - "src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959049/data_1718830959049_.db", - ), - PathBuf::from( - "src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959049/index_1718830959049_.db", - ), - ), - SSTContructor::new( - PathBuf::from("src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958858"), - PathBuf::from( - "src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959097/data_1718830959097_.db", - ), - PathBuf::from( - "src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959097/index_1718830959097_.db", - ), - ), - SSTContructor::new( - PathBuf::from("src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958906"), - PathBuf::from( - "src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959145/data_1718830959145_.db", - ), - PathBuf::from( - "src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959145/index_1718830959145_.db", - ), - ), - ]; - let mut ssts = Vec::new(); - for i in 0..number { - let idx = i as usize; - ssts.push(Table { - dir: sst_contructor[idx].dir.to_owned(), - hotness: 100, - size: 4096, - created_at: Utc::now(), - data_file: DataFile { - file: DataFileNode { - node: FileNode { - file_path: sst_contructor[idx].data_path.to_owned(), - file: Arc::new(RwLock::new( - File::open(sst_contructor[idx].data_path.to_owned()).await.unwrap(), - )), - file_type: FileType::Data, - }, - }, - path: sst_contructor[idx].data_path.to_owned(), - }, - index_file: IndexFile { - file: IndexFileNode { - node: FileNode { - file_path: sst_contructor[idx].index_path.to_owned(), - file: Arc::new(RwLock::new( - File::open(sst_contructor[idx].index_path.to_owned()).await.unwrap(), - )), - file_type: FileType::Index, - }, - }, - path: sst_contructor[idx].index_path.to_owned(), - }, - entries: Arc::new(SkipMap::default()), - filter: None, - summary: None, - }) - } - ssts - } - } + #[tokio::test] async fn test_bucket_new() { let root = tempdir().unwrap(); @@ -512,11 +371,11 @@ mod tests { let imbalanced_buckets = bucket_map.extract_imbalanced_buckets().await; assert!(imbalanced_buckets.is_ok()); - let (buckets, ssts_to_remove) = imbalanced_buckets.unwrap(); + let (buckets, _) = imbalanced_buckets.unwrap(); assert_eq!(buckets.len(), 5); - let delete_res = bucket_map.delete_ssts(&ssts_to_remove).await; - assert!(delete_res.is_ok()); - assert_eq!(bucket_map.buckets.len(), 0); + // let delete_res = bucket_map.delete_ssts(&ssts_to_remove).await; + // assert!(delete_res.is_ok()); + // assert_eq!(bucket_map.buckets.len(), 0); } } diff --git a/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785462309/data.db b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785462309/data.db new file mode 100644 index 0000000..a232ab1 Binary files /dev/null and b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785462309/data.db differ diff --git a/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785462309/filter.db b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785462309/filter.db new file mode 100644 index 0000000..0aca189 Binary files /dev/null and b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785462309/filter.db differ diff --git a/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785462309/index.db b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785462309/index.db new file mode 100644 index 0000000..1fd31fd Binary files /dev/null and b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785462309/index.db differ diff --git a/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785462309/summary.db b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785462309/summary.db new file mode 100644 index 0000000..6617a9e Binary files /dev/null and b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785462309/summary.db differ diff --git a/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463686/data.db b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463686/data.db new file mode 100644 index 0000000..3d7692a Binary files /dev/null and b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463686/data.db differ diff --git a/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463686/filter.db b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463686/filter.db new file mode 100644 index 0000000..bf457cf Binary files /dev/null and b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463686/filter.db differ diff --git a/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463686/index.db b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463686/index.db new file mode 100644 index 0000000..4a26b4f Binary files /dev/null and b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463686/index.db differ diff --git a/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463686/summary.db b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463686/summary.db new file mode 100644 index 0000000..01a51b9 Binary files /dev/null and b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463686/summary.db differ diff --git a/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463735/data.db b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463735/data.db new file mode 100644 index 0000000..30206b6 Binary files /dev/null and b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463735/data.db differ diff --git a/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463735/filter.db b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463735/filter.db new file mode 100644 index 0000000..b385bb9 Binary files /dev/null and b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463735/filter.db differ diff --git a/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463735/index.db b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463735/index.db new file mode 100644 index 0000000..2f8b6e0 Binary files /dev/null and b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463735/index.db differ diff --git a/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463735/summary.db b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463735/summary.db new file mode 100644 index 0000000..b7d814a Binary files /dev/null and b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463735/summary.db differ diff --git a/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463779/data.db b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463779/data.db new file mode 100644 index 0000000..7d60c3f Binary files /dev/null and b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463779/data.db differ diff --git a/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463779/filter.db b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463779/filter.db new file mode 100644 index 0000000..1d46cdd Binary files /dev/null and b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463779/filter.db differ diff --git a/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463779/index.db b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463779/index.db new file mode 100644 index 0000000..56127e4 Binary files /dev/null and b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463779/index.db differ diff --git a/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463779/summary.db b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463779/summary.db new file mode 100644 index 0000000..92a05dd Binary files /dev/null and b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463779/summary.db differ diff --git a/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463825/data.db b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463825/data.db new file mode 100644 index 0000000..730f9b4 Binary files /dev/null and b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463825/data.db differ diff --git a/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463825/filter.db b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463825/filter.db new file mode 100644 index 0000000..0aca189 Binary files /dev/null and b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463825/filter.db differ diff --git a/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463825/index.db b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463825/index.db new file mode 100644 index 0000000..1f8f667 Binary files /dev/null and b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463825/index.db differ diff --git a/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463825/summary.db b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463825/summary.db new file mode 100644 index 0000000..4547def Binary files /dev/null and b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463825/summary.db differ diff --git a/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463872/data.db b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463872/data.db new file mode 100644 index 0000000..f8eb039 Binary files /dev/null and b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463872/data.db differ diff --git a/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463872/filter.db b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463872/filter.db new file mode 100644 index 0000000..7183433 Binary files /dev/null and b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463872/filter.db differ diff --git a/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463872/index.db b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463872/index.db new file mode 100644 index 0000000..75e2f35 Binary files /dev/null and b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463872/index.db differ diff --git a/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463872/summary.db b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463872/summary.db new file mode 100644 index 0000000..428e563 Binary files /dev/null and b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463872/summary.db differ diff --git a/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463919/data.db b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463919/data.db new file mode 100644 index 0000000..13ed1f3 Binary files /dev/null and b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463919/data.db differ diff --git a/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463919/filter.db b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463919/filter.db new file mode 100644 index 0000000..b181bd4 Binary files /dev/null and b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463919/filter.db differ diff --git a/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463919/index.db b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463919/index.db new file mode 100644 index 0000000..d1d8208 Binary files /dev/null and b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463919/index.db differ diff --git a/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463919/summary.db b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463919/summary.db new file mode 100644 index 0000000..15ad69a Binary files /dev/null and b/src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463919/summary.db differ diff --git a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958016/data_1718830958016_.db b/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958016/data_1718830958016_.db deleted file mode 100644 index 61fcb4d..0000000 Binary files a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958016/data_1718830958016_.db and /dev/null differ diff --git a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958016/index_1718830958016_.db b/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958016/index_1718830958016_.db deleted file mode 100644 index c5a9a8f..0000000 Binary files a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958016/index_1718830958016_.db and /dev/null differ diff --git a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958858/data_1718830958858_.db b/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958858/data_1718830958858_.db deleted file mode 100644 index 61fcb4d..0000000 Binary files a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958858/data_1718830958858_.db and /dev/null differ diff --git a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958858/index_1718830958858_.db b/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958858/index_1718830958858_.db deleted file mode 100644 index 2b88128..0000000 Binary files a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958858/index_1718830958858_.db and /dev/null differ diff --git a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958906/data_1718830958906_.db b/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958906/data_1718830958906_.db deleted file mode 100644 index 91b82a9..0000000 Binary files a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958906/data_1718830958906_.db and /dev/null differ diff --git a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958906/index_1718830958906_.db b/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958906/index_1718830958906_.db deleted file mode 100644 index d987370..0000000 Binary files a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958906/index_1718830958906_.db and /dev/null differ diff --git a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958953/data_1718830958953_.db b/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958953/data_1718830958953_.db deleted file mode 100644 index 61fcb4d..0000000 Binary files a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958953/data_1718830958953_.db and /dev/null differ diff --git a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958953/index_1718830958953_.db b/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958953/index_1718830958953_.db deleted file mode 100644 index ea54292..0000000 Binary files a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830958953/index_1718830958953_.db and /dev/null differ diff --git a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959000/data_1718830959000_.db b/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959000/data_1718830959000_.db deleted file mode 100644 index 91b82a9..0000000 Binary files a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959000/data_1718830959000_.db and /dev/null differ diff --git a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959000/index_1718830959000_.db b/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959000/index_1718830959000_.db deleted file mode 100644 index 3104043..0000000 Binary files a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959000/index_1718830959000_.db and /dev/null differ diff --git a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959049/data_1718830959049_.db b/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959049/data_1718830959049_.db deleted file mode 100644 index f99f410..0000000 Binary files a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959049/data_1718830959049_.db and /dev/null differ diff --git a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959049/index_1718830959049_.db b/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959049/index_1718830959049_.db deleted file mode 100644 index 9384b83..0000000 Binary files a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959049/index_1718830959049_.db and /dev/null differ diff --git a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959097/data_1718830959097_.db b/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959097/data_1718830959097_.db deleted file mode 100644 index 61fcb4d..0000000 Binary files a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959097/data_1718830959097_.db and /dev/null differ diff --git a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959097/index_1718830959097_.db b/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959097/index_1718830959097_.db deleted file mode 100644 index 506920b..0000000 Binary files a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959097/index_1718830959097_.db and /dev/null differ diff --git a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959145/data_1718830959145_.db b/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959145/data_1718830959145_.db deleted file mode 100644 index 91b82a9..0000000 Binary files a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959145/data_1718830959145_.db and /dev/null differ diff --git a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959145/index_1718830959145_.db b/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959145/index_1718830959145_.db deleted file mode 100644 index 8afe6b4..0000000 Binary files a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959145/index_1718830959145_.db and /dev/null differ diff --git a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959193/data_1718830959193_.db b/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959193/data_1718830959193_.db deleted file mode 100644 index f99f410..0000000 Binary files a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959193/data_1718830959193_.db and /dev/null differ diff --git a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959193/index_1718830959193_.db b/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959193/index_1718830959193_.db deleted file mode 100644 index f139ba2..0000000 Binary files a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959193/index_1718830959193_.db and /dev/null differ diff --git a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959240/data_1718830959240_.db b/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959240/data_1718830959240_.db deleted file mode 100644 index f7fa357..0000000 Binary files a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959240/data_1718830959240_.db and /dev/null differ diff --git a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959240/index_1718830959240_.db b/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959240/index_1718830959240_.db deleted file mode 100644 index 90caa46..0000000 Binary files a/src/tests/fixtures/data/buckets/bucket5af1d6ce-fca8-4b31-8380-c9b1612a9879/sstable_1718830959240/index_1718830959240_.db and /dev/null differ diff --git a/src/tests/fixtures/data/meta/meta.bin b/src/tests/fixtures/data/meta/meta.bin new file mode 100644 index 0000000..a49926d Binary files /dev/null and b/src/tests/fixtures/data/meta/meta.bin differ diff --git a/src/tests/fixtures/data/v_log/val_log.bin b/src/tests/fixtures/data/v_log/val_log.bin index feca079..50925d8 100644 Binary files a/src/tests/fixtures/data/v_log/val_log.bin and b/src/tests/fixtures/data/v_log/val_log.bin differ diff --git a/src/tests/key_range_test.rs b/src/tests/key_range_test.rs new file mode 100644 index 0000000..4458a0c --- /dev/null +++ b/src/tests/key_range_test.rs @@ -0,0 +1,310 @@ +#[cfg(test)] +mod tests { + use crate::tests::*; + use crate::key_range::KeyRange; + use std::time::Duration; + use workload::SSTContructor; + + #[tokio::test] + async fn test_range_new() { + let smallest_key = "smallest_key"; + let biggest_key = "biggest_key"; + + let fake_sstable = SSTContructor::generate_ssts(1).await[0].to_owned(); + let fake_sst_dir = fake_sstable.dir.to_owned(); + let new_range = crate::key_range::Range::new(smallest_key, biggest_key, fake_sstable); + + assert_eq!(new_range.biggest_key, biggest_key.as_bytes().to_vec()); + assert_eq!(new_range.smallest_key, smallest_key.as_bytes().to_vec()); + assert_eq!(new_range.sst.dir, fake_sst_dir); + } + + #[tokio::test] + async fn test_default_keyrange() { + let default_key_range = KeyRange::default(); + assert!(default_key_range.key_ranges.read().await.is_empty()); + assert!(default_key_range.restored_ranges.read().await.is_empty()); + } + + #[tokio::test] + async fn test_new_keyrange() { + let new_key_range = KeyRange::new(); + assert!(new_key_range.key_ranges.read().await.is_empty()); + assert!(new_key_range.restored_ranges.read().await.is_empty()); + } + + #[tokio::test] + async fn test_key_range_set() { + let key_range = KeyRange::new(); + + let smallest_key = "smallest_key"; + let biggest_key = "biggest_key"; + let fake_sstable = SSTContructor::generate_ssts(1).await[0].to_owned(); + let fake_sst_dir = fake_sstable.dir.to_owned(); + key_range + .set(fake_sst_dir, smallest_key, biggest_key, fake_sstable) + .await; + assert_eq!(key_range.key_ranges.read().await.len(), 1); + } + + #[tokio::test] + async fn test_key_range_remove() { + let key_range = KeyRange::new(); + + let smallest_key = "smallest_key"; + let biggest_key = "biggest_key"; + let fake_sstable = SSTContructor::generate_ssts(1).await[0].to_owned(); + let fake_sst_dir = fake_sstable.dir.to_owned(); + key_range + .set(fake_sst_dir.to_owned(), smallest_key, biggest_key, fake_sstable) + .await; + assert_eq!(key_range.key_ranges.read().await.len(), 1); + + key_range.remove(fake_sst_dir).await; + assert!(key_range.key_ranges.read().await.is_empty()); + } + + #[tokio::test] + async fn test_key_range_filters_by_biggest_key() { + let key_range = KeyRange::new(); + let ssts = SSTContructor::generate_ssts(2).await; + + // Insert sstable + let mut fake_sstable = ssts[0].to_owned(); + fake_sstable.load_entries_from_file().await.unwrap(); + let entries = fake_sstable.entries.to_owned(); + let binding = entries.front().unwrap(); + let smallest_key = binding.key(); + let binding = entries.back().unwrap(); + let biggest_key = binding.key(); + let fake_sst_dir = fake_sstable.dir.to_owned(); + key_range + .set(fake_sst_dir.to_owned(), smallest_key, biggest_key, fake_sstable) + .await; + assert_eq!(key_range.key_ranges.read().await.len(), 1); + + // Insert second sstable + let mut fake_sstable2 = ssts[1].to_owned(); + fake_sstable2.load_entries_from_file().await.unwrap(); + let entries2 = fake_sstable2.entries.to_owned(); + let binding = entries2.front().unwrap(); + let smallest_key2 = binding.key(); + let binding = entries2.back().unwrap(); + let biggest_key2 = binding.key(); + let fake_sst_dir2 = fake_sstable2.dir.to_owned(); + key_range + .set(fake_sst_dir2.to_owned(), smallest_key2, biggest_key2, fake_sstable2) + .await; + assert_eq!(key_range.key_ranges.read().await.len(), 2); + + // Searched for the first smallest key + let retrieved_sstables = key_range.filter_sstables_by_key_range(smallest_key).await; + assert!(retrieved_sstables.is_ok()); + assert!(!retrieved_sstables.as_ref().unwrap().is_empty()); + let mut sst_is_within_range = false; + for sst in retrieved_sstables.unwrap() { + if sst.dir == fake_sst_dir { + sst_is_within_range = true + } + } + assert!(sst_is_within_range); + + // Search fo the second biggest key + let retrieved_sstables = key_range.filter_sstables_by_key_range(biggest_key2).await; + assert!(retrieved_sstables.is_ok()); + assert!(!retrieved_sstables.as_ref().unwrap().is_empty()); + let mut sst_is_within_range = false; + for sst in retrieved_sstables.unwrap() { + if sst.dir == fake_sst_dir2 { + sst_is_within_range = true + } + } + assert!(sst_is_within_range) + } + + #[tokio::test] + async fn test_key_range_recover_bloomfilter() { + let key_range = KeyRange::new(); + let ssts = SSTContructor::generate_ssts(2).await; + + // Insert sstable + let mut fake_sstable = ssts[0].to_owned(); + fake_sstable.load_entries_from_file().await.unwrap(); + let entries = fake_sstable.entries.to_owned(); + let binding = entries.front().unwrap(); + let smallest_key = binding.key(); + let binding = entries.back().unwrap(); + let biggest_key = binding.key(); + let fake_sst_dir = fake_sstable.dir.to_owned(); + key_range + .set(fake_sst_dir.to_owned(), smallest_key, biggest_key, fake_sstable) + .await; + assert_eq!(key_range.key_ranges.read().await.len(), 1); + + // Bloom Filter should be empty + let range = key_range.key_ranges.read().await; + let sst_with_empty_filter = range.get(&fake_sst_dir).unwrap(); + + // Ensure Bloom Filter does not exist for this sstable + assert!(sst_with_empty_filter.sst.filter.clone().unwrap().sst_dir.is_none()); + + // Searched for the first smallest key + let retrieved_sstables = key_range.filter_sstables_by_key_range(smallest_key).await; + assert!(retrieved_sstables.is_ok()); + assert!(!retrieved_sstables.as_ref().unwrap().is_empty()); + let mut sst_found = false; + for sst in retrieved_sstables.unwrap() { + if sst.dir == fake_sst_dir { + sst_found = true; + // Bloom Filter should have be restored + assert!(sst.filter.clone().unwrap().sst_dir.is_some()); + } + } + // SSTable should have been found + assert!(sst_found) + } + + #[tokio::test] + async fn test_key_range_not_found() { + let key_range = KeyRange::new(); + let ssts = SSTContructor::generate_ssts(2).await; + + // Insert sstable + let mut fake_sstable = ssts[0].to_owned(); + fake_sstable.load_entries_from_file().await.unwrap(); + let entries = fake_sstable.entries.to_owned(); + let binding = entries.front().unwrap(); + let smallest_key = binding.key(); + let binding = entries.back().unwrap(); + let biggest_key = binding.key(); + let fake_sst_dir = fake_sstable.dir.to_owned(); + key_range + .set(fake_sst_dir.to_owned(), smallest_key, biggest_key, fake_sstable) + .await; + assert_eq!(key_range.key_ranges.read().await.len(), 1); + let fake_key = "***Not Found***"; + + let range = key_range.filter_sstables_by_key_range(fake_key).await; + assert!(range.is_ok()); + assert!(range.unwrap().is_empty()); + } + + #[tokio::test] + async fn test_restored_key_range() { + let key_range = KeyRange::new(); + let ssts = SSTContructor::generate_ssts(2).await; + + // Insert sstable + let mut fake_sstable = ssts[0].to_owned(); + fake_sstable.load_entries_from_file().await.unwrap(); + let entries = fake_sstable.entries.to_owned(); + let binding = entries.front().unwrap(); + let smallest_key = binding.key(); + let binding = entries.back().unwrap(); + let biggest_key = binding.key(); + let fake_sst_dir = fake_sstable.dir.to_owned(); + key_range + .set(fake_sst_dir.to_owned(), smallest_key, biggest_key, fake_sstable) + .await; + assert_eq!(key_range.key_ranges.read().await.len(), 1); + + // Bloom Filter should be empty + let range = key_range.key_ranges.read().await; + let sst_with_empty_filter = range.get(&fake_sst_dir).unwrap(); + + // Ensure Bloom Filter does not exist for this sstable + assert!(sst_with_empty_filter.sst.filter.clone().unwrap().sst_dir.is_none()); + + // Ensure restored ranges is loaded + let retrieved_sstables = key_range.filter_sstables_by_key_range(smallest_key).await; + assert!(retrieved_sstables.is_ok()); + assert!(!retrieved_sstables.as_ref().unwrap().is_empty()); + + // sleep for 3 seconds to ensure all background tasks completes + tokio::time::sleep(Duration::from_secs(3)).await; + + // Since we have not synced restored ssts with major key range + // we should find the sstable in the retored key range + let retrieved_sstables = key_range.check_restored_key_ranges(smallest_key).await; + assert!(retrieved_sstables.is_ok()); + assert!(!retrieved_sstables.as_ref().unwrap().is_empty()); + let mut sst_found = false; + for sst in retrieved_sstables.unwrap() { + if sst.dir == fake_sst_dir { + sst_found = true; + // Bloom Filter should have be restored + assert!(sst.filter.clone().unwrap().sst_dir.is_some()); + } + } + // SSTable should have been found + assert!(sst_found) + } + + #[tokio::test] + async fn test_update_key_range() { + let key_range = KeyRange::new(); + let ssts = SSTContructor::generate_ssts(2).await; + + // Insert sstable + let mut fake_sstable = ssts[0].to_owned(); + fake_sstable.load_entries_from_file().await.unwrap(); + let entries = fake_sstable.entries.to_owned(); + let binding = entries.front().unwrap(); + let smallest_key = binding.key(); + let binding = entries.back().unwrap(); + let biggest_key = binding.key(); + let fake_sst_dir = fake_sstable.dir.to_owned(); + key_range + .set(fake_sst_dir.to_owned(), smallest_key, biggest_key, fake_sstable) + .await; + assert_eq!(key_range.key_ranges.read().await.len(), 1); + + // Bloom Filter should be empty + let range = key_range.key_ranges.read().await; + let sst_with_empty_filter = range.get(&fake_sst_dir).unwrap(); + + // Ensure Bloom Filter does not exist for this sstable + assert!(sst_with_empty_filter.sst.filter.clone().unwrap().sst_dir.is_none()); + + // Ensure restored ranges is loaded + let retrieved_sstables = key_range.filter_sstables_by_key_range(smallest_key).await; + assert!(retrieved_sstables.is_ok()); + assert!(!retrieved_sstables.as_ref().unwrap().is_empty()); + + // Sleep for 3 seconds to ensure all background tasks completes + tokio::time::sleep(Duration::from_secs(3)).await; + // Drop read lock + drop(range); + // Call update on key ranges to sync restored key range with main key ranges hash map + key_range.update_key_range().await; + + let retrieved_sstables = key_range.check_restored_key_ranges(smallest_key).await; + assert!(retrieved_sstables.is_ok()); + // retrieved sstables should now be empty since we synced it + // with main key ranges hash map + assert!(retrieved_sstables.as_ref().unwrap().is_empty()); + } + + #[tokio::test] + async fn test_key_range_range_query_scan() { + let key_range = KeyRange::new(); + let ssts = SSTContructor::generate_ssts(2).await; + + // Insert sstable + let mut fake_sstable = ssts[0].to_owned(); + fake_sstable.load_entries_from_file().await.unwrap(); + let entries = fake_sstable.entries.to_owned(); + let binding = entries.front().unwrap(); + let smallest_key = binding.key(); + let binding = entries.back().unwrap(); + let biggest_key = binding.key(); + let fake_sst_dir = fake_sstable.dir.to_owned(); + key_range + .set(fake_sst_dir.to_owned(), smallest_key, biggest_key, fake_sstable) + .await; + + let range = key_range.range_query_scan(smallest_key, biggest_key).await; + assert_eq!(range.len(), 1); + assert_eq!(range.first().unwrap().sst.dir, fake_sst_dir); + } +} diff --git a/src/tests/meta_test.rs b/src/tests/meta_test.rs new file mode 100644 index 0000000..c4621d3 --- /dev/null +++ b/src/tests/meta_test.rs @@ -0,0 +1,87 @@ +#[cfg(test)] +mod tests { + use crate::consts::{SIZE_OF_U32, SIZE_OF_U64}; + use crate::meta::Meta; + use tempfile::tempdir; + + #[tokio::test] + async fn test_meta_new() { + let root = tempdir().unwrap(); + let path = root.path().join("meta_new"); + + let metadata = Meta::new(path).await; + + assert!(metadata.is_ok()); + assert_eq!(metadata.as_ref().unwrap().v_log_head, 0); + assert_eq!(metadata.as_ref().unwrap().v_log_tail, 0); + } + + #[tokio::test] + async fn test_set_head_tail() { + let root = tempdir().unwrap(); + let path = root.path().join("meta_set_tail_head"); + + let mut metadata = Meta::new(path).await.unwrap(); + + let new_tail = 100; + let new_head = 50; + metadata.set_head(new_head); + metadata.set_tail(new_tail); + assert_eq!(metadata.v_log_head, new_head); + assert_eq!(metadata.v_log_tail, new_tail); + } + + #[tokio::test] + async fn test_meta_write() { + let root = tempdir().unwrap(); + let path = root.path().join("meta_new"); + + let mut metadata = Meta::new(path).await.unwrap(); + let new_tail = 100; + let new_head = 50; + metadata.set_head(new_head); + metadata.set_tail(new_tail); + + assert!(metadata.write().await.is_ok()) + } + + #[tokio::test] + async fn test_meta_recover() { + let root = tempdir().unwrap(); + let path = root.path().join("meta_new"); + + let mut metadata = Meta::new(path.to_owned()).await.unwrap(); + let new_tail = 100; + let new_head = 50; + metadata.set_head(new_head); + metadata.set_tail(new_tail); + metadata.write().await.unwrap(); + + + let mut recovered_meta = Meta::new(path).await.unwrap(); + let res = recovered_meta.recover().await; + assert!(res.is_ok()); + assert_eq!(recovered_meta.v_log_head, metadata.v_log_head); + assert_eq!(recovered_meta.v_log_tail, metadata.v_log_tail); + assert_eq!(recovered_meta.created_at.timestamp_millis(), metadata.created_at.timestamp_millis()); + assert_eq!(recovered_meta.last_modified.timestamp_millis(), metadata.last_modified.timestamp_millis()); + } + + #[tokio::test] + async fn test_meta_serialize() { + let root = tempdir().unwrap(); + let path = root.path().join("meta_serialize"); + + let mut metadata = Meta::new(path.to_owned()).await.unwrap(); + let new_tail = 100; + let new_head = 50; + metadata.set_head(new_head); + metadata.set_tail(new_tail); + + + let expected_entry_len = SIZE_OF_U32 + SIZE_OF_U32 + SIZE_OF_U64 + SIZE_OF_U64; + let serialized_entry = metadata.serialize(); + + assert_eq!(serialized_entry.len(), expected_entry_len); + } +} diff --git a/src/tests/mod.rs b/src/tests/mod.rs index 0bb5221..2528164 100644 --- a/src/tests/mod.rs +++ b/src/tests/mod.rs @@ -3,3 +3,7 @@ mod gc_test; mod store_test; #[cfg(test)] mod workload; +mod key_range_test; +mod vlog; +mod meta_test; +mod summary_test; diff --git a/src/tests/store_test.rs b/src/tests/store_test.rs index 954b79e..4c47df6 100644 --- a/src/tests/store_test.rs +++ b/src/tests/store_test.rs @@ -82,9 +82,9 @@ mod tests { assert!(tokio_res.unwrap().unwrap()); } - let read_tasks = read_workload.iter().map(|e| { + let read_tasks = read_workload.keys().map(|e| { let store_inner = Arc::clone(&store_ref); - let key = e.0.to_owned(); + let key = e.to_owned(); tokio::spawn(async move { match store_inner.read().await.get(key.to_owned()).await { Ok(entry) => Ok((key, entry)), diff --git a/src/tests/summary_test.rs b/src/tests/summary_test.rs new file mode 100644 index 0000000..498d459 --- /dev/null +++ b/src/tests/summary_test.rs @@ -0,0 +1,57 @@ +#[cfg(test)] +mod tests { + use crate::consts::{SIZE_OF_U32, SUMMARY_FILE_NAME}; + use crate::sst::Summary; + use crate::tests::workload::SSTContructor; + use tempfile::tempdir; + + #[tokio::test] + async fn test_summary_new() { + let root = tempdir().unwrap(); + let path = root.path().join("summary_new"); + + let summary = Summary::new(path.to_owned()); + + + assert_eq!(summary.smallest_key, vec![]); + assert_eq!(summary.biggest_key, vec![]); + assert_eq!(summary.path, path.join(format!("{}.db", SUMMARY_FILE_NAME))); + + } + + #[tokio::test] + async fn test_summary_recover() { + let sst = SSTContructor::generate_ssts(1).await[0].to_owned(); + + let mut recovered_summary = Summary::new(sst.dir); + let res = recovered_summary.recover().await; + assert!(res.is_ok()); + assert!(!recovered_summary.biggest_key.is_empty()); + assert!(!recovered_summary.smallest_key.is_empty()); + } + + #[tokio::test] + async fn test_summary_write() { + let sst = SSTContructor::generate_ssts(1).await[0].to_owned(); + + let mut recovered_summary = Summary::new(sst.dir.to_owned()); + let res = recovered_summary.recover().await; + assert!(res.is_ok()); + assert!(recovered_summary.write_to_file().await.is_ok()) + } + + #[tokio::test] + async fn test_summary_serialize() { + let root = tempdir().unwrap(); + let path = root.path().join("summary_write"); + + let mut summary = Summary::new(path); + summary.biggest_key = vec![1,2,3]; + summary.smallest_key = vec![0,2,3]; + + let expected_entry_len = SIZE_OF_U32 + SIZE_OF_U32 + summary.biggest_key.len() + summary.smallest_key.len(); + let serialized_entry = summary.serialize(); + + assert_eq!(serialized_entry.len(), expected_entry_len); + } +} diff --git a/src/tests/vlog.rs b/src/tests/vlog.rs new file mode 100644 index 0000000..729cf64 --- /dev/null +++ b/src/tests/vlog.rs @@ -0,0 +1,230 @@ +#[cfg(test)] +mod tests { + use crate::consts::{SIZE_OF_U32, SIZE_OF_U64, SIZE_OF_U8}; + use crate::vlog::{ValueLog, ValueLogEntry}; + use chrono::Utc; + use tempfile::tempdir; + + #[tokio::test] + async fn test_vlog_new() { + let root = tempdir().unwrap(); + let path = root.path().join("vlog_new"); + + let vlog = ValueLog::new(path).await; + + assert!(vlog.is_ok()); + assert_eq!(vlog.as_ref().unwrap().head_offset, 0); + assert_eq!(vlog.as_ref().unwrap().tail_offset, 0); + assert_eq!(vlog.unwrap().size, 0); + } + + #[tokio::test] + async fn test_append() { + let root = tempdir().unwrap(); + let path = root.path().join("vlog_append"); + + let mut vlog = ValueLog::new(path).await.unwrap(); + + let key1 = "key1"; + let val1 = "val1"; + let key2 = "key2"; + let val2 = "val2"; + let is_tombstone = false; + let offset = vlog.append(key1, val1, Utc::now(), is_tombstone).await; + assert!(offset.is_ok()); + + let offset = vlog.append(key2, val2, Utc::now(), is_tombstone).await; + assert!(offset.is_ok()); + } + + #[tokio::test] + async fn test_get() { + let root = tempdir().unwrap(); + let path = root.path().join("vlog_append"); + + let mut vlog = ValueLog::new(path).await.unwrap(); + + let key1 = "key1"; + let val1 = "val1"; + let key2 = "key2"; + let val2 = "val2"; + let time = Utc::now(); + let is_tombstone = false; + let offset = vlog.append(key1, val1, time, is_tombstone).await; + assert!(offset.is_ok()); + let start_offset1 = offset.unwrap(); + + let is_tombstone_true = true; + let offset = vlog.append(key2, val2, time, is_tombstone_true).await; + assert!(offset.is_ok()); + let start_offset2 = offset.unwrap(); + + // get first key + let recvoered = vlog.get(start_offset1).await; + assert!(recvoered.is_ok()); + assert!(recvoered.as_ref().unwrap().is_some()); + let (value, is_tomb) = recvoered.unwrap().unwrap(); + assert_eq!(value, val1.as_bytes().to_vec()); + assert_eq!(is_tomb, is_tombstone); + + // get second key + let recvoered = vlog.get(start_offset2).await; + assert!(recvoered.is_ok()); + assert!(recvoered.as_ref().unwrap().is_some()); + let (value, is_tomb) = recvoered.unwrap().unwrap(); + assert_eq!(value, val2.as_bytes().to_vec()); + assert_eq!(is_tomb, is_tombstone_true); + } + + #[tokio::test] + async fn test_sync_all() { + let root = tempdir().unwrap(); + let path = root.path().join("vlog_sync_all"); + + let mut vlog = ValueLog::new(path).await.unwrap(); + + let key1 = "key1"; + let val1 = "val1"; + let time = Utc::now(); + let is_tombstone = false; + let offset = vlog.append(key1, val1, time, is_tombstone).await; + assert!(offset.is_ok()); + + assert!(vlog.sync_to_disk().await.is_ok()); + } + + #[tokio::test] + async fn test_recover() { + let root = tempdir().unwrap(); + let path = root.path().join("vlog_recover"); + + let mut vlog = ValueLog::new(path).await.unwrap(); + + let key1 = "key1"; + let val1 = "val1"; + let key2 = "key2"; + let val2 = "val2"; + let time = Utc::now(); + let is_tombstone = false; + let offset = vlog.append(key1, val1, time, is_tombstone).await; + assert!(offset.is_ok()); + let start_offset = offset.unwrap(); + + let is_tombstone_true = true; + let offset = vlog.append(key2, val2, time, is_tombstone_true).await; + assert!(offset.is_ok()); + + let entries = vlog.recover(start_offset).await; + assert!(entries.is_ok()); + assert_eq!(entries.unwrap().len(), 2) + } + + #[tokio::test] + async fn test_read_chunk_to_garbage_collect() { + let root = tempdir().unwrap(); + let path = root.path().join("vlog_gc"); + + let mut vlog = ValueLog::new(path).await.unwrap(); + + let key1 = "key1"; + let val1 = "val1"; + let key2 = "key2"; + let val2 = "val2"; + let time = Utc::now(); + let entry_len1 = SIZE_OF_U32 + SIZE_OF_U32 + SIZE_OF_U64 + key1.len() + val1.len() + SIZE_OF_U8; + let entry_len2 = SIZE_OF_U32 + SIZE_OF_U32 + SIZE_OF_U64 + key2.len() + val2.len() + SIZE_OF_U8; + + let bytes_to_collect = entry_len1 + entry_len2; + + let is_tombstone = false; + let offset = vlog.append(key1, val1, time, is_tombstone).await; + assert!(offset.is_ok()); + + let is_tombstone_true = true; + let offset = vlog.append(key2, val2, time, is_tombstone_true).await; + assert!(offset.is_ok()); + + let entries = vlog.read_chunk_to_garbage_collect(bytes_to_collect).await; + assert!(entries.is_ok()); + let (val_entries, bytes) = entries.unwrap(); + assert_eq!(val_entries.len(), 2); + assert_eq!(bytes, bytes_to_collect); + } + + #[tokio::test] + async fn test_clear_all() { + let root = tempdir().unwrap(); + let path = root.path().join("vlog_clear_all"); + + let mut vlog = ValueLog::new(path).await.unwrap(); + + let key1 = "key1"; + let val1 = "val1"; + let key2 = "key2"; + let val2 = "val2"; + let time = Utc::now(); + let is_tombstone = false; + let mut offset = vlog.append(key1, val1, time, is_tombstone).await; + assert!(offset.is_ok()); + let is_tombstone_true = true; + offset = vlog.append(key2, val2, time, is_tombstone_true).await; + assert!(offset.is_ok()); + + vlog.clear_all().await; + + assert_eq!(vlog.head_offset, 0); + assert_eq!(vlog.tail_offset, 0); + assert_eq!(vlog.size, 0); + } + + #[tokio::test] + async fn test_set_head_tail() { + let root = tempdir().unwrap(); + let path = root.path().join("vlog_set_head_tail"); + + let mut vlog = ValueLog::new(path).await.unwrap(); + + assert_eq!(vlog.head_offset, 0); + assert_eq!(vlog.tail_offset, 0); + assert_eq!(vlog.size, 0); + + let new_head = 100; + let new_tail = 50; + + vlog.set_head(new_head); + vlog.set_tail(new_tail); + assert_eq!(vlog.head_offset, new_head); + assert_eq!(vlog.tail_offset, new_tail); + } + + #[tokio::test] + async fn test_vlog_entry_new() { + let key = "test_key"; + let val = "test_val"; + let time = Utc::now(); + let is_tombstone = false; + let entry = ValueLogEntry::new(key.len(), val.len(), key, val, time, is_tombstone); + + assert_eq!(entry.ksize, key.len()); + assert_eq!(entry.vsize, val.len()); + assert_eq!(entry.key, key.as_bytes().to_vec()); + assert_eq!(entry.value, val.as_bytes().to_vec()); + assert_eq!(entry.is_tombstone, is_tombstone); + assert_eq!(entry.created_at, time); + } + + #[tokio::test] + async fn test_vlog_entry_serialize() { + let key = "test_key"; + let val = "test_val"; + let time = Utc::now(); + let is_tombstone = false; + let entry = ValueLogEntry::new(key.len(), val.len(), key, val, time, is_tombstone); + + let expected_entry_len = SIZE_OF_U32 + SIZE_OF_U32 + SIZE_OF_U64 + key.len() + val.len() + SIZE_OF_U8; + + let serialized_entry = entry.serialize(); + + assert_eq!(serialized_entry.len(), expected_entry_len); + } +} diff --git a/src/tests/workload.rs b/src/tests/workload.rs index 3097e89..4cc2a3b 100644 --- a/src/tests/workload.rs +++ b/src/tests/workload.rs @@ -1,13 +1,18 @@ +use crate::filter::BloomFilter; use crate::memtable::SkipMapValue; +use crate::sst::{DataFile, Summary}; use crate::{ db::DataStore, err::Error, types::{Key, Value}, util, }; +use chrono::Utc; use crossbeam_skiplist::SkipMap; use futures::future::join_all; +use std::path::{Path, PathBuf}; use std::{collections::HashMap, sync::Arc}; +use tokio::fs::File; use tokio::sync::RwLock; type WriteWorkloadMap = HashMap; @@ -121,3 +126,180 @@ impl FilterWorkload { filter } } + +use crate::{ + fs::{DataFileNode, FileNode, FileType, IndexFileNode}, + index::IndexFile, + sst::Table, +}; + +pub struct SSTContructor { + pub dir: PathBuf, + pub data_path: PathBuf, + pub index_path: PathBuf, + pub filter_path: PathBuf, + pub summary_path: PathBuf, +} + +impl SSTContructor { + fn new + Send + Sync>(dir: P, data_path: P, index_path: P, filter_path: P, summary_path: P) -> Self { + return Self { + dir: dir.as_ref().to_path_buf(), + data_path: data_path.as_ref().to_path_buf(), + index_path: index_path.as_ref().to_path_buf(), + filter_path: filter_path.as_ref().to_path_buf(), + summary_path: summary_path.as_ref().to_path_buf(), + }; + } + + pub async fn generate_ssts(number: u32) -> Vec
{ + let sst_contructor: Vec = vec![ + SSTContructor::new( + PathBuf::from("src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785462309"), + PathBuf::from( + "src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785462309/data.db", + ), + PathBuf::from( + "src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785462309/index.db", + ), + PathBuf::from( + "src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785462309/filter.db", + ), + PathBuf::from( + "src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785462309/summary.db", + ), +), +SSTContructor::new( + PathBuf::from("src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463686"), + PathBuf::from( + "src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463686/data.db", + ), + PathBuf::from( + "src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463686/index.db", + ), + PathBuf::from( + "src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463686/filter.db", + ), + PathBuf::from( + "src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463686/summary.db", + ), +), +SSTContructor::new( + PathBuf::from("src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463735"), + PathBuf::from( + "src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463735/data.db", + ), + PathBuf::from( + "src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463735/index.db", + ), + PathBuf::from( + "src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463735/filter.db", + ), + PathBuf::from( + "src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463735/summary.db", + ), +), +SSTContructor::new( + PathBuf::from("src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463779"), + PathBuf::from( + "src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463779/data.db", + ), + PathBuf::from( + "src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463779/index.db", + ), + PathBuf::from( + "src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463779/filter.db", + ), + PathBuf::from( + "src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463779/summary.db", + ), +), +SSTContructor::new( + PathBuf::from("src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463825"), + PathBuf::from( + "src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463825/data.db", + ), + PathBuf::from( + "src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463825/index.db", + ), + PathBuf::from( + "src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463825/filter.db", + ), + PathBuf::from( + "src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463825/summary.db", + ), +), +SSTContructor::new( + PathBuf::from("src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463872"), + PathBuf::from( + "src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463872/data.db", + ), + PathBuf::from( + "src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463872/index.db", + ), + PathBuf::from( + "src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463872/filter.db", + ), + PathBuf::from( + "src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463872/summary.db", + ), +), +SSTContructor::new( + PathBuf::from("src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463919"), + PathBuf::from( + "src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463919/data.db", + ), + PathBuf::from( + "src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463919/index.db", + ), + PathBuf::from( + "src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463919/filter.db", + ), + PathBuf::from( + "src/tests/fixtures/data/buckets/bucket1201eb6b-8903-4557-a5bd-d87cb725f1d8/sstable_1720785463919/summary.db", + ), +) +]; + let mut ssts = Vec::new(); + for i in 0..number { + let idx = i as usize; + ssts.push(Table { + dir: sst_contructor[idx].dir.to_owned(), + hotness: 100, + size: 4096, + created_at: Utc::now(), + data_file: DataFile { + file: DataFileNode { + node: FileNode { + file_path: sst_contructor[idx].data_path.to_owned(), + file: Arc::new(RwLock::new( + File::open(sst_contructor[idx].data_path.to_owned()).await.unwrap(), + )), + file_type: FileType::Data, + }, + }, + path: sst_contructor[idx].data_path.to_owned(), + }, + index_file: IndexFile { + file: IndexFileNode { + node: FileNode { + file_path: sst_contructor[idx].index_path.to_owned(), + file: Arc::new(RwLock::new( + File::open(sst_contructor[idx].index_path.to_owned()).await.unwrap(), + )), + file_type: FileType::Index, + }, + }, + path: sst_contructor[idx].index_path.to_owned(), + }, + entries: Arc::new(SkipMap::default()), + filter: Some(BloomFilter { + file_path: Some(sst_contructor[idx].filter_path.to_owned()), + ..Default::default() + }), + summary: Some(Summary::new(sst_contructor[idx].summary_path.to_owned())), + }) + } + ssts + } +} diff --git a/src/util/mod.rs b/src/util/mod.rs index be4b6bb..20d14e1 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -48,3 +48,54 @@ pub fn float_from_le_bytes(bytes: &[u8]) -> Option { let float: f64 = f64::from_bits(bits); Some(float) } + + +#[cfg(test)] +mod tests { + use super::*; + use chrono::TimeZone; + + #[test] + fn test_generate_random_id() { + let id1 = generate_random_id(10); + let id2 = generate_random_id(10); + assert_eq!(id1.len(), 10); + assert_eq!(id2.len(), 10); + assert_ne!(id1, id2); // Ensure IDs are random and not equal + } + + #[test] + fn test_milliseconds_to_datetime() { + let datetime = milliseconds_to_datetime(1_000); + assert_eq!(datetime, Utc.with_ymd_and_hms(1970, 1, 1, 0, 0, 1).unwrap()); + + let datetime = milliseconds_to_datetime(1_000_000); + assert_eq!(datetime, Utc.with_ymd_and_hms(1970, 1, 1, 0, 16, 40).unwrap()); + } + + #[test] + fn test_default_datetime() { + let datetime = default_datetime(); + assert_eq!(datetime, Utc.with_ymd_and_hms(1970, 1, 1, 0, 0, 0).unwrap()); + } + + #[test] + fn test_float_to_le_bytes() { + let float = 1.23_f64; + let bytes = float_to_le_bytes(float); + let expected = float.to_le_bytes(); + assert_eq!(bytes, expected); + } + + #[test] + fn test_float_from_le_bytes() { + let float = 1.23_f64; + let bytes = float.to_le_bytes(); + let result = float_from_le_bytes(&bytes); + assert_eq!(result, Some(float)); + + let invalid_bytes = [0_u8; 4]; // Invalid size for f64 + let result = float_from_le_bytes(&invalid_bytes); + assert_eq!(result, None); + } +} \ No newline at end of file diff --git a/src/vlog/v_log.rs b/src/vlog/v_log.rs index 96665bc..b92a0ac 100644 --- a/src/vlog/v_log.rs +++ b/src/vlog/v_log.rs @@ -176,7 +176,7 @@ impl ValueLog { value: T, created_at: CreatedAt, is_tombstone: bool, - ) -> ValOffset { + ) -> Result { let v_log_entry = ValueLogEntry::new( key.as_ref().len(), value.as_ref().len(), @@ -190,9 +190,9 @@ impl ValueLog { // Get the current offset before writing(this will be the offset of the value stored in the memtable) let last_offset = self.size; let data_file = &self.content; - let _ = data_file.file.node.write_all(&serialized_data).await; + data_file.file.node.write_all(&serialized_data).await?; self.size += serialized_data.len(); - last_offset + Ok(last_offset) } /// Fetches value from value log @@ -217,6 +217,14 @@ impl ValueLog { } /// Fetches an entry from value log using the `start_offset` + /// + /// + /// This is used to fetch all entries that is yet to be flushed + /// before crash happened + /// + /// # Error + /// + /// Returns error in case there is an IO error pub async fn recover(&mut self, start_offset: usize) -> Result, Error> { self.content.file.recover(start_offset).await } @@ -240,9 +248,10 @@ impl ValueLog { pub async fn clear_all(&mut self) { if self.content.file.node.metadata().await.is_ok() { if let Err(err) = self.content.file.node.remove_dir_all().await { - log::error!("{}", err); + log::info!("{}", err); } } + self.size=0; self.tail_offset = 0; self.head_offset = 0; } @@ -279,7 +288,7 @@ impl ValueLogEntry { } /// Converts value log entry to a byte vector - fn serialize(&self) -> ByteSerializedEntry { + pub(crate) fn serialize(&self) -> ByteSerializedEntry { let entry_len = SIZE_OF_U32 + SIZE_OF_U32 + SIZE_OF_U64 + self.key.len() + self.value.len() + SIZE_OF_U8; let mut serialized_data = Vec::with_capacity(entry_len); @@ -299,9 +308,3 @@ impl ValueLogEntry { } } -#[cfg(test)] -mod tests { - - #[test] - fn test_serialized() {} -}