diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..99d36a1 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,54 @@ +# Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to make participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment include: + +- Using welcoming and inclusive language +- Being respectful of differing viewpoints and experiences +- Gracefully accepting constructive criticism +- Focusing on what is best for the community +- Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +- The use of sexualized language or imagery and unwelcome sexual attention or advances +- Trolling, insulting/derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or electronic address, without explicit permission +- Other conduct which could reasonably be considered inappropriate in a professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned with this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project email address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at sunkanmiadewumi1@gmail.com. All complaints will be reviewed and investigated promptly and fairly. + +All project team members are obligated to respect the privacy and security of the reporter of any incident. + +## Enforcement Guidelines + +Project maintainers will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct: + +1. **Correction**: Community Impact: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community. +2. **Warning**: Community Impact: A violation through a single incident or series of actions. +3. **Temporary Ban**: Community Impact: A serious violation of community standards, including sustained inappropriate behavior. +4. **Permanent Ban**: Community Impact: Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org), version 2.0, available at https://www.contributor-covenant.org/version/2/0/code_of_conduct.html + +For answers to common questions about this code of conduct, see https://www.contributor-covenant.org/faq diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..02749ed --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,66 @@ +# Contributing + +## License + +By contributing to this project, you agree that your contributions will be licensed under the project's license (MIT OR Apache-2.0). + +Thank you for your contribution! + +## Looking for issues? + +https://github.com/Gifted-s/bd/issues + + +### How do I contribute + +Fork the project and use the `git clone` command to download the repository to your computer. A standard procedure for working on an issue would be to: + +1. Clone the `velarixdb` repository and download it to your computer. + ```bash + git clone https://github.com/Gifted-s/velarixdb + ``` + +2. Pull all changes from the upstream `main` branch, before creating a new branch - to ensure that your `main` branch is up-to-date with the latest changes: + ```bash + git pull + ``` + +3. Create a new branch from `main` like: `bugfix-232-ensure-compaction-runs-within-time-frame-allocated`: + ```bash + git checkout -b "[the name of your branch]" + ``` + +4. Make changes to the code, and ensure all code changes are formatted correctly: + ```bash + cargo fmt + ``` +5. Ensure all clippy rules are obeyed: + ```bash + cargo clippy --tests + ``` + +6. Ensure nothing is unexpectedly broken: + ```bash + cargo test + ``` + +6. Commit your changes when finished: + ```bash + git add -A + git commit -m "[your commit message]" + ``` + +7. Push changes to GitHub: + ```bash + git push origin "[the name of your branch]" + ``` + +8. Submit your changes for review, by going to your repository on GitHub and clicking the `Compare & pull request` button. + +9. Ensure that you have entered a commit message which details the changes, and what the pull request is for. + +10. Now submit the pull request by clicking the `Create pull request` button. + +11. Wait for code review and approval. + +12. After approval, your pull request will be merged. \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index e0e2e7d..6bc399b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,10 +1,10 @@ [package] -name = "vikingsdb" +name = "velarixdb" version = "0.1.0" edition = "2021" authors = ["Adewumi Sunkanmi D."] description = "An LSM Storage Engine focused on reduced IO amplification" -keywords = ["vikingsdb","database", "lsmt", "lsm", "rocksdb", "leveldb"] +keywords = ["velarixdb","database", "lsmt", "lsm", "rocksdb", "leveldb"] categories = ["data-structures", "database-implementations", "algorithms"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/LICENSE-APACHE b/LICENSE-APACHE new file mode 100644 index 0000000..62b8c91 --- /dev/null +++ b/LICENSE-APACHE @@ -0,0 +1,176 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS \ No newline at end of file diff --git a/LICENSE-MIT b/LICENSE-MIT new file mode 100644 index 0000000..9afeb3c --- /dev/null +++ b/LICENSE-MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 fjall-rs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md index b53e10b..ff4ccd4 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,147 @@ -# BUMP STORAGE ENGINE +# VelarixDB -// In progress ... +

+ +

+VelarixDB is an LSM-based storage engine designed to significantly reduce IO amplification, resulting in better performance and durability for storage devices. + + +[![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)](code_of_conduct.md) + +## Introduction + +**VelarixDB: Efficient Data Management for High Performance** + +VelarixDB is an ongoing project (*not production ready*) designed to optimize data movement during load times, random access, and compaction. Inspired by the WiscKey paper, [WiscKey: Separating Keys from Values in SSD-conscious Storage](https://usenix.org/system/files/conference/fast16/fast16-papers-lu.pdf), velarixdb aims to significantly enhance performance over traditional key-value stores. + +## Performance Benefits + +According to the benchmarks presented in the WiscKey paper, implementations can outperform LevelDB and RocksDB by: +- **2.5x to 111x** for database loading +- **1.6x to 14x** for random lookups + +## Concurrency with Tokio + +To achieve high concurrency without the overhead of spawning new threads every time, velarixdb utilizes the Tokio runtime. This allows for efficient and scalable asynchronous operations, making the most of modern multi-core processors. + +## Disclaimer + +Please note that velarixdb is still under development and is not yet production-ready. + +### Features +- [x] Atomic `Put()`, `Get()`, `Delete()`, and `Update()` operations +- [x] 100% safe & stable Rust +- [x] Separation of keys from values, reducing the amount of data moved during compaction (i.e., reduced IO amplification) +- [x] Garbage Collector +- [x] Lock-free memtable (no `Mutex`) +- [x] Tokio Runtime for efficient thread management +- [x] Bloom Filters for fast in-memory key searches +- [x] Crash recovery using the Value Log +- [x] Index to improve searches on Sorted String Tables (SSTs) +- [x] Key Range to store the largest and smallest keys in an SST +- [x] Sized Tier Compaction Strategy (STCS) +- [ ] Batched Writes +- [ ] Snappy Compression Algorithm +- [ ] Value Buffer to keep values in memory and only flush in batches to reduce IO (under investigation) +- [ ] Checksum to detect data corruption +- [ ] Leveled Compaction (LCS), Time-Window Compaction (TCS), and Unified Compaction (UCS) +- [ ] Monitoring module to continuously monitor and generate reports + +### It is not: +- A standalone server +- A relational database +- A wide-column database: it has no notion of columns + +### Constraint +- Keys are limited to 65,536 bytes, and values are limited to 2^32 bytes. Larger keys and values have a bigger performance impact. +- Like any typical key-value store, keys are stored in lexicographic order. If you are storing integer keys (e.g., timeseries data), use the big-endian form to adhere to locality. + +# Basic usage + +```sh +cargo add velarixdb ``` - BBBBBBBB UU UU MM MM PPPPPPP DDDDDDD BBBBBBB - B BB UU UU MMM MMM P PPP D DDD B BB - BBBBBBBB UU UU MMMM MMMM PPPPPPP D DDD BBBBBBB - B BB UU UU MM MM MM MM P D DDD B BB - BBBBBBBB UUUUU MM MM MM MM P DDDDDDD BBBBBBB + +```rust +use velarixdb::db::DataStore; +# use tempfile::tempdir; + +#[tokio::main] +async fn main() { + let root = tempdir().unwrap(); + let path = root.path().join("velarix"); + let mut store = DataStore::open("big_tech", path).await.unwrap(); // handle IO error + + store.put("apple", "tim cook").await; + store.put("google", "sundar pichai").await; + store.put("nvidia", "jensen huang").await; + store.put("microsoft", "satya nadella").await; + store.put("meta", "mark zuckerberg").await; + store.put("openai", "sam altman").await; + + + let entry1 = store.get("apple").await.unwrap(); // Handle error + let entry2 = store.get("google").await.unwrap(); + let entry3 = store.get("nvidia").await.unwrap(); + let entry4 = store.get("microsoft").await.unwrap(); + let entry5 = store.get("meta").await.unwrap(); + let entry6 = store.get("openai").await.unwrap(); + let entry7 = store.get("***not_found_key**").await.unwrap(); + + assert_eq!(std::str::from_utf8(&entry1.unwrap().val).unwrap(), "tim cook"); + assert_eq!(std::str::from_utf8(&entry2.unwrap().val).unwrap(), "sundar pichai"); + assert_eq!(std::str::from_utf8(&entry3.unwrap().val).unwrap(), "jensen huang"); + assert_eq!(std::str::from_utf8(&entry4.unwrap().val).unwrap(), "satya nadella"); + assert_eq!(std::str::from_utf8(&entry5.unwrap().val).unwrap(), "mark zuckerberg"); + assert_eq!(std::str::from_utf8(&entry6.unwrap().val).unwrap(), "sam altman"); + assert!(entry7.is_none()); + + // Remove an entry + store.delete("apple").await.unwrap(); + + // Update an entry + let success = store.update("microsoft", "elon musk").await; + assert!(success.is_ok()); +} ``` -## Plans to increase read speed -* Introduce Block Cache -* Introduce Snappy Compression Algorithm to reduce offset scanned to retrieve blocks on the disk -* Introduce LCS +# Store JSON + +```rust +use serde::{Deserialize, Serialize}; +use serde_json; +use velarixdb::db::DataStore; +# use tempfile::tempdir; + +#[tokio::main] +async fn main() { + let root = tempdir().unwrap(); + let path = root.path().join("velarix"); + let mut store = DataStore::open("big_tech", path).await.unwrap(); // handle IO error + + #[derive(Serialize, Deserialize)] + struct BigTech { + name: String, + rank: i32, + } + let new_entry = BigTech { + name: String::from("Google"), + rank: 50, + }; + let json_string = serde_json::to_string(&new_entry).unwrap(); + + let res = store.put("google", json_string).await; + assert!(res.is_ok()); + + let entry = store.get("google").await.unwrap().unwrap(); + let entry_string = std::str::from_utf8(&entry.val).unwrap(); + let big_tech: BigTech = serde_json::from_str(&entry_string).unwrap(); + + assert_eq!(big_tech.name, new_entry.name); + assert_eq!(big_tech.rank, new_entry.rank); +} +``` -This will signnificantly improve the read speed for now we only have key index, bloomfilters and sized tier (grouped into buckets) \ No newline at end of file +## Examples +[See here](https://usenix.org/system/files/conference/fast16/fast16-papers-lu.pdf) for practical examples \ No newline at end of file diff --git a/compaction_algo.txt b/compaction_algo.txt index 0b7d08a..73528ea 100644 --- a/compaction_algo.txt +++ b/compaction_algo.txt @@ -6,7 +6,7 @@ Compaction algorithm (Adopted from Apache Cassandra Sized Tier Compaction Strate NOTE1: We also collect the Bucket ID and SSTable filepaths of the tables selected for compaction so that we can delete them from previous bucket where they existed NOTE2: You cannot delete the sstbales until they have been compacted and written to disk to prevent data loss in case of outage - - Compact the SS Tables and write them to buckets + - Compact the SSTables and write them to buckets - Once write is complete, you can delete the sstables from the bucket it once exisited and recalculate the average size of this bucket for the future NOTE: The deletion should include both from the disk and in-memory buckets diff --git a/examples/delete.rs b/examples/delete.rs new file mode 100644 index 0000000..9e56583 --- /dev/null +++ b/examples/delete.rs @@ -0,0 +1,22 @@ +use tempfile::tempdir; +use velarixdb::db::DataStore; + +#[tokio::main] +async fn main() { + let root = tempdir().unwrap(); + let path = root.path().join("velarix"); + let mut store = DataStore::open("big_tech", path).await.unwrap(); // handle IO error + + store.put("apple", "tim cook").await.unwrap(); // handle error + + // Retrieve entry + let entry = store.get("apple").await.unwrap(); + assert!(entry.is_some()); + + // Delete entry + store.delete("apple").await.unwrap(); + + // Entry should now be None + let entry = store.get("apple").await.unwrap(); + assert!(entry.is_none()); +} diff --git a/examples/get.rs b/examples/get.rs new file mode 100644 index 0000000..9916ede --- /dev/null +++ b/examples/get.rs @@ -0,0 +1,39 @@ +use tempfile::tempdir; +use velarixdb::db::DataStore; + +#[tokio::main] +async fn main() { + let root = tempdir().unwrap(); + let path = root.path().join("velarix"); + let mut store = DataStore::open("big_tech", path).await.unwrap(); // handle IO error + + let res1 = store.put("apple", "tim cook").await; + let res2 = store.put("google", "sundar pichai").await; + let res3 = store.put("nvidia", "jensen huang").await; + let res4 = store.put("microsoft", "satya nadella").await; + let res5 = store.put("meta", "mark zuckerberg").await; + let res6 = store.put("openai", "sam altman").await; + + assert!(res1.is_ok()); + assert!(res2.is_ok()); + assert!(res3.is_ok()); + assert!(res4.is_ok()); + assert!(res5.is_ok()); + assert!(res6.is_ok()); + + let entry1 = store.get("apple").await.unwrap(); // Handle error + let entry2 = store.get("google").await.unwrap(); + let entry3 = store.get("nvidia").await.unwrap(); + let entry4 = store.get("microsoft").await.unwrap(); + let entry5 = store.get("meta").await.unwrap(); + let entry6 = store.get("openai").await.unwrap(); + let entry7 = store.get("***not_found_key**").await.unwrap(); + + assert_eq!(std::str::from_utf8(&entry1.unwrap().val).unwrap(), "tim cook"); + assert_eq!(std::str::from_utf8(&entry2.unwrap().val).unwrap(), "sundar pichai"); + assert_eq!(std::str::from_utf8(&entry3.unwrap().val).unwrap(), "jensen huang"); + assert_eq!(std::str::from_utf8(&entry4.unwrap().val).unwrap(), "satya nadella"); + assert_eq!(std::str::from_utf8(&entry5.unwrap().val).unwrap(), "mark zuckerberg"); + assert_eq!(std::str::from_utf8(&entry6.unwrap().val).unwrap(), "sam altman"); + assert!(entry7.is_none()) +} diff --git a/examples/get_concurrent.rs b/examples/get_concurrent.rs new file mode 100644 index 0000000..fa3bf43 --- /dev/null +++ b/examples/get_concurrent.rs @@ -0,0 +1,62 @@ +use std::{collections::HashMap, sync::Arc}; + +use futures::future::join_all; +use tempfile::tempdir; +use tokio::sync::RwLock; +use velarixdb::db::DataStore; + +#[tokio::main] +async fn main() { + let root = tempdir().unwrap(); + let path = root.path().join("velarix"); + let store = DataStore::open("big_tech", path).await.unwrap(); // handle IO error + let mut entries = HashMap::new(); + entries.insert("apple", "tim cook"); + entries.insert("google", "tim cook"); + entries.insert("nvidia", "tim cook"); + entries.insert("microsoft", "tim cook"); + entries.insert("meta", "tim cook"); + entries.insert("openai", "sam altman"); + + let store_ref = Arc::new(RwLock::new(store)); + let writes = entries.iter().map(|(k, v)| { + let store_inner = Arc::clone(&store_ref); + let key = k.to_owned(); + let val = v.to_owned(); + tokio::spawn(async move { + let mut writer = store_inner.write().await; + writer.put(key, val).await + }) + }); + let all_results = join_all(writes).await; + for tokio_res in all_results { + assert!(tokio_res.is_ok()); + assert!(tokio_res.as_ref().unwrap().is_ok()); + } + + + + // Read entries concurently + let reads = entries.iter().map(|(k, _)| { + let store_inner = Arc::clone(&store_ref); + let key = k.to_owned(); + tokio::spawn(async move { + match store_inner.read().await.get(key.to_owned()).await { + Ok(entry) => Ok((key, entry)), + Err(err) => Err(err), + } + }) + }); + + let all_results = join_all(reads).await; + for tokio_res in all_results { + assert!(tokio_res.is_ok()); + assert!(tokio_res.as_ref().unwrap().is_ok()); + let (key, value) = tokio_res.unwrap().unwrap(); + assert!(value.is_some()); + assert_eq!( + std::str::from_utf8(&value.unwrap().val).unwrap(), + *entries.get(&key).unwrap() + ); + } +} diff --git a/examples/json.rs b/examples/json.rs new file mode 100644 index 0000000..5a4cbe5 --- /dev/null +++ b/examples/json.rs @@ -0,0 +1,34 @@ +use serde::{Deserialize, Serialize}; +use serde_json; +use tempfile::tempdir; +use velarixdb::db::DataStore; + +#[tokio::main] +async fn main() { + let root = tempdir().unwrap(); + let path = root.path().join("velarix"); + let mut store = DataStore::open("big_tech", path).await.unwrap(); // handle IO error + + #[derive(Serialize, Deserialize)] + struct BigTech { + name: String, + rank: i32, + } + + let new_entry = BigTech { + name: String::from("Google"), + rank: 50, + }; + + let json_string = serde_json::to_string(&new_entry).unwrap(); + let res = store.put("google", json_string).await; + + assert!(res.is_ok()); + + let entry = store.get("google").await.unwrap().unwrap(); + let entry_string = std::str::from_utf8(&entry.val).unwrap(); + let big_tech: BigTech = serde_json::from_str(&entry_string).unwrap(); + + assert_eq!(big_tech.name, new_entry.name); + assert_eq!(big_tech.rank, new_entry.rank); +} diff --git a/examples/put.rs b/examples/put.rs index fa83b43..b1e83be 100644 --- a/examples/put.rs +++ b/examples/put.rs @@ -1,13 +1,23 @@ -// use std::path::PathBuf; +use tempfile::tempdir; +use velarixdb::db::DataStore; -// use tempfile::tempdir; -// use vikingsdb::db::DataStore; +#[tokio::main] +async fn main() { + let root = tempdir().unwrap(); + let path = root.path().join("velarix"); + let mut store = DataStore::open("big_tech", path).await.unwrap(); // handle IO error -fn main() {} + let res1 = store.put("apple", "tim cook").await; + let res2 = store.put("google", "sundar pichai").await; + let res3 = store.put("nvidia", "jensen huang").await; + let res4 = store.put("microsoft", "satya nadella").await; + let res5 = store.put("meta", "mark zuckerberg").await; + let res6 = store.put("openai", "sam altman").await; -// async fn put() { -// let root = tempdir().unwrap(); -// let path = PathBuf::from(root.path().join("db")); -// let store = DataStore::new("put_db", path.clone()).await.unwrap(); -// store.put() -// } + assert!(res1.is_ok()); + assert!(res2.is_ok()); + assert!(res3.is_ok()); + assert!(res4.is_ok()); + assert!(res5.is_ok()); + assert!(res6.is_ok()); +} diff --git a/examples/put_concurrent.rs b/examples/put_concurrent.rs new file mode 100644 index 0000000..88df7be --- /dev/null +++ b/examples/put_concurrent.rs @@ -0,0 +1,38 @@ +use std::sync::Arc; + +use futures::future::join_all; +use tempfile::tempdir; +use tokio::sync::RwLock; +use velarixdb::db::DataStore; + +#[tokio::main] +async fn main() { + let root = tempdir().unwrap(); + let path = root.path().join("velarix"); + let store = DataStore::open("big_tech", path).await.unwrap(); // handle IO error + + let entries = [ + ["apple", "tim cook"], + ["google", "sundar pichai"], + ["nvidia", "jensen huang"], + ["microsoft", "satya nadella"], + ["meta", "mark zuckerberg"], + ["openai", "sam altman"], + ]; + + let store_ref = Arc::new(RwLock::new(store)); + let write_tasks = entries.iter().map(|e| { + let store_inner = Arc::clone(&store_ref); + let key = e[0]; + let val = e[1]; + tokio::spawn(async move { + let mut writer = store_inner.write().await; + writer.put(key, val).await + }) + }); + let all_results = join_all(write_tasks).await; + for tokio_res in all_results { + assert!(tokio_res.is_ok()); + assert!(tokio_res.as_ref().unwrap().is_ok()); + } +} diff --git a/examples/update.rs b/examples/update.rs new file mode 100644 index 0000000..2031bf9 --- /dev/null +++ b/examples/update.rs @@ -0,0 +1,20 @@ +use tempfile::tempdir; +use velarixdb::db::DataStore; + +#[tokio::main] +async fn main() { + let root = tempdir().unwrap(); + let path = root.path().join("velarix"); + let mut store = DataStore::open("big_tech", path).await.unwrap(); // handle IO error + + store.put("apple", "tim cook").await.unwrap(); // handle error + + // Update entry + let success = store.update("apple", "elon musk").await; + assert!(success.is_ok()); + + // Entry should now be updated + let entry = store.get("apple").await.unwrap(); // handle error + assert!(entry.is_some()); + assert_eq!(std::str::from_utf8(&entry.unwrap().val).unwrap(), "elon musk") +} diff --git a/garbage_collection.txt b/garbage_collection.txt index 797317d..3a6376e 100644 --- a/garbage_collection.txt +++ b/garbage_collection.txt @@ -1,5 +1,3 @@ - - 1. We need to keep two pointers, head and tail 2. Only the part of the vLog between the head and the tail contains valid values diff --git a/logo.png b/logo.png new file mode 100644 index 0000000..7c7a814 Binary files /dev/null and b/logo.png differ diff --git a/read_algo.txt b/read_algo.txt deleted file mode 100644 index 66a3ef3..0000000 --- a/read_algo.txt +++ /dev/null @@ -1,9 +0,0 @@ -Read algorithm - - - Check bloom filters to know which sstable contains the key - - - Read that sstable - - - Read the index that maps the key to its value_offset in the value log - - - Use this offset to read the value in value log file and return diff --git a/src/block/block.rs b/src/block/block_manager.rs similarity index 97% rename from src/block/block.rs rename to src/block/block_manager.rs index bff80b3..26a7039 100644 --- a/src/block/block.rs +++ b/src/block/block_manager.rs @@ -121,7 +121,7 @@ impl Block { let entry_size = key.as_ref().len() + SIZE_OF_U32 + SIZE_OF_U32 + SIZE_OF_U64 + SIZE_OF_U8; if self.is_full(entry_size) { - return Err(Error::BlockIsFullError); + return Err(Error::BlockIsFull); } let entry = BlockEntry { @@ -186,7 +186,7 @@ impl Block { entry_vec.push(entry.is_tombstone as u8); if entry_len != entry_vec.len() { - return Err(SerializationError("Invalid input")); + return Err(Serialization("Invalid input")); } Ok(entry_vec) @@ -237,7 +237,7 @@ mod tests { let res = block.set_entry( key.len() as u32, - key.to_owned(), + &key, value_offset, creation_date, is_tombstone, @@ -288,7 +288,7 @@ mod tests { let res = block.set_entry( key.len() as u32, - key.to_owned(), + &key, value_offset, creation_date, is_tombstone, @@ -327,7 +327,7 @@ mod tests { let res = block.set_entry( key.len() as u32, - key.to_owned(), + &key, value_offset, creation_date, is_tombstone, @@ -361,7 +361,7 @@ mod tests { block .set_entry( key.len() as u32, - key.to_owned(), + &key, value_offset, creation_date, is_tombstone, @@ -372,7 +372,7 @@ mod tests { // Attempt to set a new entry, which should result in an error let res = block.set_entry( key.len() as u32, - key.to_owned(), + &key, value_offset, creation_date, is_tombstone, diff --git a/src/block/mod.rs b/src/block/mod.rs index 973a96b..2641a64 100644 --- a/src/block/mod.rs +++ b/src/block/mod.rs @@ -1,3 +1,3 @@ -mod block; +mod block_manager; -pub use block::Block; +pub use block_manager::Block; diff --git a/src/bucket/bucket.rs b/src/bucket/bucket_manager.rs similarity index 97% rename from src/bucket/bucket.rs rename to src/bucket/bucket_manager.rs index 0d3cb84..01e12e3 100644 --- a/src/bucket/bucket.rs +++ b/src/bucket/bucket_manager.rs @@ -116,7 +116,7 @@ impl Bucket { for meta_task in fetch_files_meta { let meta_data = meta_task .await - .map_err(|err| GetFileMetaDataError(err.into()))? + .map_err(|err| GetFileMetaData(err.into()))? .unwrap(); all_sstable_size += meta_data.len() as usize; } @@ -236,7 +236,7 @@ impl BucketMap { InsertionType::New => { bucket.avarage_size = fs::metadata(sst.clone().data_file.path) .await - .map_err(GetFileMetaDataError)? + .map_err(GetFileMetaData)? .len() as usize; self.buckets.insert(bucket.id, bucket); } @@ -319,7 +319,7 @@ impl BucketMap { } else { buckets_to_delete.push(bucket_id); if let Err(err) = fs::remove_dir_all(&bucket.dir).await { - log::error!("{}", DirDeleteError(err)); + log::error!("{}", DirDelete(err)); } } } @@ -328,7 +328,7 @@ impl BucketMap { if fs::metadata(&sst.dir).await.is_ok() { if let Err(err) = fs::remove_dir_all(&sst.dir).await { all_ssts_deleted = false; - log::error!("{}", DirDeleteError(err)); + log::error!("{}", DirDelete(err)); } } } @@ -348,7 +348,7 @@ impl BucketMap { for (_, bucket) in &self.buckets { if fs::metadata(&bucket.dir).await.is_ok() { if let Err(err) = fs::remove_dir_all(&bucket.dir).await { - log::error!("{}", FileDeleteError(err)); + log::error!("{}", FileDelete(err)); } } } diff --git a/src/bucket/mod.rs b/src/bucket/mod.rs index 76e489f..965cfdd 100644 --- a/src/bucket/mod.rs +++ b/src/bucket/mod.rs @@ -1,7 +1,8 @@ -pub(crate) mod bucket; -pub use bucket::Bucket; -pub use bucket::BucketID; -pub use bucket::BucketMap; -pub use bucket::ImbalancedBuckets; -pub use bucket::InsertableToBucket; -pub use bucket::SSTablesToRemove; + +pub(crate) mod bucket_manager; +pub use bucket_manager::Bucket; +pub use bucket_manager::BucketID; +pub use bucket_manager::BucketMap; +pub use bucket_manager::ImbalancedBuckets; +pub use bucket_manager::InsertableToBucket; +pub use bucket_manager::SSTablesToRemove; diff --git a/src/cfg/config.rs b/src/cfg/config.rs index c888ad2..e81770e 100644 --- a/src/cfg/config.rs +++ b/src/cfg/config.rs @@ -75,6 +75,7 @@ fn get_open_file_limit() -> usize { } impl Config { + #![allow(clippy::too_many_arguments)] pub fn new( false_positive_rate: f64, enable_ttl: bool, @@ -163,7 +164,7 @@ impl DataStore<'static, Key> { /// The size must be at least 50 kilobytes. pub fn with_write_buffer_size(mut self, size: usize) -> Self { assert!(size >= 50, "write_buffer_size should not be less than 50 Kilobytes"); - self.config.write_buffer_size = SizeUnit::Kilobytes.to_bytes(size); + self.config.write_buffer_size = SizeUnit::Kilobytes.as_bytes(size); self } @@ -258,7 +259,7 @@ impl DataStore<'static, Key> { /// The size must be at least 50 kilobytes. pub fn with_gc_chunk_size(mut self, size: usize) -> Self { assert!(size >= 50, "gc_chunk_size should not be less than 50 Kilobyte"); - self.config.gc_chunk_size = SizeUnit::Kilobytes.to_bytes(size); + self.config.gc_chunk_size = SizeUnit::Kilobytes.as_bytes(size); self } } @@ -270,11 +271,11 @@ mod tests { use crate::cfg::Config; use super::*; - use std::{path::PathBuf, time::Duration}; + use std::time::Duration; async fn create_datastore() -> DataStore<'static, Key> { let root = tempdir().unwrap(); - let path = PathBuf::from(root.path().join("store_test_3")); + let path = root.path().join("store_test_3"); let mut store = DataStore::open("test", path).await.unwrap(); // Initialize with default or dummy values let config = Config { @@ -295,7 +296,7 @@ mod tests { open_files_limit: 150, }; store.config = config; - return store; + store } #[tokio::test] @@ -344,7 +345,7 @@ mod tests { async fn test_with_write_buffer_size() { let ds = create_datastore().await; let ds = ds.with_write_buffer_size(100); - assert_eq!(ds.config.write_buffer_size, SizeUnit::Kilobytes.to_bytes(100)); + assert_eq!(ds.config.write_buffer_size, SizeUnit::Kilobytes.as_bytes(100)); } #[tokio::test] @@ -372,7 +373,7 @@ mod tests { #[should_panic(expected = "entry_ttl_millis cannot be less than 3 days if enable_ttl is set to true")] async fn test_with_entry_ttl_invalid() { let ds = create_datastore().await.with_enable_ttl(true); - ds.with_entry_ttl(Duration::from_secs(1 * 24 * 60 * 60)); // 1 day + ds.with_entry_ttl(Duration::from_secs(24 * 60 * 60)); // 1 day } #[tokio::test] @@ -450,7 +451,7 @@ mod tests { async fn test_with_compaction_strategy() { let ds = create_datastore().await; let strategy = compactors::Strategy::STCS; // Replace with actual strategy - let ds = ds.with_compaction_strategy(strategy.clone()); + let ds = ds.with_compaction_strategy(strategy); assert_eq!(ds.config.compaction_strategy, compactors::Strategy::STCS); } @@ -479,6 +480,6 @@ mod tests { async fn test_with_gc_chunk_size() { let ds = create_datastore().await; let ds = ds.with_gc_chunk_size(100); - assert_eq!(ds.config.gc_chunk_size, SizeUnit::Kilobytes.to_bytes(100)); + assert_eq!(ds.config.gc_chunk_size, SizeUnit::Kilobytes.as_bytes(100)); } } diff --git a/src/compactors/compact.rs b/src/compactors/compact.rs index 03037b8..6759843 100644 --- a/src/compactors/compact.rs +++ b/src/compactors/compact.rs @@ -1,7 +1,8 @@ use crate::bucket::InsertableToBucket; -use crate::types::{BloomFilterHandle, Bool, BucketMapHandle, FlushReceiver, KeyRangeHandle}; +use crate::types::{Bool, BucketMapHandle, FlushReceiver, KeyRangeHandle}; use crate::{err::Error, filter::BloomFilter}; use std::sync::Arc; +use std::time; use tokio::sync::Mutex; use tokio::time::sleep; use Error::*; @@ -14,7 +15,7 @@ use Error::*; /// These expired tombstones are removed entirely during compaction, freeing up disk space. /// /// - **Unexpired Tombstones**: If a tombstone is not expired, it means the data it shadows might still be relevant in other tiers. -/// In this case, VikingsDB keeps both the tombstone and the data in the new SSTable. This ensures consistency across tiers and allows for repairs if needed. +/// In this case, velarixDB keeps both the tombstone and the data in the new SSTable. This ensures consistency across tiers and allows for repairs if needed. /// /// Currently, only the Sized-Tier Compaction Strategy (STCS) is supported. However, support for Leveled Compaction (LCS), Time-Window Compaction (TWCS), and Unified Compaction (UCS) strategies is planned. #[derive(Debug, Clone)] @@ -55,6 +56,7 @@ pub struct Config { pub(crate) filter_false_positive: f64, } impl Config { + #![allow(clippy::too_many_arguments)] pub fn new( use_ttl: Bool, entry_ttl: std::time::Duration, @@ -81,9 +83,9 @@ impl Config { #[derive(Debug, Clone, Copy, PartialEq)] pub enum Strategy { STCS, - LCS, // TODO - TCS, // TODO - UCS, // TODO + // LCS, TODO + // TCS, TODO + // UCS, TODO } #[derive(Debug, Clone)] @@ -128,7 +130,7 @@ impl MergePointer { } } -/// Merged SSTable stored here +/// Merged SSTable stored here /// before being flushed to disk #[derive(Debug)] pub struct MergedSSTable { @@ -159,14 +161,15 @@ impl MergedSSTable { } impl Compactor { + #![allow(clippy::too_many_arguments)] // Creates new `Compactor` pub fn new( use_ttl: Bool, - entry_ttl: std::time::Duration, - tombstone_ttl: std::time::Duration, - background_interval: std::time::Duration, - flush_listener_interval: std::time::Duration, - tombstone_compaction_interval: std::time::Duration, + entry_ttl: time::Duration, + tombstone_ttl: time::Duration, + background_interval: time::Duration, + flush_listener_interval: time::Duration, + tombstone_compaction_interval: time::Duration, strategy: Strategy, reason: CompactionReason, filter_false_positive: f64, @@ -192,7 +195,6 @@ impl Compactor { pub fn tombstone_compaction_condition_background_checker( &self, bucket_map: BucketMapHandle, - filter: BloomFilterHandle, key_range: KeyRangeHandle, ) { let cfg = self.config.to_owned(); @@ -210,7 +212,6 @@ impl Compactor { &self, flush_rx: FlushReceiver, bucket_map: BucketMapHandle, - filter: BloomFilterHandle, key_range: KeyRangeHandle, ) { let mut rx = flush_rx.clone(); @@ -226,10 +227,10 @@ impl Compactor { drop(state); match err { async_broadcast::TryRecvError::Overflowed(_) => { - log::error!("{}", FlushSignalChannelOverflowError) + log::error!("{}", FlushSignalChannelOverflow) } async_broadcast::TryRecvError::Closed => { - log::error!("{}", FlushSignalChannelClosedError) + log::error!("{}", FlushSignalChannelClosed) } async_broadcast::TryRecvError::Empty => {} } @@ -237,13 +238,8 @@ impl Compactor { } *state = CompState::Active; drop(state); - if let Err(err) = Compactor::handle_compaction( - Arc::clone(&bucket_map), - Arc::clone(&filter), - Arc::clone(&key_range), - &cfg, - ) - .await + if let Err(err) = + Compactor::handle_compaction(Arc::clone(&bucket_map), Arc::clone(&key_range), &cfg).await { log::info!("{}", Error::CompactionFailed(Box::new(err))); continue; @@ -257,12 +253,7 @@ impl Compactor { } /// Background compaction runner for maintenance - pub fn start_periodic_background_compaction( - &self, - buckets: BucketMapHandle, - filter: BloomFilterHandle, - key_range: KeyRangeHandle, - ) { + pub fn spawn_compaction_worker(&self, buckets: BucketMapHandle, key_range: KeyRangeHandle) { let cfg = self.config.to_owned(); let comp_state = Arc::clone(&self.is_active); tokio::spawn(async move { @@ -272,13 +263,8 @@ impl Compactor { if let CompState::Sleep = *state { *state = CompState::Active; drop(state); - if let Err(err) = Compactor::handle_compaction( - Arc::clone(&buckets), - Arc::clone(&filter), - Arc::clone(&key_range), - &cfg, - ) - .await + if let Err(err) = + Compactor::handle_compaction(Arc::clone(&buckets), Arc::clone(&key_range), &cfg).await { log::info!("{}", Error::CompactionFailed(Box::new(err))) } @@ -291,32 +277,15 @@ impl Compactor { pub async fn handle_compaction( buckets: BucketMapHandle, - filter: BloomFilterHandle, key_range: KeyRangeHandle, cfg: &Config, ) -> Result<(), Error> { match cfg.strategy { Strategy::STCS => { - let mut runner = super::sized::SizedTierRunner::new( - Arc::clone(&buckets), - Arc::clone(&filter), - Arc::clone(&key_range), - cfg, - ); + let mut runner = super::sized::SizedTierRunner::new(Arc::clone(&buckets), Arc::clone(&key_range), cfg); runner.run_compaction().await } - Strategy::LCS => { - log::info!("LCS not curently supported, try SCS instead"); - Ok(()) - } - Strategy::TCS => { - log::info!("TCS not curently supported, try SCS instead"); - Ok(()) - } - Strategy::UCS => { - log::info!("UCS not curently supported, try SCS instead"); - Ok(()) - } + // LCS, UCS and TWS will be added later } } diff --git a/src/compactors/sized.rs b/src/compactors/sized.rs index 902cfc5..88faaff 100644 --- a/src/compactors/sized.rs +++ b/src/compactors/sized.rs @@ -1,7 +1,6 @@ -use std::{cmp, collections::HashMap, path::PathBuf, sync::Arc}; +use std::{cmp, collections::HashMap, sync::Arc}; use crossbeam_skiplist::SkipMap; -use uuid::Uuid; use super::{ compact::{Config, MergePointer, WriteTracker}, @@ -12,8 +11,7 @@ use crate::{ err::Error, filter::BloomFilter, memtable::Entry, - sst::Table, - types::{BloomFilterHandle, Bool, BucketMapHandle, CreatedAt, Key, KeyRangeHandle, ValOffset}, + types::{ Bool, BucketMapHandle, CreatedAt, Key, KeyRangeHandle, ValOffset}, }; use crate::{err::Error::*, memtable::SkipMapValue}; @@ -27,9 +25,6 @@ pub struct SizedTierRunner<'a> { /// A thread-safe BucketMap with each bucket mapped to its id bucket_map: BucketMapHandle, - /// A thread-safe vector of sstable bloom filters - filters: BloomFilterHandle, - /// A thread-safe hashmap of sstables each mapped to its key range key_range: KeyRangeHandle, @@ -45,14 +40,12 @@ impl<'a> SizedTierRunner<'a> { /// creates new instance of `SizedTierRunner` pub fn new( bucket_map: BucketMapHandle, - filters: BloomFilterHandle, key_range: KeyRangeHandle, config: &'a Config, ) -> SizedTierRunner<'a> { Self { tombstones: HashMap::new(), bucket_map, - filters, key_range, config, } @@ -73,7 +66,6 @@ impl<'a> SizedTierRunner<'a> { // TODO: Handle this with multiple threads loop { let buckets: BucketMapHandle = Arc::clone(&self.bucket_map); - let filters = Arc::clone(&self.filters); let key_range = Arc::clone(&self.key_range); // Step 1: Extract imbalanced buckets let (imbalanced_buckets, ssts_to_remove) = @@ -96,64 +88,44 @@ impl<'a> SizedTierRunner<'a> { match insert_res { Ok(sst) => { if sst.summary.is_none() { - return Err(TableSummaryIsNoneError); + return Err(TableSummaryIsNone); } if sst.filter.is_none() { return Err(FilterNotProvidedForFlush); } // IMPORTANT: Don't keep sst entries in memory sst.entries.clear(); - // Step 4 Store Filter in Filters Vec - filters.write().await.push(sst.filter.to_owned().unwrap()); let summary = sst.summary.clone().unwrap(); // Step 5 Store sst key range - key_range.write().await.set( - sst.dir.to_owned(), - summary.smallest_key, - summary.biggest_key, - sst, - ); + key_range + .set(sst.dir.to_owned(), summary.smallest_key, summary.biggest_key, sst) + .await; tracker.actual += 1; } Err(err) => { - // Step 6 (Optional): Trigger recovery in case compaction failed at any point - // Ensure filters are restored to the previous state - // Remove merged sstables written to disk so far to prevent stale data - while tracker.actual > 0 { - if let Some(filter) = filters.write().await.pop() { - let sst_dir = filter.get_sst_dir().to_owned(); - if let Err(err) = tokio::fs::remove_dir_all(sst_dir.to_owned()).await { - log::error!("{}", CompactionFailed(Box::new(DirDeleteError(err)))); - tracker.actual -= 1; - continue; - } - key_range.write().await.remove(sst_dir.to_owned()); - } - tracker.actual -= 1; - } return Err(CompactionFailed(Box::new(err))); } } } if tracker.expected == tracker.actual { - // Step 7: Delete the sstables that we already merged from their previous buckets and update bloom filters + // Step 6: Delete the sstables that we already merged from their previous buckets and update bloom filters let filters_updated = self - .clean_up_after_compaction(buckets, &ssts_to_remove.clone(), filters, key_range) + .clean_up_after_compaction(buckets, &ssts_to_remove.clone(), key_range) .await; match filters_updated { Ok(None) => { return Err(Error::CompactionPartiallyFailed(Box::new( - CompactionCleanupPartialError, + CompactionCleanupPartial, ))); } Err(err) => { - return Err(Error::CompactionCleanupError(Box::new(err))); + return Err(Error::CompactionCleanup(Box::new(err))); } _ => {} } } else { - log::error!("{}", Error::CannotRemoveObsoleteSSTError) + log::error!("{}", Error::CannotRemoveObsoleteSST) } } Err(err) => return Err(CompactionFailed(Box::new(err))), @@ -173,48 +145,27 @@ impl<'a> SizedTierRunner<'a> { &self, buckets: BucketMapHandle, ssts_to_delete: &SSTablesToRemove, - filters: BloomFilterHandle, key_range: KeyRangeHandle, ) -> Result, Error> { - // Remove obsolete keys from keys range - ssts_to_delete.iter().for_each(|(_, sstables)| { - sstables.iter().for_each(|s| { - let range = Arc::clone(&key_range); - let path = s.get_data_file_path(); - tokio::spawn(async move { - range.write().await.remove(path); - }); - }) - }); - // if all obsolete sstables were not deleted then don't remove the associated filters // although this can lead to redundancy but bloom filters are in-memory and its also less costly // in terms of memory if buckets.write().await.delete_ssts(ssts_to_delete).await? { - // Step 8: Delete the filters associated with the sstables that we already merged - self.filter_out_old_filter(filters, ssts_to_delete).await; + // Step 7: Remove obsolete keys from keys range + ssts_to_delete.iter().for_each(|(_, sstables)| { + sstables.iter().for_each(|s| { + let range = Arc::clone(&key_range); + let path = s.get_data_file_path(); + tokio::spawn(async move { + range.remove(path).await; + }); + }) + }); return Ok(Some(())); } Ok(None) } - /// Remove filters of obsolete sstables from filters vector - pub async fn filter_out_old_filter(&self, filters: BloomFilterHandle, ssts_to_delete: &Vec<(Uuid, Vec)>) { - let mut filter_map: HashMap = filters - .read() - .await - .iter() - .map(|b| (b.sst_dir.to_owned().unwrap(), b.to_owned())) - .collect(); - ssts_to_delete.iter().for_each(|(_, tables)| { - tables.iter().for_each(|t| { - filter_map.remove(&t.dir); - }) - }); - filters.write().await.clear(); - filters.write().await.extend(filter_map.into_values()); - } - /// Merges the sstables in each `Bucket` to form a larger one /// /// Returns `Result` with merged sstable or error @@ -222,7 +173,7 @@ impl<'a> SizedTierRunner<'a> { /// # Errors /// /// Returns error incase an error occured during merge - async fn merge_ssts_in_buckets(&mut self, buckets: &Vec) -> Result, Error> { + async fn merge_ssts_in_buckets(&mut self, buckets: &[Bucket]) -> Result, Error> { let mut merged_ssts = Vec::new(); for bucket in buckets.iter() { let mut hotness = 0; @@ -336,7 +287,7 @@ impl<'a> SizedTierRunner<'a> { } /// Checks if an entry has been deleted or not - /// + /// /// Deleted entries are discoverd using the tombstones hashmap /// and prevented from being inserted /// diff --git a/src/consts/mod.rs b/src/consts/mod.rs index 44675bd..514fc7e 100644 --- a/src/consts/mod.rs +++ b/src/consts/mod.rs @@ -35,7 +35,7 @@ pub const SUMMARY_FILE_NAME: &str = "summary"; pub const INDEX_FILE_NAME: &str = "index"; -pub const DEFAULT_DB_NAME: &str = "vikings"; +pub const DEFAULT_DB_NAME: &str = "velarix"; pub const META_DIRECTORY_NAME: &str = "meta"; @@ -44,10 +44,10 @@ pub const TOMB_STONE_MARKER: &str = "*"; /// TODO: Many lightweight computations here, benchmark against Lazy initialization /// 1KB -pub static GC_CHUNK_SIZE: usize = SizeUnit::Kilobytes.to_bytes(1); +pub static GC_CHUNK_SIZE: usize = SizeUnit::Kilobytes.as_bytes(1); /// 50KB -pub const WRITE_BUFFER_SIZE: usize = SizeUnit::Kilobytes.to_bytes(50); +pub const WRITE_BUFFER_SIZE: usize = SizeUnit::Kilobytes.as_bytes(50); /// 5 days pub const DEFAULT_TOMBSTONE_COMPACTION_INTERVAL: Duration = Duration::from_millis(5 * 86400000); @@ -74,7 +74,7 @@ pub const BUCKET_LOW: f64 = 0.5; pub const BUCKET_HIGH: f64 = 1.5; -pub const MIN_SSTABLE_SIZE: usize = SizeUnit::Kilobytes.to_bytes(4); +pub const MIN_SSTABLE_SIZE: usize = SizeUnit::Kilobytes.as_bytes(4); pub const MIN_TRESHOLD: usize = 4; diff --git a/src/db/keyspace.rs b/src/db/keyspace.rs index efd051a..cf34100 100644 --- a/src/db/keyspace.rs +++ b/src/db/keyspace.rs @@ -34,7 +34,7 @@ mod tests { assert!(is_valid_keyspace_name("valid_keyspace-123")); // Valid but maximum length keyspace name - let max_length_name: String = "a".repeat(MAX_KEY_SPACE_SIZE as usize); + let max_length_name: String = "a".repeat(MAX_KEY_SPACE_SIZE); assert!(is_valid_keyspace_name(&max_length_name)); } @@ -44,7 +44,7 @@ mod tests { assert!(!is_valid_keyspace_name("")); // Keyspace name too long - let too_long_name: String = "a".repeat((MAX_KEY_SPACE_SIZE + 1) as usize); + let too_long_name: String = "a".repeat(MAX_KEY_SPACE_SIZE + 1); assert!(!is_valid_keyspace_name(&too_long_name)); // Keyspace name with invalid characters diff --git a/src/db/recovery.rs b/src/db/recovery.rs index 964f337..0666eb8 100644 --- a/src/db/recovery.rs +++ b/src/db/recovery.rs @@ -15,7 +15,7 @@ use crate::err::Error::*; use crate::filter::BloomFilter; use crate::flush::Flusher; use crate::fs::FileAsync; -use crate::gc::gc::GC; +use crate::gc::garbage_collector::GC; use crate::key_range::KeyRange; use crate::memtable::{Entry, MemTable}; use crate::meta::Meta; @@ -41,18 +41,17 @@ impl DataStore<'static, Key> { dir: DirPath, buckets_path: P, mut vlog: ValueLog, - mut key_range: KeyRange, + key_range: KeyRange, config: &Config, size_unit: SizeUnit, mut meta: Meta, ) -> Result, Error> { let mut recovered_buckets: IndexMap = IndexMap::new(); - let filters: Vec = Vec::new(); // Get bucket diretories streams let mut buckets_stream = open_dir_stream!(buckets_path.as_ref().to_path_buf()); // for each bucket directory - while let Some(bucket_dir) = buckets_stream.next_entry().await.map_err(|err| DirOpenError { + while let Some(bucket_dir) = buckets_stream.next_entry().await.map_err(|err| DirOpen { path: buckets_path.as_ref().to_path_buf(), error: err, })? { @@ -60,7 +59,7 @@ impl DataStore<'static, Key> { let mut sst_dir_stream = open_dir_stream!(bucket_dir.path()); // iterate over each sstable directory - while let Some(sst_dir) = sst_dir_stream.next_entry().await.map_err(|err| DirOpenError { + while let Some(sst_dir) = sst_dir_stream.next_entry().await.map_err(|err| DirOpen { path: buckets_path.as_ref().to_path_buf(), error: err, })? { @@ -69,7 +68,7 @@ impl DataStore<'static, Key> { let mut files = Vec::new(); // iterate over each file - while let Some(file) = files_stream.next_entry().await.map_err(|err| DirOpenError { + while let Some(file) = files_stream.next_entry().await.map_err(|err| DirOpen { path: buckets_path.as_ref().to_path_buf(), error: err, })? { @@ -83,7 +82,7 @@ impl DataStore<'static, Key> { let bucket_id = Self::get_bucket_id_from_full_bucket_path(sst_dir.path()); if files.len() < 4 { - return Err(InvalidSSTableDirectoryError { + return Err(InvalidSSTableDirectory { input_string: sst_dir.path().to_owned().to_string_lossy().to_string(), }); } @@ -122,11 +121,10 @@ impl DataStore<'static, Key> { table.summary = Some(summary.to_owned()); // store bloomfilter metadata in table - let mut new_filter = BloomFilter::default(); - new_filter.file_path = Some(filter_file_path); + let new_filter = BloomFilter { file_path: Some(filter_file_path), ..Default::default() }; table.filter = Some(new_filter); - key_range.set(sst_dir.path(), summary.smallest_key, summary.biggest_key, table); + key_range.set(sst_dir.path(), summary.smallest_key, summary.biggest_key, table).await; } } let mut buckets_map = BucketMap::new(buckets_path.clone()).await?; @@ -163,15 +161,13 @@ impl DataStore<'static, Key> { match recover_res { Ok((active_memtable, read_only_memtables)) => { let buckets = Arc::new(RwLock::new(buckets_map.to_owned())); - let filters = Arc::new(RwLock::new(filters)); - let key_range = Arc::new(RwLock::new(key_range.to_owned())); + let key_range = Arc::new(key_range.to_owned()); let read_only_memtables = Arc::new(read_only_memtables); let gc_table = Arc::new(RwLock::new(active_memtable.to_owned())); let gc_log = Arc::new(RwLock::new(vlog.to_owned())); let flusher = Flusher::new( read_only_memtables.clone(), buckets.clone(), - filters.clone(), key_range.clone(), ); let gc_updated_entries = Arc::new(RwLock::new(SkipMap::new())); @@ -181,7 +177,6 @@ impl DataStore<'static, Key> { val_log: vlog, dir, buckets, - filters, key_range, meta, flusher, @@ -214,7 +209,7 @@ impl DataStore<'static, Key> { flush_stream: HashSet::new(), }) } - Err(err) => Err(MemTableRecoveryError(Box::new(err))), + Err(err) => Err(MemTableRecovery(Box::new(err))), } } @@ -297,16 +292,14 @@ impl DataStore<'static, Key> { let buckets = BucketMap::new(buckets_path).await?; let (flush_signal_tx, flush_signal_rx) = broadcast(DEFAULT_FLUSH_SIGNAL_CHANNEL_SIZE); let read_only_memtables = SkipMap::new(); - let filters = Arc::new(RwLock::new(Vec::new())); let buckets = Arc::new(RwLock::new(buckets.to_owned())); - let key_range = Arc::new(RwLock::new(key_range)); + let key_range = Arc::new(key_range); let read_only_memtables = Arc::new(read_only_memtables); let gc_table = Arc::new(RwLock::new(active_memtable.to_owned())); let gc_log = Arc::new(RwLock::new(vlog.to_owned())); let flusher = Flusher::new( read_only_memtables.clone(), buckets.clone(), - filters.clone(), key_range.clone(), ); let gc_updated_entries = Arc::new(RwLock::new(SkipMap::new())); @@ -314,7 +307,6 @@ impl DataStore<'static, Key> { keyspace: DEFAULT_DB_NAME, active_memtable, val_log: vlog, - filters, buckets, dir, key_range, diff --git a/src/db/store.rs b/src/db/store.rs index d113230..c376485 100644 --- a/src/db/store.rs +++ b/src/db/store.rs @@ -6,16 +6,16 @@ use crate::consts::{ }; use crate::db::keyspace::is_valid_keyspace_name; use crate::flush::Flusher; -use crate::gc::gc::GC; +use crate::gc::garbage_collector::GC; use crate::index::Index; use crate::key_range::KeyRange; -use crate::memtable::{Entry, MemTable, K}; +use crate::memtable::{Entry, MemTable, UserEntry, K}; use crate::meta::Meta; use crate::range::RangeIterator; use crate::sst::Table; use crate::types::{ - BloomFilterHandle, Bool, BucketMapHandle, CreatedAt, FlushSignal, GCUpdatedEntries, ImmutableMemTables, Key, - KeyRangeHandle, MemtableFlushStream, Value, + Bool, BucketMapHandle, CreatedAt, FlushSignal, GCUpdatedEntries, ImmutableMemTables, Key, KeyRangeHandle, + MemtableFlushStream, }; use crate::util; use crate::vlog::ValueLog; @@ -40,9 +40,6 @@ where /// Active memtable that accepts read and writes using a lock free skipmap pub(crate) active_memtable: MemTable, - /// In memory bloom filters for fast retreival - pub(crate) filters: BloomFilterHandle, - /// Value log to perisist entries and for crash recovery pub(crate) val_log: ValueLog, @@ -112,7 +109,7 @@ pub enum SizeUnit { } impl SizeUnit { - pub(crate) const fn to_bytes(&self, value: usize) -> usize { + pub(crate) const fn as_bytes(&self, value: usize) -> usize { match self { SizeUnit::Bytes => value, SizeUnit::Kilobytes => value * KB, @@ -174,21 +171,49 @@ impl DataStore<'static, Key> { /// Should not be called, unless [`DataStore::open`] /// and should not be user-facing. pub(crate) fn start_background_tasks(&self) { - self.compactor.start_periodic_background_compaction( - Arc::clone(&self.buckets), - Arc::clone(&self.filters), - Arc::clone(&self.key_range), - ); + // NOTE: we only incrememnt the ref counter not a deep clone + self.compactor + .spawn_compaction_worker(self.buckets.clone(), self.key_range.clone()); + self.compactor.start_flush_listener( self.flush_signal_rx.clone(), - Arc::clone(&self.buckets), - Arc::clone(&self.filters), - Arc::clone(&self.key_range), + self.buckets.clone(), + self.key_range.clone(), ); + self.gc - .start_background_gc_task(Arc::clone(&self.key_range), Arc::clone(&self.read_only_memtables)); + .start_gc_worker(self.key_range.clone(), self.read_only_memtables.clone()); } + /// Inserts a new entry into the store + /// + /// # Examples + /// ``` + /// # use tempfile::tempdir; + /// use velarixdb::db::DataStore; + + /// #[tokio::main] + /// async fn main() { + /// let root = tempdir().unwrap(); + /// let path = root.path().join("velarixdb"); + /// let mut store = DataStore::open("big_tech", path).await.unwrap(); // handle IO error + /// + /// let res1 = store.put("apple", "tim cook").await; + /// let res2 = store.put("google", "sundar pichai").await; + /// let res3 = store.put("nvidia", "jensen huang").await; + /// let res4 = store.put("microsoft", "satya nadella").await; + /// let res5 = store.put("meta", "mark zuckerberg").await; + /// let res6 = store.put("openai", "sam altman").await; + /// + /// assert!(res1.is_ok()); + /// assert!(res2.is_ok()); + /// assert!(res3.is_ok()); + /// assert!(res4.is_ok()); + /// assert!(res5.is_ok()); + /// assert!(res6.is_ok()); + /// } + /// + /// ``` pub async fn put, V: AsRef<[u8]>>(&mut self, key: K, val: V) -> Result { self.validate_size(key.as_ref(), Some(val.as_ref()))?; @@ -197,7 +222,7 @@ impl DataStore<'static, Key> { } // This ensures sstables in key range whose filter is newly loaded(after crash) are mapped to the sstables - self.key_range.write().await.update_key_range().await; + self.key_range.update_key_range().await; let is_tombstone = std::str::from_utf8(val.as_ref()).unwrap() == TOMB_STONE_MARKER; let created_at = Utc::now(); let v_offset = self @@ -242,7 +267,7 @@ impl DataStore<'static, Key> { if self.read_only_memtables.len() >= self.config.max_buffer_write_number { // Background - let _ = self.flush_read_only_memtables(); + self.flush_read_only_memtables(); } self.reset_memtables(); } @@ -278,7 +303,7 @@ impl DataStore<'static, Key> { } /// Updates metadata in background - /// #[doc(hidden)] + #[doc(hidden)] pub(crate) fn update_meta_background(&self) { let meta = Arc::new(Mutex::new(self.meta.to_owned())); tokio::spawn(async move { @@ -288,6 +313,36 @@ impl DataStore<'static, Key> { }); } + /// Removes an entry from the store + /// + /// + /// # Examples + /// + /// ``` + /// # use tempfile::tempdir; + /// use velarixdb::db::DataStore; + /// + /// #[tokio::main] + /// async fn main() { + /// let root = tempdir().unwrap(); + /// let path = root.path().join("velarixdb"); + /// let mut store = DataStore::open("big_tech", path).await.unwrap(); // handle IO error + /// + /// store.put("apple", "tim cook").await.unwrap(); // handle error + /// // Retrieve entry + /// let entry = store.get("apple").await.unwrap(); + /// assert!(entry.is_some()); + /// + /// // Delete entry + /// store.delete("apple").await.unwrap(); + /// + /// // Entry should now be None + /// let entry = store.get("apple").await.unwrap(); + /// assert!(entry.is_none()); + /// } + /// + /// ``` + pub async fn delete>(&mut self, key: T) -> Result { self.validate_size(key.as_ref(), None::)?; self.get(key.as_ref()).await?; @@ -330,7 +385,56 @@ impl DataStore<'static, Key> { ))); } - pub async fn get>(&self, key: T) -> Result<(Value, CreatedAt), crate::err::Error> { + /// Reteives an entry from the [`DataStore`] + /// + /// + /// This is user facing and its asyncronous + /// + /// # Examples + /// + /// ```rust + /// # use tempfile::tempdir; + /// use velarixdb::db::DataStore; + /// + /// #[tokio::main] + /// async fn main() { + /// let root = tempdir().unwrap(); + /// let path = root.path().join("velarixdb"); + /// let mut store = DataStore::open("big_tech", path).await.unwrap(); // handle IO error + /// + /// let res1 = store.put("apple", "tim cook").await; + /// let res2 = store.put("google", "sundar pichai").await; + /// let res3 = store.put("nvidia", "jensen huang").await; + /// let res4 = store.put("microsoft", "satya nadella").await; + /// let res5 = store.put("meta", "mark zuckerberg").await; + /// let res6 = store.put("openai", "sam altman").await; + /// + /// assert!(res1.is_ok()); + /// assert!(res2.is_ok()); + /// assert!(res3.is_ok()); + /// assert!(res4.is_ok()); + /// assert!(res5.is_ok()); + /// assert!(res6.is_ok()); + /// + /// let entry1 = store.get("apple").await.unwrap(); // Handle error + /// let entry2 = store.get("google").await.unwrap(); + /// let entry3 = store.get("nvidia").await.unwrap(); + /// let entry4 = store.get("microsoft").await.unwrap(); + /// let entry5 = store.get("meta").await.unwrap(); + /// let entry6 = store.get("openai").await.unwrap(); + /// let entry7 = store.get("***not_found_key**").await.unwrap(); + /// + /// assert_eq!(std::str::from_utf8(&entry1.unwrap().val).unwrap(), "tim cook"); + /// assert_eq!(std::str::from_utf8(&entry2.unwrap().val).unwrap(), "sundar pichai"); + /// assert_eq!(std::str::from_utf8(&entry3.unwrap().val).unwrap(), "jensen huang"); + /// assert_eq!(std::str::from_utf8(&entry4.unwrap().val).unwrap(), "satya nadella"); + /// assert_eq!(std::str::from_utf8(&entry5.unwrap().val).unwrap(), "mark zuckerberg"); + /// assert_eq!(std::str::from_utf8(&entry6.unwrap().val).unwrap(), "sam altman"); + /// assert!(entry7.is_none()) + /// } + /// ``` + + pub async fn get>(&self, key: T) -> Result, crate::err::Error> { self.validate_size(key.as_ref(), None::)?; let gc_entries_reader = self.gc_updated_entries.read().await; if !gc_entries_reader.is_empty() { @@ -338,7 +442,7 @@ impl DataStore<'static, Key> { if res.is_some() { let value = res.to_owned().unwrap().value().to_owned(); if value.is_tombstone { - return Err(crate::err::Error::NotFoundInDB); + return Ok(None); } return self.get_value_from_vlog(value.val_offset, value.created_at).await; } @@ -349,7 +453,7 @@ impl DataStore<'static, Key> { let lowest_insert_time = util::default_datetime(); if let Some(value) = self.active_memtable.get(key.as_ref()) { if value.is_tombstone { - return Err(crate::err::Error::NotFoundInDB); + return Ok(None); } self.get_value_from_vlog(value.val_offset, value.created_at).await } else { @@ -366,26 +470,52 @@ impl DataStore<'static, Key> { if self.found_in_table(insert_time, lowest_insert_time) { if is_deleted { - return Err(crate::err::Error::NotFoundInDB); + return Ok(None); } self.get_value_from_vlog(offset, insert_time).await } else { - let key_range = &self.key_range.read().await; - let ssts = key_range.filter_sstables_by_biggest_key(key.as_ref()).await?; + let ssts = &self.key_range.filter_sstables_by_biggest_key(key.as_ref()).await?; if ssts.is_empty() { - return Err(crate::err::Error::NotFoundInDB); + return Ok(None); } - self.search_key_in_sstables(key, ssts).await + self.search_key_in_sstables(key, ssts.to_vec()).await } } } + /// Removes an entry from the store + /// + /// # Examples + /// + /// ``` + /// # use tempfile::tempdir; + /// use velarixdb::db::DataStore; + /// #[tokio::main] + /// async fn main() { + /// let root = tempdir().unwrap(); + /// let path = root.path().join("velarixdb"); + /// let mut store = DataStore::open("big_tech", path).await.unwrap(); // handle IO error + + /// store.put("apple", "tim cook").await.unwrap(); // handle error + + /// // Update entry + /// let success = store.update("apple", "elon musk").await; + /// assert!(success.is_ok()); + + /// // Entry should now be updated + /// let entry = store.get("apple").await.unwrap(); // handle error + /// assert!(entry.is_some()); + /// assert_eq!(std::str::from_utf8(&entry.unwrap().val).unwrap(), "elon musk") + /// } + /// ``` + pub async fn update>(&mut self, key: T, value: T) -> Result { self.validate_size(key.as_ref(), Some(value.as_ref()))?; self.get(key.as_ref()).await?; self.put(key, value).await } + /// Validate key and value sizes. /// /// Key size can be up to 65536 bytes in size, and value size can be @@ -431,7 +561,7 @@ impl DataStore<'static, Key> { &self, key: K, ssts: Vec
, - ) -> Result<(Value, CreatedAt), crate::err::Error> { + ) -> Result, crate::err::Error> { let mut insert_time = util::default_datetime(); let lowest_insert_date = util::default_datetime(); let mut offset = 0; @@ -453,7 +583,7 @@ impl DataStore<'static, Key> { } if self.found_in_table(insert_time, lowest_insert_date) { if is_deleted { - return Err(crate::err::Error::NotFoundInDB); + return Ok(None); } return self.get_value_from_vlog(offset, insert_time).await; } @@ -477,17 +607,16 @@ impl DataStore<'static, Key> { pub(crate) async fn get_value_from_vlog( &self, offset: usize, - creation_time: CreatedAt, - ) -> Result<(Value, CreatedAt), crate::err::Error> { + created_at: CreatedAt, + ) -> Result, crate::err::Error> { let res = self.val_log.get(offset).await?; - if res.is_some() { - let (value, is_tombstone) = res.unwrap(); + if let Some((value, is_tombstone)) = res { if is_tombstone { - return Err(crate::err::Error::KeyFoundAsTombstoneInValueLogError); + return Ok(None); } - return Ok((value, creation_time)); + return Ok(Some(UserEntry::new(value, created_at))); } - Err(crate::err::Error::KeyNotFoundInValueLogError) + Ok(None) } /// Flushes all memtable (active and read-only) to disk @@ -496,8 +625,9 @@ impl DataStore<'static, Key> { /// # Errors /// /// Returns error, if an IO error occurs or key was not found + #[doc(hidden)] #[cfg(test)] - pub(crate) async fn flush_all_memtables(&mut self) -> Result<(), crate::err::Error> { + pub(crate) async fn force_flush(&mut self) -> Result<(), crate::err::Error> { use crossbeam_skiplist::SkipMap; self.active_memtable.read_only = true; @@ -508,7 +638,6 @@ impl DataStore<'static, Key> { let mut flusher = Flusher::new( Arc::clone(&self.read_only_memtables), Arc::clone(&self.buckets), - Arc::clone(&self.filters), Arc::clone(&self.key_range), ); for table in immutable_tables.iter() { @@ -539,10 +668,10 @@ impl DataStore<'static, Key> { let vlog_empty = !vlog_exit || fs::metadata(vlog_path) .await - .map_err(crate::err::Error::GetFileMetaDataError)? + .map_err(crate::err::Error::GetFileMetaData)? .len() == 0; - let key_range = KeyRange::new(); + let key_range = KeyRange::default(); let vlog = ValueLog::new(vlog_path).await?; let meta = Meta::new(&dir.meta).await?; if vlog_empty { @@ -560,7 +689,6 @@ impl DataStore<'static, Key> { self.compactor.reason = CompactionReason::Manual; Compactor::handle_compaction( Arc::clone(&self.buckets), - Arc::clone(&self.filters.clone()), Arc::clone(&self.key_range), &self.compactor.config, ) @@ -568,12 +696,12 @@ impl DataStore<'static, Key> { } /// Returns length of entries in active memtable - pub async fn len_of_entries_in_memtable(&mut self) -> usize { + pub fn len_of_entries_in_memtable(&self) -> usize { self.active_memtable.entries.len() } /// Get [`DataStore`] directories - pub async fn get_dir(&mut self) -> DirPath { + pub async fn get_dir(&self) -> DirPath { self.dir.to_owned() } diff --git a/src/err/mod.rs b/src/err/mod.rs index 3721aa2..5e08d66 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -5,46 +5,46 @@ use thiserror::Error; #[non_exhaustive] pub enum Error { #[error("Failed to sync writes to file : {error}")] - FileSyncError { error: io::Error }, + FileSync { error: io::Error }, #[error("Failed to create file: `{path}`: {error}")] - FileCreationError { path: PathBuf, error: io::Error }, + FileCreation { path: PathBuf, error: io::Error }, #[error("File seek error")] - FileSeekError(#[source] io::Error), + FileSeek(#[source] io::Error), #[error("Directory deletion error")] - DirDeleteError(#[source] io::Error), + DirDelete(#[source] io::Error), #[error("Filter file path not provided")] FilterFilePathNotProvided, #[error("Filter file open error: path `{0}`")] - FilterFileOpenError(PathBuf), + FilterFileOpen(PathBuf), #[error("File deletion error")] - FileDeleteError(#[source] io::Error), + FileDelete(#[source] io::Error), #[error("Failed to open file")] - FileOpenError { path: PathBuf, error: io::Error }, + FileOpen { path: PathBuf, error: io::Error }, #[error("Failed to get file metadata")] - GetFileMetaDataError(#[source] std::io::Error), + GetFileMetaData(#[source] std::io::Error), #[error("Failed to create directory")] - DirCreationError { path: PathBuf, error: io::Error }, + DirCreation { path: PathBuf, error: io::Error }, #[error("Failed to clear file: `{path}`: {error}")] - FileClearError { path: PathBuf, error: io::Error }, + FileClear { path: PathBuf, error: io::Error }, #[error("Failed to read file `{path}`: {error}")] - FileReadError { path: PathBuf, error: io::Error }, + FileRead { path: PathBuf, error: io::Error }, #[error("Failed to write to file `{path}`: {error}")] - FileWriteError { path: PathBuf, error: io::Error }, + FileWrite{ path: PathBuf, error: io::Error }, #[error("Failed to open directory `{path}`: {error}")] - DirOpenError { path: PathBuf, error: io::Error }, + DirOpen{ path: PathBuf, error: io::Error }, #[error("File read ended unexpectedly")] UnexpectedEOF(#[source] io::Error), @@ -53,25 +53,25 @@ pub enum Error { GCErrorAttemptToRemoveUnsyncedEntries, #[error("Failed to insert sstable to bucket because no insertion condition was met")] - ConditionsToInsertToBucketNotMetError, + ConditionsToInsertToBucketNotMet, #[error("Error occured while flushing to disk")] - FlushToDiskError { + FlushToDisk{ #[source] error: Box, // Flush to disk can be caused by any of the errors above }, #[error("Error occured while inserting entry to memtable value Key: `{key}` Value: `{value_offset}`")] - InsertToMemTableFailedError { key: String, value_offset: usize }, + InsertToMemTableFailed { key: String, value_offset: usize }, #[error("Error while recovering memtable from value log")] - MemTableRecoveryError(#[source] Box), + MemTableRecovery(#[source] Box), #[error("Invalid string provided to be parsed to UUID input `{input_string}`: {error}")] InvaidUUIDParseString { input_string: String, error: uuid::Error }, #[error("Invalid sstable directory error: `{input_string}`")] - InvalidSSTableDirectoryError { input_string: String }, + InvalidSSTableDirectory { input_string: String }, #[error("Compaction failed reason : {0}")] CompactionFailed(Box), @@ -80,22 +80,22 @@ pub enum Error { CompactionPartiallyFailed(Box), #[error("No SSTable contains the searched key")] - KeyNotFoundInAnySSTableError, + KeyNotFoundInAnySSTable, #[error("Key found as tombstone in sstable")] - KeyFoundAsTombstoneInSSTableError, + KeyFoundAsTombstoneInSSTable, #[error("Key found as tombstone in memtable")] - KeyFoundAsTombstoneInMemtableError, + KeyFoundAsTombstoneInMemtable, #[error("Key found as tombstone in value log")] - KeyFoundAsTombstoneInValueLogError, + KeyFoundAsTombstoneInValueLog, #[error("Memtable does not contains the searched key")] KeyNotFoundInMemTable, #[error("Key does not exist in value log")] - KeyNotFoundInValueLogError, + KeyNotFoundInValueLog, #[error("Key not found, reason: ")] KeyNotFound(#[source] Box), @@ -107,7 +107,7 @@ pub enum Error { TombStoneCheckFailed(String), #[error("Block is full")] - BlockIsFullError, + BlockIsFull, #[error("Filter not provided, needed to flush table to disk")] FilterNotProvidedForFlush, @@ -125,19 +125,19 @@ pub enum Error { ValMaxSizeExceeded, #[error("Filter not found")] - FilterNotFoundError, + FilterNotFound, #[error("Error finding biggest key in memtable (None was returned)")] - BiggestKeyIndexError, + BiggestKeyIndex, #[error("Error finding lowest key in memtable (None was returned)")] - LowestKeyIndexError, + LowestKeyIndex, #[error("SSTable summary field is None")] - TableSummaryIsNoneError, + TableSummaryIsNone, #[error("All bloom filters return false for all sstables")] - KeyNotFoundByAnyBloomFilterError, + KeyNotFoundByAnyBloomFilter, #[error("Failed to insert to a bucket, reason `{0}`")] FailedToInsertToBucket(String), @@ -148,41 +148,35 @@ pub enum Error { #[error("Unsuported OS for garbage collection, err message `{0}`")] GCErrorUnsupportedPlatform(String), - #[error("GC Error `{0}`")] - GCError(String), - #[error("Range scan error `{0}`")] - RangeScanError(Box), + RangeScan(Box), #[error("Flush signal channel was overloaded with signals, please check all signal consumers or try again later")] - FlushSignalChannelOverflowError, + FlushSignalChannelOverflow, #[error("GC update channel was overloaded with data, please check all consumers")] - GCUpdateChannelOverflowError, + GCUpdateChannelOverflow, #[error("Flush signal channel has been closed")] - FlushSignalChannelClosedError, + FlushSignalChannelClosed, #[error("Serializartion error: {0} ")] - SerializationError(&'static str), - - #[error("Flush error: {0} ")] - FlushError(Box), + Serialization(&'static str), #[error("Partial failure, sstable merge was successful but obsolete sstables not deleted ")] - CompactionCleanupPartialError, + CompactionCleanupPartial, #[error("Compaction cleanup failed but sstable merge was successful : {0} ")] - CompactionCleanupError(Box), + CompactionCleanup(Box), #[error("Cannot remove obsolete sstables from disk because not every merged sstable was written to disk")] - CannotRemoveObsoleteSSTError, + CannotRemoveObsoleteSST, #[error("Error, merged sstables has empty entries")] MergeSSTContainsZeroEntries, #[error("Tokio join tasks error")] - TokioJoinError, + TokioJoin, #[error("Entries cannot be empty during flush")] EntriesCannotBeEmptyDuringFlush, diff --git a/src/flush/flusher.rs b/src/flush/flusher.rs index bb22041..3c9dd79 100644 --- a/src/flush/flusher.rs +++ b/src/flush/flusher.rs @@ -1,8 +1,7 @@ use crate::consts::FLUSH_SIGNAL; use crate::flush::flusher::Error::FilterNotProvidedForFlush; -use crate::flush::flusher::Error::FlushError; -use crate::flush::flusher::Error::TableSummaryIsNoneError; -use crate::types::{self, BloomFilterHandle, BucketMapHandle, FlushSignal, ImmutableMemTables, KeyRangeHandle}; +use crate::flush::flusher::Error::TableSummaryIsNone; +use crate::types::{self, BucketMapHandle, FlushSignal, ImmutableMemTables, KeyRangeHandle}; use crate::{err::Error, memtable::MemTable}; use std::fmt::Debug; use std::sync::Arc; @@ -15,7 +14,6 @@ pub type InActiveMemtable = Arc>; pub struct Flusher { pub(crate) read_only_memtable: ImmutableMemTables, pub(crate) bucket_map: BucketMapHandle, - pub(crate) filters: BloomFilterHandle, pub(crate) key_range: KeyRangeHandle, } @@ -23,13 +21,11 @@ impl Flusher { pub fn new( read_only_memtable: ImmutableMemTables, bucket_map: BucketMapHandle, - filters: BloomFilterHandle, key_range: KeyRangeHandle, ) -> Self { Self { read_only_memtable, bucket_map, - filters, key_range, } } @@ -50,20 +46,17 @@ impl Flusher { .await?; drop(table_reader); if sst.summary.is_none() { - return Err(TableSummaryIsNoneError); + return Err(TableSummaryIsNone); } if sst.filter.is_none() { return Err(FilterNotProvidedForFlush); } //IMPORTANT: Don't keep sst entries in memory sst.entries.clear(); - flush_data.filters.write().await.push(sst.filter.to_owned().unwrap()); let summary = sst.summary.clone().unwrap(); flush_data .key_range - .write() - .await - .set(sst.dir.to_owned(), summary.smallest_key, summary.biggest_key, sst); + .set(sst.dir.to_owned(), summary.smallest_key, summary.biggest_key, sst).await; Ok(()) } @@ -81,25 +74,24 @@ impl Flusher { ) { let tx = flush_tx.clone(); let buckets = self.bucket_map.clone(); - let filters = self.filters.clone(); let key_range = self.key_range.clone(); let read_only_memtable = self.read_only_memtable.clone(); tokio::spawn(async move { - let mut flusher = Flusher::new(read_only_memtable.clone(), buckets, filters, key_range); + let mut flusher = Flusher::new(read_only_memtable.clone(), buckets, key_range); match flusher.flush(table_to_flush).await { Ok(_) => { read_only_memtable.remove(&table_id.as_ref().to_vec()); if let Err(err) = tx.try_broadcast(FLUSH_SIGNAL) { match err { async_broadcast::TrySendError::Full(_) => { - log::info!("{}", Error::FlushSignalChannelOverflowError.to_string()) + log::info!("{}", Error::FlushSignalChannelOverflow.to_string()) } _ => log::error!("{}", err), } } } Err(err) => { - log::error!("{}", FlushError(Box::new(err))) + log::error!("{}", err.to_string()) } } }); diff --git a/src/fs/mod.rs b/src/fs/mod.rs index 53b9d7c..01a6d35 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -165,7 +165,7 @@ impl FileAsync for FileNode { .create(true) .open(path.as_ref()) .await - .map_err(|err| FileCreationError { + .map_err(|err| FileCreation { path: path.as_ref().to_path_buf(), error: err, })?) @@ -174,7 +174,7 @@ impl FileAsync for FileNode { async fn create_dir_all + Send + Sync>(dir: P) -> Result<(), Error> { let dir = dir.as_ref(); if !dir.exists() { - return fs::create_dir_all(&dir).await.map_err(|err| DirCreationError { + return fs::create_dir_all(&dir).await.map_err(|err| DirCreation { path: dir.to_path_buf(), error: err, }); @@ -184,11 +184,11 @@ impl FileAsync for FileNode { async fn metadata(&self) -> Result { let file = self.r_lock().await; - Ok(file.metadata().await.map_err(GetFileMetaDataError)?) + Ok(file.metadata().await.map_err(GetFileMetaData)?) } async fn open + Send + Sync>(path: P) -> Result { - Ok(File::open(path.as_ref()).await.map_err(|err| FileOpenError { + Ok(File::open(path.as_ref()).await.map_err(|err| FileOpen { path: path.as_ref().to_path_buf(), error: err, })?) @@ -196,7 +196,7 @@ impl FileAsync for FileNode { async fn read_buf(&self, buf: &mut Buf) -> Result { let mut file = self.w_lock().await; - Ok(file.read(buf).await.map_err(|err| FileReadError { + Ok(file.read(buf).await.map_err(|err| FileRead { path: self.file_path.clone(), error: err, })?) @@ -204,7 +204,7 @@ impl FileAsync for FileNode { async fn write_all(&self, buf: &Buf) -> Result<(), Error> { let mut file = self.w_lock().await; - Ok(file.write_all(buf).await.map_err(|err| FileWriteError { + Ok(file.write_all(buf).await.map_err(|err| FileWrite { path: self.file_path.clone(), error: err, })?) @@ -212,7 +212,7 @@ impl FileAsync for FileNode { async fn clear(&self) -> Result<(), Error> { let file = self.w_lock().await; - Ok(file.set_len(0).await.map_err(|err| FileClearError { + Ok(file.set_len(0).await.map_err(|err| FileClear{ path: self.file_path.clone(), error: err, })?) @@ -223,12 +223,12 @@ impl FileAsync for FileNode { Ok(file .sync_all() .await - .map_err(|err| Error::FileSyncError { error: err })?) + .map_err(|err| Error::FileSync { error: err })?) } async fn flush(&self) -> Result<(), Error> { let mut file = self.w_lock().await; - Ok(file.flush().await.map_err(|err| Error::FileSyncError { error: err })?) + Ok(file.flush().await.map_err(|err| Error::FileSync { error: err })?) } async fn seek(&self, start_offset: u64) -> Result { @@ -236,13 +236,13 @@ impl FileAsync for FileNode { Ok(file .seek(SeekFrom::Start(start_offset)) .await - .map_err(FileSeekError)?) + .map_err(FileSeek)?) } async fn remove_dir_all(&self) -> Result<(), Error> { Ok(fs::remove_dir_all(&self.file_path) .await - .map_err(DirDeleteError)?) + .map_err(DirDelete)?) } async fn w_lock(&self) -> WGuard { @@ -272,7 +272,7 @@ impl DataFs for DataFileNode { let mut file = self.node.file.write().await; file.seek(std::io::SeekFrom::Start(0)) .await - .map_err(FileSeekError)?; + .map_err(FileSeek)?; loop { let mut key_len_bytes = [0; SIZE_OF_U32]; @@ -335,7 +335,7 @@ impl DataFs for DataFileNode { let mut file = self.node.file.write().await; file.seek(std::io::SeekFrom::Start(offset.into())) .await - .map_err(FileSeekError)?; + .map_err(FileSeek)?; loop { let mut key_len_bytes = [0; SIZE_OF_U32]; @@ -390,7 +390,7 @@ impl DataFs for DataFileNode { let mut file = self.node.file.write().await; file.seek(std::io::SeekFrom::Start((range_offset.start_offset) as u64)) .await - .map_err(FileSeekError)?; + .map_err(FileSeek)?; loop { let mut key_len_bytes = [0; SIZE_OF_U32]; @@ -462,7 +462,7 @@ impl VLogFs for VLogFileNode { let mut file = self.node.file.write().await; file.seek(std::io::SeekFrom::Start((start_offset) as u64)) .await - .map_err(FileSeekError)?; + .map_err(FileSeek)?; let mut key_len_bytes = [0; SIZE_OF_U32]; let mut bytes_read = load_buffer!(file, &mut key_len_bytes, path.to_owned())?; @@ -511,7 +511,7 @@ impl VLogFs for VLogFileNode { let mut file = self.node.file.write().await; file.seek(std::io::SeekFrom::Start((start_offset) as u64)) .await - .map_err(FileSeekError)?; + .map_err(FileSeek)?; loop { let mut key_len_bytes = [0; SIZE_OF_U32]; @@ -574,7 +574,7 @@ impl VLogFs for VLogFileNode { let mut file = self.node.file.write().await; file.seek(std::io::SeekFrom::Start(offset)) .await - .map_err(FileSeekError)?; + .map_err(FileSeek)?; let mut total_bytes_read: usize = 0; loop { let mut key_len_bytes = [0; SIZE_OF_U32]; @@ -655,7 +655,7 @@ impl IndexFs for IndexFileNode { let mut file = self.node.file.write().await; file.seek(std::io::SeekFrom::Start(0_u64)) .await - .map_err(FileSeekError)?; + .map_err(FileSeek)?; loop { let mut key_len_bytes = [0; SIZE_OF_U32]; @@ -700,7 +700,7 @@ impl IndexFs for IndexFileNode { let mut file = self.node.file.write().await; file.seek(std::io::SeekFrom::Start(0_u64)) .await - .map_err(FileSeekError)?; + .map_err(FileSeek)?; loop { let mut key_len_bytes = [0; SIZE_OF_U32]; @@ -753,7 +753,7 @@ impl FilterFs for FilterFileNode { ) -> Result<(FalsePositive, NoHashFunc, NoOfElements), Error> { let mut file = FileNode::open(path.as_ref()) .await - .map_err(|_| return FilterFileOpenError(path.as_ref().to_owned()))?; + .map_err(|_| return FilterFileOpen(path.as_ref().to_owned()))?; let mut no_hash_func_bytes = [0; SIZE_OF_U32]; let mut bytes_read = load_buffer!(file, &mut no_hash_func_bytes, path.as_ref().to_path_buf())?; if bytes_read == 0 { @@ -797,7 +797,7 @@ impl MetaFs for MetaFileNode { ) -> Result<(VLogHead, VLogTail, CreatedAt, LastModified), Error> { let mut file = FileNode::open(path.as_ref()) .await - .map_err(|_| return FilterFileOpenError(path.as_ref().to_owned()))?; + .map_err(|_| return FilterFileOpen(path.as_ref().to_owned()))?; let mut head_offset_bytes = [0; SIZE_OF_U32]; let mut bytes_read = load_buffer!(file, &mut head_offset_bytes, path.as_ref().to_path_buf())?; if bytes_read == 0 { @@ -848,7 +848,7 @@ impl SummaryFs for SummaryFileNode { async fn recover + Send + Sync>(path: P) -> Result<(SmallestKey, BiggestKey), Error> { let mut file = FileNode::open(path.as_ref()) .await - .map_err(|_| return FilterFileOpenError(path.as_ref().to_owned()))?; + .map_err(|_| return FilterFileOpen(path.as_ref().to_owned()))?; let mut smallest_key_len_bytes = [0; SIZE_OF_U32]; let mut bytes_read = load_buffer!(file, &mut smallest_key_len_bytes, path.as_ref().to_owned())?; if bytes_read == 0 { diff --git a/src/gc/gc.rs b/src/gc/garbage_collector.rs similarity index 93% rename from src/gc/gc.rs rename to src/gc/garbage_collector.rs index 00ee6a2..0be4c2b 100644 --- a/src/gc/gc.rs +++ b/src/gc/garbage_collector.rs @@ -85,8 +85,7 @@ pub(crate) struct Config { /// Marks area of value log file /// to be punched -#[derive(Clone, Debug)] -#[derive(Default)] +#[derive(Clone, Debug, Default)] pub struct PunchMarker { /// Offset in value log to start punching hole pub(crate) punch_hole_start_offset: usize, @@ -95,7 +94,6 @@ pub struct PunchMarker { pub(crate) punch_hole_length: usize, } - impl GC { /// Creates `GC` instance pub fn new( @@ -118,7 +116,7 @@ impl GC { } /// Continues to check if it's time to run GC (works in background) - pub fn start_background_gc_task(&self, key_range: KeyRangeHandle, read_only_memtables: ImmutableMemTables) { + pub fn start_gc_worker(&self, key_range: KeyRangeHandle, read_only_memtables: ImmutableMemTables) { let cfg = self.config.to_owned(); let memtable = self.table.clone(); let vlog = self.vlog.clone(); @@ -151,7 +149,7 @@ impl GC { log::info!("GC successful, awaiting sync") } Err(err) => { - log::error!("{}", GCError(err.to_string())) + log::error!("GC Error {}", err.to_string()); } } } @@ -217,13 +215,9 @@ impl GC { let all_results = join_all(tasks).await; for tokio_res in all_results { match tokio_res { - Ok(res) => { - if let Err(err) = res { - return Err(GCError(err.to_string())); - } - } - Err(err) => { - return Err(GCError(err.to_string())); + Ok(res) => res?, + Err(_) => { + return Err(TokioJoin); } } } @@ -258,7 +252,7 @@ impl GC { marker_lock.punch_hole_start_offset = vlog.read().await.tail_offset; marker_lock.punch_hole_length = total_bytes_read; } - Err(err) => return Err(GCError(err.to_string())), + Err(err) => return Err(err), }; Ok(()) } @@ -453,9 +447,8 @@ impl GC { GC::get_value_from_vlog(&vlog, offset, insert_time).await } else { // Step 3: Check sstables - let key_range = &key_range.read().await; - let ssts = key_range.filter_sstables_by_biggest_key(&key).await?; - GC::search_key_in_sstables(key, ssts, &vlog).await + let ssts = &key_range.filter_sstables_by_biggest_key(&key).await?; + GC::search_key_in_sstables(key, ssts.to_vec(), &vlog).await } } } @@ -517,14 +510,13 @@ impl GC { creation_at: CreatedAt, ) -> Result<(Value, CreatedAt), Error> { let res = val_log.read().await.get(offset).await?; - if res.is_some() { - let (value, is_tombstone) = res.unwrap(); + if let Some((value, is_tombstone)) = res { if is_tombstone { - return Err(KeyFoundAsTombstoneInValueLogError); + return Err(NotFoundInDB); } return Ok((value, creation_at)); } - Err(KeyNotFoundInValueLogError) + Err(NotFoundInDB) } /// Handles entries marked as tombstone @@ -534,13 +526,7 @@ impl GC { err: Error, ) -> std::result::Result<(), Error> { match err { - KeyFoundAsTombstoneInMemtableError - | KeyNotFoundInAnySSTableError - | KeyNotFoundByAnyBloomFilterError - | KeyFoundAsTombstoneInSSTableError - | KeyFoundAsTombstoneInValueLogError - | KeyNotFoundInValueLogError - | NotFoundInDB => { + NotFoundInDB => { invalid_entries.write().await.push(entry); Ok(()) } diff --git a/src/gc/mod.rs b/src/gc/mod.rs index 5238903..19c7ce0 100644 --- a/src/gc/mod.rs +++ b/src/gc/mod.rs @@ -1 +1 @@ -pub(crate) mod gc; +pub(crate) mod garbage_collector; diff --git a/src/index/index.rs b/src/index/indexer.rs similarity index 98% rename from src/index/index.rs rename to src/index/indexer.rs index e315839..2ea6c9e 100644 --- a/src/index/index.rs +++ b/src/index/indexer.rs @@ -161,7 +161,7 @@ impl Index { // block offset entry_vec.extend_from_slice(&e.block_handle.to_le_bytes()); if entry_len != entry_vec.len() { - return Err(SerializationError("Invalid entry size")); + return Err(Serialization("Invalid entry size")); } Ok(entry_vec) } diff --git a/src/index/mod.rs b/src/index/mod.rs index 3585f9c..1a7005b 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -1,4 +1,4 @@ -mod index; -pub use index::Index; -pub use index::IndexFile; -pub use index::RangeOffset; +mod indexer; +pub use indexer::Index; +pub use indexer::IndexFile; +pub use indexer::RangeOffset; diff --git a/src/key_range/range.rs b/src/key_range/range.rs index 8437872..e8c23f2 100644 --- a/src/key_range/range.rs +++ b/src/key_range/range.rs @@ -18,7 +18,7 @@ pub type SmallestKey = types::Key; #[derive(Clone, Debug)] pub struct KeyRange { /// HashMap to map SSTable directory path to its key range - pub key_ranges: HashMap, + pub key_ranges: Arc>>, /// Maps SSTable path to its key range (for sstables /// whose filters are just restored yet to be move to @@ -43,23 +43,30 @@ impl Range { } } } +impl Default for KeyRange { + fn default() -> Self { + Self::new() + } +} impl KeyRange { // Creates new `KeyRange`` pub fn new() -> Self { Self { - key_ranges: HashMap::new(), + key_ranges: Arc::new(RwLock::new(HashMap::new())), restored_ranges: Arc::new(RwLock::new(HashMap::new())), } } /// Maps SSTable path to its key range - pub fn set + Send + Sync, T: AsRef<[u8]>>( - &mut self, + pub async fn set + Send + Sync, T: AsRef<[u8]>>( + &self, sst_dir: P, smallest_key: T, biggest_key: T, table: Table, ) -> bool { self.key_ranges + .write() + .await .insert( sst_dir.as_ref().to_path_buf(), Range::new(smallest_key.as_ref(), biggest_key.as_ref(), table), @@ -68,8 +75,8 @@ impl KeyRange { } /// Removes an entry from the `key_ranges` hash map - pub fn remove + Send + Sync>(&mut self, sst_path: P) -> bool { - self.key_ranges.remove(sst_path.as_ref()).is_some() + pub async fn remove + Send + Sync>(&self, sst_path: P) -> bool { + self.key_ranges.write().await.remove(sst_path.as_ref()).is_some() } /// Returns `Table` vector whose last key is greater than the @@ -86,7 +93,7 @@ impl KeyRange { } let mut restored_range_map: HashMap = HashMap::new(); - for (_, range) in self.key_ranges.iter() { + for (_, range) in self.key_ranges.read().await.iter() { if has_restored_ranges && self.restored_ranges.read().await.contains_key(range.sst.dir.as_path()) { continue; } @@ -99,6 +106,7 @@ impl KeyRange { if range.sst.filter.as_ref().unwrap().sst_dir.is_none() { let mut mut_range = range.to_owned(); let mut filter = mut_range.sst.filter.as_ref().unwrap().to_owned(); + filter.recover_meta().await?; filter.sst_dir = Some(mut_range.sst.dir.to_owned()); @@ -120,8 +128,9 @@ impl KeyRange { } if !restored_range_map.is_empty() { // store the key ranges with sstables that contains - // bloom filters just restored to disk in the restored_ranges map we are not - // updating key_ranges immediatlely to prevent a mutable reference on get operations + // bloom filters just restored to disk in the restored_ranges map. We are not + // updating key_ranges immediatlely to prevent write locks on key_ranges for + // get operations let restored_ranges = self.restored_ranges.clone(); tokio::spawn(async move { *(restored_ranges.write().await) = restored_range_map; @@ -142,7 +151,10 @@ impl KeyRange { let mut filtered_ssts: Vec
= Vec::new(); let key_ranges = self.restored_ranges.read().await; for (_, range) in key_ranges.iter() { - if (range.biggest_key.as_slice().cmp(key.as_ref()) == Ordering::Greater || range.biggest_key.as_slice().cmp(key.as_ref()) == Ordering::Equal) && range.sst.filter.as_ref().unwrap().contains(key.as_ref()) { + if (range.biggest_key.as_slice().cmp(key.as_ref()) == Ordering::Greater + || range.biggest_key.as_slice().cmp(key.as_ref()) == Ordering::Equal) + && range.sst.filter.as_ref().unwrap().contains(key.as_ref()) + { filtered_ssts.push(range.sst.to_owned()) } } @@ -151,11 +163,11 @@ impl KeyRange { /// Moves entries in `restored_ranges` with sstables whose filters are just restored /// to `key_ranges` - pub async fn update_key_range(&mut self) { + pub async fn update_key_range(&self) { let restored_ranges = self.restored_ranges.read().await; if !restored_ranges.is_empty() { for (path, range) in restored_ranges.iter() { - self.key_ranges.insert(path.to_owned(), range.to_owned()); + self.key_ranges.write().await.insert(path.to_owned(), range.to_owned()); } drop(restored_ranges); self.restored_ranges.write().await.clear(); @@ -163,8 +175,10 @@ impl KeyRange { } /// Returns SSTables whose keys overlap with the key range supplied - pub fn range_scan>(&self, start_key: T, end_key: T) -> Vec<&Range> { + pub async fn range_scan>(&self, start_key: T, end_key: T) -> Vec { self.key_ranges + .read() + .await .iter() .filter(|(_, range)| { // Check minimum range @@ -175,7 +189,7 @@ impl KeyRange { || (range.biggest_key.as_slice().cmp(end_key.as_ref()) == Ordering::Greater || range.biggest_key.as_slice().cmp(end_key.as_ref()) == Ordering::Equal) }) - .map(|(_, path)| path) + .map(|(_, path)| path.to_owned()) .collect() } } diff --git a/src/lib.rs b/src/lib.rs index b06aa3c..a98264c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,28 +1,44 @@ -//! VikingsDB is an LSM-based storage engine designed to significantly reduce IO amplification, resulting in better performance and durability for storage devices. +//! Velarixdb is an LSM-based storage engine designed to significantly reduce IO amplification, resulting in better performance and durability for storage devices. //! //! ## Introduction //! -//! VikingsDB is an ongoing project (**not production ready**) that aims to reduce the amount of data moved during load times, random access, and compaction. It is based on the WiscKey paper (). -//! According to the paper, benchmarks shows that WiscKey implementations can outperform LevelDB by a factor of **2.5x to 111x** for loading a database and **1.6x to 14x** for random lookups. VikingsDB uses the Tokio Runtime so we can achieve concurrency without necessarily spawning new threads. +//! # Velarixdb: Efficient Data Management for High Performance //! +//! velarixdb is an ongoing project (**not production ready**) designed to optimize data movement during load times, random access, and compaction. Inspired by the WiscKey paper, [WiscKey: Separating Keys from Values in SSD-conscious Storage](https://usenix.org/system/files/conference/fast16/fast16-papers-lu.pdf), velarixdb aims to significantly enhance performance over traditional key-value stores. +//! +//! ## Performance Benefits +//! +//! According to the benchmarks presented in the WiscKey paper, implementations can outperform LevelDB and RocksDB by: +//! - **2.5x to 111x** for database loading +//! - **1.6x to 14x** for random lookups +//! +//! ## Concurrency with Tokio +//! +//! To achieve high concurrency without the overhead of spawning new threads every time, velarixdb utilizes the Tokio runtime. This allows for efficient and scalable asynchronous operations, making the most of modern multi-core processors. +//! +//! ## Disclaimer +//! +//! Please note that velarixdb is still under development and is not yet production-ready. + //! ### Features -//! - [x] Atomic `Put()`, `Get()`, `Delete()`, `Update()` operations +//! - [x] Atomic `Put()`, `Get()`, `Delete()`, and `Update()` operations //! - [x] 100% safe & stable Rust -//! - [x] Seperates keys from values, reducing amount of data physically moved during compaction (i.e Reduced IO amplification) -//! - [x] Garbage Collector to physically free up space occupied by obsolete entries on the storage device -//! - [x] Lock-Free Skipmap from crossbeam as memtable data structure (no Mutex on memtable) -//! - [x] Tokio Asynchronous Runtime to achieve concurrency +//! - [x] Separation of keys from values, reducing the amount of data moved during compaction (i.e., reduced IO amplification) +//! - [x] Garbage Collector +//! - [x] Lock-free memtable (no `Mutex`) +//! - [x] Tokio Runtime for efficient thread management //! - [x] Bloom Filters for fast in-memory key searches //! - [x] Crash recovery using the Value Log //! - [x] Index to improve searches on Sorted String Tables (SSTs) -//! - [x] Key Range to store the largest and smallest keys in a String Table (SST) +//! - [x] Key Range to store the largest and smallest keys in an SST //! - [x] Sized Tier Compaction Strategy (STCS) +//! - [ ] Batched Writes //! - [ ] Snappy Compression Algorithm //! - [ ] Value Buffer to keep values in memory and only flush in batches to reduce IO (under investigation) //! - [ ] Checksum to detect data corruption -//! - [ ] Batched Writes //! - [ ] Leveled Compaction (LCS), Time-Window Compaction (TCS), and Unified Compaction (UCS) -//! - [ ] Monitoring module to constantly monitor and generate reports +//! - [ ] Monitoring module to continuously monitor and generate reports +//! //! //! ### It is not: //! - A standalone server @@ -31,9 +47,97 @@ //! //! ### Constraint //! - Keys are limited to 65,536 bytes, and values are limited to 2^32 bytes. Larger keys and values have a bigger performance impact. -//! +//! //! - Like any typical key-value store, keys are stored in lexicographic order. //! If you are storing integer keys (e.g., timeseries data), use the big-endian form to adhere to locality. +//! +//! +//! # Basic usage +//! +//! +//! ```rust +//! use velarixdb::db::DataStore; +//! # use tempfile::tempdir; +//! +//! #[tokio::main] +//! async fn main() { +//! let root = tempdir().unwrap(); +//! let path = root.path().join("velarix"); +//! let mut store = DataStore::open("big_tech", path).await.unwrap(); // handle IO error +//! +//! store.put("apple", "tim cook").await; +//! store.put("google", "sundar pichai").await; +//! store.put("nvidia", "jensen huang").await; +//! store.put("microsoft", "satya nadella").await; +//! store.put("meta", "mark zuckerberg").await; +//! store.put("openai", "sam altman").await; +//! +//! +//! let entry1 = store.get("apple").await.unwrap(); // Handle error +//! let entry2 = store.get("google").await.unwrap(); +//! let entry3 = store.get("nvidia").await.unwrap(); +//! let entry4 = store.get("microsoft").await.unwrap(); +//! let entry5 = store.get("meta").await.unwrap(); +//! let entry6 = store.get("openai").await.unwrap(); +//! let entry7 = store.get("***not_found_key**").await.unwrap(); +//! +//! assert_eq!(std::str::from_utf8(&entry1.unwrap().val).unwrap(), "tim cook"); +//! assert_eq!(std::str::from_utf8(&entry2.unwrap().val).unwrap(), "sundar pichai"); +//! assert_eq!(std::str::from_utf8(&entry3.unwrap().val).unwrap(), "jensen huang"); +//! assert_eq!(std::str::from_utf8(&entry4.unwrap().val).unwrap(), "satya nadella"); +//! assert_eq!(std::str::from_utf8(&entry5.unwrap().val).unwrap(), "mark zuckerberg"); +//! assert_eq!(std::str::from_utf8(&entry6.unwrap().val).unwrap(), "sam altman"); +//! assert!(entry7.is_none()); +//! +//! // Remove an entry +//! store.delete("apple").await.unwrap(); +//! +//! // Update an entry +//! let success = store.update("microsoft", "elon musk").await; +//! assert!(success.is_ok()); +//! } +//! ``` +//! +//! +//! ## Store JSON +//! +//! ```rust +//! use serde::{Deserialize, Serialize}; +//! use serde_json; +//! use velarixdb::db::DataStore; +//! # use tempfile::tempdir; +//! +//! #[tokio::main] +//! async fn main() { +//! let root = tempdir().unwrap(); +//! let path = root.path().join("velarix"); +//! let mut store = DataStore::open("big_tech", path).await.unwrap(); // handle IO error +//! +//! #[derive(Serialize, Deserialize)] +//! struct BigTech { +//! name: String, +//! rank: i32, +//! } +//! let new_entry = BigTech { +//! name: String::from("Google"), +//! rank: 50, +//! }; +//! let json_string = serde_json::to_string(&new_entry).unwrap(); +//! +//! let res = store.put("google", json_string).await; +//! assert!(res.is_ok()); +//! +//! let entry = store.get("google").await.unwrap().unwrap(); +//! let entry_string = std::str::from_utf8(&entry.val).unwrap(); +//! let big_tech: BigTech = serde_json::from_str(&entry_string).unwrap(); +//! +//! assert_eq!(big_tech.name, new_entry.name); +//! assert_eq!(big_tech.rank, new_entry.rank); +//! } +//! ``` + +#![doc(html_logo_url = "https://raw.githubusercontent.com/fjall-rs/fjall/main/logo.png")] +#![doc(html_favicon_url = "https://raw.githubusercontent.com/fjall-rs/fjall/main/logo.png")] mod block; mod bucket; diff --git a/src/macro/mod.rs b/src/macro/mod.rs index a9e0484..eb601c1 100644 --- a/src/macro/mod.rs +++ b/src/macro/mod.rs @@ -4,7 +4,7 @@ pub mod fs_macros { ($file:expr, $buffer:expr, $file_path:expr) => { match $file.read($buffer).await { Ok(bytes_read) => Ok(bytes_read), - Err(err) => Err(FileReadError { + Err(err) => Err(FileRead { path: $file_path, error: err, }), @@ -14,7 +14,7 @@ pub mod fs_macros { #[macro_export] macro_rules! open_dir_stream { ($path:expr) => {{ - let stream = read_dir($path.to_owned()).await.map_err(|err| DirOpenError { + let stream = read_dir($path.to_owned()).await.map_err(|err| DirOpen { path: $path, error: err, })?; diff --git a/src/memtable/memtable.rs b/src/memtable/mem.rs similarity index 93% rename from src/memtable/memtable.rs rename to src/memtable/mem.rs index f9e908f..94b31f3 100644 --- a/src/memtable/memtable.rs +++ b/src/memtable/mem.rs @@ -34,6 +34,19 @@ pub struct Entry { pub is_tombstone: bool, } +/// Entry returned to user upon retreival +pub struct UserEntry { + pub val: Value, + pub created_at: CreatedAt, +} + +impl UserEntry { + /// Creates new `UserEntry` + pub fn new(val: Value, created_at: CreatedAt) -> Self { + Self { val, created_at } + } +} + /// Value in SkipMap #[derive(Clone, Debug, PartialEq)] pub struct SkipMapValue { @@ -43,6 +56,7 @@ pub struct SkipMapValue { } impl SkipMapValue { + /// Creates new `SkipMapValue` pub(crate) fn new(val_offset: V, created_at: CreatedAt, is_tombstone: IsTombStone) -> Self { SkipMapValue { val_offset, @@ -105,7 +119,7 @@ impl Config { #[allow(dead_code)] pub enum ValueOption { /// TODO: Value will be cached in memory if the size is small, this will reduce - /// number of Disk IO + /// number of Disk IO Raw(Value), /// Value offset gotten from value position in value log @@ -165,7 +179,7 @@ impl MemTable { ); assert!(capacity > 0, "Capacity should be greater than 0"); - let capacity_to_bytes = size_unit.to_bytes(capacity); + let capacity_to_bytes = size_unit.as_bytes(capacity); let avg_entry_size = 100; let max_no_of_entries = capacity_to_bytes / avg_entry_size as usize; let bf = BloomFilter::new(false_positive_rate, max_no_of_entries); @@ -219,11 +233,11 @@ impl MemTable { } /// Updates an entry in `entries` map - /// + /// /// # Error - /// + /// /// Returns error if key was not found - /// + /// pub fn update(&mut self, entry: &Entry) -> Result<(), Error> { if !self.bloom_filter.contains(&entry.key) { return Err(KeyNotFoundInMemTable); @@ -248,9 +262,9 @@ impl MemTable { } /// Inserts an entry with tombstone to `entries` map - /// + /// /// # Errors - /// + /// /// Returns error if key was not found pub fn delete(&mut self, entry: &Entry) -> Result<(), Error> { if !self.bloom_filter.contains(&entry.key) { @@ -312,7 +326,7 @@ impl MemTable { /// Clears all key-value entries in the MemTable. pub fn clear(&mut self) { - let capacity_to_bytes = self.config.size_unit.to_bytes(self.config.capacity); + let capacity_to_bytes = self.config.size_unit.as_bytes(self.config.capacity); let avg_entry_size = 100; let max_no_of_entries = capacity_to_bytes / avg_entry_size as usize; @@ -339,10 +353,10 @@ mod tests { assert_eq!(memtable.config.size_unit, SizeUnit::Bytes); assert_eq!( memtable.config.capacity, - memtable.config.size_unit.to_bytes(buffer_size) + memtable.config.size_unit.as_bytes(buffer_size) ); assert_eq!(memtable.config.false_pos_rate, false_pos_rate); - assert_eq!(memtable.read_only, false); + assert!(!memtable.read_only); } #[test] @@ -356,10 +370,10 @@ mod tests { assert_eq!(memtable.config.size_unit, SizeUnit::Bytes); assert_eq!( memtable.config.capacity, - memtable.config.size_unit.to_bytes(buffer_size) + memtable.config.size_unit.as_bytes(buffer_size) ); assert_eq!(memtable.config.false_pos_rate, false_pos_rate); - assert_eq!(memtable.read_only, false); + assert!(!memtable.read_only); } #[test] @@ -378,14 +392,14 @@ mod tests { let entry = Entry::new(key, val_offset, created_at, is_tombstone); let expected_len = entry.key.len() + SIZE_OF_U32 + SIZE_OF_U64 + SIZE_OF_U8; - let _ = memtable.insert(&entry); + memtable.insert(&entry); println!("{}", memtable.size); assert_eq!(memtable.size, expected_len); - let _ = memtable.insert(&entry); + memtable.insert(&entry); assert_eq!(memtable.size, expected_len + expected_len); - let _ = memtable.insert(&entry); + memtable.insert(&entry); assert_eq!(memtable.size, expected_len + expected_len + expected_len); } @@ -402,14 +416,14 @@ mod tests { let entry = Entry::new(key.to_owned(), val_offset, created_at, is_tombstone); let expected_len = entry.key.len() + SIZE_OF_U32 + SIZE_OF_U64 + SIZE_OF_U8; - let _ = memtable.insert(&entry); + memtable.insert(&entry); assert_eq!(memtable.size, expected_len); // get key let res = memtable.get(&key); assert!(res.is_some()); // get key the was not inserted let invalid_key = vec![8, 2, 3, 4]; - let res = memtable.get(&invalid_key); + let res = memtable.get(invalid_key); assert!(res.is_none()); } @@ -421,7 +435,7 @@ mod tests { let false_pos_rate = 1e-300; let memtable = MemTable::new(buffer_size, false_pos_rate); let memtable = Arc::new(Mutex::new(memtable)); - let mut handlers = Vec::with_capacity(5 as usize); + let mut handlers = Vec::with_capacity(5_usize); let keys = vec![ vec![1, 2, 3, 4], vec![2, 2, 3, 4], @@ -501,7 +515,7 @@ mod tests { let created_at = Utc::now(); let mut entry = Entry::new(key, val_offset, created_at, is_tombstone); - let _ = memtable.insert(&entry); + memtable.insert(&entry); let e = memtable.get(&entry.key); assert!(e.is_some()); @@ -517,7 +531,7 @@ mod tests { let _ = memtable.update(&entry); let e = memtable.get(&entry.key); - assert_eq!(e.unwrap().is_tombstone, true); + assert!(e.unwrap().is_tombstone); entry.key = vec![2, 2, 3, 4]; let e = memtable.update(&entry); @@ -540,7 +554,7 @@ mod tests { let created_at = Utc::now(); let mut entry = Entry::new(key, val_offset, created_at, is_tombstone); - let _ = memtable.insert(&entry); + memtable.insert(&entry); let e = memtable.get(&entry.key); assert!(e.is_some()); @@ -549,7 +563,7 @@ mod tests { let _ = memtable.delete(&entry); let e = memtable.get(&entry.key); - assert_eq!(e.unwrap().is_tombstone, true); + assert!(e.unwrap().is_tombstone); entry.key = vec![2, 2, 3, 4]; let e = memtable.delete(&entry); @@ -588,17 +602,17 @@ mod tests { let within_range = MemTable::is_entry_within_range(&map.get(&keys[0]).unwrap(), keys[0].to_owned(), keys[3].to_owned()); - assert_eq!(within_range, true); + assert!(within_range); let start_invalid = vec![10, 20, 30, 40]; let end_invalid = vec![0, 0, 0, 0]; let within_range = MemTable::is_entry_within_range(&map.get(&keys[0]).unwrap(), start_invalid, end_invalid); - assert_eq!(within_range, false); + assert!(!within_range); let start_valid = &keys[0]; let end_invalid = vec![0, 0, 0, 0]; let within_range = MemTable::is_entry_within_range(&map.get(&keys[0]).unwrap(), start_valid, &end_invalid); - assert_eq!(within_range, true); + assert!(within_range); } #[test] @@ -606,10 +620,10 @@ mod tests { let buffer_size = 51200; let false_pos_rate = 1e-300; let memtable = MemTable::new(buffer_size, false_pos_rate); - let key = vec![1, 2, 3, 4]; + let key = [1, 2, 3, 4]; let is_full = memtable .to_owned() .is_full(key.len() + SIZE_OF_U32 + SIZE_OF_U64 + SIZE_OF_U8 + memtable.capacity()); - assert_eq!(is_full, true); + assert!(is_full); } } diff --git a/src/memtable/mod.rs b/src/memtable/mod.rs index 47a2bad..a6acffc 100644 --- a/src/memtable/mod.rs +++ b/src/memtable/mod.rs @@ -1,5 +1,6 @@ -mod memtable; -pub use memtable::Entry; -pub use memtable::MemTable; -pub use memtable::SkipMapValue; -pub use memtable::K; +mod mem; +pub use mem::Entry; +pub use mem::UserEntry; +pub use mem::MemTable; +pub use mem::SkipMapValue; +pub use mem::K; diff --git a/src/meta/meta.rs b/src/meta/meta_manager.rs similarity index 100% rename from src/meta/meta.rs rename to src/meta/meta_manager.rs diff --git a/src/meta/mod.rs b/src/meta/mod.rs index 0f61f67..040244e 100644 --- a/src/meta/mod.rs +++ b/src/meta/mod.rs @@ -1,2 +1,2 @@ -mod meta; -pub use meta::Meta; +mod meta_manager; +pub use meta_manager::Meta; diff --git a/src/range/mod.rs b/src/range/mod.rs index 7d4eb9a..a5d67af 100644 --- a/src/range/mod.rs +++ b/src/range/mod.rs @@ -1,2 +1,2 @@ -mod range; -pub use range::RangeIterator; +mod range_iterator; +pub use range_iterator::RangeIterator; diff --git a/src/range/range.rs b/src/range/range_iterator.rs similarity index 95% rename from src/range/range.rs rename to src/range/range_iterator.rs index 5951e2f..cb8f2e9 100644 --- a/src/range/range.rs +++ b/src/range/range_iterator.rs @@ -64,12 +64,9 @@ impl<'a> RangeIterator<'a> { if self.current_is_at_last_key() { return None; } - match self.prefetch_entries().await { - Err(err) => { - error!("{}", Error::RangeScanError(Box::new(err))); - return None; - } - _ => {} + if let Err(err) = self.prefetch_entries().await { + error!("{}", Error::RangeScan(Box::new(err))); + return None; } } let entry = self.current_entry(); @@ -96,18 +93,17 @@ impl<'a> RangeIterator<'a> { } pub async fn prefetch_entries(&mut self) -> Result<(), Error> { - let keys: Vec>; - if self.current + self.prefetch_entries_size <= self.keys.len() { - keys = self.keys[self.current..self.current + self.prefetch_entries_size].to_vec(); + let keys: Vec> = if self.current + self.prefetch_entries_size <= self.keys.len() { + self.keys[self.current..self.current + self.prefetch_entries_size].to_vec() } else { - keys = self.keys[self.current..].to_vec(); - } + self.keys[self.current..].to_vec() + }; let entries = self.fetch_entries_in_parralel(&keys).await; match entries { Ok(e) => { self.prefetch_entries.extend(e); Ok(()) - }, + } Err(err) => Err(err), } } @@ -138,9 +134,7 @@ impl<'a> RangeIterator<'a> { match entry_from_vlog { Ok(val_opt) => match val_opt { Some((val, is_deleted)) => Ok((entry.key, val, is_deleted)), - None => { - Err(Error::KeyNotFoundInValueLogError) - } + None => Err(Error::KeyNotFoundInValueLog), }, Err(err) => Err(err), } diff --git a/src/sst/table.rs b/src/sst/table.rs index feecaa7..ba4b845 100644 --- a/src/sst/table.rs +++ b/src/sst/table.rs @@ -35,14 +35,6 @@ //! - TODO: In the future we will introduce Snappy Compression to reduce the size on the disk and also //! introduce checksum to ensure the data has not been corrupted -use chrono::Utc; -use crossbeam_skiplist::SkipMap; -use std::{ - cmp::Ordering, - path::{Path, PathBuf}, - sync::Arc, - time::SystemTime, -}; use crate::{ block::Block, bucket::InsertableToBucket, @@ -56,6 +48,13 @@ use crate::{ types::{ByteSerializedEntry, CreatedAt, IsTombStone, Key, SkipMapEntries, ValOffset}, util, }; +use chrono::Utc; +use crossbeam_skiplist::SkipMap; +use std::{ + path::{Path, PathBuf}, + sync::Arc, + time::SystemTime, +}; use Error::*; /// DataFile @@ -136,12 +135,12 @@ impl Table { Ok(Self { dir: dir.as_ref().to_path_buf(), - hotness: 0, + hotness: Default::default(), index_file: IndexFile::new(index_file_path, index_file), data_file: DataFile::new(data_file_path, data_file), created_at, entries: Arc::new(SkipMap::new()), - size: 0, + size: Default::default(), filter: None, summary: None, }) @@ -271,10 +270,7 @@ impl Table { // write filter to disk self.filter.as_mut().unwrap().write(self.dir.to_owned()).await?; - self.filter - .as_mut() - .unwrap() - .set_sstable_path(&self.data_file.path); + self.filter.as_mut().unwrap().set_sstable_path(&self.data_file.path); // write data blocks let mut current_block = Block::new(); if self.size > 0 { @@ -349,17 +345,6 @@ impl Table { self.entries.get(key.as_ref()).map(|entry| entry.value().to_owned()) } - /// Compares two offsets - fn compare_offsets(offset_a: usize, offset_b: usize) -> Ordering { - if offset_a < offset_b { - Ordering::Less - } else if offset_a == offset_b { - Ordering::Equal - } else { - Ordering::Greater - } - } - /// Returns `size` of `Table` pub fn size(&self) -> usize { self.size diff --git a/src/tests/bucket_test.rs b/src/tests/bucket_test.rs index ac46be2..ad15810 100644 --- a/src/tests/bucket_test.rs +++ b/src/tests/bucket_test.rs @@ -115,52 +115,53 @@ mod tests { ]; let mut ssts = Vec::new(); for i in 0..number { + let idx = i as usize; ssts.push(Table { - dir: sst_contructor[i as usize].dir.to_owned(), + dir: sst_contructor[idx].dir.to_owned(), hotness: 100, size: 4096, created_at: Utc::now(), data_file: DataFile { file: DataFileNode { node: FileNode { - file_path: sst_contructor[i as usize].data_path.to_owned(), + file_path: sst_contructor[idx].data_path.to_owned(), file: Arc::new(RwLock::new( - File::open(sst_contructor[i as usize].data_path.to_owned()) + File::open(sst_contructor[idx].data_path.to_owned()) .await .unwrap(), )), file_type: FileType::Data, }, }, - path: sst_contructor[i as usize].data_path.to_owned(), + path: sst_contructor[idx].data_path.to_owned(), }, index_file: IndexFile { file: IndexFileNode { node: FileNode { - file_path: sst_contructor[i as usize].index_path.to_owned(), + file_path: sst_contructor[idx].index_path.to_owned(), file: Arc::new(RwLock::new( - File::open(sst_contructor[i as usize].index_path.to_owned()) + File::open(sst_contructor[idx].index_path.to_owned()) .await .unwrap(), )), file_type: FileType::Index, }, }, - path: sst_contructor[i as usize].index_path.to_owned(), + path: sst_contructor[idx].index_path.to_owned(), }, entries: Arc::new(SkipMap::default()), filter: None, summary: None, }) } - return ssts; + ssts } } #[tokio::test] async fn test_bucket_new() { let root = tempdir().unwrap(); - let path = PathBuf::from(root.path().join(".")); + let path = root.path().join("."); let new_bucket = Bucket::new(path.to_owned()).await; assert!(new_bucket.is_ok()); let new_bucket = new_bucket.unwrap(); @@ -175,7 +176,7 @@ mod tests { #[tokio::test] async fn test_bucket_from_with_empty() { let root = tempdir().unwrap(); - let path = PathBuf::from(root.path().join(".")); + let path = root.path().join("."); let id = Uuid::new_v4(); let average_size = 0; let sstables = Vec::new(); @@ -191,7 +192,7 @@ mod tests { #[tokio::test] async fn test_bucket_from_with_sstables() { let root = tempdir().unwrap(); - let path = PathBuf::from(root.path().join(".")); + let path = root.path().join("."); let id = Uuid::new_v4(); let sst_count = 3; let sst_samples = SSTContructor::generate_ssts(sst_count).await; @@ -202,7 +203,7 @@ mod tests { for meta_task in sst_meta { let meta_data = meta_task .await - .map_err(|err| Error::GetFileMetaDataError(err.into())) + .map_err(|err| Error::GetFileMetaData(err.into())) .unwrap(); all_sstable_size += meta_data.unwrap().len() as usize; } @@ -227,7 +228,7 @@ mod tests { for meta_task in sst_meta { let meta_data = meta_task .await - .map_err(|err| Error::GetFileMetaDataError(err.into())) + .map_err(|err| Error::GetFileMetaData(err.into())) .unwrap(); all_sstable_size += meta_data.unwrap().len() as usize; } @@ -240,24 +241,24 @@ mod tests { #[tokio::test] async fn test_sstcount_exceed_threshold() { let root = tempdir().unwrap(); - let path = PathBuf::from(root.path().join(".")); + let path = root.path().join("."); let new_bucket = Bucket::new(path.to_owned()).await.unwrap(); let sst_count = 5; let sst_samples = SSTContructor::generate_ssts(sst_count).await; for s in sst_samples { new_bucket.sstables.write().await.push(s) } - assert_eq!(new_bucket.sstable_count_exceeds_threshhold().await, true); + assert!(new_bucket.sstable_count_exceeds_threshhold().await); new_bucket.sstables.write().await.clear(); - assert_eq!(new_bucket.sstable_count_exceeds_threshhold().await, false); + assert!(!(new_bucket.sstable_count_exceeds_threshhold().await)); } #[tokio::test] async fn test_extract_sstable_to_compact() { let root = tempdir().unwrap(); - let path = PathBuf::from(root.path().join(".")); + let path = root.path().join("."); let new_bucket = Bucket::new(path.to_owned()).await.unwrap(); let sst_count = 5; let sst_samples = SSTContructor::generate_ssts(sst_count).await; @@ -268,7 +269,7 @@ mod tests { for meta_task in sst_meta { let meta_data = meta_task .await - .map_err(|err| Error::GetFileMetaDataError(err.into())) + .map_err(|err| Error::GetFileMetaData(err.into())) .unwrap(); all_sstable_size += meta_data.unwrap().len() as usize; } @@ -286,7 +287,7 @@ mod tests { #[tokio::test] async fn table_fits_into_bucket() { let root = tempdir().unwrap(); - let path = PathBuf::from(root.path().join(".")); + let path =root.path().join("."); let mut new_bucket = Bucket::new(path.to_owned()).await.unwrap(); let sst_sample = SSTContructor::generate_ssts(2).await; for s in sst_sample { @@ -296,23 +297,23 @@ mod tests { new_bucket.avarage_size = sst_within_size_range.size(); let fits_into_bucket = new_bucket.fits_into_bucket(Arc::new(Box::new(sst_within_size_range.to_owned()))); // size of sstable is not less than bucket low - assert_eq!(fits_into_bucket, true); + assert!(fits_into_bucket); // increase sstable size to be greater than bucket high range sst_within_size_range.size = ((new_bucket.avarage_size as f64 * BUCKET_HIGH) * 2.0) as usize; let fits_into_bucket = new_bucket.fits_into_bucket(Arc::new(Box::new(sst_within_size_range.to_owned()))); // sstable size is greater than bucket high range - assert_eq!(fits_into_bucket, false); + assert!(!fits_into_bucket); // increase bucket average new_bucket.avarage_size = ((new_bucket.avarage_size as f64 * BUCKET_HIGH) * 2.0) as usize; let fits_into_bucket = new_bucket.fits_into_bucket(Arc::new(Box::new(sst_within_size_range.to_owned()))); // sstable size is within bucket range - assert_eq!(fits_into_bucket, true); + assert!(fits_into_bucket); } #[tokio::test] async fn test_bucket_map_new() { let root = tempdir().unwrap(); - let path = PathBuf::from(root.path().join(".")); + let path = root.path().join("."); let bucket_map = BucketMap::new(path.to_owned()).await; assert!(bucket_map.is_ok()); let bucket_map = bucket_map.unwrap(); @@ -323,31 +324,31 @@ mod tests { #[tokio::test] async fn test_bucket_map_extract_imbalanced_buckets() { let root = tempdir().unwrap(); - let path = PathBuf::from(root.path().join(".")); + let path = root.path().join("."); let new_bucket1 = Bucket::new(path.to_owned()).await.unwrap(); let sst_count = 6; let sst_samples = SSTContructor::generate_ssts(sst_count).await; - for s in sst_samples.to_owned() { + for s in sst_samples.iter().cloned() { new_bucket1.sstables.write().await.push(s) } let new_bucket2 = Bucket::new(path.to_owned()).await.unwrap(); - for s in sst_samples.to_owned() { + for s in sst_samples.iter().cloned() { new_bucket2.sstables.write().await.push(s) } let new_bucket3 = Bucket::new(path.to_owned()).await.unwrap(); - for s in sst_samples.to_owned() { + for s in sst_samples.iter().cloned() { new_bucket3.sstables.write().await.push(s) } let new_bucket4 = Bucket::new(path.to_owned()).await.unwrap(); - for s in sst_samples.to_owned() { + for s in sst_samples.iter().cloned() { new_bucket4.sstables.write().await.push(s) } let root = tempdir().unwrap(); - let path = PathBuf::from(root.path().join(".")); + let path = root.path().join("."); let mut bucket_map = BucketMap::new(path.to_owned()).await.unwrap(); bucket_map.buckets.insert(new_bucket1.id, new_bucket1.to_owned()); bucket_map.buckets.insert(new_bucket2.id, new_bucket2); @@ -359,7 +360,7 @@ mod tests { let (buckets, ssts_to_remove) = imbalanced_buckets.unwrap(); let mut expected_ssts_to_remove_in_buckets = 0; assert_eq!(buckets.len(), 4); - for bucket in buckets.to_owned() { + for bucket in buckets.iter().cloned() { let sst_len = bucket.sstables.read().await.len(); assert!(sst_len == sst_count as usize); assert!(sst_len > MIN_TRESHOLD); @@ -393,31 +394,31 @@ mod tests { #[tokio::test] async fn test_bucket_map_is_balanced() { let root = tempdir().unwrap(); - let path = PathBuf::from(root.path().join(".")); + let path = root.path().join("."); let new_bucket1 = Bucket::new(path.to_owned()).await.unwrap(); let sst_count = 6; let sst_samples = SSTContructor::generate_ssts(sst_count).await; - for s in sst_samples.to_owned() { + for s in sst_samples.iter().cloned() { new_bucket1.sstables.write().await.push(s) } let new_bucket2 = Bucket::new(path.to_owned()).await.unwrap(); - for s in sst_samples.to_owned() { + for s in sst_samples.iter().cloned() { new_bucket2.sstables.write().await.push(s) } let new_bucket3 = Bucket::new(path.to_owned()).await.unwrap(); - for s in sst_samples.to_owned() { + for s in sst_samples.iter().cloned() { new_bucket3.sstables.write().await.push(s) } let new_bucket4 = Bucket::new(path.to_owned()).await.unwrap(); - for s in sst_samples.to_owned() { + for s in sst_samples.iter().cloned() { new_bucket4.sstables.write().await.push(s) } let root = tempdir().unwrap(); - let path = PathBuf::from(root.path().join(".")); + let path = root.path().join("."); let mut bucket_map = BucketMap::new(path.to_owned()).await.unwrap(); bucket_map.buckets.insert(new_bucket1.id, new_bucket1.to_owned()); bucket_map.buckets.insert(new_bucket2.id, new_bucket2); @@ -425,30 +426,33 @@ mod tests { bucket_map.buckets.insert(new_bucket4.id, new_bucket4); let is_balanced = bucket_map.is_balanced().await; - assert_eq!(is_balanced, false); + assert!(!is_balanced); // test empty map bucket_map.buckets.clear(); let is_balanced = bucket_map.is_balanced().await; - assert_eq!(is_balanced, true); + assert!(is_balanced); // Should not return false if all buckets are balanced new_bucket1.sstables.write().await.clear(); new_bucket1.sstables.write().await.push(sst_samples[0].to_owned()); bucket_map.buckets.insert(new_bucket1.id, new_bucket1); let is_balanced = bucket_map.is_balanced().await; - assert_eq!(is_balanced, true); + assert!(is_balanced); } #[tokio::test] async fn table_insert_to_appropriate_bucket() { let root = tempdir().unwrap(); - let path = PathBuf::from(root.path().join(".")); + let path =root.path().join("."); let mut bucket_map = BucketMap::new(path.to_owned()).await.unwrap(); let false_pos = 0.1; let mut sst_within_size_range = SSTContructor::generate_ssts(1).await[0].to_owned(); sst_within_size_range.load_entries_from_file().await.unwrap(); - sst_within_size_range.filter = Some(FilterWorkload::new(false_pos, sst_within_size_range.entries.to_owned())); + sst_within_size_range.filter = Some(FilterWorkload::from( + false_pos, + sst_within_size_range.entries.to_owned(), + )); // bucket insertion is succeeds let insert_res = bucket_map .insert_to_appropriate_bucket(Arc::new(Box::new(sst_within_size_range.to_owned()))) @@ -473,36 +477,36 @@ mod tests { #[tokio::test] async fn test_delete_sstables() { let root = tempdir().unwrap(); - let path = PathBuf::from(root.path().join(".")); + let path =root.path().join("."); let new_bucket1 = Bucket::new(path.to_owned()).await.unwrap(); let sst_count = 6; let sst_samples = SSTContructor::generate_ssts(sst_count).await; - for s in sst_samples.to_owned() { + for s in sst_samples.iter().cloned() { new_bucket1.sstables.write().await.push(s) } let new_bucket2 = Bucket::new(path.to_owned()).await.unwrap(); - for s in sst_samples.to_owned() { + for s in sst_samples.iter().cloned() { new_bucket2.sstables.write().await.push(s) } let new_bucket3 = Bucket::new(path.to_owned()).await.unwrap(); - for s in sst_samples.to_owned() { + for s in sst_samples.iter().cloned() { new_bucket3.sstables.write().await.push(s) } let new_bucket4 = Bucket::new(path.to_owned()).await.unwrap(); - for s in sst_samples.to_owned() { + for s in sst_samples.iter().cloned() { new_bucket4.sstables.write().await.push(s) } let new_bucket5 = Bucket::new(path.to_owned()).await.unwrap(); - for s in sst_samples.to_owned() { + for s in sst_samples.iter().cloned() { new_bucket5.sstables.write().await.push(s) } let root = tempdir().unwrap(); - let path = PathBuf::from(root.path().join(".")); + let path = root.path().join("."); let mut bucket_map = BucketMap::new(path.to_owned()).await.unwrap(); bucket_map.buckets.insert(new_bucket1.id, new_bucket1.to_owned()); bucket_map.buckets.insert(new_bucket2.id, new_bucket2); diff --git a/src/tests/gc_test.rs b/src/tests/gc_test.rs index e6f7c95..0244190 100644 --- a/src/tests/gc_test.rs +++ b/src/tests/gc_test.rs @@ -3,9 +3,8 @@ mod tests{ use crate::consts::{SIZE_OF_U32, SIZE_OF_U64, SIZE_OF_U8}; use crate::db::{DataStore, SizeUnit}; use crate::err::Error; - use crate::gc::gc::GC; + use crate::gc::garbage_collector::GC; use crate::types::Key; - use std::path::PathBuf; use std::sync::Arc; use tempfile::tempdir; use tokio::sync::RwLock; @@ -24,13 +23,13 @@ mod tests{ let _ = store.write().await.put("test_key", "test").await?; let _ = store.write().await.delete("test_key").await?; } - return Ok(()); + Ok(()) } // Generate test to find keys after compaction #[tokio::test] async fn datastore_gc_test_success() { let root = tempdir().unwrap(); - let path = PathBuf::from(root.path().join("gc_test_1")); + let path = root.path().join("gc_test_1"); let s_engine = DataStore::open_without_background("test", path.clone()).await.unwrap(); let store = Arc::new(RwLock::new(s_engine)); let workload_size = 15000; @@ -65,7 +64,7 @@ mod tests{ #[tokio::test] async fn datastore_gc_test_unsupported_platform() { let root = tempdir().unwrap(); - let path = PathBuf::from(root.path().join("gc_test_2")); + let path = root.path().join("gc_test_2"); let s_engine = DataStore::open_without_background("test", path.clone()).await.unwrap(); let store = Arc::new(RwLock::new(s_engine)); let workload_size = 15000; @@ -99,7 +98,7 @@ mod tests{ #[tokio::test] async fn datastore_gc_test_tail_shifted() { let root = tempdir().unwrap(); - let path = PathBuf::from(root.path().join("gc_test_3")); + let path = root.path().join("gc_test_3"); let s_engine = DataStore::open_without_background("test", path.clone()).await.unwrap(); let store = Arc::new(RwLock::new(s_engine)); let workload_size = 15000; @@ -135,7 +134,7 @@ mod tests{ #[tokio::test] async fn datastore_gc_test_free_before_synchronization() { let root = tempdir().unwrap(); - let path = PathBuf::from(root.path().join("gc_test_free")); + let path = root.path().join("gc_test_free"); let s_engine = DataStore::open_without_background("test", path.clone()).await.unwrap(); let store = Arc::new(RwLock::new(s_engine)); let workload_size = 15000; @@ -168,9 +167,9 @@ mod tests{ #[tokio::test] async fn datastore_gc_test_tail_shifted_to_correct_position() { - let bytes_to_scan_for_garbage_colection = SizeUnit::Bytes.to_bytes(100); + let bytes_to_scan_for_garbage_colection = SizeUnit::Bytes.as_bytes(100); let root = tempdir().unwrap(); - let path = PathBuf::from(root.path().join("gc_test_4")); + let path = root.path().join("gc_test_4"); let s_engine = DataStore::open_without_background("test", path.clone()).await.unwrap(); let store = Arc::new(RwLock::new(s_engine)); let workload_size = 5; @@ -216,9 +215,9 @@ mod tests{ #[tokio::test] async fn datastore_gc_test_head_shifted() { - let bytes_to_scan_for_garbage_colection = SizeUnit::Bytes.to_bytes(100); + let bytes_to_scan_for_garbage_colection = SizeUnit::Bytes.as_bytes(100); let root = tempdir().unwrap(); - let path = PathBuf::from(root.path().join("gc_test_5")); + let path = root.path().join("gc_test_5"); let s_engine = DataStore::open_without_background("test", path.clone()).await.unwrap(); let store = Arc::new(RwLock::new(s_engine)); let _ = store.write().await.put("test_key", "test_val").await; @@ -255,7 +254,7 @@ mod tests{ async fn datastore_gc_test_no_entry_to_collect() { let prepare_delete = false; let root = tempdir().unwrap(); - let path = PathBuf::from(root.path().join("gc_test_no_delete")); + let path = root.path().join("gc_test_no_delete"); let s_engine = DataStore::open_without_background("test", path.clone()).await.unwrap(); let store = Arc::new(RwLock::new(s_engine)); let workload_size = 15000; diff --git a/src/tests/store_test.rs b/src/tests/store_test.rs index 5ba66ed..2940f38 100644 --- a/src/tests/store_test.rs +++ b/src/tests/store_test.rs @@ -1,16 +1,13 @@ #[cfg(test)] mod tests { use crate::db::DataStore; - use crate::err::Error; use crate::tests::*; use futures::future::join_all; - use workload::Workload; - use std::path::PathBuf; use std::sync::Arc; use tempfile::tempdir; use tokio::sync::RwLock; - - + use workload::Workload; + fn setup() { let _ = env_logger::builder().is_test(true).try_init(); } @@ -18,16 +15,16 @@ mod tests { async fn datastore_create_new() { setup(); let root = tempdir().unwrap(); - let path = PathBuf::from(root.path().join("store_test_1")); + let path = root.path().join("store_test_1"); let store = DataStore::open_without_background("test", path.clone()).await; assert!(store.is_ok()) } - + #[tokio::test] async fn datastore_put_test() { setup(); let root = tempdir().unwrap(); - let path = PathBuf::from(root.path().join("store_test_2")); + let path = root.path().join("store_test_2"); let store = DataStore::open_without_background("test", path.clone()).await.unwrap(); let workload_size = 20000; let key_len = 5; @@ -35,7 +32,7 @@ mod tests { let write_read_ratio = 0.5; let workload = Workload::new(workload_size, key_len, val_len, write_read_ratio); let (_, write_workload) = workload.generate_workload_data_as_map(); - + let store_ref = Arc::new(RwLock::new(store)); let write_tasks = write_workload.iter().map(|e| { let store_inner = Arc::clone(&store_ref); @@ -46,22 +43,22 @@ mod tests { writer.put(key, val).await }) }); - + let all_results = join_all(write_tasks).await; for tokio_res in all_results { assert!(tokio_res.is_ok()); assert!(tokio_res.as_ref().unwrap().is_ok()); - assert_eq!(tokio_res.unwrap().unwrap(), true); + assert!(tokio_res.unwrap().unwrap()); } } - + #[tokio::test] async fn datastore_test_put_and_get() { setup(); let root = tempdir().unwrap(); - let path = PathBuf::new().join(root.path().join("store_test_3")); + let path = root.path().join("store_test_get_put"); let store = DataStore::open_without_background("test", path).await.unwrap(); - let workload_size = 50000; + let workload_size = 20000; let key_len = 5; let val_len = 5; let write_read_ratio = 1.0; @@ -77,40 +74,41 @@ mod tests { value.put(key, val).await }) }); - + let all_results = join_all(write_tasks).await; for tokio_res in all_results { assert!(tokio_res.is_ok()); assert!(tokio_res.as_ref().unwrap().is_ok()); - assert_eq!(tokio_res.unwrap().unwrap(), true); + assert!(tokio_res.unwrap().unwrap()); } - + let read_tasks = read_workload.iter().map(|e| { let store_inner = Arc::clone(&store_ref); let key = e.0.to_owned(); tokio::spawn(async move { match store_inner.read().await.get(key.to_owned()).await { - Ok((value, _)) => Ok((key, value)), - Err(err) => return Err(err), + Ok(entry) => Ok((key, entry)), + Err(err) => Err(err), } }) }); - + let all_results = join_all(read_tasks).await; - + for tokio_res in all_results { assert!(tokio_res.is_ok()); assert!(tokio_res.as_ref().unwrap().is_ok()); let (key, value) = tokio_res.unwrap().unwrap(); - assert_eq!(value, *write_workload.get(&key).unwrap()); + assert!(value.is_some()); + assert_eq!(value.unwrap().val, *write_workload.get(&key).unwrap()); } } - + #[tokio::test] async fn datastore_test_put_and_get_concurrent() { setup(); let root = tempdir().unwrap(); - let path = PathBuf::from(root.path().join("store_test_4")); + let path = root.path().join("store_test_4"); let store = DataStore::open_without_background("test", path.clone()).await.unwrap(); let workload_size = 1; let key_len = 5; @@ -128,10 +126,10 @@ mod tests { entry3.val = b"val3".to_vec(); entry4.val = b"val4".to_vec(); entry5.val = b"val5".to_vec(); - + let concurrent_write_workload = vec![entry1, entry2, entry3, entry4, entry5.to_owned()]; let store_ref = Arc::new(RwLock::new(store)); - + let concurrent_write_tasks = concurrent_write_workload.iter().map(|e| { let store_inner = Arc::clone(&store_ref); let key = e.key.to_owned(); @@ -141,25 +139,25 @@ mod tests { value.put(key, val).await }) }); - + let all_results = join_all(concurrent_write_tasks).await; for tokio_res in all_results { assert!(tokio_res.is_ok()); assert!(tokio_res.as_ref().unwrap().is_ok()); - assert_eq!(tokio_res.unwrap().unwrap(), true); + assert!(tokio_res.unwrap().unwrap()); } - - let res = store_ref.read().await.get(std::str::from_utf8(&key).unwrap()).await; + + let res = store_ref.read().await.get(std::str::from_utf8(key).unwrap()).await; assert!(res.is_ok()); // Even though the write of thesame key happened concurrently, we expect the last entry to reflect - assert_eq!(res.unwrap().0, entry5.val); + assert_eq!(res.unwrap().unwrap().val, entry5.val); } - + #[tokio::test] async fn datastore_test_seqential_put() { setup(); let root = tempdir().unwrap(); - let path = PathBuf::from(root.path().join("store_test_5")); + let path = root.path().join("store_test_5"); let mut store = DataStore::open_without_background("test", path.clone()).await.unwrap(); let workload_size = 10000; let key_len = 5; @@ -172,15 +170,15 @@ mod tests { let val = e.1.to_owned(); let res = store.put(key, val).await; assert!(res.is_ok()); - assert_eq!(res.unwrap(), true); + assert!(res.unwrap()); } } - + #[tokio::test] async fn datastore_test_sequential_put_and_get() { setup(); let root = tempdir().unwrap(); - let path = PathBuf::from(root.path().join("store_test_6")); + let path = root.path().join("store_test_6"); let mut store = DataStore::open_without_background("test", path.clone()).await.unwrap(); let workload_size = 5000; let key_len = 5; @@ -188,25 +186,25 @@ mod tests { let write_read_ratio = 1.0; let workload = Workload::new(workload_size, key_len, val_len, write_read_ratio); let (read_workload, write_workload) = workload.generate_workload_data_as_vec(); - + for e in write_workload.iter() { let res = store.put(e.key.to_owned(), e.val.to_owned()).await; assert!(res.is_ok()); - assert_eq!(res.unwrap(), true); + assert!(res.unwrap()); } for e in read_workload.iter() { let res = store.get(&e.key).await; assert!(res.is_ok()); let w = write_workload.iter().find(|e1| e1.key == e.key).unwrap(); - assert_eq!(res.unwrap().0, w.val); + assert_eq!(res.unwrap().unwrap().val, w.val); } } - + #[tokio::test] async fn datastore_not_found() { setup(); let root = tempdir().unwrap(); - let path = PathBuf::from(root.path().join("store_test_7")); + let path = root.path().join("store_test_7"); let store = DataStore::open_without_background("test", path.clone()).await.unwrap(); let workload_size = 10000; let key_len = 5; @@ -216,22 +214,22 @@ mod tests { let (_, write_workload) = workload.generate_workload_data_as_vec(); let store_ref = Arc::new(RwLock::new(store)); let res = workload.insert_parallel(&write_workload, store_ref.clone()).await; - if !res.is_ok() { + if res.is_err() { log::error!("Insert failed {:?}", res.err()); return; } - + let not_found_key = "**_not_found_**"; let res = store_ref.read().await.get(not_found_key).await; - assert!(res.is_err()); - assert_eq!(res.err().unwrap().to_string(), Error::NotFoundInDB.to_string()); + assert!(res.is_ok()); + assert!(res.unwrap().is_none()); } - + #[tokio::test] async fn datastore_compaction() { setup(); let root = tempdir().unwrap(); - let path = PathBuf::from(root.path().join("store_test_8")); + let path = root.path().join("store_test_8"); let store = DataStore::open_without_background("test", path.clone()).await.unwrap(); let workload_size = 20000; let key_len = 5; @@ -241,7 +239,7 @@ mod tests { let (_, write_workload) = workload.generate_workload_data_as_vec(); let store_ref = Arc::new(RwLock::new(store)); let res = workload.insert_parallel(&write_workload, store_ref.clone()).await; - if !res.is_ok() { + if res.is_err() { log::error!("Insert failed {:?}", res.err()); return; } @@ -253,45 +251,45 @@ mod tests { let res2 = store_ref.read().await.get(key2).await; let res3 = store_ref.read().await.get(key3).await; let res4 = store_ref.read().await.get(key4).await; - + assert!(res1.is_ok()); assert!(res2.is_ok()); assert!(res3.is_ok()); assert!(res4.is_ok()); - assert_eq!(res1.unwrap().0, write_workload[0].val); - assert_eq!(res2.unwrap().0, write_workload[1].val); - assert_eq!(res3.unwrap().0, write_workload[2].val); - assert_eq!(res4.unwrap().0, write_workload[3].val); - + assert_eq!(res1.unwrap().unwrap().val, write_workload[0].val); + assert_eq!(res2.unwrap().unwrap().val, write_workload[1].val); + assert_eq!(res3.unwrap().unwrap().val, write_workload[2].val); + assert_eq!(res4.unwrap().unwrap().val, write_workload[3].val); + let res = store_ref.write().await.delete(key1).await; assert!(res.is_ok()); - assert_eq!(res.unwrap(), true); - + assert!(res.unwrap()); + let res = store_ref.read().await.get(key1).await; - assert!(res.is_err()); - assert_eq!(Error::NotFoundInDB.to_string(), res.err().unwrap().to_string()); - - let _ = store_ref.write().await.flush_all_memtables().await; - + assert!(res.is_ok()); + assert!(res.unwrap().is_none()); + + let _ = store_ref.write().await.force_flush().await; + // We expect tombstone to be flushed to an sstable at this point let res = store_ref.read().await.get(key1).await; - assert!(res.is_err()); - assert_eq!(Error::NotFoundInDB.to_string(), res.err().unwrap().to_string()); - + assert!(res.is_ok()); + assert!(res.unwrap().is_none()); + let comp_res = store_ref.write().await.run_compaction().await; assert!(comp_res.is_ok()); - + // Fetch after compaction let res = store_ref.read().await.get(key1).await; - assert!(res.is_err()); - assert_eq!(Error::NotFoundInDB.to_string(), res.err().unwrap().to_string()); + assert!(res.is_ok()); + assert!(res.unwrap().is_none()); } - + #[tokio::test] async fn datastore_update() { setup(); let root = tempdir().unwrap(); - let path = PathBuf::from(root.path().join("store_test_9")); + let path = root.path().join("store_test_9"); let store = DataStore::open_without_background("test", path.clone()).await.unwrap(); let workload_size = 20000; let key_len = 5; @@ -301,48 +299,48 @@ mod tests { let (_, write_workload) = workload.generate_workload_data_as_vec(); let store_ref = Arc::new(RwLock::new(store)); let res = workload.insert_parallel(&write_workload, store_ref.clone()).await; - if !res.is_ok() { + if res.is_err() { log::error!("Insert failed {:?}", res.err()); return; } - + let key1 = &write_workload[0].key; let updated_value = "updated_key".as_bytes().to_vec(); - + let res = store_ref.read().await.get(key1).await; assert!(res.is_ok()); - assert_eq!(res.unwrap().0, write_workload[0].val); - + assert_eq!(res.unwrap().unwrap().val, write_workload[0].val); + let res = store_ref.write().await.update(key1, &updated_value).await; assert!(res.is_ok()); - assert_eq!(res.unwrap(), true); - + assert!(res.unwrap()); + let res = store_ref.read().await.get(key1).await; assert!(res.is_ok()); - assert_ne!(res.as_ref().unwrap().0, write_workload[0].val); - assert_eq!(res.as_ref().unwrap().0, updated_value); - - let res = store_ref.write().await.flush_all_memtables().await; + assert_ne!(res.as_ref().unwrap().as_ref().unwrap().val, write_workload[0].val); + assert_eq!(res.as_ref().unwrap().as_ref().unwrap().val, updated_value); + + let res = store_ref.write().await.force_flush().await; assert!(res.is_ok()); - + let res = store_ref.read().await.get(key1).await; assert!(res.is_ok()); - assert_eq!(res.unwrap().0, updated_value); - + assert_eq!(res.unwrap().unwrap().val, updated_value); + // Run compaction let comp_res = store_ref.write().await.run_compaction().await; assert!(comp_res.is_ok()); - + let res = store_ref.read().await.get(key1).await; assert!(res.is_ok()); - assert_eq!(res.unwrap().0, updated_value); + assert_eq!(res.unwrap().unwrap().val, updated_value); } - + #[tokio::test] async fn datastore_deletion() { setup(); let root = tempdir().unwrap(); - let path = PathBuf::from(root.path().join("store_test_10")); + let path = root.path().join("store_test_10"); let store = DataStore::open_without_background("test", path.clone()).await.unwrap(); let workload_size = 20000; let key_len = 5; @@ -352,33 +350,32 @@ mod tests { let (_, write_workload) = workload.generate_workload_data_as_vec(); let store_ref = Arc::new(RwLock::new(store)); let res = workload.insert_parallel(&write_workload, store_ref.clone()).await; - if !res.is_ok() { + if res.is_err() { log::error!("Insert failed {:?}", res.err()); return; } let key1 = &write_workload[0].key; - + let res = store_ref.read().await.get(key1).await; assert!(res.is_ok()); - assert_eq!(res.unwrap().0, write_workload[0].val); - + assert_eq!(res.unwrap().unwrap().val, write_workload[0].val); + let res = store_ref.write().await.delete(key1).await; assert!(res.is_ok()); - assert_eq!(res.unwrap(), true); - - let res = store_ref.write().await.flush_all_memtables().await; + assert!(res.unwrap()); + + let res = store_ref.write().await.force_flush().await; assert!(res.is_ok()); - + let res = store_ref.read().await.get(key1).await; - assert!(res.is_err()); - assert_eq!(Error::NotFoundInDB.to_string(), res.err().unwrap().to_string()); - + assert!(res.is_ok()); + assert!(res.unwrap().is_none()); + let comp_opt = store_ref.write().await.run_compaction().await; assert!(comp_opt.is_ok()); - + let res = store_ref.read().await.get(key1).await; - assert!(res.is_err()); - assert_eq!(Error::NotFoundInDB.to_string(), res.err().unwrap().to_string()); + assert!(res.is_ok()); + assert!(res.unwrap().is_none()); } - } diff --git a/src/tests/workload.rs b/src/tests/workload.rs index f17a7d7..2f26693 100644 --- a/src/tests/workload.rs +++ b/src/tests/workload.rs @@ -1,5 +1,4 @@ use crate::memtable::SkipMapValue; -use crate::tests::workload::Error::TokioJoinError; use crate::{ db::DataStore, err::Error, @@ -85,7 +84,7 @@ impl Workload { pub async fn insert_parallel( &self, - entries: &Vec, + entries: &[Entry], store: Arc>>, ) -> Result<(), Error> { let tasks = entries.iter().map(|e| { @@ -104,14 +103,12 @@ impl Workload { for tokio_response in all_results { match tokio_response { Ok(entry) => { - if let Err(err) = entry { - return Err(err); - } + entry?; } - Err(_) => return Err(TokioJoinError), + Err(_) => return Err(Error::TokioJoin), } } - return Ok(()); + Ok(()) } } @@ -121,9 +118,9 @@ pub struct FilterWorkload { } impl FilterWorkload { - pub fn new(false_pos: f64, entries: Arc, SkipMapValue>>) -> crate::filter::BloomFilter { + pub fn from(false_pos: f64, entries: Arc, SkipMapValue>>) -> crate::filter::BloomFilter { let mut filter = crate::filter::BloomFilter::new(false_pos, entries.len()); filter.build_filter_from_entries(&entries); - return filter; + filter } } diff --git a/src/types/mod.rs b/src/types/mod.rs index a42bc77..d9d5d6e 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -1,6 +1,5 @@ use crate::{ bucket::BucketMap, - filter::BloomFilter, key_range::KeyRange, memtable::{MemTable, SkipMapValue}, }; @@ -43,11 +42,8 @@ pub type FlushReceiver = async_broadcast::Receiver; /// Thread-safe BucketMap pub type BucketMapHandle = Arc>; -/// Thread-safe vector of BloomFilters -pub type BloomFilterHandle = Arc>>; - /// Thread-safe KeyRange type -pub type KeyRangeHandle = Arc>; +pub type KeyRangeHandle = Arc; /// Represents read-only MemTables pub type ImmutableMemTables = Arc>>>; diff --git a/src/util/mod.rs b/src/util/mod.rs index 5285594..be4b6bb 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -30,7 +30,7 @@ pub fn float_to_le_bytes(f: f64) -> [u8; 8] { let bits: u64 = f.to_bits(); // Convert the u64 to an array of 8 bytes in little-endian order - let bytes: [u8; 8] = bits.to_le_bytes().try_into().unwrap(); + let bytes: [u8; 8] = bits.to_le_bytes(); bytes } diff --git a/tests/benchmarks/.DS_Store b/tests/benchmarks/.DS_Store deleted file mode 100644 index 08dd1cc..0000000 Binary files a/tests/benchmarks/.DS_Store and /dev/null differ diff --git a/tests/benchmarks/raw/random_write.fio b/tests/benchmarks/raw/random_write.fio deleted file mode 100644 index 1ca7baa..0000000 --- a/tests/benchmarks/raw/random_write.fio +++ /dev/null @@ -1,10 +0,0 @@ -[random_write] -ioengine=sync -direct=1 -bs=4k -rw=randwrite -size=1g -numjobs=1 -runtime=60 -time_based -name=rand_write_test diff --git a/tests/benchmarks/raw/sequential_write.fio b/tests/benchmarks/raw/sequential_write.fio deleted file mode 100644 index 56b9a5b..0000000 --- a/tests/benchmarks/raw/sequential_write.fio +++ /dev/null @@ -1,10 +0,0 @@ -[sequential_write] -ioengine=sync -direct=1 -bs=1m -rw=write -size=1g -numjobs=1 -runtime=60 -time_based -name=seq_write_test diff --git a/tests/delete.rs b/tests/delete.rs new file mode 100644 index 0000000..5f75471 --- /dev/null +++ b/tests/delete.rs @@ -0,0 +1,22 @@ +use tempfile::tempdir; +use velarixdb::db::DataStore; + +#[tokio::test] +async fn test_delete() { + let root = tempdir().unwrap(); + let path = root.path().join("velarix"); + let mut store = DataStore::open("big_tech", path).await.unwrap(); // handle IO error + + store.put("apple", "tim cook").await.unwrap(); // handle error + + // Retrieve entry + let entry = store.get("apple").await.unwrap(); + assert!(entry.is_some()); + + // Delete entry + store.delete("apple").await.unwrap(); + + // Entry should now be None + let entry = store.get("apple").await.unwrap(); + assert!(entry.is_none()); +} diff --git a/tests/get.rs b/tests/get.rs new file mode 100644 index 0000000..132a635 --- /dev/null +++ b/tests/get.rs @@ -0,0 +1,40 @@ +use tempfile::tempdir; +use velarixdb::db::DataStore; + + +#[tokio::test] +async fn test_get() { + let root = tempdir().unwrap(); + let path = root.path().join("velarixdb"); + let mut store = DataStore::open("big_tech", path).await.unwrap(); // handle IO error + + let res1 = store.put("apple", "tim cook").await; + let res2 = store.put("google", "sundar pichai").await; + let res3 = store.put("nvidia", "jensen huang").await; + let res4 = store.put("microsoft", "satya nadella").await; + let res5 = store.put("meta", "mark zuckerberg").await; + let res6 = store.put("openai", "sam altman").await; + + assert!(res1.is_ok()); + assert!(res2.is_ok()); + assert!(res3.is_ok()); + assert!(res4.is_ok()); + assert!(res5.is_ok()); + assert!(res6.is_ok()); + + let entry1 = store.get("apple").await.unwrap(); // Handle error + let entry2 = store.get("google").await.unwrap(); + let entry3 = store.get("nvidia").await.unwrap(); + let entry4 = store.get("microsoft").await.unwrap(); + let entry5 = store.get("meta").await.unwrap(); + let entry6 = store.get("openai").await.unwrap(); + let entry7 = store.get("***not_found_key**").await.unwrap(); + + assert_eq!(std::str::from_utf8(&entry1.unwrap().val).unwrap(), "tim cook"); + assert_eq!(std::str::from_utf8(&entry2.unwrap().val).unwrap(), "sundar pichai"); + assert_eq!(std::str::from_utf8(&entry3.unwrap().val).unwrap(), "jensen huang"); + assert_eq!(std::str::from_utf8(&entry4.unwrap().val).unwrap(), "satya nadella"); + assert_eq!(std::str::from_utf8(&entry5.unwrap().val).unwrap(), "mark zuckerberg"); + assert_eq!(std::str::from_utf8(&entry6.unwrap().val).unwrap(), "sam altman"); + assert!(entry7.is_none()) +} diff --git a/tests/get_concurrent.rs b/tests/get_concurrent.rs new file mode 100644 index 0000000..a906a7f --- /dev/null +++ b/tests/get_concurrent.rs @@ -0,0 +1,63 @@ +use std::{collections::HashMap, sync::Arc}; + +use futures::future::join_all; +use tempfile::tempdir; +use tokio::sync::RwLock; +use velarixdb::db::DataStore; + + +#[tokio::test] +async fn test_get_concurrent() { + let root = tempdir().unwrap(); + let path = root.path().join("velarix"); + let store = DataStore::open("big_tech", path).await.unwrap(); // handle IO error + let mut entries = HashMap::new(); + entries.insert("apple", "tim cook"); + entries.insert("google", "sundar pichai"); + entries.insert("nvidia", "jensen huang"); + entries.insert("microsoft", "satya nadella"); + entries.insert("meta", "mark zuckerberg"); + entries.insert("openai", "sam altman"); + + let store_ref = Arc::new(RwLock::new(store)); + let writes = entries.iter().map(|(k, v)| { + let store_inner = Arc::clone(&store_ref); + let key = k.to_owned(); + let val = v.to_owned(); + tokio::spawn(async move { + let mut writer = store_inner.write().await; + writer.put(key, val).await + }) + }); + let all_results = join_all(writes).await; + for tokio_res in all_results { + assert!(tokio_res.is_ok()); + assert!(tokio_res.as_ref().unwrap().is_ok()); + } + + + + // Read entries concurently + let reads = entries.keys().map(|k| { + let store_inner = Arc::clone(&store_ref); + let key = k.to_owned(); + tokio::spawn(async move { + match store_inner.read().await.get(key.to_owned()).await { + Ok(entry) => Ok((key, entry)), + Err(err) => Err(err), + } + }) + }); + + let all_results = join_all(reads).await; + for tokio_res in all_results { + assert!(tokio_res.is_ok()); + assert!(tokio_res.as_ref().unwrap().is_ok()); + let (key, value) = tokio_res.unwrap().unwrap(); + assert!(value.is_some()); + assert_eq!( + std::str::from_utf8(&value.unwrap().val).unwrap(), + *entries.get(&key).unwrap() + ); + } +} diff --git a/tests/put_concurrent.rs b/tests/put_concurrent.rs new file mode 100644 index 0000000..8476701 --- /dev/null +++ b/tests/put_concurrent.rs @@ -0,0 +1,39 @@ +use std::sync::Arc; + +use futures::future::join_all; +use tempfile::tempdir; +use tokio::sync::RwLock; +use velarixdb::db::DataStore; + + +#[tokio::test] +async fn test_put_concurrent() { + let root = tempdir().unwrap(); + let path = root.path().join("velarixdb"); + let store = DataStore::open("big_tech", path).await.unwrap(); // handle IO error + + let entries = [ + ["apple", "tim cook"], + ["google", "sundar pichai"], + ["nvidia", "jensen huang"], + ["microsoft", "satya nadella"], + ["meta", "mark zuckerberg"], + ["openai", "sam altman"], + ]; + + let store_ref = Arc::new(RwLock::new(store)); + let write_tasks = entries.iter().map(|e| { + let store_inner = Arc::clone(&store_ref); + let key = e[0]; + let val = e[1]; + tokio::spawn(async move { + let mut writer = store_inner.write().await; + writer.put(key, val).await + }) + }); + let all_results = join_all(write_tasks).await; + for tokio_res in all_results { + assert!(tokio_res.is_ok()); + assert!(tokio_res.as_ref().unwrap().is_ok()); + } +} diff --git a/tests/update.rs b/tests/update.rs new file mode 100644 index 0000000..24acd26 --- /dev/null +++ b/tests/update.rs @@ -0,0 +1,20 @@ +use tempfile::tempdir; +use velarixdb::db::DataStore; + +#[tokio::test] +async fn test_update() { + let root = tempdir().unwrap(); + let path = root.path().join("velarixdb"); + let mut store = DataStore::open("big_tech", path).await.unwrap(); // handle IO error + + store.put("apple", "tim cook").await.unwrap(); // handle error + + // Update entry + let success = store.update("apple", "elon musk").await; + assert!(success.is_ok()); + + // Entry should now be updated + let entry = store.get("apple").await.unwrap(); // handle error + assert!(entry.is_some()); + assert_eq!(std::str::from_utf8(&entry.unwrap().val).unwrap(), "elon musk") +} diff --git a/write_algo.txt b/write_algo.txt deleted file mode 100644 index 0a58d7f..0000000 --- a/write_algo.txt +++ /dev/null @@ -1,16 +0,0 @@ -Write algorithm - - - Get the size of the sstable - - - Get which bucket will be the best to store the sstable by using the bucket average size - - - If bucket exist, then write sstable to its directory and return bucket url - - - Else, create new bucket, write sstable to its directory, append to list of buckets and return sstable file path - - - Append the sstable file path to bloom filter and append bloom filter to existing bloom filters - - - Clear MemTable - - -