feat: add metric manager (#1621)

Author: jiacai2050

Cargo.lock

@@ -31,12 +31,14 @@ members = [
     "src/common",
     "src/metric_engine",
     "src/pb_types",
-    "src/server"
+    "src/server",
+    "src/storage"
 ]
 
 [workspace.dependencies]
 anyhow = { version = "1.0" }
 metric_engine = { path = "src/metric_engine" }
+horaedb_storage = { package = "storage", path = "src/storage" }
 common = { path = "src/common" }
 thiserror = "1"
 bytes = "1"

Cargo.toml

@@ -47,7 +47,8 @@ udeps:
 	cd $(DIR); cargo udeps --all-targets --all-features --workspace
 
 clippy:
-	cd $(DIR); cargo clippy --all-targets --all-features --workspace -- -D warnings -D clippy::dbg-macro -A clippy::too-many-arguments
+	cd $(DIR); cargo clippy --all-targets --all-features --workspace -- -D warnings -D clippy::dbg-macro -A clippy::too-many-arguments \
+	-A dead_code
 
 ensure-disk-quota:
 	bash ./scripts/free-disk-space.sh

Makefile

@@ -28,7 +28,7 @@ description.workspace = true
 [dependencies]
 bytes = { workspace = true }
 common = { workspace = true }
-metric_engine = { workspace = true }
+horaedb_storage = { workspace = true }
 pb_types = { workspace = true }
 prost = { workspace = true }
 serde = { workspace = true }

src/benchmarks/Cargo.toml

@@ -18,7 +18,7 @@
 //! encoding bench.
 
 use bytes::Bytes;
-use metric_engine::{
+use horaedb_storage::{
     manifest::Snapshot,
     sst::{FileMeta, SstFile},
 };

src/benchmarks/src/encoding_bench.rs

@@ -26,5 +26,7 @@ homepage.workspace = true
 description.workspace = true
 
 [dependencies]
+anyhow = { workspace = true }
 serde = { workspace = true }
+thiserror = { workspace = true }
 toml = { workspace = true }

src/common/Cargo.toml

@@ -15,8 +15,10 @@
 // specific language governing permissions and limitations
 // under the License.
 
+mod error;
 mod size_ext;
 mod time_ext;
 
+pub use error::{AnyhowError, Error, Result};
 pub use size_ext::ReadableSize;
 pub use time_ext::{now, ReadableDuration};

src/metric_engine/src/error.rs renamed to src/common/src/error.rs

@@ -27,26 +27,11 @@ description.workspace = true
 
 [dependencies]
 anyhow = { workspace = true }
-arrow = { workspace = true }
-async-scoped = { workspace = true }
-async-trait = { workspace = true }
-byteorder = { workspace = true }
-bytes = { workspace = true }
-bytesize = { workspace = true }
 common = { workspace = true }
-datafusion = { workspace = true }
-futures = { workspace = true }
-itertools = { workspace = true }
-lazy_static = { workspace = true }
-object_store = { workspace = true }
-parquet = { workspace = true, features = ["object_store"] }
-pb_types = { workspace = true }
-prost = { workspace = true }
-serde = { workspace = true }
+horaedb_storage = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true }
 tracing = { workspace = true }
-uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
 
 [dev-dependencies]
 temp-dir = { workspace = true }

src/common/src/lib.rs

@@ -0,0 +1,40 @@
+* Metric Engine
+
+The basic write process is as follows:
+
+```plaintext
+  +----------------+
+  |     start      |
+  +----------------+
+    |
+    |
+    v
++ - - - - - - - - - -+
+'   Metric Engine    '
+'                    '
+' +----------------+ '
+' | metric_manager | '
+' +----------------+ '
+'   |                '
+'   |                '
+'   v                '
+' +----------------+ '
+' | index_manager  | '
+' +----------------+ '
+'   |                '
+'   |                '
+'   v                '
+' +----------------+ '
+' |  data_manager  | '
+' +----------------+ '
+'                    '
++ - - - - - - - - - -+
+    |
+    |
+    v
+  +----------------+
+  |      end       |
+  +----------------+
+```
+
+The structure pass between different module is `Sample`, modeled after [data model](https://prometheus.io/docs/concepts/data_model/) used in prometheus.