feat(storage): upgrade tokio to 1.38.0 and simplify memory limiter notify

Cargo.toml

@@ -42,6 +42,7 @@ members = [
     "src/storage",
     "src/storage/backup",
     "src/storage/compactor",
+    "src/storage/hummock_memory_limiter",
     "src/storage/hummock_sdk",
     "src/storage/hummock_test",
     "src/stream",
@@ -217,6 +218,7 @@ risingwave_error = { path = "./src/error" }
 risingwave_expr = { path = "./src/expr/core" }
 risingwave_expr_impl = { path = "./src/expr/impl" }
 risingwave_frontend = { path = "./src/frontend" }
+risingwave_hummock_memory_limiter = { path = "./src/storage/hummock_memory_limiter" }
 risingwave_hummock_sdk = { path = "./src/storage/hummock_sdk" }
 risingwave_hummock_test = { path = "./src/storage/hummock_test" }
 risingwave_hummock_trace = { path = "./src/storage/hummock_trace" }
@@ -257,6 +259,7 @@ unexpected_cfgs = { level = "warn", check-cfg = [
     'cfg(madsim)',
     'cfg(coverage)',
     'cfg(dashboard_built)',
+    'cfg(loom)',
 ] }
 
 [workspace.lints.clippy]

ci/scripts/run-unit-test.sh

@@ -8,3 +8,6 @@ REPO_ROOT=${PWD}
 echo "+++ Run unit tests"
 # use tee to disable progress bar
 NEXTEST_PROFILE=ci cargo nextest run --features failpoints,sync_point --workspace --exclude risingwave_simulation
+
+echo "+++ Run hummock memory limiter loom test"
+LOOM_MAX_BRANCHES=100000 RUSTFLAGS="--cfg loom" cargo test -p risingwave_hummock_memory_limiter

src/storage/Cargo.toml

@@ -48,6 +48,7 @@ risingwave_common = { workspace = true }
 risingwave_common_estimate_size = { workspace = true }
 risingwave_common_service = { workspace = true }
 risingwave_expr = { workspace = true }
+risingwave_hummock_memory_limiter = { workspace = true }
 risingwave_hummock_sdk = { workspace = true }
 risingwave_hummock_trace = { workspace = true }
 risingwave_object_store = { workspace = true }

src/storage/hummock_memory_limiter/Cargo.toml

@@ -0,0 +1,38 @@
+[package]
+name = "risingwave_hummock_memory_limiter"
+version = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+keywords = { workspace = true }
+license = { workspace = true }
+repository = { workspace = true }
+
+[package.metadata.cargo-machete]
+ignored = ["workspace-hack", "num-traits"]
+
+[package.metadata.cargo-udeps.ignore]
+normal = ["workspace-hack"]
+
+[dev-dependencies]
+futures = { version = "0.3" }
+rand = { workspace = true }
+
+[target.'cfg(madsim)'.dependencies]
+tokio = { version = "0.2", package = "madsim-tokio", features = ["sync"] }
+
+[target.'cfg(not(madsim))'.dependencies]
+tokio = { version = "1.38.0", features = ["sync", "rt", "rt-multi-thread", "time", "macros"] }
+workspace-hack = { path = "../../workspace-hack" }
+
+[target."cfg(not(loom))".dependencies]
+parking_lot = { version = "0.12", features = ["deadlock_detection"] }
+
+[target."cfg(loom)".dependencies]
+loom = { version = "0.7", features = ["futures", "checkpoint"] }
+
+[lints]
+workspace = true
+
+[lib]
+doctest = false
+