refactor: tune disk cache size configuration

src/query/config/src/config.rs

@@ -129,7 +129,7 @@ const CATALOG_HIVE: &str = "hive";
 /// It's forbidden to do any breaking changes on this struct.
 /// Only adding new fields is allowed.
 /// This same rules should be applied to all fields of this struct.
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Args)]
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Args)]
 #[serde(default)]
 pub struct Config {
     // Query engine config.
@@ -3189,7 +3189,7 @@ impl TryInto<InnerLocalConfig> for LocalConfig {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Args)]
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Args)]
 #[serde(default)]
 pub struct CacheConfig {
     /// The data in meta-service using key `TenantOwnershipObjectIdent`
@@ -3533,7 +3533,7 @@ impl Default for DiskCacheKeyReloadPolicy {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Args)]
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Args)]
 #[serde(default)]
 pub struct DiskCacheConfig {
     /// Max bytes of cached raw table data. Default 20GB, set it to 0 to disable it.
@@ -3570,6 +3570,10 @@ pub struct DiskCacheConfig {
     )]
     #[serde(default = "bool_true")]
     pub sync_data: bool,
+
+    #[clap(flatten)]
+    #[serde(default)]
+    pub ratios: DiskCacheRatioConfig,
 }
 
 impl Default for DiskCacheConfig {
@@ -3578,6 +3582,72 @@ impl Default for DiskCacheConfig {
     }
 }
 
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Args)]
+#[serde(default)]
+pub struct DiskCacheRatioConfig {
+    /// Weight used to derive on-disk column data cache capacity when no explicit size is provided.
+    #[clap(
+        long = "cache-disk-column-data-ratio",
+        value_name = "VALUE",
+        default_value = "1"
+    )]
+    pub column_data: f64,
+
+    /// Weight used to derive on-disk bloom index filter cache capacity when no explicit size is provided.
+    #[clap(
+        long = "cache-disk-bloom-index-filter-ratio",
+        value_name = "VALUE",
+        default_value = "0"
+    )]
+    pub bloom_index_filter: f64,
+
+    /// Weight used to derive on-disk bloom index meta cache capacity when no explicit size is provided.
+    #[clap(
+        long = "cache-disk-bloom-index-meta-ratio",
+        value_name = "VALUE",
+        default_value = "0"
+    )]
+    pub bloom_index_meta: f64,
+
+    /// Weight used to derive on-disk inverted index filter cache capacity when no explicit size is provided.
+    #[clap(
+        long = "cache-disk-inverted-index-filter-ratio",
+        value_name = "VALUE",
+        default_value = "0"
+    )]
+    pub inverted_index_filter: f64,
+
+    /// Weight used to derive on-disk inverted index meta cache capacity when no explicit size is provided.
+    #[clap(
+        long = "cache-disk-inverted-index-meta-ratio",
+        value_name = "VALUE",
+        default_value = "0"
+    )]
+    pub inverted_index_meta: f64,
+
+    /// Weight used to derive on-disk vector index filter cache capacity when no explicit size is provided.
+    #[clap(
+        long = "cache-disk-vector-index-filter-ratio",
+        value_name = "VALUE",
+        default_value = "0"
+    )]
+    pub vector_index_filter: f64,
+
+    /// Weight used to derive on-disk vector index meta cache capacity when no explicit size is provided.
+    #[clap(
+        long = "cache-disk-vector-index-meta-ratio",
+        value_name = "VALUE",
+        default_value = "0"
+    )]
+    pub vector_index_meta: f64,
+}
+
+impl Default for DiskCacheRatioConfig {
+    fn default() -> Self {
+        inner::DiskCacheRatioConfig::default().into()
+    }
+}
+
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Args)]
 #[serde(default)]
 pub struct SpillConfig {
@@ -3850,6 +3920,7 @@ mod cache_config_converters {
                 max_bytes: value.max_bytes,
                 path: value.path,
                 sync_data: value.sync_data,
+                ratios: value.ratios.into(),
             })
         }
     }
@@ -3860,6 +3931,35 @@ mod cache_config_converters {
                 max_bytes: value.max_bytes,
                 path: value.path,
                 sync_data: value.sync_data,
+                ratios: value.ratios.into(),
+            }
+        }
+    }
+
+    impl From<DiskCacheRatioConfig> for inner::DiskCacheRatioConfig {
+        fn from(value: DiskCacheRatioConfig) -> Self {
+            Self {
+                column_data: value.column_data,
+                bloom_index_filter: value.bloom_index_filter,
+                bloom_index_meta: value.bloom_index_meta,
+                inverted_index_filter: value.inverted_index_filter,
+                inverted_index_meta: value.inverted_index_meta,
+                vector_index_filter: value.vector_index_filter,
+                vector_index_meta: value.vector_index_meta,
+            }
+        }
+    }
+
+    impl From<inner::DiskCacheRatioConfig> for DiskCacheRatioConfig {
+        fn from(value: inner::DiskCacheRatioConfig) -> Self {
+            Self {
+                column_data: value.column_data,
+                bloom_index_filter: value.bloom_index_filter,
+                bloom_index_meta: value.bloom_index_meta,
+                inverted_index_filter: value.inverted_index_filter,
+                inverted_index_meta: value.inverted_index_meta,
+                vector_index_filter: value.vector_index_filter,
+                vector_index_meta: value.vector_index_meta,
             }
         }
     }

src/query/config/src/inner.rs

@@ -44,7 +44,7 @@ use crate::BuiltInConfig;
 /// Inner config for query.
 ///
 /// All function should implement based on this Config.
-#[derive(Clone, Default, PartialEq, Eq)]
+#[derive(Clone, Default, PartialEq)]
 pub struct InnerConfig {
     // Query engine config.
     pub query: QueryConfig,
@@ -554,7 +554,7 @@ impl Default for LocalConfig {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Eq)]
+#[derive(Clone, Debug, PartialEq)]
 pub struct CacheConfig {
     /// The data in meta-service using key `TenantOwnershipObjectIdent`
     pub meta_service_ownership_cache: bool,
@@ -726,7 +726,7 @@ impl Display for DiskCacheKeyReloadPolicy {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Eq)]
+#[derive(Clone, Debug, PartialEq)]
 pub struct DiskCacheConfig {
     /// Max bytes of cached raw table data. Default 20GB, set it to 0 to disable it.
     pub max_bytes: u64,
@@ -739,6 +739,9 @@ pub struct DiskCacheConfig {
     /// it's recommended to set this to true to prevent the container from
     /// being killed due to high dirty page memory usage.
     pub sync_data: bool,
+
+    /// Ratio weights used to derive per-cache disk capacities when explicit sizes are not specified.
+    pub ratios: DiskCacheRatioConfig,
 }
 
 impl Default for DiskCacheConfig {
@@ -747,6 +750,32 @@ impl Default for DiskCacheConfig {
             max_bytes: 21474836480,
             path: "./.databend/_cache".to_owned(),
             sync_data: true,
+            ratios: DiskCacheRatioConfig::default(),
+        }
+    }
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct DiskCacheRatioConfig {
+    pub column_data: f64,
+    pub bloom_index_filter: f64,
+    pub bloom_index_meta: f64,
+    pub inverted_index_filter: f64,
+    pub inverted_index_meta: f64,
+    pub vector_index_filter: f64,
+    pub vector_index_meta: f64,
+}
+
+impl Default for DiskCacheRatioConfig {
+    fn default() -> Self {
+        Self {
+            column_data: 1.0,
+            bloom_index_filter: 0.0,
+            bloom_index_meta: 0.0,
+            inverted_index_filter: 0.0,
+            inverted_index_meta: 0.0,
+            vector_index_filter: 0.0,
+            vector_index_meta: 0.0,
         }
     }
 }

src/query/config/src/lib.rs

@@ -47,6 +47,7 @@ pub use inner::CatalogConfig;
 pub use inner::CatalogHiveConfig;
 pub use inner::DiskCacheConfig as DiskCacheInnerConfig;
 pub use inner::DiskCacheKeyReloadPolicy;
+pub use inner::DiskCacheRatioConfig as DiskCacheRatioInnerConfig;
 pub use inner::InnerConfig;
 pub use inner::MetaConfig;
 pub use inner::QueryConfig;