feat(query-engine): add hash functions fnv, murmur3, md5, sha1, sha512, xxh3, xxh128

rust/otap-dataflow/Cargo.toml

@@ -212,6 +212,7 @@ weaver_forge = { git = "https://github.com/open-telemetry/weaver.git", rev = "37
 weaver_resolved_schema = { git = "https://github.com/open-telemetry/weaver.git", rev = "37c645a5ebc9e0d2a68f9228205f1d7d6a32ccbc"}
 weaver_resolver = { git = "https://github.com/open-telemetry/weaver.git", rev = "37c645a5ebc9e0d2a68f9228205f1d7d6a32ccbc"}
 weaver_semconv = { git = "https://github.com/open-telemetry/weaver.git", rev = "37c645a5ebc9e0d2a68f9228205f1d7d6a32ccbc"}
+sha1 = { version = "0.10" }
 xxhash-rust = { version = "0.8", features = ["xxh3"] }
 zip = "=8.6.0"
 byte-unit = { version = "5.2.0", features = ["serde"] }

rust/otap-dataflow/crates/query-engine/Cargo.toml

@@ -19,6 +19,8 @@ datafusion = { workspace = true, features = [
 ] }
 futures-core = { workspace = true }
 memchr = { workspace = true }
+sha1 = { workspace = true, optional = true }
+xxhash-rust = { workspace = true }
 parking_lot = { workspace = true }
 smallvec = { workspace = true }
 thiserror = { workspace = true }
@@ -29,6 +31,9 @@ uuid = { workspace = true }
 otap-df-config = { workspace = true }
 otap-df-pdata = { workspace = true, features = ["testing"] }
 
+[features]
+sha1-hash = ["dep:sha1"]
+
 [dev-dependencies]
 criterion = { workspace = true, features = ["async_tokio"] }
 data_engine_kql_parser = { workspace = true }

rust/otap-dataflow/crates/query-engine/src/consts.rs

@@ -14,6 +14,14 @@ pub(crate) const LTRIM_FUNC_NAME: &str = "ltrim";
 pub(crate) const REGEXP_SUBSTR_FUNC_NAME: &str = "regexp_substr";
 pub(crate) const RTRIM_FUNC_NAME: &str = "rtrim";
 pub(crate) const SHA256_FUNC_NAME: &str = "sha256";
+pub(crate) const MD5_FUNC_NAME: &str = "md5";
+pub(crate) const FNV_FUNC_NAME: &str = "fnv";
+pub(crate) const MURMUR3_FUNC_NAME: &str = "murmur3";
+#[cfg(feature = "sha1-hash")]
+pub(crate) const SHA1_FUNC_NAME: &str = "sha1";
+pub(crate) const SHA512_FUNC_NAME: &str = "sha512";
+pub(crate) const XXH3_FUNC_NAME: &str = "xxh3";
+pub(crate) const XXH128_FUNC_NAME: &str = "xxh128";
 pub(crate) const UUID_FUNC_NAME: &str = "uuid";
 pub(crate) const UUIDV7_FUNC_NAME: &str = "uuidv7";
 pub(crate) const LOWER_CASE_FUNC_NAME: &str = "lower_case";

rust/otap-dataflow/crates/query-engine/src/parser.rs

@@ -10,17 +10,20 @@ use data_engine_expressions::{
 };
 use data_engine_parser_abstractions::ParserOptions;
 
+#[cfg(feature = "sha1-hash")]
+use crate::consts::SHA1_FUNC_NAME;
 use crate::consts::{
-    ENCODE_FUNC_NAME, FORMAT_DATETIME_FUNC_NAME, LOG_FUNC_NAME, LOWER_CASE_FUNC_NAME,
-    LTRIM_FUNC_NAME, REGEXP_SUBSTR_FUNC_NAME, RTRIM_FUNC_NAME, SHA256_FUNC_NAME,
-    UPPER_CASE_FUNC_NAME, UUID_FUNC_NAME, UUIDV7_FUNC_NAME,
+    ENCODE_FUNC_NAME, FNV_FUNC_NAME, FORMAT_DATETIME_FUNC_NAME, LOG_FUNC_NAME,
+    LOWER_CASE_FUNC_NAME, LTRIM_FUNC_NAME, MD5_FUNC_NAME, MURMUR3_FUNC_NAME,
+    REGEXP_SUBSTR_FUNC_NAME, RTRIM_FUNC_NAME, SHA256_FUNC_NAME, SHA512_FUNC_NAME,
+    UPPER_CASE_FUNC_NAME, UUID_FUNC_NAME, UUIDV7_FUNC_NAME, XXH3_FUNC_NAME, XXH128_FUNC_NAME,
 };
 
 /// Create parser options that can be used when parsing an expression that will be executed with
 /// this query engine
 #[must_use]
 pub fn default_parser_options() -> ParserOptions {
-    ParserOptions::new()
+    let opts = ParserOptions::new()
         // Add placeholders for scalar UDFs supported by this engine - these are needed because
         // the invoke function expression in our expression AST references the function by an ID,
         // adding these will make a named function with some ID available in the parser. Only the
@@ -36,6 +39,12 @@ pub fn default_parser_options() -> ParserOptions {
         //
         .with_external_function(FORMAT_DATETIME_FUNC_NAME, param_placeholders(2), None)
         .with_external_function(SHA256_FUNC_NAME, param_placeholders(1), None)
+        .with_external_function(MD5_FUNC_NAME, param_placeholders(1), None)
+        .with_external_function(FNV_FUNC_NAME, param_placeholders(1), None)
+        .with_external_function(MURMUR3_FUNC_NAME, param_placeholders(1), None)
+        .with_external_function(SHA512_FUNC_NAME, param_placeholders(1), None)
+        .with_external_function(XXH3_FUNC_NAME, param_placeholders(1), None)
+        .with_external_function(XXH128_FUNC_NAME, param_placeholders(1), None)
         .with_external_function(ENCODE_FUNC_NAME, param_placeholders(2), None)
         .with_external_function(UUID_FUNC_NAME, param_placeholders(0), None)
         .with_external_function(UUIDV7_FUNC_NAME, param_placeholders(0), None)
@@ -105,7 +114,12 @@ pub fn default_parser_options() -> ParserOptions {
                 ),
             ],
             None,
-        )
+        );
+
+    #[cfg(feature = "sha1-hash")]
+    let opts = opts.with_external_function(SHA1_FUNC_NAME, param_placeholders(1), None);
+
+    opts
 }
 
 fn param_placeholders(