diff --git a/.github/actions/setup_build_tool/action.yml b/.github/actions/setup_build_tool/action.yml index b8728afb5b9e8..e6ef22825e023 100644 --- a/.github/actions/setup_build_tool/action.yml +++ b/.github/actions/setup_build_tool/action.yml @@ -42,7 +42,7 @@ runs: aws) echo "setting up sccache for AWS S3..." cat <>$BIN_LOCAL/build-tool - export ENABLE_SCCACHE=true + export ENABLE_SCCACHE=false export SCCACHE_IDLE_TIMEOUT=0 export SCCACHE_BUCKET=databend-ci export SCCACHE_REGION=us-east-2 @@ -56,7 +56,7 @@ runs: gcp) echo "setting up sccache for Google Cloud Storage..." cat <>$BIN_LOCAL/build-tool - export ENABLE_SCCACHE=true + export ENABLE_SCCACHE=false export SCCACHE_IDLE_TIMEOUT=0 export SCCACHE_GCS_RW_MODE=READ_WRITE export SCCACHE_GCS_BUCKET=databend-ci diff --git a/.gitignore b/.gitignore index 5957a12a03de3..a58e7de17276f 100644 --- a/.gitignore +++ b/.gitignore @@ -83,5 +83,7 @@ benchmark/clickbench/results # lychee .lycheecache +.claude + # tmp tmp diff --git a/.typos.toml b/.typos.toml index 6288b2c44f386..50c5b6177fa0c 100644 --- a/.typos.toml +++ b/.typos.toml @@ -13,6 +13,7 @@ extend-ignore-re = [ Nd = "Nd" typ = "typ" aranges = "aranges" +pendings = "pendings" [default.extend-identifiers] diff --git a/Cargo.lock b/Cargo.lock index 758356029b890..8f07c5b62a3e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -144,12 +144,6 @@ version = "0.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9d4ee0d472d1cd2e28c97dfa124b3d8d992e10eb0a035f33f5d12e3a177ba3b" -[[package]] -name = "android-tzdata" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" - [[package]] name = "android_system_properties" version = "0.1.5" @@ -220,9 +214,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.98" +version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" dependencies = [ "backtrace", ] @@ -1532,9 +1526,9 @@ dependencies = [ [[package]] name = "backon" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd0b50b1b78dbadd44ab18b3c794e496f3a139abb9fbc27d9c94c4eebbb96496" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" dependencies = [ "fastrand", "gloo-timers", @@ -1602,7 +1596,7 @@ version = "0.1.0" dependencies = [ "arrow 56.2.0", "arrow-schema 56.2.0", - "ctor", + "ctor 0.2.9", "databend-common-base", "databend-common-catalog", "databend-common-config", @@ -1689,7 +1683,7 @@ dependencies = [ "bitflags 2.9.0", "cexpr", "clang-sys", - "itertools 0.12.1", + "itertools 0.11.0", "lazy_static", "lazycell", "log", @@ -2311,17 +2305,16 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.41" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" +checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" dependencies = [ - "android-tzdata", "iana-time-zone", "js-sys", "num-traits", "serde", "wasm-bindgen", - "windows-link 0.1.1", + "windows-link 0.2.1", ] [[package]] @@ -3127,6 +3120,22 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "ctor" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "424e0138278faeb2b401f174ad17e715c829512d74f3d1e81eb43365c2e0590e" +dependencies = [ + "ctor-proc-macro", + "dtor", +] + +[[package]] +name = "ctor-proc-macro" +version = "0.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52560adf09603e58c9a7ee1fe1dcb95a16927b17c127f0ac02d6e768a0e25bc1" + [[package]] name = "ctr" version = "0.9.2" @@ -3728,7 +3737,7 @@ dependencies = [ "bumpalo", "comfy-table", "crc32fast", - "ctor", + "ctor 0.2.9", "databend-common-ast", "databend-common-base", "databend-common-column", @@ -4456,7 +4465,7 @@ dependencies = [ "chrono-tz 0.8.6", "cidr", "cron", - "ctor", + "ctor 0.2.9", "dashmap 6.1.0", "databend-common-ast", "databend-common-base", @@ -4537,6 +4546,8 @@ dependencies = [ "log", "lru", "opendal", + "opendal-layer-immutable-index", + "opendal-layer-observe-metrics-common", "parquet 56.2.0", "prometheus-client 0.22.3", "regex", @@ -5479,7 +5490,7 @@ dependencies = [ "chrono-tz 0.8.6", "concurrent-queue", "cron", - "ctor", + "ctor 0.2.9", "dashmap 6.1.0", "databend-common-ast", "databend-common-base", @@ -6338,6 +6349,21 @@ version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6add3b8cff394282be81f3fc1a0605db594ed69890078ca6e2cab1c408bcf04" +[[package]] +name = "dtor" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "404d02eeb088a82cfd873006cb713fe411306c7d182c344905e101fb1167d301" +dependencies = [ + "dtor-proc-macro", +] + +[[package]] +name = "dtor-proc-macro" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f678cf4a922c215c63e0de95eb1ff08a958a81d47e485cf9da1e27bf6305cfa5" + [[package]] name = "dtparse" version = "2.0.0" @@ -9059,7 +9085,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.61.0", + "windows-core 0.57.0", ] [[package]] @@ -9074,7 +9100,7 @@ dependencies = [ [[package]] name = "iceberg" version = "0.4.0" -source = "git+https://github.com/databendlabs/iceberg-rust?rev=32b1403#32b1403eef8b00d7f2a526c551aa35b8fc31927e" +source = "git+https://github.com/databendlabs/iceberg-rust?rev=6536f9c#6536f9ccac1ce56a05ad2e738a2fa6760dea8cb5" dependencies = [ "anyhow", "apache-avro", @@ -9123,7 +9149,7 @@ dependencies = [ [[package]] name = "iceberg-catalog-glue" version = "0.4.0" -source = "git+https://github.com/databendlabs/iceberg-rust?rev=32b1403#32b1403eef8b00d7f2a526c551aa35b8fc31927e" +source = "git+https://github.com/databendlabs/iceberg-rust?rev=6536f9c#6536f9ccac1ce56a05ad2e738a2fa6760dea8cb5" dependencies = [ "anyhow", "async-trait", @@ -9140,7 +9166,7 @@ dependencies = [ [[package]] name = "iceberg-catalog-hms" version = "0.4.0" -source = "git+https://github.com/databendlabs/iceberg-rust?rev=32b1403#32b1403eef8b00d7f2a526c551aa35b8fc31927e" +source = "git+https://github.com/databendlabs/iceberg-rust?rev=6536f9c#6536f9ccac1ce56a05ad2e738a2fa6760dea8cb5" dependencies = [ "anyhow", "async-trait", @@ -9164,7 +9190,7 @@ dependencies = [ [[package]] name = "iceberg-catalog-rest" version = "0.4.0" -source = "git+https://github.com/databendlabs/iceberg-rust?rev=32b1403#32b1403eef8b00d7f2a526c551aa35b8fc31927e" +source = "git+https://github.com/databendlabs/iceberg-rust?rev=6536f9c#6536f9ccac1ce56a05ad2e738a2fa6760dea8cb5" dependencies = [ "async-trait", "chrono", @@ -9184,7 +9210,7 @@ dependencies = [ [[package]] name = "iceberg-catalog-s3tables" version = "0.4.0" -source = "git+https://github.com/databendlabs/iceberg-rust?rev=32b1403#32b1403eef8b00d7f2a526c551aa35b8fc31927e" +source = "git+https://github.com/databendlabs/iceberg-rust?rev=6536f9c#6536f9ccac1ce56a05ad2e738a2fa6760dea8cb5" dependencies = [ "anyhow", "async-trait", @@ -9740,10 +9766,12 @@ dependencies = [ "jiff-static", "jiff-tzdb", "jiff-tzdb-platform", + "js-sys", "log", "portable-atomic", "portable-atomic-util", "serde_core", + "wasm-bindgen", "windows-sys 0.61.2", ] @@ -10513,6 +10541,15 @@ dependencies = [ "digest", ] +[[package]] +name = "mea" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1a78f54a189049e2f554d43d2021e3010036ed65a8f5376ab12cc0432d9a341" +dependencies = [ + "slab", +] + [[package]] name = "measure_time" version = "0.9.0" @@ -11267,13 +11304,14 @@ dependencies = [ [[package]] name = "object_store_opendal" -version = "0.54.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0b88fc0e0c4890c1d99e2b8c519c5db40f7d9b69a0f562ff1ad4967a4c8bbc6" +version = "0.55.0" +source = "git+https://github.com/apache/opendal.git?rev=02953ef#02953ef90c475eb592596f1a0b68370188a80128" dependencies = [ "async-trait", "bytes", + "chrono", "futures", + "mea", "object_store", "opendal", "pin-project", @@ -11304,39 +11342,246 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "opendal" -version = "0.54.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42afda58fa2cf50914402d132cc1caacff116a85d10c72ab2082bb7c50021754" +version = "0.55.0" +source = "git+https://github.com/apache/opendal.git?rev=02953ef#02953ef90c475eb592596f1a0b68370188a80128" +dependencies = [ + "opendal-core", + "opendal-layer-async-backtrace", + "opendal-layer-fastrace", + "opendal-layer-prometheus-client", + "opendal-service-azblob", + "opendal-service-azdls", + "opendal-service-gcs", + "opendal-service-ipfs", + "opendal-service-moka", + "opendal-service-obs", + "opendal-service-oss", + "opendal-service-s3", +] + +[[package]] +name = "opendal-core" +version = "0.55.0" +source = "git+https://github.com/apache/opendal.git?rev=02953ef#02953ef90c475eb592596f1a0b68370188a80128" dependencies = [ "anyhow", - "async-backtrace", "backon", "base64 0.22.1", "bytes", - "chrono", - "crc32c", - "fastrace", + "ctor 0.6.3", "futures", "getrandom 0.2.16", "hdrs", "http 1.3.1", "http-body 1.0.1", + "jiff", "log", "md-5", - "moka", + "mea", "percent-encoding", - "prometheus-client 0.23.1", - "prost", "quick-xml 0.38.4", "reqsign", "reqwest", "serde", "serde_json", - "sha2", "tokio", + "url", + "uuid", + "web-time", +] + +[[package]] +name = "opendal-layer-async-backtrace" +version = "0.55.0" +source = "git+https://github.com/apache/opendal.git?rev=02953ef#02953ef90c475eb592596f1a0b68370188a80128" +dependencies = [ + "async-backtrace", + "opendal-core", +] + +[[package]] +name = "opendal-layer-fastrace" +version = "0.55.0" +source = "git+https://github.com/apache/opendal.git?rev=02953ef#02953ef90c475eb592596f1a0b68370188a80128" +dependencies = [ + "fastrace", + "opendal-core", +] + +[[package]] +name = "opendal-layer-immutable-index" +version = "0.55.0" +source = "git+https://github.com/apache/opendal.git?rev=02953ef#02953ef90c475eb592596f1a0b68370188a80128" +dependencies = [ + "opendal-core", +] + +[[package]] +name = "opendal-layer-observe-metrics-common" +version = "0.55.0" +source = "git+https://github.com/apache/opendal.git?rev=02953ef#02953ef90c475eb592596f1a0b68370188a80128" +dependencies = [ + "futures", + "http 1.3.1", + "opendal-core", +] + +[[package]] +name = "opendal-layer-prometheus-client" +version = "0.55.0" +source = "git+https://github.com/apache/opendal.git?rev=02953ef#02953ef90c475eb592596f1a0b68370188a80128" +dependencies = [ + "opendal-core", + "opendal-layer-observe-metrics-common", + "prometheus-client 0.24.0", +] + +[[package]] +name = "opendal-service-azblob" +version = "0.55.0" +source = "git+https://github.com/apache/opendal.git?rev=02953ef#02953ef90c475eb592596f1a0b68370188a80128" +dependencies = [ + "base64 0.22.1", + "bytes", + "ctor 0.6.3", + "http 1.3.1", + "log", + "opendal-core", + "opendal-service-azure-common", + "quick-xml 0.38.4", + "reqsign", + "serde", + "sha2", "uuid", ] +[[package]] +name = "opendal-service-azdls" +version = "0.55.0" +source = "git+https://github.com/apache/opendal.git?rev=02953ef#02953ef90c475eb592596f1a0b68370188a80128" +dependencies = [ + "bytes", + "ctor 0.6.3", + "http 1.3.1", + "log", + "opendal-core", + "opendal-service-azure-common", + "quick-xml 0.38.4", + "reqsign", + "serde", + "serde_json", +] + +[[package]] +name = "opendal-service-azure-common" +version = "0.55.0" +source = "git+https://github.com/apache/opendal.git?rev=02953ef#02953ef90c475eb592596f1a0b68370188a80128" +dependencies = [ + "http 1.3.1", + "opendal-core", + "reqsign", +] + +[[package]] +name = "opendal-service-gcs" +version = "0.55.0" +source = "git+https://github.com/apache/opendal.git?rev=02953ef#02953ef90c475eb592596f1a0b68370188a80128" +dependencies = [ + "backon", + "base64 0.22.1", + "bytes", + "ctor 0.6.3", + "http 1.3.1", + "log", + "opendal-core", + "percent-encoding", + "quick-xml 0.38.4", + "reqsign", + "reqwest", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "opendal-service-ipfs" +version = "0.55.0" +source = "git+https://github.com/apache/opendal.git?rev=02953ef#02953ef90c475eb592596f1a0b68370188a80128" +dependencies = [ + "bytes", + "ctor 0.6.3", + "http 1.3.1", + "log", + "opendal-core", + "prost", + "serde", +] + +[[package]] +name = "opendal-service-moka" +version = "0.55.0" +source = "git+https://github.com/apache/opendal.git?rev=02953ef#02953ef90c475eb592596f1a0b68370188a80128" +dependencies = [ + "ctor 0.6.3", + "log", + "moka", + "opendal-core", + "serde", +] + +[[package]] +name = "opendal-service-obs" +version = "0.55.0" +source = "git+https://github.com/apache/opendal.git?rev=02953ef#02953ef90c475eb592596f1a0b68370188a80128" +dependencies = [ + "bytes", + "ctor 0.6.3", + "http 1.3.1", + "log", + "opendal-core", + "quick-xml 0.38.4", + "reqsign", + "serde", +] + +[[package]] +name = "opendal-service-oss" +version = "0.55.0" +source = "git+https://github.com/apache/opendal.git?rev=02953ef#02953ef90c475eb592596f1a0b68370188a80128" +dependencies = [ + "bytes", + "ctor 0.6.3", + "http 1.3.1", + "log", + "opendal-core", + "quick-xml 0.38.4", + "reqsign", + "reqwest", + "serde", +] + +[[package]] +name = "opendal-service-s3" +version = "0.55.0" +source = "git+https://github.com/apache/opendal.git?rev=02953ef#02953ef90c475eb592596f1a0b68370188a80128" +dependencies = [ + "base64 0.22.1", + "bytes", + "crc32c", + "ctor 0.6.3", + "http 1.3.1", + "log", + "md-5", + "opendal-core", + "quick-xml 0.38.4", + "reqsign-aws-v4", + "reqsign-core", + "reqsign-file-read-tokio", + "reqsign-http-send-reqwest", + "reqwest", + "serde", +] + [[package]] name = "openraft" version = "0.10.0" @@ -12516,19 +12761,19 @@ dependencies = [ "dtoa", "itoa", "parking_lot 0.12.3", - "prometheus-client-derive-encode", + "prometheus-client-derive-encode 0.4.2", ] [[package]] name = "prometheus-client" -version = "0.23.1" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf41c1a7c32ed72abe5082fb19505b969095c12da9f5732a4bc9878757fd087c" +checksum = "e4500adecd7af8e0e9f4dbce15cfee07ce913fbf6ad605cc468b83f2d531ee94" dependencies = [ "dtoa", "itoa", "parking_lot 0.12.3", - "prometheus-client-derive-encode", + "prometheus-client-derive-encode 0.5.0", ] [[package]] @@ -12542,6 +12787,17 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "prometheus-client-derive-encode" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9adf1691c04c0a5ff46ff8f262b58beb07b0dbb61f96f9f54f6cbd82106ed87f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + [[package]] name = "prometheus-parse" version = "0.2.5" @@ -12587,7 +12843,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck 0.5.0", - "itertools 0.14.0", + "itertools 0.11.0", "log", "multimap", "once_cell", @@ -12607,7 +12863,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.11.0", "proc-macro2", "quote", "syn 2.0.106", @@ -12879,9 +13135,9 @@ dependencies = [ [[package]] name = "python3-dll-a" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49fe4227a288cf9493942ad0220ea3f185f4d1f2a14f197f7344d6d02f4ed4ed" +checksum = "d381ef313ae70b4da5f95f8a4de773c6aa5cd28f73adec4b4a31df70b66780d8" dependencies = [ "cc", ] @@ -12901,16 +13157,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "quick-xml" -version = "0.37.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "quick-xml" version = "0.38.4" @@ -13389,18 +13635,88 @@ dependencies = [ "log", "once_cell", "percent-encoding", - "quick-xml 0.37.5", "rand 0.8.5", "reqwest", "rsa", + "serde", + "serde_json", + "sha1", + "sha2", +] + +[[package]] +name = "reqsign-aws-v4" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4510c2a3e42b653cf788d560a3d54b0ae4cc315a62aaba773554f18319c0db0b" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "form_urlencoded", + "http 1.3.1", + "log", + "percent-encoding", + "quick-xml 0.38.4", + "reqsign-core", "rust-ini", "serde", "serde_json", + "serde_urlencoded", + "sha1", +] + +[[package]] +name = "reqsign-core" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39da118ccf3bdb067ac6cc40136fec99bc5ba418cbd388dc88e4ce0e5d0b1423" +dependencies = [ + "anyhow", + "async-trait", + "base64 0.22.1", + "bytes", + "form_urlencoded", + "hex", + "hmac", + "http 1.3.1", + "jiff", + "log", + "percent-encoding", "sha1", "sha2", + "windows-sys 0.61.2", +] + +[[package]] +name = "reqsign-file-read-tokio" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "669ea66036266a9ac371d2e63cc7d345e69994da0168b4e6f3487fe21e126f76" +dependencies = [ + "anyhow", + "async-trait", + "reqsign-core", "tokio", ] +[[package]] +name = "reqsign-http-send-reqwest" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46186bce769674f9200ad01af6f2ca42de3e819ddc002fff1edae135bfb6cd9c" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "futures-channel", + "http 1.3.1", + "http-body-util", + "reqsign-core", + "reqwest", + "wasm-bindgen-futures", +] + [[package]] name = "reqwest" version = "0.12.24" @@ -14561,12 +14877,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.9" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" [[package]] name = "sled" @@ -17505,7 +17818,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.48.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 8143f848e0d7e..677ab022778e4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ version = "0.1.0" authors = ["Databend Authors "] license = "Apache-2.0" publish = false -edition = "2021" +edition = "2024" [workspace] resolver = "2" @@ -81,26 +81,7 @@ members = [ "src/query/ee_features/resources_management", "src/query/service", "src/query/ee", - "src/meta/api", - "src/meta/app", - "src/meta/app-types", - "src/meta/binaries", - "src/meta/cache", - "src/meta/client", - "src/meta/control", - "src/meta/ee", - "src/meta/kvapi", - "src/meta/kvapi-test-suite", - "src/meta/process", - "src/meta/proto-conv", - "src/meta/protos", - "src/meta/raft-store", - "src/meta/semaphore", - "src/meta/sled-store", - "src/meta/stoerr", - "src/meta/store", - "src/meta/types", - "src/meta/service", + "src/meta/*", "tests/sqllogictests", "src/tests/sqlsmith", "src/bendsave", @@ -332,13 +313,13 @@ hyper-util = { version = "0.1.9", features = ["client", "client-legacy", "tokio" lru = "0.12" ## in branch dev -iceberg = { version = "0.4.0", git = "https://github.com/databendlabs/iceberg-rust", rev = "32b1403", features = [ +iceberg = { version = "0.4.0", git = "https://github.com/databendlabs/iceberg-rust", rev = "6536f9c", features = [ "storage-all", ] } -iceberg-catalog-glue = { version = "0.4.0", git = "https://github.com/databendlabs/iceberg-rust", rev = "32b1403" } -iceberg-catalog-hms = { version = "0.4.0", git = "https://github.com/databendlabs/iceberg-rust", rev = "32b1403" } -iceberg-catalog-rest = { version = "0.4.0", git = "https://github.com/databendlabs/iceberg-rust", rev = "32b1403" } -iceberg-catalog-s3tables = { version = "0.4.0", git = "https://github.com/databendlabs/iceberg-rust", rev = "32b1403" } +iceberg-catalog-glue = { version = "0.4.0", git = "https://github.com/databendlabs/iceberg-rust", rev = "6536f9c" } +iceberg-catalog-hms = { version = "0.4.0", git = "https://github.com/databendlabs/iceberg-rust", rev = "6536f9c" } +iceberg-catalog-rest = { version = "0.4.0", git = "https://github.com/databendlabs/iceberg-rust", rev = "6536f9c" } +iceberg-catalog-s3tables = { version = "0.4.0", git = "https://github.com/databendlabs/iceberg-rust", rev = "6536f9c" } # Explicitly specify compatible AWS SDK versions aws-config = "1.5.18" @@ -387,9 +368,9 @@ num-derive = "0.4.2" num-traits = "0.2.19" num_cpus = "1.17" object = "0.36.5" -object_store_opendal = { version = "0.54.1" } +object_store_opendal = { git = "https://github.com/apache/opendal.git", rev = "02953ef" } once_cell = "1.15.0" -opendal = { version = "0.54.1", features = [ +opendal = { git = "https://github.com/apache/opendal.git", rev = "02953ef", features = [ "layers-fastrace", "layers-prometheus-client", "layers-async-backtrace", @@ -407,6 +388,8 @@ opendal = { version = "0.54.1", features = [ "services-webhdfs", "services-huggingface", ] } +opendal-layer-immutable-index = { git = "https://github.com/apache/opendal.git", rev = "02953ef" } +opendal-layer-observe-metrics-common = { git = "https://github.com/apache/opendal.git", rev = "02953ef" } openraft = { version = "0.10.0", features = [ "serde", "tracing-log", diff --git a/clippy.toml b/clippy.toml index 21de708a92a5d..32ca663469a8d 100644 --- a/clippy.toml +++ b/clippy.toml @@ -23,11 +23,8 @@ disallowed-methods = [ disallowed-types = [ { path = "once_cell::sync::Lazy", reason = "Please use `std::sync::LazyLock` instead." }, - - { path = "tokio::task::Builder", reason = "Disallowed types, please remove it." }, { path = "tokio::task::LocalSet", reason = "Disallowed types, please remove it." }, { path = "tokio::task::join_set::JoinSet", reason = "Disallowed types, please remove it." }, - { path = "tokio::task::join_set::Builder", reason = "Disallowed types, please remove it." }, ] disallowed-macros = [ diff --git a/rust-toolchain.toml b/rust-toolchain.toml index ba51651a29f2f..70ac8ab72394e 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2025-04-04" +channel = "nightly-2025-12-11" components = ["rustfmt", "clippy", "rust-src", "miri", "rust-analyzer"] diff --git a/rustfmt.toml b/rustfmt.toml index 18a287f768f90..6c45530a92184 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -1,5 +1,4 @@ -edition = "2021" -style_edition = "2021" +edition = "2024" reorder_imports = true imports_granularity = "Item" group_imports = "StdExternalCrate" diff --git a/src/bendpy/src/context.rs b/src/bendpy/src/context.rs index db26acd43a841..9a51bbc0a5590 100644 --- a/src/bendpy/src/context.rs +++ b/src/bendpy/src/context.rs @@ -23,10 +23,10 @@ use databend_query::sessions::Session; use databend_query::sql::Planner; use pyo3::prelude::*; -use crate::dataframe::default_box_size; use crate::dataframe::PyDataFrame; -use crate::utils::wait_for_future; +use crate::dataframe::default_box_size; use crate::utils::RUNTIME; +use crate::utils::wait_for_future; #[pyclass(name = "SessionContext", module = "databend", subclass)] #[derive(Clone)] @@ -223,7 +223,7 @@ impl PySessionContext { .map(|r| format!(" region = '{}'", r)) .unwrap_or_default(); let sql = format!( - "CREATE OR REPLACE CONNECTION {} STORAGE_TYPE = 'S3' access_key_id = '{}' secret_access_key = '{}'{}{}", + "CREATE OR REPLACE CONNECTION {} STORAGE_TYPE = 'S3' access_key_id = '{}' secret_access_key = '{}'{}{}", name, access_key_id, secret_access_key, endpoint_clause, region_clause ); let _ = self.sql(&sql, py)?.collect(py)?; diff --git a/src/bendpy/src/datablock.rs b/src/bendpy/src/datablock.rs index 7d37d75294f49..cb355a22d534e 100644 --- a/src/bendpy/src/datablock.rs +++ b/src/bendpy/src/datablock.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_expression::block_debug::box_render; use databend_common_expression::DataBlock; use databend_common_expression::DataSchemaRef; +use databend_common_expression::block_debug::box_render; use pyo3::prelude::*; use crate::dataframe::PyBoxSize; diff --git a/src/bendsave/Cargo.toml b/src/bendsave/Cargo.toml index 69dd96569ef71..dafe89608cd82 100644 --- a/src/bendsave/Cargo.toml +++ b/src/bendsave/Cargo.toml @@ -4,7 +4,7 @@ description = "databend backup and restore tools" version = "0.1.0" authors = { workspace = true } license = "Elastic-2.0" -edition = "2021" +edition = "2024" [dependencies] databend-common-base = { workspace = true } diff --git a/src/bendsave/src/backup.rs b/src/bendsave/src/backup.rs index a49320b762420..cbb47bca25881 100644 --- a/src/bendsave/src/backup.rs +++ b/src/bendsave/src/backup.rs @@ -25,8 +25,8 @@ use crate::storage::load_databend_meta; use crate::storage::load_query_config; use crate::storage::load_query_storage; use crate::storage::verify_query_license; -use crate::utils::storage_copy; use crate::utils::DATABEND_META_BACKUP_PATH; +use crate::utils::storage_copy; pub async fn backup(from: &str, to: &str) -> Result<()> { let query_cfg = load_query_config(from)?; diff --git a/src/bendsave/src/lib.rs b/src/bendsave/src/lib.rs index b50235bbe6d9b..a57c93c423044 100644 --- a/src/bendsave/src/lib.rs +++ b/src/bendsave/src/lib.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![allow(clippy::collapsible_if, clippy::to_string_in_format_args)] + mod storage; pub mod utils; diff --git a/src/bendsave/src/restore.rs b/src/bendsave/src/restore.rs index df68d0f0842c8..297bb82cae53c 100644 --- a/src/bendsave/src/restore.rs +++ b/src/bendsave/src/restore.rs @@ -25,8 +25,8 @@ use crate::storage::load_bendsave_storage; use crate::storage::load_meta_config; use crate::storage::load_query_config; use crate::storage::load_query_storage; -use crate::utils::storage_copy; use crate::utils::DATABEND_META_BACKUP_PATH; +use crate::utils::storage_copy; pub async fn restore(from: &str, to_query: &str, to_meta: &str) -> Result<()> { let bendsave_storage = load_bendsave_storage(from).await?; diff --git a/src/bendsave/src/storage.rs b/src/bendsave/src/storage.rs index 217d6535c7a25..d9affd2871989 100644 --- a/src/bendsave/src/storage.rs +++ b/src/bendsave/src/storage.rs @@ -16,9 +16,9 @@ use std::collections::HashMap; use std::str::FromStr; use std::sync::Arc; -use anyhow::anyhow; use anyhow::Ok; use anyhow::Result; +use anyhow::anyhow; use bytes::BufMut; use bytes::Bytes; use bytes::BytesMut; @@ -36,8 +36,8 @@ use databend_common_meta_client::ClientHandle; use databend_common_meta_client::MetaGrpcClient; use databend_common_meta_types::protobuf::ExportRequest; use databend_common_storage::init_operator; -use databend_common_users::builtin::BuiltIn; use databend_common_users::UserApiProvider; +use databend_common_users::builtin::BuiltIn; use databend_common_version::BUILD_INFO; use databend_enterprise_query::license::RealLicenseManager; use databend_query::sessions::BuildInfoRef; @@ -45,9 +45,9 @@ use databend_query::sessions::SessionManager; use futures::TryStream; use futures::TryStreamExt; use log::debug; +use opendal::Operator; use opendal::layers::LoggingLayer; use opendal::layers::RetryLayer; -use opendal::Operator; /// Load the configuration file of databend query. /// @@ -219,7 +219,7 @@ mod tests { use std::path::Path; use databend_common_base::base::tokio; - use opendal::Scheme; + use databend_common_storage::Scheme; use super::*; @@ -237,12 +237,12 @@ mod tests { #[tokio::test] async fn test_load_epochfs_storage() -> Result<()> { let op = load_bendsave_storage("s3://bendsave/tmp?region=us-east-1").await?; - assert_eq!(op.info().scheme(), Scheme::S3); + assert_eq!(op.info().scheme(), Scheme::S3.to_string()); assert_eq!(op.info().name(), "bendsave"); assert_eq!(op.info().root(), "/tmp/"); let op = load_bendsave_storage("fs://opt").await?; - assert_eq!(op.info().scheme(), Scheme::Fs); + assert_eq!(op.info().scheme(), Scheme::Fs.to_string()); assert_eq!(op.info().root(), "/opt"); Ok(()) } diff --git a/src/binaries/query/ee_main.rs b/src/binaries/query/ee_main.rs index b13be358ebb90..175a61ec1ce40 100644 --- a/src/binaries/query/ee_main.rs +++ b/src/binaries/query/ee_main.rs @@ -24,9 +24,9 @@ use databend_common_base::runtime::Runtime; use databend_common_base::runtime::ThreadTracker; use databend_common_exception::Result; use databend_common_exception::ResultExt; +use databend_common_tracing::SignalListener; use databend_common_tracing::pipe_file; use databend_common_tracing::set_crash_hook; -use databend_common_tracing::SignalListener; use databend_common_version::BUILD_INFO; use databend_common_version::DATABEND_COMMIT_VERSION; use databend_enterprise_query::enterprise_services::EnterpriseServices; diff --git a/src/binaries/query/entry.rs b/src/binaries/query/entry.rs index 949d2fb06dac6..3c11f30bb70af 100644 --- a/src/binaries/query/entry.rs +++ b/src/binaries/query/entry.rs @@ -12,12 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![allow(clippy::collapsible_if)] + use std::env; use std::time::Duration; use databend_common_base::mem_allocator::TrackingGlobalAllocator; -use databend_common_base::runtime::set_alloc_error_hook; use databend_common_base::runtime::GLOBAL_MEM_STAT; +use databend_common_base::runtime::set_alloc_error_hook; use databend_common_config::InnerConfig; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -31,11 +33,9 @@ use databend_common_version::DATABEND_COMMIT_VERSION; use databend_common_version::DATABEND_GIT_SEMVER; use databend_common_version::DATABEND_SEMVER; use databend_common_version::VERGEN_GIT_SHA; +use databend_query::GlobalServices; use databend_query::clusters::ClusterDiscovery; use databend_query::history_tables::GlobalHistoryLog; -use databend_query::servers::admin::AdminService; -use databend_query::servers::flight::FlightService; -use databend_query::servers::metrics::MetricService; use databend_query::servers::FlightSQLServer; use databend_query::servers::HttpHandler; use databend_query::servers::HttpHandlerKind; @@ -43,8 +43,10 @@ use databend_query::servers::MySQLHandler; use databend_query::servers::MySQLTlsConfig; use databend_query::servers::Server; use databend_query::servers::ShutdownHandle; +use databend_query::servers::admin::AdminService; +use databend_query::servers::flight::FlightService; +use databend_query::servers::metrics::MetricService; use databend_query::task::TaskService; -use databend_query::GlobalServices; use log::info; use super::cmd::Cmd; diff --git a/src/binaries/query/oss_main.rs b/src/binaries/query/oss_main.rs index bf5be80dd8996..93283f328efb6 100644 --- a/src/binaries/query/oss_main.rs +++ b/src/binaries/query/oss_main.rs @@ -26,9 +26,9 @@ use databend_common_exception::Result; use databend_common_exception::ResultExt; use databend_common_license::license_manager::LicenseManager; use databend_common_license::license_manager::OssLicenseManager; +use databend_common_tracing::SignalListener; use databend_common_tracing::pipe_file; use databend_common_tracing::set_crash_hook; -use databend_common_tracing::SignalListener; use databend_common_version::DATABEND_COMMIT_VERSION; use entry::MainError; diff --git a/src/binaries/tool/table_meta_inspector.rs b/src/binaries/tool/table_meta_inspector.rs index 1e953dda3caa9..902a0f8f4ad96 100644 --- a/src/binaries/tool/table_meta_inspector.rs +++ b/src/binaries/tool/table_meta_inspector.rs @@ -16,27 +16,27 @@ use std::collections::BTreeMap; use std::env; use std::fs::File; use std::io; -use std::io::stdout; use std::io::BufWriter; use std::io::Read; use std::io::Write; +use std::io::stdout; use clap::Parser; use databend_common_config::Config; use databend_common_config::InnerConfig; use databend_common_exception::Result; -use databend_common_storage::init_operator; use databend_common_storage::StorageConfig; -use databend_common_tracing::init_logging; +use databend_common_storage::init_operator; use databend_common_tracing::Config as LogConfig; +use databend_common_tracing::init_logging; use databend_common_version::BUILD_INFO; use databend_common_version::DATABEND_COMMIT_VERSION; use databend_query::GlobalServices; use databend_storages_common_table_meta::meta::SegmentInfo; use databend_storages_common_table_meta::meta::TableSnapshot; use log::info; -use opendal::services::Fs; use opendal::Operator; +use opendal::services::Fs; use serde::Deserialize; use serde::Serialize; use serfig::collectors::from_file; diff --git a/src/common/base/src/base/dma.rs b/src/common/base/src/base/dma.rs index 16ea13434ec36..3e484599a4269 100644 --- a/src/common/base/src/base/dma.rs +++ b/src/common/base/src/base/dma.rs @@ -129,7 +129,7 @@ unsafe impl Allocator for DmaAllocator { } unsafe fn deallocate(&self, ptr: std::ptr::NonNull, layout: Layout) { - Global {}.deallocate(ptr, self.real_layout(layout)) + unsafe { Global {}.deallocate(ptr, self.real_layout(layout)) } } unsafe fn grow( @@ -138,11 +138,13 @@ unsafe impl Allocator for DmaAllocator { old_layout: Layout, new_layout: Layout, ) -> Result, AllocError> { - Global {}.grow( - ptr, - self.real_layout(old_layout), - self.real_layout(new_layout), - ) + unsafe { + Global {}.grow( + ptr, + self.real_layout(old_layout), + self.real_layout(new_layout), + ) + } } unsafe fn grow_zeroed( @@ -151,11 +153,13 @@ unsafe impl Allocator for DmaAllocator { old_layout: Layout, new_layout: Layout, ) -> Result, AllocError> { - Global {}.grow_zeroed( - ptr, - self.real_layout(old_layout), - self.real_layout(new_layout), - ) + unsafe { + Global {}.grow_zeroed( + ptr, + self.real_layout(old_layout), + self.real_layout(new_layout), + ) + } } unsafe fn shrink( @@ -164,11 +168,13 @@ unsafe impl Allocator for DmaAllocator { old_layout: Layout, new_layout: Layout, ) -> Result, AllocError> { - Global {}.shrink( - ptr, - self.real_layout(old_layout), - self.real_layout(new_layout), - ) + unsafe { + Global {}.shrink( + ptr, + self.real_layout(old_layout), + self.real_layout(new_layout), + ) + } } } @@ -371,20 +377,22 @@ impl AsyncDmaFile { let fd = self.fd.as_raw_fd(); let alignment = self.alignment; - let buf = asyncify(move || loop { - let remain = buf.capacity() - buf.len(); - let mut file = DmaFile { - fd: unsafe { BorrowedFd::borrow_raw(fd) }, - alignment, - offset: 0, - }; - let offset = align_start + buf.len(); - let n = file.pread_direct(&mut buf, remain, offset as _)?; - if align_start + buf.len() >= range.end as usize { - return Ok(buf); - } - if n == 0 { - return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "")); + let buf = asyncify(move || { + loop { + let remain = buf.capacity() - buf.len(); + let mut file = DmaFile { + fd: unsafe { BorrowedFd::borrow_raw(fd) }, + alignment, + offset: 0, + }; + let offset = align_start + buf.len(); + let n = file.pread_direct(&mut buf, remain, offset as _)?; + if align_start + buf.len() >= range.end as usize { + return Ok(buf); + } + if n == 0 { + return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "")); + } } }) .await?; @@ -396,11 +404,7 @@ impl AsyncDmaFile { #[cfg(target_os = "linux")] fn flags_direct_or_empty(dio: bool) -> OFlags { - if dio { - OFlags::DIRECT - } else { - OFlags::empty() - } + if dio { OFlags::DIRECT } else { OFlags::empty() } } #[cfg(not(target_os = "linux"))] diff --git a/src/common/base/src/base/drop_callback.rs b/src/common/base/src/base/drop_callback.rs index 6ac664e175a42..714843852cf01 100644 --- a/src/common/base/src/base/drop_callback.rs +++ b/src/common/base/src/base/drop_callback.rs @@ -43,9 +43,9 @@ impl Drop for DropCallback { #[cfg(test)] mod tests { + use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; - use std::sync::Arc; use super::*; diff --git a/src/common/base/src/base/mod.rs b/src/common/base/src/base/mod.rs index f97911ead9c00..e11bacaa1993a 100644 --- a/src/common/base/src/base/mod.rs +++ b/src/common/base/src/base/mod.rs @@ -40,13 +40,13 @@ pub use profiling::Profiling; pub use progress::Progress; pub use progress::ProgressValues; pub use progress::SpillProgress; -pub use select::select3; pub use select::Select3Output; +pub use select::select3; pub use semaphore::Semaphore; -pub use shutdown_signal::signal_stream; pub use shutdown_signal::DummySignalStream; pub use shutdown_signal::SignalStream; pub use shutdown_signal::SignalType; +pub use shutdown_signal::signal_stream; pub use singleton_instance::GlobalInstance; pub use stop_handle::StopHandle; pub use stoppable::Stoppable; diff --git a/src/common/base/src/base/ordered_float.rs b/src/common/base/src/base/ordered_float.rs index aaa103192431f..5b9447ceb27dc 100644 --- a/src/common/base/src/base/ordered_float.rs +++ b/src/common/base/src/base/ordered_float.rs @@ -45,7 +45,6 @@ use std::error::Error; use bytemuck::Pod; use bytemuck::Zeroable; use micromarshal::Unmarshal; -use num_traits::float::FloatCore; use num_traits::AsPrimitive; use num_traits::Bounded; use num_traits::Float; @@ -58,6 +57,7 @@ use num_traits::Pow; use num_traits::Signed; use num_traits::ToPrimitive; use num_traits::Zero; +use num_traits::float::FloatCore; // masks for the parts of the IEEE 754 float const SIGN_MASK: u64 = 0x8000000000000000u64; @@ -1113,6 +1113,7 @@ impl Num for OrderedFloat { /// // This will panic: /// let c = a + b; /// ``` +#[allow(clippy::derive_ord_xor_partial_ord)] #[derive(PartialOrd, PartialEq, Default, Clone, Copy)] #[repr(transparent)] pub struct NotNan(T); @@ -1157,7 +1158,7 @@ impl NotNan { )] #[inline] pub const unsafe fn unchecked_new(val: T) -> Self { - Self::new_unchecked(val) + unsafe { Self::new_unchecked(val) } } } @@ -1845,7 +1846,7 @@ impl NumCast for NotNan { } macro_rules! impl_float_const_method { - ($wrapper:expr, $method:ident) => { + ($wrapper:expr_2021, $method:ident) => { #[allow(non_snake_case)] #[allow(clippy::redundant_closure_call)] fn $method() -> Self { @@ -1855,7 +1856,7 @@ macro_rules! impl_float_const_method { } macro_rules! impl_float_const { - ($type:ident, $wrapper:expr) => { + ($type:ident, $wrapper:expr_2021) => { impl FloatConst for $type { impl_float_const_method!($wrapper, E); impl_float_const_method!($wrapper, FRAC_1_PI); @@ -1888,25 +1889,25 @@ mod impl_serde { use num_traits::float::FloatCore; use serde::de::IntoDeserializer; - use self::serde::de::Error; - use self::serde::de::Unexpected; use self::serde::Deserialize; use self::serde::Deserializer; use self::serde::Serialize; use self::serde::Serializer; + use self::serde::de::Error; + use self::serde::de::Unexpected; use super::NotNan; use super::OrderedFloat; #[cfg(test)] extern crate serde_test; #[cfg(test)] - use self::serde_test::assert_de_tokens_error; - #[cfg(test)] - use self::serde_test::assert_tokens; - #[cfg(test)] use self::serde_test::Configure; #[cfg(test)] use self::serde_test::Token; + #[cfg(test)] + use self::serde_test::assert_de_tokens_error; + #[cfg(test)] + use self::serde_test::assert_tokens; impl Serialize for OrderedFloat { #[inline] @@ -2099,12 +2100,12 @@ mod impl_borsh { } mod impl_rand { - use rand::distributions::uniform::*; + use rand::Rng; use rand::distributions::Distribution; use rand::distributions::Open01; use rand::distributions::OpenClosed01; use rand::distributions::Standard; - use rand::Rng; + use rand::distributions::uniform::*; use serde::Deserialize; use serde::Serialize; diff --git a/src/common/base/src/base/semaphore.rs b/src/common/base/src/base/semaphore.rs index 6dee75f51e27f..a69926c71fff1 100644 --- a/src/common/base/src/base/semaphore.rs +++ b/src/common/base/src/base/semaphore.rs @@ -94,7 +94,7 @@ impl Semaphore { /// /// This function is semantically equivalent to an `acquire` followed by a /// `release` when the guard returned is dropped. - pub fn access(&self) -> SemaphoreGuard { + pub fn access(&self) -> SemaphoreGuard<'_> { self.acquire(); SemaphoreGuard { sem: self } } diff --git a/src/common/base/src/base/shutdown_signal.rs b/src/common/base/src/base/shutdown_signal.rs index ea1bf3c3c6057..354003250d725 100644 --- a/src/common/base/src/base/shutdown_signal.rs +++ b/src/common/base/src/base/shutdown_signal.rs @@ -19,15 +19,15 @@ use std::task::Poll; use databend_common_exception::Result; use futures::Stream; #[cfg(not(target_os = "windows"))] -use tokio::signal::unix::signal; -#[cfg(not(target_os = "windows"))] use tokio::signal::unix::Signal; #[cfg(not(target_os = "windows"))] use tokio::signal::unix::SignalKind; -#[cfg(target_os = "windows")] -use tokio::signal::windows::ctrl_c; +#[cfg(not(target_os = "windows"))] +use tokio::signal::unix::signal; #[cfg(target_os = "windows")] use tokio::signal::windows::CtrlC; +#[cfg(target_os = "windows")] +use tokio::signal::windows::ctrl_c; pub enum SignalType { Hangup, diff --git a/src/common/base/src/base/stop_handle.rs b/src/common/base/src/base/stop_handle.rs index 19819343b202c..d33a445f66e6e 100644 --- a/src/common/base/src/base/stop_handle.rs +++ b/src/common/base/src/base/stop_handle.rs @@ -13,9 +13,9 @@ // limitations under the License. use std::error::Error; +use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; -use std::sync::Arc; use databend_common_exception::ErrorCode; use futures::Future; diff --git a/src/common/base/src/base/stoppable.rs b/src/common/base/src/base/stoppable.rs index 4429ad108aa40..c0ad4aec6c680 100644 --- a/src/common/base/src/base/stoppable.rs +++ b/src/common/base/src/base/stoppable.rs @@ -34,5 +34,5 @@ pub trait Stoppable { /// /// Calling `stop()` twice should get an error. async fn stop(&mut self, mut force: Option>) - -> Result<(), Self::Error>; + -> Result<(), Self::Error>; } diff --git a/src/common/base/src/containers/fixed_heap.rs b/src/common/base/src/containers/fixed_heap.rs index 5a85251ae3859..8cd2773842d58 100644 --- a/src/common/base/src/containers/fixed_heap.rs +++ b/src/common/base/src/containers/fixed_heap.rs @@ -291,43 +291,47 @@ impl FixedHeap { /// assert_eq!(heap.pop_at(1), Some(1)); /// ``` pub fn pop_at(&mut self, index: usize) -> Option { - if let Some(removed_node) = self.swap_remove(index) { - let mut node_index: usize = index; - loop { - let lchild_index = (node_index << 1) + 1; - let rchild_index = (node_index << 1) + 2; - // # Safety - // These indices are initialized because they are in `0..high` - let node = unsafe { self.data.get_unchecked(node_index).assume_init_ref() }; - // Determine which child to sift upwards by comparing - let swap = if rchild_index < self.high { - let lchild = unsafe { self.data.get_unchecked(lchild_index).assume_init_ref() }; - let rchild = unsafe { self.data.get_unchecked(rchild_index).assume_init_ref() }; - if lchild > rchild { + match self.swap_remove(index) { + Some(removed_node) => { + let mut node_index: usize = index; + loop { + let lchild_index = (node_index << 1) + 1; + let rchild_index = (node_index << 1) + 2; + // # Safety + // These indices are initialized because they are in `0..high` + let node = unsafe { self.data.get_unchecked(node_index).assume_init_ref() }; + // Determine which child to sift upwards by comparing + let swap = if rchild_index < self.high { + let lchild = + unsafe { self.data.get_unchecked(lchild_index).assume_init_ref() }; + let rchild = + unsafe { self.data.get_unchecked(rchild_index).assume_init_ref() }; + if lchild > rchild { + (lchild > node, lchild_index) + } else { + (rchild > node, rchild_index) + } + } else if lchild_index < self.high { + let lchild = + unsafe { self.data.get_unchecked(lchild_index).assume_init_ref() }; (lchild > node, lchild_index) } else { - (rchild > node, rchild_index) - } - } else if lchild_index < self.high { - let lchild = unsafe { self.data.get_unchecked(lchild_index).assume_init_ref() }; - (lchild > node, lchild_index) - } else { - (false, 0) - }; - // Sift upwards if the `compared_index` is higher priority - if let (true, compared_index) = swap { - unsafe { - self.data.swap_unchecked(node_index, compared_index); + (false, 0) + }; + // Sift upwards if the `compared_index` is higher priority + if let (true, compared_index) = swap { + unsafe { + self.data.swap_unchecked(node_index, compared_index); + } + node_index = compared_index; + } else { + break; } - node_index = compared_index; - } else { - break; } - } - Some(removed_node) - } else { - None + Some(removed_node) + } + _ => None, } } diff --git a/src/common/base/src/http_client.rs b/src/common/base/src/http_client.rs index b27864356dc2d..dd179c6eb733c 100644 --- a/src/common/base/src/http_client.rs +++ b/src/common/base/src/http_client.rs @@ -123,10 +123,10 @@ impl HttpClient { builder = builder.connect_timeout(Duration::from_secs(connect_timeout)); // Enable TCP keepalive if set. - if let Ok(v) = env::var("_DATABEND_INTERNAL_TCP_KEEPALIVE") { - if let Ok(v) = v.parse::() { - builder = builder.tcp_keepalive(Duration::from_secs(v)); - } + if let Ok(v) = env::var("_DATABEND_INTERNAL_TCP_KEEPALIVE") + && let Ok(v) = v.parse::() + { + builder = builder.tcp_keepalive(Duration::from_secs(v)); } let client = builder.build().expect("http client must be created"); diff --git a/src/common/base/src/lib.rs b/src/common/base/src/lib.rs index 43c5bf225ab66..8746d14ca452e 100644 --- a/src/common/base/src/lib.rs +++ b/src/common/base/src/lib.rs @@ -17,7 +17,6 @@ #![feature(allocator_api)] #![feature(thread_local)] #![feature(ptr_metadata)] -#![feature(result_flattening)] #![feature(try_trait_v2)] #![feature(thread_id_value)] #![feature(backtrace_frames)] @@ -25,10 +24,8 @@ #![feature(slice_swap_unchecked)] #![feature(variant_count)] #![feature(ptr_alignment_type)] -#![feature(vec_into_raw_parts)] #![feature(slice_ptr_get)] #![feature(alloc_layout_extra)] -#![feature(let_chains)] pub mod base; pub mod containers; @@ -42,9 +39,9 @@ pub mod rangemap; pub mod runtime; pub mod vec_ext; +pub use runtime::AsyncTaskItem; +pub use runtime::GLOBAL_TASK; +pub use runtime::JoinHandle; pub use runtime::dump_backtrace; pub use runtime::get_all_tasks; pub use runtime::set_alloc_error_hook; -pub use runtime::AsyncTaskItem; -pub use runtime::JoinHandle; -pub use runtime::GLOBAL_TASK; diff --git a/src/common/base/src/mem_allocator/global.rs b/src/common/base/src/mem_allocator/global.rs index 7c6e3f5acc827..97151417331ab 100644 --- a/src/common/base/src/mem_allocator/global.rs +++ b/src/common/base/src/mem_allocator/global.rs @@ -16,11 +16,11 @@ use std::alloc::AllocError; use std::alloc::Allocator; use std::alloc::GlobalAlloc; use std::alloc::Layout; -use std::ptr::null_mut; use std::ptr::NonNull; +use std::ptr::null_mut; -use crate::mem_allocator::tracker::MetaTrackerAllocator; use crate::mem_allocator::DefaultAllocator; +use crate::mem_allocator::tracker::MetaTrackerAllocator; pub type DefaultGlobalAllocator = GlobalAllocator; pub type TrackingGlobalAllocator = GlobalAllocator>; @@ -77,7 +77,7 @@ unsafe impl Allocator for GlobalAllocator { #[inline(always)] unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { - self.inner.deallocate(ptr, layout) + unsafe { self.inner.deallocate(ptr, layout) } } #[inline(always)] @@ -87,7 +87,7 @@ unsafe impl Allocator for GlobalAllocator { old_layout: Layout, new_layout: Layout, ) -> Result, AllocError> { - self.inner.grow(ptr, old_layout, new_layout) + unsafe { self.inner.grow(ptr, old_layout, new_layout) } } #[inline(always)] @@ -97,7 +97,7 @@ unsafe impl Allocator for GlobalAllocator { old_layout: Layout, new_layout: Layout, ) -> Result, AllocError> { - self.inner.grow_zeroed(ptr, old_layout, new_layout) + unsafe { self.inner.grow_zeroed(ptr, old_layout, new_layout) } } #[inline(always)] @@ -107,7 +107,7 @@ unsafe impl Allocator for GlobalAllocator { old_layout: Layout, new_layout: Layout, ) -> Result, AllocError> { - self.inner.shrink(ptr, old_layout, new_layout) + unsafe { self.inner.shrink(ptr, old_layout, new_layout) } } } @@ -122,8 +122,8 @@ unsafe impl GlobalAlloc for GlobalAllocator { #[inline] unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { - let ptr = NonNull::new(ptr).unwrap_unchecked(); - self.deallocate(ptr, layout); + let ptr = unsafe { NonNull::new(ptr).unwrap_unchecked() }; + unsafe { self.deallocate(ptr, layout) }; } #[inline] @@ -138,15 +138,15 @@ unsafe impl GlobalAlloc for GlobalAllocator { unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { use std::cmp::Ordering::*; - let ptr = NonNull::new(ptr).unwrap_unchecked(); + let ptr = unsafe { NonNull::new(ptr).unwrap_unchecked() }; let new_layout = Layout::from_size_align(new_size, layout.align()).unwrap(); match layout.size().cmp(&new_size) { Equal => ptr.as_ptr(), - Less => match self.grow(ptr, layout, new_layout) { + Less => match unsafe { self.grow(ptr, layout, new_layout) } { Ok(ptr) => ptr.as_ptr() as *mut u8, Err(_) => null_mut(), }, - Greater => match self.shrink(ptr, layout, new_layout) { + Greater => match unsafe { self.shrink(ptr, layout, new_layout) } { Ok(ptr) => ptr.as_ptr() as *mut u8, Err(_) => null_mut(), }, diff --git a/src/common/base/src/mem_allocator/jemalloc.rs b/src/common/base/src/mem_allocator/jemalloc.rs index ba21d7e1be522..cffdc5dcd1c02 100644 --- a/src/common/base/src/mem_allocator/jemalloc.rs +++ b/src/common/base/src/mem_allocator/jemalloc.rs @@ -111,7 +111,7 @@ pub mod linux { debug_assert_eq!(ptr.as_ptr() as usize, layout.align()); } else { let flags = layout_to_flags(layout.align(), layout.size()); - ffi::sdallocx(ptr.as_ptr() as *mut _, layout.size(), flags); + unsafe { ffi::sdallocx(ptr.as_ptr() as *mut _, layout.size(), flags) }; } } @@ -125,14 +125,18 @@ pub mod linux { debug_assert!(old_layout.size() <= new_layout.size()); let data_address = if new_layout.size() == 0 { - NonNull::new(new_layout.align() as *mut ()).unwrap_unchecked() + unsafe { NonNull::new(new_layout.align() as *mut ()).unwrap_unchecked() } } else if old_layout.size() == 0 { let flags = layout_to_flags(new_layout.align(), new_layout.size()); - NonNull::new(ffi::mallocx(new_layout.size(), flags) as *mut ()).ok_or(AllocError)? + NonNull::new(unsafe { ffi::mallocx(new_layout.size(), flags) } as *mut ()) + .ok_or(AllocError)? } else { let flags = layout_to_flags(new_layout.align(), new_layout.size()); - NonNull::new(ffi::rallocx(ptr.cast().as_ptr(), new_layout.size(), flags) as *mut ()) - .unwrap() + NonNull::new( + unsafe { ffi::rallocx(ptr.cast().as_ptr(), new_layout.size(), flags) } + as *mut (), + ) + .unwrap() }; Ok(NonNull::<[u8]>::from_raw_parts( @@ -151,22 +155,26 @@ pub mod linux { debug_assert!(old_layout.size() <= new_layout.size()); let data_address = if new_layout.size() == 0 { - NonNull::new(new_layout.align() as *mut ()).unwrap_unchecked() + unsafe { NonNull::new(new_layout.align() as *mut ()).unwrap_unchecked() } } else if old_layout.size() == 0 { let flags = layout_to_flags(new_layout.align(), new_layout.size()) | ffi::MALLOCX_ZERO; - NonNull::new(ffi::mallocx(new_layout.size(), flags) as *mut ()).ok_or(AllocError)? + NonNull::new(unsafe { ffi::mallocx(new_layout.size(), flags) } as *mut ()) + .ok_or(AllocError)? } else { let flags = layout_to_flags(new_layout.align(), new_layout.size()); // Jemalloc doesn't support `grow_zeroed`, so it might be better to use // mmap allocator for large frequent memory allocation and take jemalloc // as fallback. - let raw = ffi::rallocx(ptr.cast().as_ptr(), new_layout.size(), flags) as *mut u8; + let raw = unsafe { ffi::rallocx(ptr.cast().as_ptr(), new_layout.size(), flags) } + as *mut u8; if raw.is_null() { return Err(AllocError); } else { - raw.add(old_layout.size()) - .write_bytes(0, new_layout.size() - old_layout.size()); + unsafe { + raw.add(old_layout.size()) + .write_bytes(0, new_layout.size() - old_layout.size()) + }; NonNull::new(raw as *mut ()).unwrap() } }; @@ -188,24 +196,26 @@ pub mod linux { if old_layout.size() == 0 { debug_assert_eq!(ptr.as_ptr() as usize, old_layout.align()); - let slice = std::slice::from_raw_parts_mut(ptr.as_ptr(), 0); - let ptr = NonNull::new(slice).unwrap_unchecked(); + let slice = unsafe { std::slice::from_raw_parts_mut(ptr.as_ptr(), 0) }; + let ptr = unsafe { NonNull::new(slice).unwrap_unchecked() }; return Ok(ptr); } let flags = layout_to_flags(new_layout.align(), new_layout.size()); let new_ptr = if new_layout.size() == 0 { - ffi::sdallocx(ptr.as_ptr() as *mut c_void, new_layout.size(), flags); - let slice = std::slice::from_raw_parts_mut(new_layout.align() as *mut u8, 0); - NonNull::new(slice).unwrap_unchecked() + unsafe { ffi::sdallocx(ptr.as_ptr() as *mut c_void, new_layout.size(), flags) }; + let slice = + unsafe { std::slice::from_raw_parts_mut(new_layout.align() as *mut u8, 0) }; + unsafe { NonNull::new(slice).unwrap_unchecked() } } else { let data_address = - ffi::rallocx(ptr.cast().as_ptr(), new_layout.size(), flags) as *mut u8; + unsafe { ffi::rallocx(ptr.cast().as_ptr(), new_layout.size(), flags) } + as *mut u8; if data_address.is_null() { return Err(AllocError); } else { let metadata = new_layout.size(); - let slice = std::slice::from_raw_parts_mut(data_address, metadata); + let slice = unsafe { std::slice::from_raw_parts_mut(data_address, metadata) }; NonNull::new(slice).ok_or(AllocError)? } }; diff --git a/src/common/base/src/mem_allocator/std_.rs b/src/common/base/src/mem_allocator/std_.rs index ab7d286696815..9e6eeb6f81427 100644 --- a/src/common/base/src/mem_allocator/std_.rs +++ b/src/common/base/src/mem_allocator/std_.rs @@ -49,7 +49,7 @@ unsafe impl Allocator for StdAllocator { #[inline(always)] unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { - System.deallocate(ptr, layout) + unsafe { System.deallocate(ptr, layout) } } #[inline(always)] @@ -59,7 +59,7 @@ unsafe impl Allocator for StdAllocator { old_layout: Layout, new_layout: Layout, ) -> Result, AllocError> { - System.grow(ptr, old_layout, new_layout) + unsafe { System.grow(ptr, old_layout, new_layout) } } #[inline(always)] @@ -69,7 +69,7 @@ unsafe impl Allocator for StdAllocator { old_layout: Layout, new_layout: Layout, ) -> Result, AllocError> { - System.grow_zeroed(ptr, old_layout, new_layout) + unsafe { System.grow_zeroed(ptr, old_layout, new_layout) } } #[inline(always)] @@ -79,6 +79,6 @@ unsafe impl Allocator for StdAllocator { old_layout: Layout, new_layout: Layout, ) -> Result, AllocError> { - System.shrink(ptr, old_layout, new_layout) + unsafe { System.shrink(ptr, old_layout, new_layout) } } } diff --git a/src/common/base/src/mem_allocator/tracker.rs b/src/common/base/src/mem_allocator/tracker.rs index 0bebf09db2bc7..850d9c9a09882 100644 --- a/src/common/base/src/mem_allocator/tracker.rs +++ b/src/common/base/src/mem_allocator/tracker.rs @@ -16,8 +16,8 @@ use std::alloc::AllocError; use std::alloc::Allocator; use std::alloc::Layout; use std::mem::ManuallyDrop; -use std::ptr::slice_from_raw_parts_mut; use std::ptr::NonNull; +use std::ptr::slice_from_raw_parts_mut; use std::sync::Arc; use crate::runtime::GlobalStatBuffer; @@ -102,15 +102,15 @@ impl MetaTrackerAllocator { unsafe fn dealloc(&self, ptr: NonNull, layout: Layout) -> Option { let adjusted_layout = Self::adjusted_layout(layout); - let mem_stat_address = ptr.add(layout.size()).cast::().read_unaligned(); + let mem_stat_address = unsafe { ptr.add(layout.size()).cast::().read_unaligned() }; if mem_stat_address == 0 { return Some(adjusted_layout); } - let mem_stat = Arc::from_raw(mem_stat_address as *const MemStat); + let mem_stat = unsafe { Arc::from_raw(mem_stat_address as *const MemStat) }; MemStatBuffer::current().dealloc(&mem_stat, adjusted_layout.size() as i64); - self.inner.deallocate(ptr, adjusted_layout); + unsafe { self.inner.deallocate(ptr, adjusted_layout) }; None } @@ -125,7 +125,8 @@ impl MetaTrackerAllocator { MemStatBuffer::current().alloc(stat, new_adjusted_layout.size() as i64)?; GlobalStatBuffer::current().dealloc(old_layout.size() as i64); - let Ok(grow_ptr) = self.inner.grow(ptr, old_layout, new_adjusted_layout) else { + let Ok(grow_ptr) = (unsafe { self.inner.grow(ptr, old_layout, new_adjusted_layout) }) + else { GlobalStatBuffer::current().force_alloc(old_layout.size() as i64); MemStatBuffer::current().dealloc(stat, new_adjusted_layout.size() as i64); return Err(AllocError); @@ -143,16 +144,16 @@ impl MetaTrackerAllocator { ) -> Result, AllocError> { let old_adjusted_layout = Self::adjusted_layout(old_layout); let new_adjusted_layout = Self::adjusted_layout(new_layout); - let address = ptr.add(old_layout.size()).cast::().read_unaligned(); + let address = unsafe { ptr.add(old_layout.size()).cast::().read_unaligned() }; if address == 0 { let diff = new_adjusted_layout.size() - old_adjusted_layout.size(); GlobalStatBuffer::current().alloc(diff as i64)?; - let Ok(grow_ptr) = self - .inner - .grow(ptr, old_adjusted_layout, new_adjusted_layout) - else { + let Ok(grow_ptr) = (unsafe { + self.inner + .grow(ptr, old_adjusted_layout, new_adjusted_layout) + }) else { GlobalStatBuffer::current().dealloc(diff as i64); return Err(AllocError); }; @@ -161,14 +162,14 @@ impl MetaTrackerAllocator { } let diff = new_adjusted_layout.size() - old_adjusted_layout.size(); - let stat = ManuallyDrop::new(Arc::from_raw(address as *const MemStat)); + let stat = ManuallyDrop::new(unsafe { Arc::from_raw(address as *const MemStat) }); MemStatBuffer::current().alloc(&stat, diff as i64)?; - let Ok(grow_ptr) = self - .inner - .grow(ptr, old_adjusted_layout, new_adjusted_layout) - else { + let Ok(grow_ptr) = (unsafe { + self.inner + .grow(ptr, old_adjusted_layout, new_adjusted_layout) + }) else { MemStatBuffer::current().dealloc(&stat, diff as i64); return Err(AllocError); }; @@ -188,7 +189,9 @@ impl MetaTrackerAllocator { MemStatBuffer::current().alloc(stat, new_adjusted_layout.size() as i64)?; GlobalStatBuffer::current().dealloc(old_layout.size() as i64); - let Ok(grow_ptr) = self.inner.grow_zeroed(ptr, old_layout, new_adjusted_layout) else { + let Ok(grow_ptr) = + (unsafe { self.inner.grow_zeroed(ptr, old_layout, new_adjusted_layout) }) + else { GlobalStatBuffer::current().force_alloc(old_layout.size() as i64); MemStatBuffer::current().dealloc(stat, new_adjusted_layout.size() as i64); return Err(AllocError); @@ -207,16 +210,16 @@ impl MetaTrackerAllocator { let old_adjusted_layout = Self::adjusted_layout(old_layout); let new_adjusted_layout = Self::adjusted_layout(new_layout); - let address = ptr.add(old_layout.size()).cast::().read_unaligned(); + let address = unsafe { ptr.add(old_layout.size()).cast::().read_unaligned() }; if address == 0 { let diff = new_adjusted_layout.size() - old_adjusted_layout.size(); GlobalStatBuffer::current().alloc(diff as i64)?; - let Ok(grow_ptr) = + let Ok(grow_ptr) = (unsafe { self.inner .grow_zeroed(ptr, old_adjusted_layout, new_adjusted_layout) - else { + }) else { GlobalStatBuffer::current().dealloc(diff as i64); return Err(AllocError); }; @@ -225,23 +228,25 @@ impl MetaTrackerAllocator { } let alloc_size = new_adjusted_layout.size() - old_adjusted_layout.size(); - let stat = ManuallyDrop::new(Arc::from_raw(address as *const MemStat)); + let stat = ManuallyDrop::new(unsafe { Arc::from_raw(address as *const MemStat) }); MemStatBuffer::current().alloc(&stat, alloc_size as i64)?; - let Ok(grow_ptr) = self - .inner - .grow_zeroed(ptr, old_adjusted_layout, new_adjusted_layout) - else { + let Ok(grow_ptr) = (unsafe { + self.inner + .grow_zeroed(ptr, old_adjusted_layout, new_adjusted_layout) + }) else { MemStatBuffer::current().dealloc(&stat, alloc_size as i64); return Err(AllocError); }; - grow_ptr - .as_non_null_ptr() - .add(old_layout.size()) - .cast::() - .write_unaligned(0); + unsafe { + grow_ptr + .as_non_null_ptr() + .add(old_layout.size()) + .cast::() + .write_unaligned(0); + } Ok(Self::with_meta(grow_ptr, new_layout, address)) } @@ -254,10 +259,13 @@ impl MetaTrackerAllocator { ) -> Result, AllocError> { let old_adjusted_layout = Self::adjusted_layout(old_layout); - let mem_stat_address = ptr.add(old_layout.size()).cast::().read_unaligned(); + let mem_stat_address = + unsafe { ptr.add(old_layout.size()).cast::().read_unaligned() }; if mem_stat_address == 0 { - let Ok(reduced_ptr) = self.inner.shrink(ptr, old_adjusted_layout, new_layout) else { + let Ok(reduced_ptr) = + (unsafe { self.inner.shrink(ptr, old_adjusted_layout, new_layout) }) + else { return Err(AllocError); }; @@ -266,11 +274,12 @@ impl MetaTrackerAllocator { return Ok(reduced_ptr); } - let Ok(reduced_ptr) = self.inner.shrink(ptr, old_adjusted_layout, new_layout) else { + let Ok(reduced_ptr) = (unsafe { self.inner.shrink(ptr, old_adjusted_layout, new_layout) }) + else { return Err(AllocError); }; - let mem_stat = Arc::from_raw(mem_stat_address as *const MemStat); + let mem_stat = unsafe { Arc::from_raw(mem_stat_address as *const MemStat) }; MemStatBuffer::current().dealloc(&mem_stat, old_adjusted_layout.size() as i64); GlobalStatBuffer::current().force_alloc(new_layout.size() as i64); Ok(reduced_ptr) @@ -285,15 +294,15 @@ impl MetaTrackerAllocator { let old_adjusted_layout = Self::adjusted_layout(old_layout); let new_adjusted_layout = Self::adjusted_layout(new_layout); - let address = ptr.add(old_layout.size()).cast::().read_unaligned(); + let address = unsafe { ptr.add(old_layout.size()).cast::().read_unaligned() }; if address == 0 { let diff = old_adjusted_layout.size() - new_adjusted_layout.size(); GlobalStatBuffer::current().dealloc(diff as i64); - let Ok(ptr) = self - .inner - .shrink(ptr, old_adjusted_layout, new_adjusted_layout) - else { + let Ok(ptr) = (unsafe { + self.inner + .shrink(ptr, old_adjusted_layout, new_adjusted_layout) + }) else { GlobalStatBuffer::current().force_alloc(diff as i64); return Err(AllocError); }; @@ -302,13 +311,13 @@ impl MetaTrackerAllocator { } let alloc_size = old_adjusted_layout.size() - new_adjusted_layout.size(); - let stat = ManuallyDrop::new(Arc::from_raw(address as *const MemStat)); + let stat = ManuallyDrop::new(unsafe { Arc::from_raw(address as *const MemStat) }); MemStatBuffer::current().dealloc(&stat, alloc_size as i64); - let Ok(ptr) = self - .inner - .shrink(ptr, old_adjusted_layout, new_adjusted_layout) - else { + let Ok(ptr) = (unsafe { + self.inner + .shrink(ptr, old_adjusted_layout, new_adjusted_layout) + }) else { MemStatBuffer::current().force_alloc(&stat, alloc_size as i64); return Err(AllocError); }; @@ -375,16 +384,16 @@ unsafe impl Allocator for MetaTrackerAllocator { #[inline(always)] unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { if layout.size() >= META_TRACKER_THRESHOLD { - if let Some(layout) = self.dealloc(ptr, layout) { + if let Some(layout) = unsafe { self.dealloc(ptr, layout) } { GlobalStatBuffer::current().dealloc(layout.size() as i64); - self.inner.deallocate(ptr, layout); + unsafe { self.inner.deallocate(ptr, layout) }; } return; } GlobalStatBuffer::current().dealloc(layout.size() as i64); - self.inner.deallocate(ptr, layout); + unsafe { self.inner.deallocate(ptr, layout) }; } #[inline(always)] @@ -395,23 +404,22 @@ unsafe impl Allocator for MetaTrackerAllocator { new_layout: Layout, ) -> Result, AllocError> { if old_layout.size() == new_layout.size() { - return Ok(NonNull::new_unchecked(slice_from_raw_parts_mut( - ptr.as_mut(), - new_layout.size(), - ))); + return Ok(unsafe { + NonNull::new_unchecked(slice_from_raw_parts_mut(ptr.as_mut(), new_layout.size())) + }); } if old_layout.size() >= META_TRACKER_THRESHOLD && new_layout.size() >= META_TRACKER_THRESHOLD { - self.grow_impl(ptr, old_layout, new_layout) + unsafe { self.grow_impl(ptr, old_layout, new_layout) } } else if old_layout.size() < META_TRACKER_THRESHOLD && new_layout.size() < META_TRACKER_THRESHOLD { let diff = new_layout.size() - old_layout.size(); GlobalStatBuffer::current().alloc(diff as i64)?; - let Ok(grow_ptr) = self.inner.grow(ptr, old_layout, new_layout) else { + let Ok(grow_ptr) = (unsafe { self.inner.grow(ptr, old_layout, new_layout) }) else { GlobalStatBuffer::current().dealloc(diff as i64); return Err(AllocError); }; @@ -419,7 +427,7 @@ unsafe impl Allocator for MetaTrackerAllocator { Ok(grow_ptr) } else { if let Some(mem_stat) = ThreadTracker::mem_stat() { - return self.move_grow(ptr, mem_stat, old_layout, new_layout); + return unsafe { self.move_grow(ptr, mem_stat, old_layout, new_layout) }; } let new_adjusted_layout = Self::adjusted_layout(new_layout); @@ -427,7 +435,8 @@ unsafe impl Allocator for MetaTrackerAllocator { let diff = new_adjusted_layout.size() - old_layout.size(); GlobalStatBuffer::current().alloc(diff as i64)?; - let Ok(grow_ptr) = self.inner.grow(ptr, old_layout, new_adjusted_layout) else { + let Ok(grow_ptr) = (unsafe { self.inner.grow(ptr, old_layout, new_adjusted_layout) }) + else { GlobalStatBuffer::current().dealloc(diff as i64); return Err(AllocError); }; @@ -444,22 +453,22 @@ unsafe impl Allocator for MetaTrackerAllocator { new_layout: Layout, ) -> Result, AllocError> { if old_layout.size() == new_layout.size() { - return Ok(NonNull::new_unchecked(slice_from_raw_parts_mut( - ptr.as_mut(), - new_layout.size(), - ))); + return Ok(unsafe { + NonNull::new_unchecked(slice_from_raw_parts_mut(ptr.as_mut(), new_layout.size())) + }); } if old_layout.size() >= META_TRACKER_THRESHOLD && new_layout.size() >= META_TRACKER_THRESHOLD { - self.grow_zeroed_impl(ptr, old_layout, new_layout) + unsafe { self.grow_zeroed_impl(ptr, old_layout, new_layout) } } else if old_layout.size() < META_TRACKER_THRESHOLD && new_layout.size() < META_TRACKER_THRESHOLD { let diff = new_layout.size() - old_layout.size(); GlobalStatBuffer::current().alloc(diff as i64)?; - let Ok(grow_ptr) = self.inner.grow_zeroed(ptr, old_layout, new_layout) else { + let Ok(grow_ptr) = (unsafe { self.inner.grow_zeroed(ptr, old_layout, new_layout) }) + else { GlobalStatBuffer::current().dealloc(diff as i64); return Err(AllocError); }; @@ -467,22 +476,26 @@ unsafe impl Allocator for MetaTrackerAllocator { Ok(grow_ptr) } else { if let Some(mem_stat) = ThreadTracker::mem_stat() { - return self.move_grow_zeroed(ptr, mem_stat, old_layout, new_layout); + return unsafe { self.move_grow_zeroed(ptr, mem_stat, old_layout, new_layout) }; } let new_adjusted_layout = Self::adjusted_layout(new_layout); let diff = new_adjusted_layout.size() - old_layout.size(); GlobalStatBuffer::current().alloc(diff as i64)?; - let Ok(grow_ptr) = self.inner.grow_zeroed(ptr, old_layout, new_adjusted_layout) else { + let Ok(grow_ptr) = + (unsafe { self.inner.grow_zeroed(ptr, old_layout, new_adjusted_layout) }) + else { GlobalStatBuffer::current().dealloc(diff as i64); return Err(AllocError); }; - Ok(NonNull::new_unchecked(slice_from_raw_parts_mut( - grow_ptr.as_non_null_ptr().as_mut(), - new_layout.size(), - ))) + Ok(unsafe { + NonNull::new_unchecked(slice_from_raw_parts_mut( + grow_ptr.as_non_null_ptr().as_mut(), + new_layout.size(), + )) + }) } } @@ -494,29 +507,28 @@ unsafe impl Allocator for MetaTrackerAllocator { new_layout: Layout, ) -> Result, AllocError> { if old_layout.size() == new_layout.size() { - return Ok(NonNull::new_unchecked(slice_from_raw_parts_mut( - ptr.as_mut(), - new_layout.size(), - ))); + return Ok(unsafe { + NonNull::new_unchecked(slice_from_raw_parts_mut(ptr.as_mut(), new_layout.size())) + }); } if old_layout.size() >= META_TRACKER_THRESHOLD && new_layout.size() >= META_TRACKER_THRESHOLD { - self.shrink_impl(ptr, old_layout, new_layout) + unsafe { self.shrink_impl(ptr, old_layout, new_layout) } } else if old_layout.size() < META_TRACKER_THRESHOLD && new_layout.size() < META_TRACKER_THRESHOLD { let diff = old_layout.size() - new_layout.size(); GlobalStatBuffer::current().dealloc(diff as i64); - let Ok(shrink_ptr) = self.inner.shrink(ptr, old_layout, new_layout) else { + let Ok(shrink_ptr) = (unsafe { self.inner.shrink(ptr, old_layout, new_layout) }) else { GlobalStatBuffer::current().force_alloc(diff as i64); return Err(AllocError); }; Ok(shrink_ptr) } else { - self.move_shrink(ptr, old_layout, new_layout) + unsafe { self.move_shrink(ptr, old_layout, new_layout) } } } } @@ -527,20 +539,20 @@ mod tests { use std::alloc::Allocator; use std::alloc::Layout; use std::ptr::NonNull; - use std::sync::atomic::Ordering; use std::sync::Arc; + use std::sync::atomic::Ordering; use crate::base::GlobalUniqName; - use crate::mem_allocator::tracker::MetaTrackerAllocator; - use crate::mem_allocator::tracker::META_TRACKER_THRESHOLD; use crate::mem_allocator::DefaultAllocator; + use crate::mem_allocator::tracker::META_TRACKER_THRESHOLD; + use crate::mem_allocator::tracker::MetaTrackerAllocator; + use crate::runtime::GLOBAL_QUERIES_MANAGER; use crate::runtime::GlobalStatBuffer; use crate::runtime::MemStat; use crate::runtime::MemStatBuffer; use crate::runtime::ParentMemStat; use crate::runtime::Thread; use crate::runtime::ThreadTracker; - use crate::runtime::GLOBAL_QUERIES_MANAGER; fn with_mock_env< T: Allocator + Send + Sync + 'static, @@ -786,7 +798,7 @@ mod tests { } unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { - self.0.deallocate(ptr, layout) + unsafe { self.0.deallocate(ptr, layout) } } unsafe fn grow( @@ -864,7 +876,7 @@ mod tests { } unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { - self.0.deallocate(ptr, layout) + unsafe { self.0.deallocate(ptr, layout) } } unsafe fn grow_zeroed( @@ -941,7 +953,7 @@ mod tests { } unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { - self.0.deallocate(ptr, layout) + unsafe { self.0.deallocate(ptr, layout) } } unsafe fn shrink( @@ -1034,7 +1046,7 @@ mod tests { } unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { - self.inner.deallocate(ptr, layout) + unsafe { self.inner.deallocate(ptr, layout) } } unsafe fn grow( @@ -1043,10 +1055,12 @@ mod tests { old_layout: Layout, new_layout: Layout, ) -> Result, AllocError> { - if rand::random::() < self.failure_rate { - Err(AllocError) - } else { - self.inner.grow(ptr, old_layout, new_layout) + unsafe { + if rand::random::() < self.failure_rate { + Err(AllocError) + } else { + self.inner.grow(ptr, old_layout, new_layout) + } } } @@ -1056,10 +1070,12 @@ mod tests { old_layout: Layout, new_layout: Layout, ) -> Result, AllocError> { - if rand::random::() < self.failure_rate { - Err(AllocError) - } else { - self.inner.grow_zeroed(ptr, old_layout, new_layout) + unsafe { + if rand::random::() < self.failure_rate { + Err(AllocError) + } else { + self.inner.grow_zeroed(ptr, old_layout, new_layout) + } } } @@ -1069,10 +1085,12 @@ mod tests { old_layout: Layout, new_layout: Layout, ) -> Result, AllocError> { - if rand::random::() < self.failure_rate { - Err(AllocError) - } else { - self.inner.shrink(ptr, old_layout, new_layout) + unsafe { + if rand::random::() < self.failure_rate { + Err(AllocError) + } else { + self.inner.shrink(ptr, old_layout, new_layout) + } } } } diff --git a/src/common/base/src/rangemap/range_merger.rs b/src/common/base/src/rangemap/range_merger.rs index 20aedafc6b402..da6ca73f4cf4d 100644 --- a/src/common/base/src/rangemap/range_merger.rs +++ b/src/common/base/src/rangemap/range_merger.rs @@ -64,15 +64,15 @@ impl RangeMerger { ranges: Vec::with_capacity(raw_ranges.len()), }; - if let Some(whole_read_size) = whole_read_size { - if !raw_ranges.is_empty() { - let max_end = raw_ranges.iter().map(|r| r.end).max().unwrap_or(0); + if let Some(whole_read_size) = whole_read_size + && !raw_ranges.is_empty() + { + let max_end = raw_ranges.iter().map(|r| r.end).max().unwrap_or(0); - if max_end - raw_ranges[0].start <= whole_read_size { - let r = raw_ranges.first().unwrap().start..max_end; - rs.ranges = vec![r]; - return rs; - } + if max_end - raw_ranges[0].start <= whole_read_size { + let r = raw_ranges.first().unwrap().start..max_end; + rs.ranges = vec![r]; + return rs; } } @@ -99,11 +99,11 @@ impl RangeMerger { } fn add(&mut self, range: &Range) { - if let Some(last) = self.ranges.last_mut() { - if overlaps(last, range, self.max_gap_size, self.max_range_size) { - merge(last, range); - return; - } + if let Some(last) = self.ranges.last_mut() + && overlaps(last, range, self.max_gap_size, self.max_range_size) + { + merge(last, range); + return; } self.ranges.push(range.clone()); diff --git a/src/common/base/src/runtime/memory/mem_stat.rs b/src/common/base/src/runtime/memory/mem_stat.rs index 3294d487376f9..1b7c853c63dd9 100644 --- a/src/common/base/src/runtime/memory/mem_stat.rs +++ b/src/common/base/src/runtime/memory/mem_stat.rs @@ -14,19 +14,19 @@ use std::fmt::Debug; use std::fmt::Formatter; -use std::sync::atomic::AtomicBool; -use std::sync::atomic::AtomicI64; -use std::sync::atomic::Ordering; use std::sync::Arc; use std::sync::Mutex; use std::sync::PoisonError; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::AtomicI64; +use std::sync::atomic::Ordering; use bytesize::ByteSize; use crate::base::GlobalSequence; -use crate::runtime::memory::memory_manager::QueriesMemoryManager; -use crate::runtime::memory::memory_manager::GLOBAL_QUERIES_MANAGER; use crate::runtime::LimitMemGuard; +use crate::runtime::memory::memory_manager::GLOBAL_QUERIES_MANAGER; +use crate::runtime::memory::memory_manager::QueriesMemoryManager; /// The program mem stat /// @@ -434,9 +434,9 @@ mod tests { use databend_common_exception::Result; - use crate::runtime::memory::mem_stat::ParentMemStat; - use crate::runtime::memory::mem_stat::MINIMUM_MEMORY_LIMIT; use crate::runtime::MemStat; + use crate::runtime::memory::mem_stat::MINIMUM_MEMORY_LIMIT; + use crate::runtime::memory::mem_stat::ParentMemStat; #[test] fn test_single_level_mem_stat() -> Result<()> { @@ -457,9 +457,11 @@ mod tests { mem_stat.set_limit(MINIMUM_MEMORY_LIMIT, false); mem_stat.record_memory::(1, 1).unwrap(); - assert!(mem_stat - .record_memory::(MINIMUM_MEMORY_LIMIT, MINIMUM_MEMORY_LIMIT) - .is_err()); + assert!( + mem_stat + .record_memory::(MINIMUM_MEMORY_LIMIT, MINIMUM_MEMORY_LIMIT) + .is_err() + ); assert_eq!( mem_stat.used.load(Ordering::Relaxed), 1 + MINIMUM_MEMORY_LIMIT @@ -530,9 +532,11 @@ mod tests { child_mem_stat.set_limit(MINIMUM_MEMORY_LIMIT, false); mem_stat.record_memory::(1, 1).unwrap(); - assert!(mem_stat - .record_memory::(MINIMUM_MEMORY_LIMIT, MINIMUM_MEMORY_LIMIT) - .is_ok()); + assert!( + mem_stat + .record_memory::(MINIMUM_MEMORY_LIMIT, MINIMUM_MEMORY_LIMIT) + .is_ok() + ); assert_eq!( mem_stat.used.load(Ordering::Relaxed), 1 + MINIMUM_MEMORY_LIMIT @@ -540,9 +544,11 @@ mod tests { assert_eq!(child_mem_stat.used.load(Ordering::Relaxed), 0); child_mem_stat.record_memory::(1, 1).unwrap(); - assert!(child_mem_stat - .record_memory::(MINIMUM_MEMORY_LIMIT, MINIMUM_MEMORY_LIMIT) - .is_err()); + assert!( + child_mem_stat + .record_memory::(MINIMUM_MEMORY_LIMIT, MINIMUM_MEMORY_LIMIT) + .is_err() + ); assert_eq!( mem_stat.used.load(Ordering::Relaxed), 1 + MINIMUM_MEMORY_LIMIT + 1 + MINIMUM_MEMORY_LIMIT @@ -562,9 +568,11 @@ mod tests { ); child_mem_stat.set_limit(MINIMUM_MEMORY_LIMIT * 2, false); - assert!(child_mem_stat - .record_memory::(1 + MINIMUM_MEMORY_LIMIT, 1 + MINIMUM_MEMORY_LIMIT) - .is_err()); + assert!( + child_mem_stat + .record_memory::(1 + MINIMUM_MEMORY_LIMIT, 1 + MINIMUM_MEMORY_LIMIT) + .is_err() + ); assert_eq!(mem_stat.used.load(Ordering::Relaxed), 0); assert_eq!(child_mem_stat.used.load(Ordering::Relaxed), 0); @@ -578,9 +586,11 @@ mod tests { ); child_mem_stat.set_limit(MINIMUM_MEMORY_LIMIT, false); - assert!(child_mem_stat - .record_memory::(1 + MINIMUM_MEMORY_LIMIT, 1 + MINIMUM_MEMORY_LIMIT) - .is_err()); + assert!( + child_mem_stat + .record_memory::(1 + MINIMUM_MEMORY_LIMIT, 1 + MINIMUM_MEMORY_LIMIT) + .is_err() + ); assert_eq!(mem_stat.used.load(Ordering::Relaxed), 0); // assert_eq!(mem_stat.peak_used.load(Ordering::Relaxed), 0); assert_eq!(child_mem_stat.used.load(Ordering::Relaxed), 0); diff --git a/src/common/base/src/runtime/memory/memory_manager.rs b/src/common/base/src/runtime/memory/memory_manager.rs index ca74d85446f04..5e0e285a1ba12 100644 --- a/src/common/base/src/runtime/memory/memory_manager.rs +++ b/src/common/base/src/runtime/memory/memory_manager.rs @@ -12,25 +12,25 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::hash_map::Entry; use std::collections::HashMap; use std::collections::HashSet; +use std::collections::hash_map::Entry; use std::hash::Hash; use std::hash::Hasher; -use std::sync::atomic::AtomicU8; -use std::sync::atomic::Ordering; use std::sync::Arc; use std::sync::Condvar; use std::sync::Mutex; use std::sync::MutexGuard; use std::sync::OnceLock; use std::sync::PoisonError; +use std::sync::atomic::AtomicU8; +use std::sync::atomic::Ordering; use std::time::Duration; use std::time::Instant; -use crate::runtime::memory::mem_stat::ParentMemStat; use crate::runtime::MemStat; use crate::runtime::OutOfLimit; +use crate::runtime::memory::mem_stat::ParentMemStat; pub static GLOBAL_QUERIES_MANAGER: QueriesMemoryManager = QueriesMemoryManager::create(); @@ -360,11 +360,11 @@ impl QueriesMemoryManager { return Ok(()); } - if let Some(waiting_resource) = state.resources.get(waiting_id) { - if waiting_resource.state.load(Ordering::Acquire) != RUNNING { - // kill by other while in wait - return Ok(()); - } + if let Some(waiting_resource) = state.resources.get(waiting_id) + && waiting_resource.state.load(Ordering::Acquire) != RUNNING + { + // kill by other while in wait + return Ok(()); } if terminate_state.load(Ordering::Acquire) != KILLED { @@ -415,22 +415,22 @@ impl QueriesMemoryManager { #[cfg(test)] mod tests { + use std::sync::Arc; + use std::sync::Barrier; use std::sync::atomic::AtomicBool; - use std::sync::atomic::AtomicU64; use std::sync::atomic::AtomicU8; + use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; - use std::sync::Arc; - use std::sync::Barrier; use std::time::Duration; - use crate::runtime::memory::memory_manager::Group; - use crate::runtime::memory::memory_manager::QueriesMemoryManager; - use crate::runtime::memory::memory_manager::KILLING; - use crate::runtime::memory::memory_manager::RUNNING; use crate::runtime::MemStat; use crate::runtime::OutOfLimit; use crate::runtime::ParentMemStat; use crate::runtime::Thread; + use crate::runtime::memory::memory_manager::Group; + use crate::runtime::memory::memory_manager::KILLING; + use crate::runtime::memory::memory_manager::QueriesMemoryManager; + use crate::runtime::memory::memory_manager::RUNNING; fn mock_exceeded_memory(mem_stat: &MemStat, set_water_height: bool) { mem_stat.set_limit(257 * 1024 * 1024, set_water_height); diff --git a/src/common/base/src/runtime/memory/mod.rs b/src/common/base/src/runtime/memory/mod.rs index ea9e2b99fcb88..8ff97dd94e449 100644 --- a/src/common/base/src/runtime/memory/mod.rs +++ b/src/common/base/src/runtime/memory/mod.rs @@ -19,10 +19,10 @@ mod stat_buffer_global; mod stat_buffer_mem_stat; pub use alloc_error_hook::set_alloc_error_hook; +pub use mem_stat::GLOBAL_MEM_STAT; pub use mem_stat::MemStat; pub use mem_stat::OutOfLimit; pub use mem_stat::ParentMemStat; -pub use mem_stat::GLOBAL_MEM_STAT; pub use memory_manager::GLOBAL_QUERIES_MANAGER; pub use stat_buffer_global::GlobalStatBuffer; pub use stat_buffer_mem_stat::MemStatBuffer; diff --git a/src/common/base/src/runtime/memory/stat_buffer_global.rs b/src/common/base/src/runtime/memory/stat_buffer_global.rs index 3022c568e0b3a..bbf7ddbae2c44 100644 --- a/src/common/base/src/runtime/memory/stat_buffer_global.rs +++ b/src/common/base/src/runtime/memory/stat_buffer_global.rs @@ -14,15 +14,15 @@ use std::alloc::AllocError; use std::ptr::addr_of_mut; -use std::sync::atomic::Ordering; #[cfg(test)] use std::sync::Arc; +use std::sync::atomic::Ordering; -use crate::runtime::memory::mem_stat::OutOfLimit; -use crate::runtime::memory::MemStat; +use crate::runtime::GLOBAL_MEM_STAT; use crate::runtime::LimitMemGuard; use crate::runtime::ThreadTracker; -use crate::runtime::GLOBAL_MEM_STAT; +use crate::runtime::memory::MemStat; +use crate::runtime::memory::mem_stat::OutOfLimit; #[thread_local] static mut GLOBAL_STAT_BUFFER: GlobalStatBuffer = GlobalStatBuffer::empty(&GLOBAL_MEM_STAT); @@ -209,10 +209,10 @@ mod tests { use databend_common_exception::Result; - use crate::runtime::memory::stat_buffer_global::MEM_STAT_BUFFER_SIZE; + use crate::runtime::GLOBAL_QUERIES_MANAGER; use crate::runtime::memory::GlobalStatBuffer; use crate::runtime::memory::MemStat; - use crate::runtime::GLOBAL_QUERIES_MANAGER; + use crate::runtime::memory::stat_buffer_global::MEM_STAT_BUFFER_SIZE; #[test] fn test_alloc() -> Result<()> { diff --git a/src/common/base/src/runtime/memory/stat_buffer_mem_stat.rs b/src/common/base/src/runtime/memory/stat_buffer_mem_stat.rs index 88c66979cbfc5..7ffffac8409c1 100644 --- a/src/common/base/src/runtime/memory/stat_buffer_mem_stat.rs +++ b/src/common/base/src/runtime/memory/stat_buffer_mem_stat.rs @@ -14,15 +14,15 @@ use std::alloc::AllocError; use std::ptr::addr_of_mut; -use std::sync::atomic::Ordering; use std::sync::Arc; +use std::sync::atomic::Ordering; -use crate::runtime::memory::stat_buffer_global::MEM_STAT_BUFFER_SIZE; -use crate::runtime::memory::OutOfLimit; +use crate::runtime::GLOBAL_MEM_STAT; use crate::runtime::LimitMemGuard; use crate::runtime::MemStat; use crate::runtime::ThreadTracker; -use crate::runtime::GLOBAL_MEM_STAT; +use crate::runtime::memory::OutOfLimit; +use crate::runtime::memory::stat_buffer_global::MEM_STAT_BUFFER_SIZE; #[thread_local] static mut MEM_STAT_BUFFER: MemStatBuffer = MemStatBuffer::empty(&GLOBAL_MEM_STAT); @@ -91,12 +91,13 @@ impl MemStatBuffer { } if mem_stat.id != self.cur_mem_stat_id { - if let Err(out_of_limit) = self.flush::(0) { - if !std::thread::panicking() && !self.unlimited_flag { - let _guard = LimitMemGuard::enter_unlimited(); - ThreadTracker::replace_error_message(Some(format!("{:?}", out_of_limit))); - return Err(AllocError); - } + if let Err(out_of_limit) = self.flush::(0) + && !std::thread::panicking() + && !self.unlimited_flag + { + let _guard = LimitMemGuard::enter_unlimited(); + ThreadTracker::replace_error_message(Some(format!("{:?}", out_of_limit))); + return Err(AllocError); } self.cur_mem_stat = Some(mem_stat.clone()); @@ -232,12 +233,12 @@ mod tests { use std::alloc::AllocError; use std::sync::atomic::Ordering; + use crate::runtime::GLOBAL_QUERIES_MANAGER; + use crate::runtime::GlobalStatBuffer; + use crate::runtime::MemStat; use crate::runtime::memory::mem_stat::ParentMemStat; use crate::runtime::memory::stat_buffer_global::MEM_STAT_BUFFER_SIZE; use crate::runtime::memory::stat_buffer_mem_stat::MemStatBuffer; - use crate::runtime::GlobalStatBuffer; - use crate::runtime::MemStat; - use crate::runtime::GLOBAL_QUERIES_MANAGER; #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn test_alloc_with_same_allocator() -> Result<(), AllocError> { diff --git a/src/common/base/src/runtime/metrics/counter.rs b/src/common/base/src/runtime/metrics/counter.rs index 9668792756f29..13521c350e53b 100644 --- a/src/common/base/src/runtime/metrics/counter.rs +++ b/src/common/base/src/runtime/metrics/counter.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::Arc; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; -use std::sync::Arc; use prometheus_client::encoding::EncodeMetric; use prometheus_client::encoding::MetricEncoder; diff --git a/src/common/base/src/runtime/metrics/family.rs b/src/common/base/src/runtime/metrics/family.rs index 295fbfeb4da4c..4dc4fdc37f411 100644 --- a/src/common/base/src/runtime/metrics/family.rs +++ b/src/common/base/src/runtime/metrics/family.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::hash_map::Entry; use std::collections::HashMap; +use std::collections::hash_map::Entry; use std::fmt::Debug; use std::fmt::Error; use std::fmt::Formatter; @@ -30,13 +30,13 @@ use prometheus_client::metrics::MetricType; use prometheus_client::metrics::TypedMetric; use prometheus_client::registry::Registry; +use crate::runtime::metrics::ScopedRegistry; use crate::runtime::metrics::family_metrics::FamilyCounter; use crate::runtime::metrics::family_metrics::FamilyGauge; use crate::runtime::metrics::family_metrics::FamilyHistogram; use crate::runtime::metrics::registry::DatabendMetric; use crate::runtime::metrics::registry::Metric; use crate::runtime::metrics::sample::MetricSample; -use crate::runtime::metrics::ScopedRegistry; pub trait FamilyMetric: Metric + TypedMetric {} @@ -196,10 +196,10 @@ mod prometheus_parse { use prometheus_parse::LineInfo; use prometheus_parse::SampleType; - use crate::runtime::metrics::sample::SummaryCount; use crate::runtime::metrics::HistogramCount; use crate::runtime::metrics::MetricSample; use crate::runtime::metrics::MetricValue; + use crate::runtime::metrics::sample::SummaryCount; fn parse_golang_float(s: &str) -> Result::Err> { match s.to_lowercase().as_str() { diff --git a/src/common/base/src/runtime/metrics/family_metrics/counter.rs b/src/common/base/src/runtime/metrics/family_metrics/counter.rs index 6953ac4986ca8..912c1a9d4e6a7 100644 --- a/src/common/base/src/runtime/metrics/family_metrics/counter.rs +++ b/src/common/base/src/runtime/metrics/family_metrics/counter.rs @@ -13,22 +13,22 @@ // limitations under the License. use std::fmt::Debug; +use std::sync::Arc; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; -use std::sync::Arc; use prometheus_client::encoding::EncodeMetric; use prometheus_client::encoding::MetricEncoder; use prometheus_client::metrics::MetricType; use prometheus_client::metrics::TypedMetric; +use crate::runtime::metrics::MetricSample; +use crate::runtime::metrics::MetricValue; +use crate::runtime::metrics::ScopedRegistry; use crate::runtime::metrics::family::Family; use crate::runtime::metrics::family::FamilyLabels; use crate::runtime::metrics::family::FamilyMetric; use crate::runtime::metrics::registry::DatabendMetric; -use crate::runtime::metrics::MetricSample; -use crate::runtime::metrics::MetricValue; -use crate::runtime::metrics::ScopedRegistry; #[derive(Debug)] pub struct FamilyCounter { diff --git a/src/common/base/src/runtime/metrics/family_metrics/gauge.rs b/src/common/base/src/runtime/metrics/family_metrics/gauge.rs index e09527e43b375..08923d67049cc 100644 --- a/src/common/base/src/runtime/metrics/family_metrics/gauge.rs +++ b/src/common/base/src/runtime/metrics/family_metrics/gauge.rs @@ -14,17 +14,17 @@ use prometheus_client::encoding::EncodeMetric; use prometheus_client::encoding::MetricEncoder; -use prometheus_client::metrics::gauge::Gauge as PGauge; use prometheus_client::metrics::MetricType; use prometheus_client::metrics::TypedMetric; +use prometheus_client::metrics::gauge::Gauge as PGauge; +use crate::runtime::metrics::ScopedRegistry; use crate::runtime::metrics::family::Family; use crate::runtime::metrics::family::FamilyLabels; use crate::runtime::metrics::family::FamilyMetric; use crate::runtime::metrics::registry::DatabendMetric; use crate::runtime::metrics::sample::MetricSample; use crate::runtime::metrics::sample::MetricValue; -use crate::runtime::metrics::ScopedRegistry; #[derive(Debug)] pub struct FamilyGauge { diff --git a/src/common/base/src/runtime/metrics/family_metrics/histogram.rs b/src/common/base/src/runtime/metrics/family_metrics/histogram.rs index e0a509a204a7d..da8defded5a08 100644 --- a/src/common/base/src/runtime/metrics/family_metrics/histogram.rs +++ b/src/common/base/src/runtime/metrics/family_metrics/histogram.rs @@ -23,6 +23,7 @@ use prometheus_client::encoding::MetricEncoder; use prometheus_client::metrics::MetricType; use prometheus_client::metrics::TypedMetric; +use crate::runtime::metrics::ScopedRegistry; use crate::runtime::metrics::family::Family; use crate::runtime::metrics::family::FamilyLabels; use crate::runtime::metrics::family::FamilyMetric; @@ -31,7 +32,6 @@ use crate::runtime::metrics::registry::MAX_HISTOGRAM_BOUND; use crate::runtime::metrics::sample::HistogramCount; use crate::runtime::metrics::sample::MetricSample; use crate::runtime::metrics::sample::MetricValue; -use crate::runtime::metrics::ScopedRegistry; /// Histogram is a port of prometheus-client's Histogram. The only difference is that /// we can reset the histogram. @@ -106,7 +106,7 @@ impl FamilyHistogram { } } - pub(crate) fn get(&self) -> (f64, u64, MappedRwLockReadGuard>) { + pub(crate) fn get(&self) -> (f64, u64, MappedRwLockReadGuard<'_, Vec<(f64, u64)>>) { let inner = self.inner.read(); let sum = inner.sum; let count = inner.count; diff --git a/src/common/base/src/runtime/metrics/gauge.rs b/src/common/base/src/runtime/metrics/gauge.rs index e1062f814245f..e58c22696146d 100644 --- a/src/common/base/src/runtime/metrics/gauge.rs +++ b/src/common/base/src/runtime/metrics/gauge.rs @@ -14,14 +14,14 @@ use prometheus_client::encoding::EncodeMetric; use prometheus_client::encoding::MetricEncoder; -use prometheus_client::metrics::gauge::Gauge as PGauge; use prometheus_client::metrics::MetricType; use prometheus_client::metrics::TypedMetric; +use prometheus_client::metrics::gauge::Gauge as PGauge; +use crate::runtime::metrics::ScopedRegistry; use crate::runtime::metrics::registry::DatabendMetric; use crate::runtime::metrics::sample::MetricSample; use crate::runtime::metrics::sample::MetricValue; -use crate::runtime::metrics::ScopedRegistry; #[derive(Debug)] pub struct Gauge { diff --git a/src/common/base/src/runtime/metrics/histogram.rs b/src/common/base/src/runtime/metrics/histogram.rs index f5feaad7e55b7..b8a5bfa290b97 100644 --- a/src/common/base/src/runtime/metrics/histogram.rs +++ b/src/common/base/src/runtime/metrics/histogram.rs @@ -23,12 +23,12 @@ use prometheus_client::encoding::MetricEncoder; use prometheus_client::metrics::MetricType; use prometheus_client::metrics::TypedMetric; +use crate::runtime::metrics::ScopedRegistry; use crate::runtime::metrics::registry::DatabendMetric; use crate::runtime::metrics::registry::MAX_HISTOGRAM_BOUND; use crate::runtime::metrics::sample::HistogramCount; use crate::runtime::metrics::sample::MetricSample; use crate::runtime::metrics::sample::MetricValue; -use crate::runtime::metrics::ScopedRegistry; pub static BUCKET_SECONDS: [f64; 16] = [ 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 300.0, 600.0, 1800.0, 3600.0, 7200.0, @@ -115,7 +115,7 @@ impl Histogram { } } - pub(crate) fn get(&self) -> (f64, u64, MappedRwLockReadGuard>) { + pub(crate) fn get(&self) -> (f64, u64, MappedRwLockReadGuard<'_, Vec<(f64, u64)>>) { let inner = self.inner.read(); let sum = inner.sum; let count = inner.count; diff --git a/src/common/base/src/runtime/metrics/mod.rs b/src/common/base/src/runtime/metrics/mod.rs index cc1e5e3a36708..f30da611ac4c4 100644 --- a/src/common/base/src/runtime/metrics/mod.rs +++ b/src/common/base/src/runtime/metrics/mod.rs @@ -23,10 +23,17 @@ mod sample; pub use counter::Counter; pub use gauge::Gauge; -pub use histogram::Histogram; pub use histogram::BUCKET_MILLISECONDS; pub use histogram::BUCKET_SECONDS; +pub use histogram::Histogram; pub use process_collector::dump_process_stat; +pub use registry::FamilyCounter; +pub use registry::FamilyGauge; +pub use registry::FamilyHistogram; +pub use registry::GLOBAL_METRICS_REGISTRY; +pub use registry::MAX_HISTOGRAM_BOUND; +pub use registry::MIN_HISTOGRAM_BOUND; +pub use registry::ScopedRegistry; pub use registry::register_counter; pub use registry::register_counter_family; pub use registry::register_gauge; @@ -38,13 +45,6 @@ pub use registry::register_histogram_family_in_rows; pub use registry::register_histogram_family_in_seconds; pub use registry::register_histogram_in_milliseconds; pub use registry::register_histogram_in_seconds; -pub use registry::FamilyCounter; -pub use registry::FamilyGauge; -pub use registry::FamilyHistogram; -pub use registry::ScopedRegistry; -pub use registry::GLOBAL_METRICS_REGISTRY; -pub use registry::MAX_HISTOGRAM_BOUND; -pub use registry::MIN_HISTOGRAM_BOUND; pub use sample::HistogramCount; pub use sample::MetricSample; pub use sample::MetricValue; diff --git a/src/common/base/src/runtime/metrics/registry.rs b/src/common/base/src/runtime/metrics/registry.rs index deba672ec3751..e5ea3996e290a 100644 --- a/src/common/base/src/runtime/metrics/registry.rs +++ b/src/common/base/src/runtime/metrics/registry.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::sync::atomic::AtomicBool; -use std::sync::atomic::Ordering; use std::sync::Arc; use std::sync::LazyLock; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -26,6 +26,7 @@ use parking_lot::RwLock; use prometheus_client::registry::Metric as PMetrics; use prometheus_client::registry::Registry; +use crate::runtime::ThreadTracker; use crate::runtime::metrics::counter::Counter; use crate::runtime::metrics::family::Family; use crate::runtime::metrics::family::FamilyCounterCreator as InnerFamilyCounterCreator; @@ -36,13 +37,12 @@ use crate::runtime::metrics::family_metrics::FamilyCounter as InnerFamilyCounter use crate::runtime::metrics::family_metrics::FamilyGauge as InnerFamilyGauge; use crate::runtime::metrics::family_metrics::FamilyHistogram as InnerFamilyHistogram; use crate::runtime::metrics::gauge::Gauge; -use crate::runtime::metrics::histogram::Histogram; use crate::runtime::metrics::histogram::BUCKET_MILLISECONDS; use crate::runtime::metrics::histogram::BUCKET_ROWS; use crate::runtime::metrics::histogram::BUCKET_SECONDS; +use crate::runtime::metrics::histogram::Histogram; use crate::runtime::metrics::process_collector::ProcessCollector; use crate::runtime::metrics::sample::MetricSample; -use crate::runtime::ThreadTracker; pub const MIN_HISTOGRAM_BOUND: f64 = f64::MIN; pub const MAX_HISTOGRAM_BOUND: f64 = f64::MAX; @@ -114,7 +114,10 @@ impl GlobalRegistry { self.inner.lock().registry.register_collector(collector); } - pub(crate) fn new_scoped_metric(&self, index: usize) -> impl Iterator { + pub(crate) fn new_scoped_metric( + &self, + index: usize, + ) -> impl Iterator + use<> { let global_registry = self.inner.lock(); let mut scoped_metrics = Vec::with_capacity(global_registry.metrics.len() - index); diff --git a/src/common/base/src/runtime/mod.rs b/src/common/base/src/runtime/mod.rs index 83780ce530ae6..3608d01978df4 100644 --- a/src/common/base/src/runtime/mod.rs +++ b/src/common/base/src/runtime/mod.rs @@ -29,28 +29,33 @@ mod thread; mod time_series; pub mod workload_group; +pub use backtrace::AsyncTaskItem; pub use backtrace::dump_backtrace; pub use backtrace::get_all_tasks; -pub use backtrace::AsyncTaskItem; +pub use catch_unwind::CatchUnwindFuture; pub use catch_unwind::catch_unwind; pub use catch_unwind::drop_guard; -pub use catch_unwind::CatchUnwindFuture; pub use defer::defer; pub use executor_stats::ExecutorStats; pub use executor_stats::ExecutorStatsSlot; pub use executor_stats::ExecutorStatsSnapshot; pub use global_runtime::GlobalIORuntime; pub use global_runtime::GlobalQueryRuntime; -pub use memory::set_alloc_error_hook; +pub use memory::GLOBAL_MEM_STAT; +pub use memory::GLOBAL_QUERIES_MANAGER; pub use memory::GlobalStatBuffer; pub use memory::MemStat; pub use memory::MemStatBuffer; pub use memory::OutOfLimit; pub use memory::ParentMemStat; -pub use memory::GLOBAL_MEM_STAT; -pub use memory::GLOBAL_QUERIES_MANAGER; +pub use memory::set_alloc_error_hook; pub use perf::QueryPerf; pub use perf::QueryPerfGuard; +pub use runtime::Dropper; +pub use runtime::GLOBAL_TASK; +pub use runtime::JoinHandle; +pub use runtime::Runtime; +pub use runtime::TrySpawn; pub use runtime::block_on; pub use runtime::execute_futures_in_parallel; pub use runtime::spawn; @@ -59,11 +64,6 @@ pub use runtime::spawn_local; pub use runtime::spawn_named; pub use runtime::try_block_on; pub use runtime::try_spawn_blocking; -pub use runtime::Dropper; -pub use runtime::JoinHandle; -pub use runtime::Runtime; -pub use runtime::TrySpawn; -pub use runtime::GLOBAL_TASK; pub use runtime_tracker::CaptureLogSettings; pub use runtime_tracker::LimitMemGuard; pub use runtime_tracker::ThreadTracker; @@ -72,11 +72,11 @@ pub use runtime_tracker::TrackingPayload; pub use runtime_tracker::UnlimitedFuture; pub use thread::Thread; pub use thread::ThreadJoinHandle; -pub use time_series::compress_time_point; -pub use time_series::get_time_series_profile_desc; pub use time_series::ProfilePoints; pub use time_series::QueryTimeSeriesProfile; pub use time_series::QueryTimeSeriesProfileBuilder; pub use time_series::TimeSeriesProfileDesc; pub use time_series::TimeSeriesProfileName; pub use time_series::TimeSeriesProfiles; +pub use time_series::compress_time_point; +pub use time_series::get_time_series_profile_desc; diff --git a/src/common/base/src/runtime/profile/mod.rs b/src/common/base/src/runtime/profile/mod.rs index 5826464fd555f..e0cdd01a5486f 100644 --- a/src/common/base/src/runtime/profile/mod.rs +++ b/src/common/base/src/runtime/profile/mod.rs @@ -18,6 +18,6 @@ mod profiles; pub use profile::Profile; pub use profile::ProfileLabel; -pub use profiles::get_statistics_desc; pub use profiles::ProfileDesc; pub use profiles::ProfileStatisticsName; +pub use profiles::get_statistics_desc; diff --git a/src/common/base/src/runtime/profile/profile.rs b/src/common/base/src/runtime/profile/profile.rs index 1dc288e7303f0..43724ddeff0b5 100644 --- a/src/common/base/src/runtime/profile/profile.rs +++ b/src/common/base/src/runtime/profile/profile.rs @@ -12,16 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::Arc; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; -use std::sync::Arc; use parking_lot::Mutex; +use crate::runtime::ThreadTracker; use crate::runtime::error_info::NodeErrorType; use crate::runtime::metrics::ScopedRegistry; use crate::runtime::profile::ProfileStatisticsName; -use crate::runtime::ThreadTracker; #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct ProfileLabel { diff --git a/src/common/base/src/runtime/profile/profiles.rs b/src/common/base/src/runtime/profile/profiles.rs index 443fdbda95e4b..ed2a1e4e8828f 100644 --- a/src/common/base/src/runtime/profile/profiles.rs +++ b/src/common/base/src/runtime/profile/profiles.rs @@ -125,8 +125,8 @@ pub static PROFILES_INDEX: OnceCell< Arc<[Option; std::mem::variant_count::()]>, > = OnceCell::new(); -pub fn get_statistics_name_index( -) -> Arc<[Option; std::mem::variant_count::()]> { +pub fn get_statistics_name_index() +-> Arc<[Option; std::mem::variant_count::()]> { PROFILES_INDEX .get_or_init(|| { let statistics_desc = get_statistics_desc(); diff --git a/src/common/base/src/runtime/runtime.rs b/src/common/base/src/runtime/runtime.rs index 71f2c8082e8ac..3c9a57933f1f2 100644 --- a/src/common/base/src/runtime/runtime.rs +++ b/src/common/base/src/runtime/runtime.rs @@ -25,20 +25,20 @@ use std::time::Instant; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_exception::ResultExt; -use futures::future; use futures::FutureExt; +use futures::future; use log::warn; use tokio::runtime::Builder; use tokio::runtime::Handle; -use tokio::sync::oneshot; use tokio::sync::OwnedSemaphorePermit; use tokio::sync::Semaphore; +use tokio::sync::oneshot; -use crate::runtime::catch_unwind::CatchUnwindFuture; -use crate::runtime::drop_guard; use crate::runtime::Thread; use crate::runtime::ThreadJoinHandle; use crate::runtime::ThreadTracker; +use crate::runtime::catch_unwind::CatchUnwindFuture; +use crate::runtime::drop_guard; pub struct JoinHandle { inner: tokio::task::JoinHandle, @@ -181,10 +181,10 @@ impl Runtime { #[cfg(debug_assertions)] { // We need to pass the thread name in the unit test, because the thread name is the test name - if matches!(std::env::var("UNIT_TEST"), Ok(var_value) if var_value == "TRUE") { - if let Some(thread_name) = std::thread::current().name() { - runtime_builder.thread_name(thread_name); - } + if matches!(std::env::var("UNIT_TEST"), Ok(var_value) if var_value == "TRUE") + && let Some(thread_name) = std::thread::current().name() + { + runtime_builder.thread_name(thread_name); } runtime_builder.thread_stack_size(20 * 1024 * 1024); @@ -205,10 +205,10 @@ impl Runtime { #[cfg(debug_assertions)] { // We need to pass the thread name in the unit test, because the thread name is the test name - if matches!(std::env::var("UNIT_TEST"), Ok(var_value) if var_value == "TRUE") { - if let Some(cur_thread_name) = std::thread::current().name() { - thread_name = Some(cur_thread_name.to_string()); - } + if matches!(std::env::var("UNIT_TEST"), Ok(var_value) if var_value == "TRUE") + && let Some(cur_thread_name) = std::thread::current().name() + { + thread_name = Some(cur_thread_name.to_string()); } runtime_builder.thread_stack_size(20 * 1024 * 1024); @@ -371,22 +371,22 @@ impl Drop for Dropper { fn drop(&mut self) { drop_guard(move || { // Send a signal to say i am dropping. - if let Some(close_sender) = self.close.take() { - if close_sender.send(()).is_ok() { - match self.join_handler.take().unwrap().join() { - Err(e) => warn!("Runtime dropper panic, {:?}", e), - Ok(true) => { - // When the runtime shutdown is blocked for more than 3 seconds, - // we will print the backtrace in the warn log, which will help us debug. - warn!( - "Runtime dropper is blocked 3 seconds, runtime name: {:?}, drop backtrace: {:?}", - self.name, - Backtrace::capture() - ); - } - _ => {} - }; - } + if let Some(close_sender) = self.close.take() + && close_sender.send(()).is_ok() + { + match self.join_handler.take().unwrap().join() { + Err(e) => warn!("Runtime dropper panic, {:?}", e), + Ok(true) => { + // When the runtime shutdown is blocked for more than 3 seconds, + // we will print the backtrace in the warn log, which will help us debug. + warn!( + "Runtime dropper is blocked 3 seconds, runtime name: {:?}, drop backtrace: {:?}", + self.name, + Backtrace::capture() + ); + } + _ => {} + }; } }) } @@ -518,7 +518,7 @@ fn location_future( future: F, frame_location: &'static Location, frame_name: Option, -) -> impl Future +) -> impl Future + use where F: Future, { diff --git a/src/common/base/src/runtime/runtime_tracker.rs b/src/common/base/src/runtime/runtime_tracker.rs index 4a1f0d95f45a2..62860dd4a383d 100644 --- a/src/common/base/src/runtime/runtime_tracker.rs +++ b/src/common/base/src/runtime/runtime_tracker.rs @@ -45,8 +45,8 @@ use std::cell::RefCell; use std::future::Future; use std::pin::Pin; -use std::sync::atomic::AtomicUsize; use std::sync::Arc; +use std::sync::atomic::AtomicUsize; use std::task::Context; use std::task::Poll; @@ -54,16 +54,16 @@ use concurrent_queue::ConcurrentQueue; use log::LevelFilter; use pin_project_lite::pin_project; +use crate::runtime::MemStatBuffer; +use crate::runtime::OutOfLimit; +use crate::runtime::QueryPerf; +use crate::runtime::TimeSeriesProfiles; use crate::runtime::memory::GlobalStatBuffer; use crate::runtime::memory::MemStat; use crate::runtime::metrics::ScopedRegistry; use crate::runtime::profile::Profile; use crate::runtime::time_series::QueryTimeSeriesProfile; use crate::runtime::workload_group::WorkloadGroupResource; -use crate::runtime::MemStatBuffer; -use crate::runtime::OutOfLimit; -use crate::runtime::QueryPerf; -use crate::runtime::TimeSeriesProfiles; // For implemented and needs to call drop, we cannot use the attribute tag thread local. // https://play.rust-lang.org/?version=nightly&mode=debug&edition=2021&gist=ea33533387d401e86423df1a764b5609 diff --git a/src/common/base/src/runtime/thread.rs b/src/common/base/src/runtime/thread.rs index 9cfadc6b21efa..82f92211ee005 100644 --- a/src/common/base/src/runtime/thread.rs +++ b/src/common/base/src/runtime/thread.rs @@ -57,10 +57,10 @@ impl Thread { #[cfg(debug_assertions)] { // We need to pass the thread name in the unit test, because the thread name is the test name - if matches!(std::env::var("UNIT_TEST"), Ok(var_value) if var_value == "TRUE") { - if let Some(thread_name) = std::thread::current().name() { - name = Some(thread_name.to_string()); - } + if matches!(std::env::var("UNIT_TEST"), Ok(var_value) if var_value == "TRUE") + && let Some(thread_name) = std::thread::current().name() + { + name = Some(thread_name.to_string()); } thread_builder = thread_builder.stack_size(5 * 1024 * 1024); diff --git a/src/common/base/src/runtime/time_series/mod.rs b/src/common/base/src/runtime/time_series/mod.rs index 8fe3e1455557b..f437df0eb3714 100644 --- a/src/common/base/src/runtime/time_series/mod.rs +++ b/src/common/base/src/runtime/time_series/mod.rs @@ -15,11 +15,11 @@ mod profile; mod query_profile; -pub use profile::compress_time_point; -pub use profile::get_time_series_profile_desc; pub use profile::ProfilePoints; pub use profile::TimeSeriesProfileDesc; pub use profile::TimeSeriesProfileName; pub use profile::TimeSeriesProfiles; +pub use profile::compress_time_point; +pub use profile::get_time_series_profile_desc; pub use query_profile::QueryTimeSeriesProfile; pub use query_profile::QueryTimeSeriesProfileBuilder; diff --git a/src/common/base/src/runtime/time_series/profile.rs b/src/common/base/src/runtime/time_series/profile.rs index aaa85f58ce65f..ad2e7394a44ff 100644 --- a/src/common/base/src/runtime/time_series/profile.rs +++ b/src/common/base/src/runtime/time_series/profile.rs @@ -13,9 +13,9 @@ // limitations under the License. use std::mem; +use std::sync::Arc; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering::SeqCst; -use std::sync::Arc; use std::time::SystemTime; use std::time::UNIX_EPOCH; diff --git a/src/common/base/src/runtime/time_series/query_profile.rs b/src/common/base/src/runtime/time_series/query_profile.rs index a47a34e13e2d2..98b76babb53fc 100644 --- a/src/common/base/src/runtime/time_series/query_profile.rs +++ b/src/common/base/src/runtime/time_series/query_profile.rs @@ -13,18 +13,18 @@ // limitations under the License. use std::collections::BTreeMap; +use std::sync::Arc; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering::SeqCst; -use std::sync::Arc; use log::info; use serde::Serialize; +use crate::runtime::ThreadTracker; +use crate::runtime::TimeSeriesProfileDesc; use crate::runtime::get_time_series_profile_desc; use crate::runtime::time_series::profile::TimeSeriesProfileName; use crate::runtime::time_series::profile::TimeSeriesProfiles; -use crate::runtime::ThreadTracker; -use crate::runtime::TimeSeriesProfileDesc; const DEFAULT_BATCH_SIZE: usize = 1024; @@ -40,14 +40,13 @@ impl QueryTimeSeriesProfile { |x| match x.borrow().payload.local_time_series_profile.as_ref() { None => {} Some(profile) => { - if profile.record(name, value) { - if let Some(global_profile) = + if profile.record(name, value) + && let Some(global_profile) = x.borrow().payload.time_series_profile.as_ref() - { - let should_flush = Self::should_flush(&global_profile.global_count); - if should_flush { - global_profile.flush(false); - } + { + let should_flush = Self::should_flush(&global_profile.global_count); + if should_flush { + global_profile.flush(false); } } } diff --git a/src/common/base/src/runtime/workload_group.rs b/src/common/base/src/runtime/workload_group.rs index 1c6c4654884ff..1201f0ea00073 100644 --- a/src/common/base/src/runtime/workload_group.rs +++ b/src/common/base/src/runtime/workload_group.rs @@ -15,8 +15,8 @@ use std::collections::HashMap; use std::fmt::Display; use std::fmt::Formatter; -use std::sync::atomic::AtomicUsize; use std::sync::Arc; +use std::sync::atomic::AtomicUsize; use std::time::Duration; use tokio::sync::Mutex; diff --git a/src/common/base/src/vec_ext.rs b/src/common/base/src/vec_ext.rs index 62ed9fdf029a0..8814bf8229397 100644 --- a/src/common/base/src/vec_ext.rs +++ b/src/common/base/src/vec_ext.rs @@ -38,16 +38,16 @@ impl VecExt for Vec { #[inline] unsafe fn push_unchecked(&mut self, value: T) { debug_assert!(self.capacity() > self.len()); - let end = self.as_mut_ptr().add(self.len()); - std::ptr::write(end, value); - self.set_len(self.len() + 1); + let end = unsafe { self.as_mut_ptr().add(self.len()) }; + unsafe { std::ptr::write(end, value) }; + unsafe { self.set_len(self.len() + 1) }; } unsafe fn extend_from_slice_unchecked(&mut self, val: &[T]) { debug_assert!(self.capacity() >= self.len() + val.len()); - let end = self.as_mut_ptr().add(self.len()); - std::ptr::copy_nonoverlapping(val.as_ptr(), end, val.len()); - self.set_len(self.len() + val.len()); + let end = unsafe { self.as_mut_ptr().add(self.len()) }; + unsafe { std::ptr::copy_nonoverlapping(val.as_ptr(), end, val.len()) }; + unsafe { self.set_len(self.len() + val.len()) }; } } @@ -60,8 +60,14 @@ pub trait VecU8Ext { impl VecU8Ext for Vec { unsafe fn store_value_uncheckd(&mut self, val: &T) { debug_assert!(self.capacity() >= self.len() + std::mem::size_of::()); - let end = self.as_mut_ptr().add(self.len()); - std::ptr::copy_nonoverlapping(val as *const T as *const u8, end, std::mem::size_of::()); - self.set_len(self.len() + std::mem::size_of::()); + let end = unsafe { self.as_mut_ptr().add(self.len()) }; + unsafe { + std::ptr::copy_nonoverlapping( + val as *const T as *const u8, + end, + std::mem::size_of::(), + ) + }; + unsafe { self.set_len(self.len() + std::mem::size_of::()) }; } } diff --git a/src/common/base/tests/it/metrics/registry.rs b/src/common/base/tests/it/metrics/registry.rs index d48f3c3b345e3..f3b707509ed49 100644 --- a/src/common/base/tests/it/metrics/registry.rs +++ b/src/common/base/tests/it/metrics/registry.rs @@ -15,20 +15,20 @@ use std::collections::HashMap; use databend_common_base::base::GlobalUniqName; +use databend_common_base::runtime::ThreadTracker; +use databend_common_base::runtime::metrics::BUCKET_MILLISECONDS; +use databend_common_base::runtime::metrics::BUCKET_SECONDS; +use databend_common_base::runtime::metrics::GLOBAL_METRICS_REGISTRY; +use databend_common_base::runtime::metrics::HistogramCount; +use databend_common_base::runtime::metrics::MAX_HISTOGRAM_BOUND; +use databend_common_base::runtime::metrics::MetricSample; +use databend_common_base::runtime::metrics::MetricValue; +use databend_common_base::runtime::metrics::ScopedRegistry; use databend_common_base::runtime::metrics::register_counter; use databend_common_base::runtime::metrics::register_counter_family; use databend_common_base::runtime::metrics::register_gauge; use databend_common_base::runtime::metrics::register_histogram_in_milliseconds; use databend_common_base::runtime::metrics::register_histogram_in_seconds; -use databend_common_base::runtime::metrics::HistogramCount; -use databend_common_base::runtime::metrics::MetricSample; -use databend_common_base::runtime::metrics::MetricValue; -use databend_common_base::runtime::metrics::ScopedRegistry; -use databend_common_base::runtime::metrics::BUCKET_MILLISECONDS; -use databend_common_base::runtime::metrics::BUCKET_SECONDS; -use databend_common_base::runtime::metrics::GLOBAL_METRICS_REGISTRY; -use databend_common_base::runtime::metrics::MAX_HISTOGRAM_BOUND; -use databend_common_base::runtime::ThreadTracker; use databend_common_exception::Result; fn assert_contain_metric(samples: Vec, expected: MetricSample) { diff --git a/src/common/base/tests/it/pool.rs b/src/common/base/tests/it/pool.rs index 50490cb2972b5..6ac8fbfec347f 100644 --- a/src/common/base/tests/it/pool.rs +++ b/src/common/base/tests/it/pool.rs @@ -15,8 +15,8 @@ use std::time::Duration; use async_trait::async_trait; -use databend_common_base::base::tokio; use databend_common_base::base::GlobalSequence; +use databend_common_base::base::tokio; use databend_common_base::containers::ItemManager; use databend_common_base::containers::Pool; use log::info; diff --git a/src/common/base/tests/it/time_series/profile.rs b/src/common/base/tests/it/time_series/profile.rs index 8af6576b7bfdc..ffee0df9b6dff 100644 --- a/src/common/base/tests/it/time_series/profile.rs +++ b/src/common/base/tests/it/time_series/profile.rs @@ -3,11 +3,11 @@ use std::sync::atomic::Ordering::SeqCst; use std::thread; use std::time::Duration; -use databend_common_base::runtime::compress_time_point; use databend_common_base::runtime::ProfilePoints; use databend_common_base::runtime::QueryTimeSeriesProfile; use databend_common_base::runtime::TimeSeriesProfileName; use databend_common_base::runtime::TimeSeriesProfiles; +use databend_common_base::runtime::compress_time_point; // Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/src/common/building/src/git.rs b/src/common/building/src/git.rs index 6d560fbb22f0f..c05ad0c3f0081 100644 --- a/src/common/building/src/git.rs +++ b/src/common/building/src/git.rs @@ -16,8 +16,8 @@ use std::collections::BTreeMap; use anyhow::Context; use anyhow::Result; -use gix::commit::describe::SelectRef; use gix::Repository; +use gix::commit::describe::SelectRef; // Get the latest tag: // git describe --tags --abbrev=0 diff --git a/src/common/building/src/license.rs b/src/common/building/src/license.rs index 6007cc8d85a33..d64e58187c912 100644 --- a/src/common/building/src/license.rs +++ b/src/common/building/src/license.rs @@ -16,7 +16,6 @@ use std::collections::HashMap; use std::collections::HashSet; use anyhow::Result; -use cargo_metadata::semver::Version; use cargo_metadata::DepKindInfo; use cargo_metadata::DependencyKind; use cargo_metadata::Metadata; @@ -25,6 +24,7 @@ use cargo_metadata::Node; use cargo_metadata::NodeDep; use cargo_metadata::Package; use cargo_metadata::PackageId; +use cargo_metadata::semver::Version; fn normalize(license_string: &str) -> String { let mut list: Vec<&str> = license_string diff --git a/src/common/cache/src/cache/lru.rs b/src/common/cache/src/cache/lru.rs index d9b8c7c46865f..bccc699ed320f 100644 --- a/src/common/cache/src/cache/lru.rs +++ b/src/common/cache/src/cache/lru.rs @@ -52,8 +52,8 @@ use std::borrow::Borrow; use std::fmt; use std::hash::Hash; -use hashlink::linked_hash_map; use hashlink::LinkedHashMap; +use hashlink::linked_hash_map; use crate::cache::Cache; use crate::mem_sized::MemSized; diff --git a/src/common/cache/src/lib.rs b/src/common/cache/src/lib.rs index 52fc8f3a9a39b..4ab8895a6c7da 100644 --- a/src/common/cache/src/lib.rs +++ b/src/common/cache/src/lib.rs @@ -18,7 +18,7 @@ mod cache; mod mem_sized; -pub use cache::lru::LruCache; pub use cache::Cache; +pub use cache::lru::LruCache; pub use hashbrown::DefaultHashBuilder; pub use mem_sized::MemSized; diff --git a/src/common/cloud_control/build.rs b/src/common/cloud_control/build.rs index f97a402f68c3f..23a76f501109a 100644 --- a/src/common/cloud_control/build.rs +++ b/src/common/cloud_control/build.rs @@ -71,11 +71,9 @@ fn build_proto() -> Result<()> { // Version before 3.12 doesn't support allow_proto3_optional if version < Version::new(3, 12, 0) { - return Err(Error::other( - format!( - "protoc version is outdated, expect: >= 3.12.0, actual: {version}, reason: need feature --experimental_allow_proto3_optional" - ), - )); + return Err(Error::other(format!( + "protoc version is outdated, expect: >= 3.12.0, actual: {version}, reason: need feature --experimental_allow_proto3_optional" + ))); } // allow_proto3_optional has been enabled by default since 3.15.0 if version < Version::new(3, 15, 0) { diff --git a/src/common/cloud_control/src/notification_client.rs b/src/common/cloud_control/src/notification_client.rs index 3490eae6bf3d1..21677ad60d7d0 100644 --- a/src/common/cloud_control/src/notification_client.rs +++ b/src/common/cloud_control/src/notification_client.rs @@ -14,10 +14,9 @@ use std::sync::Arc; -use tonic::transport::Channel; use tonic::Request; +use tonic::transport::Channel; -use crate::pb::notification_service_client::NotificationServiceClient; use crate::pb::CreateNotificationRequest; use crate::pb::CreateNotificationResponse; use crate::pb::DropNotificationRequest; @@ -26,6 +25,7 @@ use crate::pb::GetNotificationRequest; use crate::pb::GetNotificationResponse; use crate::pb::ListNotificationRequest; use crate::pb::ListNotificationResponse; +use crate::pb::notification_service_client::NotificationServiceClient; pub(crate) const NOTIFICATION_CLIENT_VERSION: &str = "v1"; pub(crate) const NOTIFICATION_CLIENT_VERSION_NAME: &str = "NOTIFICATION_CLIENT_VERSION"; diff --git a/src/common/cloud_control/src/task_client.rs b/src/common/cloud_control/src/task_client.rs index df7b7c7f84653..d917ba1fced43 100644 --- a/src/common/cloud_control/src/task_client.rs +++ b/src/common/cloud_control/src/task_client.rs @@ -15,12 +15,11 @@ use std::sync::Arc; use databend_common_exception::Result; -use tonic::transport::Channel; use tonic::Request; +use tonic::transport::Channel; -use crate::client_config::make_request; use crate::client_config::ClientConfig; -use crate::pb::task_service_client::TaskServiceClient; +use crate::client_config::make_request; use crate::pb::AlterTaskRequest; use crate::pb::AlterTaskResponse; use crate::pb::CreateTaskRequest; @@ -33,6 +32,7 @@ use crate::pb::ExecuteTaskRequest; use crate::pb::ExecuteTaskResponse; use crate::pb::ShowTasksRequest; use crate::pb::ShowTasksResponse; +use crate::pb::task_service_client::TaskServiceClient; pub(crate) const TASK_CLIENT_VERSION: &str = "v1"; pub(crate) const TASK_CLIENT_VERSION_NAME: &str = "TASK_CLIENT_VERSION"; diff --git a/src/common/cloud_control/src/task_utils.rs b/src/common/cloud_control/src/task_utils.rs index 312553230d447..190052d703487 100644 --- a/src/common/cloud_control/src/task_utils.rs +++ b/src/common/cloud_control/src/task_utils.rs @@ -21,9 +21,9 @@ use chrono::Utc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use crate::pb::schedule_options::ScheduleType; use crate::pb::ScheduleOptions; use crate::pb::WarehouseOptions; +use crate::pb::schedule_options::ScheduleType; #[derive(Debug, Clone, PartialEq)] pub enum Status { diff --git a/src/common/cloud_control/tests/it/task_client.rs b/src/common/cloud_control/tests/it/task_client.rs index 46f996b409286..8aa627e621007 100644 --- a/src/common/cloud_control/tests/it/task_client.rs +++ b/src/common/cloud_control/tests/it/task_client.rs @@ -14,10 +14,6 @@ use databend_common_base::base::tokio; use databend_common_base::runtime; -use databend_common_cloud_control::pb::task::Status::Suspended; -use databend_common_cloud_control::pb::task_service_client::TaskServiceClient; -use databend_common_cloud_control::pb::task_service_server::TaskService; -use databend_common_cloud_control::pb::task_service_server::TaskServiceServer; use databend_common_cloud_control::pb::AlterTaskRequest; use databend_common_cloud_control::pb::AlterTaskResponse; use databend_common_cloud_control::pb::CreateTaskRequest; @@ -37,15 +33,19 @@ use databend_common_cloud_control::pb::ShowTaskRunsResponse; use databend_common_cloud_control::pb::ShowTasksRequest; use databend_common_cloud_control::pb::ShowTasksResponse; use databend_common_cloud_control::pb::Task; +use databend_common_cloud_control::pb::task::Status::Suspended; +use databend_common_cloud_control::pb::task_service_client::TaskServiceClient; +use databend_common_cloud_control::pb::task_service_server::TaskService; +use databend_common_cloud_control::pb::task_service_server::TaskServiceServer; use databend_common_exception::Result; use hyper_util::rt::TokioIo; +use tonic::Request; +use tonic::Response; +use tonic::Status; use tonic::codegen::tokio_stream; use tonic::transport::Endpoint; use tonic::transport::Server; use tonic::transport::Uri; -use tonic::Request; -use tonic::Response; -use tonic::Status; use tower::service_fn; #[derive(Default)] diff --git a/src/common/column/src/binary/builder.rs b/src/common/column/src/binary/builder.rs index 40e7ab51da3c9..bb371ac658d91 100644 --- a/src/common/column/src/binary/builder.rs +++ b/src/common/column/src/binary/builder.rs @@ -178,11 +178,13 @@ impl BinaryColumnBuilder { /// /// Calling this method with an out-of-bounds index is *[undefined behavior]* pub unsafe fn index_unchecked(&self, row: usize) -> &[u8] { - debug_assert!(row + 1 < self.offsets.len()); + unsafe { + debug_assert!(row + 1 < self.offsets.len()); - let start = *self.offsets.get_unchecked(row) as usize; - let end = *self.offsets.get_unchecked(row + 1) as usize; - self.data.get_unchecked(start..end) + let start = *self.offsets.get_unchecked(row) as usize; + let end = *self.offsets.get_unchecked(row + 1) as usize; + self.data.get_unchecked(start..end) + } } pub fn push_repeat(&mut self, item: &[u8], n: usize) { diff --git a/src/common/column/src/binary/iterator.rs b/src/common/column/src/binary/iterator.rs index 74a35f2ce6f72..c0c78ad7ffab8 100644 --- a/src/common/column/src/binary/iterator.rs +++ b/src/common/column/src/binary/iterator.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::builder::BinaryColumnBuilder; use super::BinaryColumn; +use super::builder::BinaryColumnBuilder; use crate::iterator::ColumnAccessor; use crate::iterator::ColumnValuesIter; @@ -22,7 +22,7 @@ unsafe impl<'a> ColumnAccessor<'a> for BinaryColumn { #[inline] unsafe fn value_unchecked(&'a self, index: usize) -> Self::Item { - self.index_unchecked(index) + unsafe { self.index_unchecked(index) } } #[inline] @@ -48,7 +48,7 @@ unsafe impl<'a> ColumnAccessor<'a> for BinaryColumnBuilder { #[inline] unsafe fn value_unchecked(&'a self, index: usize) -> Self::Item { - self.index_unchecked(index) + unsafe { self.index_unchecked(index) } } #[inline] diff --git a/src/common/column/src/binary/mod.rs b/src/common/column/src/binary/mod.rs index 7592831dc1270..9de550aa68c86 100644 --- a/src/common/column/src/binary/mod.rs +++ b/src/common/column/src/binary/mod.rs @@ -26,9 +26,9 @@ pub use builder::BinaryColumnBuilder; pub use iterator::BinaryColumnBuilderIter; pub use iterator::BinaryColumnIter; +use crate::bitmap::Bitmap; use crate::bitmap::utils::BitmapIter; use crate::bitmap::utils::ZipValidity; -use crate::bitmap::Bitmap; use crate::buffer::Buffer; use crate::error::Error; use crate::error::Result; @@ -89,15 +89,17 @@ impl BinaryColumn { /// Calling this method with an out-of-bounds index is *[undefined behavior]* #[inline] pub unsafe fn index_unchecked(&self, index: usize) -> &[u8] { - let start = *self.offsets.get_unchecked(index) as usize; - let end = *self.offsets.get_unchecked(index + 1) as usize; - // here we use checked slice to avoid UB - // Less regressed perfs: - // bench_kernels/binary_sum_len_unchecked/20 - // time: [45.234 µs 45.278 µs 45.312 µs] - // change: [+1.4430% +1.5796% +1.7344%] (p = 0.00 < 0.05) - // Performance has regressed. - &self.data[start..end] + unsafe { + let start = *self.offsets.get_unchecked(index) as usize; + let end = *self.offsets.get_unchecked(index + 1) as usize; + // here we use checked slice to avoid UB + // Less regressed perfs: + // bench_kernels/binary_sum_len_unchecked/20 + // time: [45.234 µs 45.278 µs 45.312 µs] + // change: [+1.4430% +1.5796% +1.7344%] (p = 0.00 < 0.05) + // Performance has regressed. + &self.data[start..end] + } } pub fn slice(&self, range: Range) -> Self { @@ -111,7 +113,7 @@ impl BinaryColumn { } } - pub fn iter(&self) -> BinaryColumnIter { + pub fn iter(&self) -> BinaryColumnIter<'_> { BinaryColumnIter::new(self) } diff --git a/src/common/column/src/binview/builder.rs b/src/common/column/src/binview/builder.rs index 77fa758d8fe11..8518d56713a9d 100644 --- a/src/common/column/src/binview/builder.rs +++ b/src/common/column/src/binview/builder.rs @@ -20,11 +20,11 @@ use std::sync::Arc; use super::view::CheckUTF8; use crate::binary::BinaryColumn; -use crate::binview::iterator::BinaryViewBuilderIter; -use crate::binview::view::validate_utf8_only; use crate::binview::BinaryViewColumnGeneric; use crate::binview::View; use crate::binview::ViewType; +use crate::binview::iterator::BinaryViewBuilderIter; +use crate::binview::view::validate_utf8_only; use crate::buffer::Buffer; use crate::error::Result; use crate::types::NativeType; @@ -133,16 +133,18 @@ impl BinaryViewColumnBuilder { /// - caller must ensure the view and buffers match. #[inline] pub(crate) unsafe fn push_view_unchecked(&mut self, v: View, buffers: &[Buffer]) { - let len = v.length; - if len <= 12 { - self.total_bytes_len += len as usize; - self.views.push(v) - } else { - let data = buffers.get_unchecked(v.buffer_idx as usize); - let offset = v.offset as usize; - let bytes = data.get_unchecked(offset..offset + len as usize); - let t = T::from_bytes_unchecked(bytes); - self.push_value(t) + unsafe { + let len = v.length; + if len <= 12 { + self.total_bytes_len += len as usize; + self.views.push(v) + } else { + let data = buffers.get_unchecked(v.buffer_idx as usize); + let offset = v.offset as usize; + let bytes = data.get_unchecked(offset..offset + len as usize); + let t = T::from_bytes_unchecked(bytes); + self.push_value(t) + } } } @@ -296,40 +298,42 @@ impl BinaryViewColumnBuilder { /// Assumes that the `i < self.len`. #[inline] pub unsafe fn value_unchecked(&self, i: usize) -> &T { - let v = *self.views.get_unchecked(i); - let len = v.length; + unsafe { + let v = *self.views.get_unchecked(i); + let len = v.length; + + // view layout: + // for no-inlined layout: + // length: 4 bytes + // prefix: 4 bytes + // buffer_index: 4 bytes + // offset: 4 bytes + + // for inlined layout: + // length: 4 bytes + // data: 12 bytes + let bytes = if len <= 12 { + let ptr = self.views.as_ptr() as *const u8; + std::slice::from_raw_parts(ptr.add(i * 16 + 4), len as usize) + } else { + let buffer_idx = v.buffer_idx as usize; + let offset = v.offset; - // view layout: - // for no-inlined layout: - // length: 4 bytes - // prefix: 4 bytes - // buffer_index: 4 bytes - // offset: 4 bytes - - // for inlined layout: - // length: 4 bytes - // data: 12 bytes - let bytes = if len <= 12 { - let ptr = self.views.as_ptr() as *const u8; - std::slice::from_raw_parts(ptr.add(i * 16 + 4), len as usize) - } else { - let buffer_idx = v.buffer_idx as usize; - let offset = v.offset; + let data = if buffer_idx == self.completed_buffers.len() { + self.in_progress_buffer.as_slice() + } else { + self.completed_buffers.get_unchecked(buffer_idx) + }; - let data = if buffer_idx == self.completed_buffers.len() { - self.in_progress_buffer.as_slice() - } else { - self.completed_buffers.get_unchecked(buffer_idx) + let offset = offset as usize; + data.get_unchecked(offset..offset + len as usize) }; - - let offset = offset as usize; - data.get_unchecked(offset..offset + len as usize) - }; - T::from_bytes_unchecked(bytes) + T::from_bytes_unchecked(bytes) + } } /// Returns an iterator of `&[u8]` over every element of this array - pub fn iter(&self) -> BinaryViewBuilderIter { + pub fn iter(&self) -> BinaryViewBuilderIter<'_, T> { BinaryViewBuilderIter::new(self) } diff --git a/src/common/column/src/binview/iterator.rs b/src/common/column/src/binview/iterator.rs index 874d0e15ce072..123847a82b459 100644 --- a/src/common/column/src/binview/iterator.rs +++ b/src/common/column/src/binview/iterator.rs @@ -13,8 +13,8 @@ // limitations under the License. use super::BinaryViewColumnGeneric; -use crate::binview::builder::BinaryViewColumnBuilder; use crate::binview::ViewType; +use crate::binview::builder::BinaryViewColumnBuilder; use crate::iterator::ColumnAccessor; use crate::iterator::ColumnValuesIter; @@ -23,7 +23,7 @@ unsafe impl<'a, T: ViewType + ?Sized> ColumnAccessor<'a> for BinaryViewColumnGen #[inline] unsafe fn value_unchecked(&'a self, index: usize) -> Self::Item { - self.value_unchecked(index) + unsafe { self.value_unchecked(index) } } #[inline] @@ -49,7 +49,7 @@ unsafe impl<'a, T: ViewType + ?Sized> ColumnAccessor<'a> for BinaryViewColumnBui #[inline] unsafe fn value_unchecked(&'a self, index: usize) -> Self::Item { - self.value_unchecked(index) + unsafe { self.value_unchecked(index) } } #[inline] diff --git a/src/common/column/src/binview/mod.rs b/src/common/column/src/binview/mod.rs index 609f696933cfe..a9fa2abcc0374 100644 --- a/src/common/column/src/binview/mod.rs +++ b/src/common/column/src/binview/mod.rs @@ -31,15 +31,15 @@ pub use builder::BinaryViewColumnBuilder; use either::Either; pub use iterator::BinaryViewColumnIter; use private::Sealed; -use view::validate_utf8_only; pub use view::CheckUTF8; pub use view::View; +use view::validate_utf8_only; use crate::binary::BinaryColumn; use crate::binary::BinaryColumnBuilder; +use crate::bitmap::Bitmap; use crate::bitmap::utils::BitmapIter; use crate::bitmap::utils::ZipValidity; -use crate::bitmap::Bitmap; use crate::buffer::Buffer; use crate::error::Result; use crate::impl_sliced; @@ -79,7 +79,7 @@ impl ViewType for str { #[inline(always)] unsafe fn from_bytes_unchecked(slice: &[u8]) -> &Self { - std::str::from_utf8_unchecked(slice) + unsafe { std::str::from_utf8_unchecked(slice) } } #[inline(always)] @@ -255,8 +255,10 @@ impl BinaryViewColumnGeneric { /// Assumes that the `i < self.len`. #[inline] pub unsafe fn value_unchecked(&self, i: usize) -> &T { - let v = self.views.get_unchecked(i); - T::from_bytes_unchecked(v.get_slice_unchecked(&self.buffers)) + unsafe { + let v = self.views.get_unchecked(i); + T::from_bytes_unchecked(v.get_slice_unchecked(&self.buffers)) + } } /// same as value_unchecked @@ -264,8 +266,10 @@ impl BinaryViewColumnGeneric { /// Assumes that the `i < self.len`. #[inline] pub unsafe fn index_unchecked(&self, i: usize) -> &T { - let v = self.views.get_unchecked(i); - T::from_bytes_unchecked(v.get_slice_unchecked(&self.buffers)) + unsafe { + let v = self.views.get_unchecked(i); + T::from_bytes_unchecked(v.get_slice_unchecked(&self.buffers)) + } } /// same as value_unchecked, yet it will return bytes @@ -273,12 +277,14 @@ impl BinaryViewColumnGeneric { /// Assumes that the `i < self.len`. #[inline] pub unsafe fn index_unchecked_bytes(&self, i: usize) -> &[u8] { - let v = self.views.get_unchecked(i); - v.get_slice_unchecked(&self.buffers) + unsafe { + let v = self.views.get_unchecked(i); + v.get_slice_unchecked(&self.buffers) + } } /// Returns an iterator of `&[u8]` over every element of this array, ignoring the validity - pub fn iter(&self) -> BinaryViewColumnIter { + pub fn iter(&self) -> BinaryViewColumnIter<'_, T> { BinaryViewColumnIter::new(self) } @@ -419,9 +425,11 @@ impl BinaryViewColumnGeneric { } unsafe fn slice_unchecked(&mut self, offset: usize, length: usize) { - debug_assert!(offset + length <= self.len()); - self.views.slice_unchecked(offset, length); - self.total_bytes_len = OnceLock::new(); + unsafe { + debug_assert!(offset + length <= self.len()); + self.views.slice_unchecked(offset, length); + self.total_bytes_len = OnceLock::new(); + } } impl_sliced!(); diff --git a/src/common/column/src/binview/view.rs b/src/common/column/src/binview/view.rs index 2b1c881965957..b9d844576dd66 100644 --- a/src/common/column/src/binview/view.rs +++ b/src/common/column/src/binview/view.rs @@ -273,23 +273,25 @@ pub(super) fn validate_utf8_view(views: &[View], buffers: &[Buffer]) -> Resu /// # Safety /// The views and buffers must uphold the invariants of BinaryView otherwise we will go OOB. pub(super) unsafe fn validate_utf8_only(views: &[View], buffers: &[Buffer]) -> Result<()> { - for view in views { - let len = view.length; - if len <= 12 { - validate_utf8(view.to_le_bytes().get_unchecked(4..4 + len as usize))?; - } else { - let buffer_idx = view.buffer_idx; - let offset = view.offset; - let data = buffers.get_unchecked(buffer_idx as usize); + unsafe { + for view in views { + let len = view.length; + if len <= 12 { + validate_utf8(view.to_le_bytes().get_unchecked(4..4 + len as usize))?; + } else { + let buffer_idx = view.buffer_idx; + let offset = view.offset; + let data = buffers.get_unchecked(buffer_idx as usize); + + let start = offset as usize; + let end = start + len as usize; + let b = &data.as_slice().get_unchecked(start..end); + validate_utf8(b)?; + }; + } - let start = offset as usize; - let end = start + len as usize; - let b = &data.as_slice().get_unchecked(start..end); - validate_utf8(b)?; - }; + Ok(()) } - - Ok(()) } pub trait CheckUTF8 { diff --git a/src/common/column/src/bitmap/bitmap_ops.rs b/src/common/column/src/bitmap/bitmap_ops.rs index 5a18ae9186dc3..f055bae91ffc2 100644 --- a/src/common/column/src/bitmap/bitmap_ops.rs +++ b/src/common/column/src/bitmap/bitmap_ops.rs @@ -19,10 +19,10 @@ use std::ops::BitOr; use std::ops::BitXor; use std::ops::Not; +use super::Bitmap; use super::utils::BitChunk; use super::utils::BitChunkIterExact; use super::utils::BitChunksExact; -use super::Bitmap; use crate::bitmap::MutableBitmap; /// Creates a [Vec] from an [`Iterator`] of [`BitChunk`]. @@ -31,27 +31,29 @@ use crate::bitmap::MutableBitmap; pub unsafe fn from_chunk_iter_unchecked>( iterator: I, ) -> Vec { - let (_, upper) = iterator.size_hint(); - let upper = upper.expect("try_from_trusted_len_iter requires an upper limit"); - let len = upper * std::mem::size_of::(); - - let mut buffer = Vec::with_capacity(len); - - let mut dst = buffer.as_mut_ptr(); - for item in iterator { - let bytes = item.to_ne_bytes(); - for i in 0..std::mem::size_of::() { - std::ptr::write(dst, bytes[i]); - dst = dst.add(1); + unsafe { + let (_, upper) = iterator.size_hint(); + let upper = upper.expect("try_from_trusted_len_iter requires an upper limit"); + let len = upper * std::mem::size_of::(); + + let mut buffer = Vec::with_capacity(len); + + let mut dst = buffer.as_mut_ptr(); + for item in iterator { + let bytes = item.to_ne_bytes(); + for i in 0..std::mem::size_of::() { + std::ptr::write(dst, bytes[i]); + dst = dst.add(1); + } } + assert_eq!( + dst.offset_from(buffer.as_ptr()) as usize, + len, + "Trusted iterator length was not accurately reported" + ); + buffer.set_len(len); + buffer } - assert_eq!( - dst.offset_from(buffer.as_ptr()) as usize, - len, - "Trusted iterator length was not accurately reported" - ); - buffer.set_len(len); - buffer } /// Creates a [`Vec`] from a [`TrustedLen`] of [`BitChunk`]. diff --git a/src/common/column/src/bitmap/immutable.rs b/src/common/column/src/bitmap/immutable.rs index 72b3deab7d4d0..00fbb2a0851dc 100644 --- a/src/common/column/src/bitmap/immutable.rs +++ b/src/common/column/src/bitmap/immutable.rs @@ -22,17 +22,17 @@ use arrow_data::ArrayData; use arrow_data::ArrayDataBuilder; use either::Either; +use super::IntoIter; +use super::MutableBitmap; use super::chunk_iter_to_vec; -use super::utils::count_zeros; -use super::utils::fmt; -use super::utils::get_bit; -use super::utils::get_bit_unchecked; use super::utils::BitChunk; use super::utils::BitChunks; use super::utils::BitmapIter; use super::utils::ZipValidity; -use super::IntoIter; -use super::MutableBitmap; +use super::utils::count_zeros; +use super::utils::fmt; +use super::utils::get_bit; +use super::utils::get_bit_unchecked; use crate::buffer::Bytes; use crate::error::Error; @@ -143,7 +143,7 @@ impl Bitmap { } /// Returns a new iterator of `bool` over this bitmap - pub fn iter(&self) -> BitmapIter { + pub fn iter(&self) -> BitmapIter<'_> { BitmapIter::new(&self.bytes, self.offset, self.length) } @@ -158,7 +158,7 @@ impl Bitmap { /// Returns an iterator over bits in bit chunks [`BitChunk`]. /// /// This iterator is useful to operate over multiple bits via e.g. bitwise. - pub fn chunks(&self) -> BitChunks { + pub fn chunks(&self) -> BitChunks<'_, T> { BitChunks::new(&self.bytes, self.offset, self.length) } @@ -249,8 +249,10 @@ impl Bitmap { #[inline] #[must_use] pub unsafe fn sliced_unchecked(mut self, offset: usize, length: usize) -> Self { - self.slice_unchecked(offset, length); - self + unsafe { + self.slice_unchecked(offset, length); + self + } } /// Returns whether the bit at position `i` is set. @@ -266,7 +268,7 @@ impl Bitmap { /// Unsound iff `i >= self.len()`. #[inline] pub unsafe fn get_bit_unchecked(&self, i: usize) -> bool { - get_bit_unchecked(&self.bytes, self.offset + i) + unsafe { get_bit_unchecked(&self.bytes, self.offset + i) } } /// Returns a pointer to the start of this [`Bitmap`] (ignores `offsets`) @@ -455,7 +457,7 @@ impl Bitmap { /// The iterator must report an accurate length. #[inline] pub unsafe fn from_trusted_len_iter_unchecked>(iterator: I) -> Self { - MutableBitmap::from_trusted_len_iter_unchecked(iterator).into() + unsafe { MutableBitmap::from_trusted_len_iter_unchecked(iterator).into() } } /// Creates a new [`Bitmap`] from an iterator of booleans. diff --git a/src/common/column/src/bitmap/mutable.rs b/src/common/column/src/bitmap/mutable.rs index 8015f06b36277..666b524f061d3 100644 --- a/src/common/column/src/bitmap/mutable.rs +++ b/src/common/column/src/bitmap/mutable.rs @@ -21,15 +21,15 @@ use std::sync::Arc; use databend_common_base::vec_ext::VecExt; +use super::Bitmap; +use super::utils::BitChunk; +use super::utils::BitChunksExactMut; +use super::utils::BitmapIter; use super::utils::count_zeros; use super::utils::fmt; use super::utils::get_bit; use super::utils::set; use super::utils::set_bit; -use super::utils::BitChunk; -use super::utils::BitChunksExactMut; -use super::utils::BitmapIter; -use super::Bitmap; use crate::bitmap::utils::merge_reversed; use crate::bitmap::utils::set_bit_unchecked; use crate::error::Error; @@ -137,7 +137,7 @@ impl MutableBitmap { /// Pushes a new bit to the [`MutableBitmap`], re-sizing it if necessary. #[inline] pub fn push(&mut self, value: bool) { - if self.length % 8 == 0 { + if self.length.is_multiple_of(8) { self.buffer.push(0); } let byte = self.buffer.as_mut_slice().last_mut().unwrap(); @@ -155,7 +155,7 @@ impl MutableBitmap { self.length -= 1; let value = self.get(self.length); - if self.length % 8 == 0 { + if self.length.is_multiple_of(8) { self.buffer.pop(); } Some(value) @@ -178,7 +178,7 @@ impl MutableBitmap { } /// constructs a new iterator over the bits of [`MutableBitmap`]. - pub fn iter(&self) -> BitmapIter { + pub fn iter(&self) -> BitmapIter<'_> { BitmapIter::new(&self.buffer, 0, self.length) } @@ -267,12 +267,14 @@ impl MutableBitmap { /// The caller must ensure that the [`MutableBitmap`] has sufficient capacity. #[inline] pub unsafe fn push_unchecked(&mut self, value: bool) { - if self.length % 8 == 0 { - self.buffer.push_unchecked(0); + unsafe { + if self.length.is_multiple_of(8) { + self.buffer.push_unchecked(0); + } + let byte = self.buffer.as_mut_slice().last_mut().unwrap(); + *byte = set(*byte, self.length % 8, value); + self.length += 1; } - let byte = self.buffer.as_mut_slice().last_mut().unwrap(); - *byte = set(*byte, self.length % 8, value); - self.length += 1; } /// Returns the number of unset bits on this [`MutableBitmap`]. @@ -296,8 +298,10 @@ impl MutableBitmap { /// The caller must ensure that the [`MutableBitmap`] was properly initialized up to `len`. #[inline] pub(crate) unsafe fn set_len(&mut self, len: usize) { - self.buffer.set_len(len.saturating_add(7) / 8); - self.length = len; + unsafe { + self.buffer.set_len(len.saturating_add(7) / 8); + self.length = len; + } } fn extend_set(&mut self, mut additional: usize) { @@ -354,7 +358,7 @@ impl MutableBitmap { /// Caller must ensure that `index < self.len()` #[inline] pub unsafe fn set_unchecked(&mut self, index: usize, value: bool) { - set_bit_unchecked(self.buffer.as_mut_slice(), index, value) + unsafe { set_bit_unchecked(self.buffer.as_mut_slice(), index, value) } } /// Shrinks the capacity of the [`MutableBitmap`] to fit its current length. @@ -363,7 +367,7 @@ impl MutableBitmap { } /// Returns an iterator over mutable slices, [`BitChunksExactMut`] - pub(crate) fn bitchunks_exact_mut(&mut self) -> BitChunksExactMut { + pub(crate) fn bitchunks_exact_mut(&mut self) -> BitChunksExactMut<'_, T> { BitChunksExactMut::new(&mut self.buffer, self.length) } @@ -505,37 +509,39 @@ unsafe fn extend_aligned_trusted_iter_unchecked( buffer: &mut Vec, mut iterator: impl Iterator, ) -> usize { - let additional_bits = iterator.size_hint().1.unwrap(); - let chunks = additional_bits / 64; - let remainder = additional_bits % 64; - - let additional = additional_bits.div_ceil(8); - debug_assert_eq!( - additional, - // a hint of how the following calculation will be done - chunks * 8 + remainder / 8 + (remainder % 8 > 0) as usize - ); - buffer.reserve(additional); + unsafe { + let additional_bits = iterator.size_hint().1.unwrap(); + let chunks = additional_bits / 64; + let remainder = additional_bits % 64; + + let additional = additional_bits.div_ceil(8); + debug_assert_eq!( + additional, + // a hint of how the following calculation will be done + chunks * 8 + remainder / 8 + !remainder.is_multiple_of(8) as usize + ); + buffer.reserve(additional); - // chunks of 64 bits - for _ in 0..chunks { - let chunk = get_chunk_unchecked(&mut iterator); - buffer.extend_from_slice_unchecked(&chunk.to_le_bytes()); - } + // chunks of 64 bits + for _ in 0..chunks { + let chunk = get_chunk_unchecked(&mut iterator); + buffer.extend_from_slice_unchecked(&chunk.to_le_bytes()); + } - // remaining complete bytes - for _ in 0..(remainder / 8) { - let byte = unsafe { get_byte_unchecked(8, &mut iterator) }; - buffer.push_unchecked(byte) - } + // remaining complete bytes + for _ in 0..(remainder / 8) { + let byte = get_byte_unchecked(8, &mut iterator); + buffer.push_unchecked(byte) + } - // remaining bits - let remainder = remainder % 8; - if remainder > 0 { - let byte = unsafe { get_byte_unchecked(remainder, &mut iterator) }; - buffer.push_unchecked(byte) + // remaining bits + let remainder = remainder % 8; + if remainder > 0 { + let byte = get_byte_unchecked(remainder, &mut iterator); + buffer.push_unchecked(byte) + } + additional_bits } - additional_bits } impl MutableBitmap { @@ -604,11 +610,13 @@ impl MutableBitmap { #[inline] pub unsafe fn from_trusted_len_iter_unchecked(iterator: I) -> Self where I: Iterator { - let mut buffer = Vec::::new(); + unsafe { + let mut buffer = Vec::::new(); - let length = extend_aligned_trusted_iter_unchecked(&mut buffer, iterator); + let length = extend_aligned_trusted_iter_unchecked(&mut buffer, iterator); - Self { buffer, length } + Self { buffer, length } + } } /// Creates a new [`MutableBitmap`] from an iterator of booleans. @@ -684,8 +692,8 @@ impl MutableBitmap { if length == 0 { return; }; - let is_aligned = self.length % 8 == 0; - let other_is_aligned = offset % 8 == 0; + let is_aligned = self.length.is_multiple_of(8); + let other_is_aligned = offset.is_multiple_of(8); match (is_aligned, other_is_aligned) { (true, true) => self.extend_aligned(slice, offset, length), (false, true) => self.extend_unaligned(slice, offset, length), diff --git a/src/common/column/src/bitmap/utils/mod.rs b/src/common/column/src/bitmap/utils/mod.rs index 8ecc575d839eb..ff6a69fc9ac61 100644 --- a/src/common/column/src/bitmap/utils/mod.rs +++ b/src/common/column/src/bitmap/utils/mod.rs @@ -23,11 +23,11 @@ mod zip_validity; use std::convert::TryInto; -pub(crate) use chunk_iterator::merge_reversed; pub use chunk_iterator::BitChunk; pub use chunk_iterator::BitChunkIterExact; pub use chunk_iterator::BitChunks; pub use chunk_iterator::BitChunksExact; +pub(crate) use chunk_iterator::merge_reversed; pub use chunks_exact_mut::BitChunksExactMut; pub use fmt::fmt; pub use iterator::BitmapIter; @@ -76,8 +76,10 @@ pub fn set_bit(data: &mut [u8], i: usize, value: bool) { /// caller must ensure that `i / 8 < data.len()` #[inline] pub unsafe fn set_bit_unchecked(data: &mut [u8], i: usize, value: bool) { - let byte = data.get_unchecked_mut(i / 8); - *byte = set(*byte, i % 8, value); + unsafe { + let byte = data.get_unchecked_mut(i / 8); + *byte = set(*byte, i % 8, value); + } } /// Returns whether bit at position `i` in `data` is set @@ -94,7 +96,7 @@ pub fn get_bit(bytes: &[u8], i: usize) -> bool { /// `i / 8 >= data.len()` results in undefined behavior #[inline] pub unsafe fn get_bit_unchecked(data: &[u8], i: usize) -> bool { - (*data.as_ptr().add(i >> 3) & BIT_MASK[i & 7]) != 0 + unsafe { (*data.as_ptr().add(i >> 3) & BIT_MASK[i & 7]) != 0 } } /// Returns the number of bytes required to hold `bits` bits. @@ -132,7 +134,7 @@ pub fn count_zeros(slice: &[u8], offset: usize, len: usize) -> usize { set_count += (slice[0] >> offset).count_ones() as usize; slice = &slice[1..]; } - if (offset + len) % 8 != 0 { + if !(offset + len).is_multiple_of(8) { let end_offset = (offset + len) % 8; // i.e. 3 + 4 = 7 let last_index = slice.len() - 1; // count all ignoring the last `offset` bits diff --git a/src/common/column/src/bitmap/utils/zip_validity.rs b/src/common/column/src/bitmap/utils/zip_validity.rs index a2150cc862c20..af44ff9a75904 100644 --- a/src/common/column/src/bitmap/utils/zip_validity.rs +++ b/src/common/column/src/bitmap/utils/zip_validity.rs @@ -15,8 +15,8 @@ use std::iter::TrustedLen; -use crate::bitmap::utils::BitmapIter; use crate::bitmap::Bitmap; +use crate::bitmap::utils::BitmapIter; /// An [`Iterator`] over validity and values. #[derive(Debug, Clone)] diff --git a/src/common/column/src/buffer/immutable.rs b/src/common/column/src/buffer/immutable.rs index 62c2a49117ecd..0e467ae1dd92c 100644 --- a/src/common/column/src/buffer/immutable.rs +++ b/src/common/column/src/buffer/immutable.rs @@ -24,9 +24,9 @@ use either::Either; use num_traits::Zero; use super::Bytes; +use crate::bitmap::Bitmap; use crate::bitmap::utils::BitmapIter; use crate::bitmap::utils::ZipValidity; -use crate::bitmap::Bitmap; use crate::types::NativeType; /// [`Buffer`] is a contiguous memory region that can be shared across @@ -174,8 +174,10 @@ impl Buffer { #[inline] #[must_use] pub unsafe fn sliced_unchecked(mut self, offset: usize, length: usize) -> Self { - self.slice_unchecked(offset, length); - self + unsafe { + self.slice_unchecked(offset, length); + self + } } /// Slices this buffer starting at `offset`. @@ -183,8 +185,10 @@ impl Buffer { /// The caller must ensure `offset + length <= self.len()` #[inline] pub unsafe fn slice_unchecked(&mut self, offset: usize, length: usize) { - self.ptr = self.ptr.add(offset); - self.length = length; + unsafe { + self.ptr = self.ptr.add(offset); + self.length = length; + } } /// Returns a pointer to the start of this buffer. @@ -266,8 +270,10 @@ impl Buffer { /// # Safety /// Callers must ensure all invariants of this struct are upheld. pub unsafe fn from_inner_unchecked(data: Arc>, offset: usize, length: usize) -> Self { - let ptr = data.as_ptr().add(offset); - Self { data, ptr, length } + unsafe { + let ptr = data.as_ptr().add(offset); + Self { data, ptr, length } + } } } diff --git a/src/common/column/src/buffer/mod.rs b/src/common/column/src/buffer/mod.rs index c76d93a027388..74ee264e428a1 100644 --- a/src/common/column/src/buffer/mod.rs +++ b/src/common/column/src/buffer/mod.rs @@ -20,8 +20,8 @@ mod iterator; use std::ops::Deref; -pub use immutable::buffer_to_array_data; pub use immutable::Buffer; +pub use immutable::buffer_to_array_data; #[allow(dead_code)] pub(crate) enum BytesAllocator { @@ -44,7 +44,7 @@ impl Bytes { /// the region `[ptr, ptr+length[` when dropped. #[inline] pub(crate) unsafe fn from_foreign(ptr: *const T, length: usize, owner: BytesAllocator) -> Self { - Self(BytesInner::from_foreign(ptr, length, owner)) + unsafe { Self(BytesInner::from_foreign(ptr, length, owner)) } } /// Returns a `Some` mutable reference of [`Vec`] iff this was initialized diff --git a/src/common/column/src/offset.rs b/src/common/column/src/offset.rs index ea9cba3382bd3..eaca26a0496fb 100644 --- a/src/common/column/src/offset.rs +++ b/src/common/column/src/offset.rs @@ -184,10 +184,12 @@ impl Offsets { /// `index` must be `< self.len()` #[inline] pub unsafe fn start_end_unchecked(&self, index: usize) -> (usize, usize) { - // soundness: the invariant of the function - let start = self.0.get_unchecked(index).to_usize(); - let end = self.0.get_unchecked(index + 1).to_usize(); - (start, end) + unsafe { + // soundness: the invariant of the function + let start = self.0.get_unchecked(index).to_usize(); + let end = self.0.get_unchecked(index + 1).to_usize(); + (start, end) + } } /// Returns the length an array with these offsets would be. @@ -469,10 +471,12 @@ impl OffsetsBuffer { /// `index` must be `< self.len()` #[inline] pub unsafe fn start_end_unchecked(&self, index: usize) -> (usize, usize) { - // soundness: the invariant of the function - let start = self.0.get_unchecked(index).to_usize(); - let end = self.0.get_unchecked(index + 1).to_usize(); - (start, end) + unsafe { + // soundness: the invariant of the function + let start = self.0.get_unchecked(index).to_usize(); + let end = self.0.get_unchecked(index + 1).to_usize(); + (start, end) + } } /// Slices this [`OffsetsBuffer`]. @@ -490,7 +494,9 @@ impl OffsetsBuffer { /// The caller must ensure `offset + length <= self.len()` #[inline] pub unsafe fn slice_unchecked(&mut self, offset: usize, length: usize) { - self.0.slice_unchecked(offset, length); + unsafe { + self.0.slice_unchecked(offset, length); + } } /// Returns an iterator with the lengths of the offsets diff --git a/src/common/column/src/types/native.rs b/src/common/column/src/types/native.rs index 74a4aa8b30295..b8d93bd2e4841 100644 --- a/src/common/column/src/types/native.rs +++ b/src/common/column/src/types/native.rs @@ -30,9 +30,9 @@ use borsh::BorshSerialize; use bytemuck::Pod; use bytemuck::Zeroable; use databend_common_base::base::OrderedFloat; +use jiff::Timestamp; use jiff::fmt::strtime; use jiff::tz; -use jiff::Timestamp; use log::error; use serde_derive::Deserialize; use serde_derive::Serialize; @@ -86,7 +86,7 @@ pub trait NativeType: } macro_rules! native_type { - ($type:ty, $primitive_type:expr) => { + ($type:ty, $primitive_type:expr_2021) => { impl NativeType for $type { const PRIMITIVE: PrimitiveType = $primitive_type; diff --git a/src/common/column/src/types/simd/mod.rs b/src/common/column/src/types/simd/mod.rs index 8fbe6e611609b..593dd75039207 100644 --- a/src/common/column/src/types/simd/mod.rs +++ b/src/common/column/src/types/simd/mod.rs @@ -16,13 +16,13 @@ //! Contains traits and implementations of multi-data used in SIMD. //! The actual representation is driven by the feature flag `"simd"`, which, if set, //! uses [`std::simd`]. +use super::BitChunk; +use super::BitChunkIter; +use super::NativeType; use super::days_ms; use super::f16; use super::i256; use super::months_days_micros; -use super::BitChunk; -use super::BitChunkIter; -use super::NativeType; /// Describes the ability to convert itself from a [`BitChunk`]. pub trait FromMaskChunk { @@ -79,7 +79,7 @@ mod packed; pub use packed::*; macro_rules! native_simd { - ($name:tt, $type:ty, $lanes:expr, $mask:ty) => { + ($name:tt, $type:ty, $lanes:expr_2021, $mask:ty) => { /// Multi-Data correspondence of the native type #[allow(non_camel_case_types)] #[derive(Copy, Clone)] @@ -144,8 +144,6 @@ macro_rules! native_simd { }; } -pub(super) use native_simd; - // Types do not have specific intrinsics and thus SIMD can't be specialized. // Therefore, we can declare their MD representation as `[$t; 8]` irrespectively // of how they are represented in the different channels. diff --git a/src/common/column/src/types/simd/packed.rs b/src/common/column/src/types/simd/packed.rs index 0b414678cbce0..d180b83c7729f 100644 --- a/src/common/column/src/types/simd/packed.rs +++ b/src/common/column/src/types/simd/packed.rs @@ -13,27 +13,27 @@ // See the License for the specific language governing permissions and // limitations under the License. -pub use std::simd::f32x16; pub use std::simd::f32x8; +pub use std::simd::f32x16; pub use std::simd::f64x8; -pub use std::simd::i16x32; +pub use std::simd::i8x8; +pub use std::simd::i8x64; pub use std::simd::i16x8; -pub use std::simd::i32x16; +pub use std::simd::i16x32; pub use std::simd::i32x8; +pub use std::simd::i32x16; pub use std::simd::i64x8; -pub use std::simd::i8x64; -pub use std::simd::i8x8; +pub use std::simd::mask8x64 as m8x64; pub use std::simd::mask32x16 as m32x16; pub use std::simd::mask64x8 as m64x8; -pub use std::simd::mask8x64 as m8x64; pub use std::simd::prelude::SimdPartialEq; -pub use std::simd::u16x32; +pub use std::simd::u8x8; +pub use std::simd::u8x64; pub use std::simd::u16x8; -pub use std::simd::u32x16; +pub use std::simd::u16x32; pub use std::simd::u32x8; +pub use std::simd::u32x16; pub use std::simd::u64x8; -pub use std::simd::u8x64; -pub use std::simd::u8x8; /// Vector of 32 16-bit masks #[allow(non_camel_case_types)] @@ -42,7 +42,7 @@ pub type m16x32 = std::simd::Mask; use super::*; macro_rules! simd { - ($name:tt, $type:ty, $lanes:expr, $chunk:ty, $mask:tt) => { + ($name:tt, $type:ty, $lanes:expr_2021, $chunk:ty, $mask:tt) => { unsafe impl NativeSimd for $name { const LANES: usize = $lanes; type Native = $type; @@ -86,7 +86,7 @@ simd!(f32x16, f32, 16, u16, m32x16); simd!(f64x8, f64, 8, u8, m64x8); macro_rules! chunk_macro { - ($type:ty, $chunk:ty, $simd:ty, $mask:tt, $m:expr) => { + ($type:ty, $chunk:ty, $simd:ty, $mask:tt, $m:expr_2021) => { impl FromMaskChunk<$chunk> for $mask { #[inline] fn from_chunk(chunk: $chunk) -> Self { diff --git a/src/common/column/src/utils.rs b/src/common/column/src/utils.rs index 99a771dc02cb7..95776d72b7bfa 100644 --- a/src/common/column/src/utils.rs +++ b/src/common/column/src/utils.rs @@ -39,8 +39,10 @@ macro_rules! impl_sliced { #[inline] #[must_use] pub unsafe fn sliced_unchecked(mut self, offset: usize, length: usize) -> Self { - self.slice_unchecked(offset, length); - self + unsafe { + self.slice_unchecked(offset, length); + self + } } }; } diff --git a/src/common/column/tests/it/bitmap/assign_ops.rs b/src/common/column/tests/it/bitmap/assign_ops.rs index 79fafa06f2246..e2705367376d3 100644 --- a/src/common/column/tests/it/bitmap/assign_ops.rs +++ b/src/common/column/tests/it/bitmap/assign_ops.rs @@ -13,10 +13,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_column::bitmap::binary_assign; -use databend_common_column::bitmap::unary_assign; use databend_common_column::bitmap::Bitmap; use databend_common_column::bitmap::MutableBitmap; +use databend_common_column::bitmap::binary_assign; +use databend_common_column::bitmap::unary_assign; use proptest::prelude::*; use crate::bitmap::bitmap_strategy; diff --git a/src/common/column/tests/it/bitmap/bitmap_ops.rs b/src/common/column/tests/it/bitmap/bitmap_ops.rs index 1b20810a7ace4..ef164886d6450 100644 --- a/src/common/column/tests/it/bitmap/bitmap_ops.rs +++ b/src/common/column/tests/it/bitmap/bitmap_ops.rs @@ -13,10 +13,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_column::bitmap::Bitmap; use databend_common_column::bitmap::and; use databend_common_column::bitmap::or; use databend_common_column::bitmap::xor; -use databend_common_column::bitmap::Bitmap; use proptest::prelude::*; use crate::bitmap::bitmap_strategy; diff --git a/src/common/column/tests/it/bitmap/utils/iterator.rs b/src/common/column/tests/it/bitmap/utils/iterator.rs index b06f506f699b7..9d0f33f676d96 100644 --- a/src/common/column/tests/it/bitmap/utils/iterator.rs +++ b/src/common/column/tests/it/bitmap/utils/iterator.rs @@ -50,9 +50,11 @@ fn rev() { let result = iter.rev().collect::>(); assert_eq!( result, - vec![false, true, true, false, true, false, true, true, false, true, true, false, true] - .into_iter() - .rev() - .collect::>() + vec![ + false, true, true, false, true, false, true, true, false, true, true, false, true + ] + .into_iter() + .rev() + .collect::>() ) } diff --git a/src/common/column/tests/it/bitmap/utils/slice_iterator.rs b/src/common/column/tests/it/bitmap/utils/slice_iterator.rs index a971c5bbea74c..25304fb80249d 100644 --- a/src/common/column/tests/it/bitmap/utils/slice_iterator.rs +++ b/src/common/column/tests/it/bitmap/utils/slice_iterator.rs @@ -13,8 +13,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_column::bitmap::utils::SlicesIterator; use databend_common_column::bitmap::Bitmap; +use databend_common_column::bitmap::utils::SlicesIterator; use proptest::prelude::*; use crate::bitmap::bitmap_strategy; diff --git a/src/common/column/tests/it/bitmap/utils/zip_validity.rs b/src/common/column/tests/it/bitmap/utils/zip_validity.rs index de7043ea5d5ea..30e74cc9ede32 100644 --- a/src/common/column/tests/it/bitmap/utils/zip_validity.rs +++ b/src/common/column/tests/it/bitmap/utils/zip_validity.rs @@ -13,9 +13,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_column::bitmap::Bitmap; use databend_common_column::bitmap::utils::BitmapIter; use databend_common_column::bitmap::utils::ZipValidity; -use databend_common_column::bitmap::Bitmap; #[test] fn basic() { diff --git a/src/common/compress/src/decode.rs b/src/common/compress/src/decode.rs index 32b0a31150d82..59c71bdf21562 100644 --- a/src/common/compress/src/decode.rs +++ b/src/common/compress/src/decode.rs @@ -32,10 +32,10 @@ use async_compression::util::PartialBuffer; use bytes::Buf; use bytes::BytesMut; use databend_common_exception::ErrorCode; -use futures::io::BufReader; -use futures::ready; use futures::AsyncBufRead; use futures::AsyncRead; +use futures::io::BufReader; +use futures::ready; use log::trace; use pin_project::pin_project; use zip::ZipArchive; @@ -374,14 +374,13 @@ impl DecompressDecoder { "Zip only supports single file", )); } - if memory_limit > 0 { - if let Some(size) = zip.decompressed_size() { - if size + compressed.len() as u128 > memory_limit as u128 / 2 { - return Err(ErrorCode::BadBytes(format!( - "zip file {path} is too large, decompressed_size = {size}", - ))); - } - } + if memory_limit > 0 + && let Some(size) = zip.decompressed_size() + && size + compressed.len() as u128 > memory_limit as u128 / 2 + { + return Err(ErrorCode::BadBytes(format!( + "zip file {path} is too large, decompressed_size = {size}", + ))); } let mut file = zip.by_index(0).map_err(|e| { ErrorCode::InvalidCompressionData(format!("compression data invalid: {e}")) @@ -439,8 +438,8 @@ mod tests { use async_compression::futures::bufread::GzipEncoder; use async_compression::futures::bufread::ZlibEncoder; - use futures::io::Cursor; use futures::AsyncReadExt; + use futures::io::Cursor; use rand::prelude::*; use super::*; diff --git a/src/common/compress/src/encode.rs b/src/common/compress/src/encode.rs index 6e62a5e999b7c..51e5ed902870a 100644 --- a/src/common/compress/src/encode.rs +++ b/src/common/compress/src/encode.rs @@ -16,6 +16,7 @@ use std::io::Cursor; use std::io::Result; use std::io::Write; +use async_compression::Level; use async_compression::codec::BrotliEncoder; use async_compression::codec::BzEncoder; use async_compression::codec::DeflateEncoder; @@ -26,11 +27,10 @@ use async_compression::codec::XzEncoder; use async_compression::codec::ZlibEncoder; use async_compression::codec::ZstdEncoder; use async_compression::util::PartialBuffer; -use async_compression::Level; use brotli::enc::backward_references::BrotliEncoderParams; use databend_common_exception::ErrorCode; -use zip::write::FileOptions; use zip::ZipWriter; +use zip::write::FileOptions; use crate::CompressAlgorithm; diff --git a/src/common/exception/src/elf/dwarf.rs b/src/common/exception/src/elf/dwarf.rs index 1dafec886c9c1..322421980fc1e 100644 --- a/src/common/exception/src/elf/dwarf.rs +++ b/src/common/exception/src/elf/dwarf.rs @@ -35,9 +35,9 @@ use object::CompressionFormat; use object::Object; use object::ObjectSection; +use crate::elf::ElfFile; use crate::elf::dwarf_unit::Unit; use crate::elf::dwarf_unit::UnitAttrs; -use crate::elf::ElfFile; #[derive(Debug)] pub struct CallLocation { @@ -164,10 +164,10 @@ impl Dwarf { let head = self.debug_info.header_from_offset(debug_info_offset)?; let type_ = head.type_(); - if matches!(type_, UnitType::Compilation | UnitType::Skeleton(_)) { - if let Some(unit) = self.get_unit(head)? { - return Ok(Some(unit.find_frames(probe)?)); - } + if matches!(type_, UnitType::Compilation | UnitType::Skeleton(_)) + && let Some(unit) = self.get_unit(head)? + { + return Ok(Some(unit.find_frames(probe)?)); } } @@ -177,12 +177,11 @@ impl Dwarf { fn slow_find_frames(&self, probe: u64) -> gimli::Result> { let mut units = self.debug_info.units(); while let Some(head) = units.next()? { - if matches!(head.type_(), UnitType::Compilation | UnitType::Skeleton(_)) { - if let Some(unit) = self.get_unit(head)? { - if unit.match_pc(probe) { - return unit.find_frames(probe); - } - } + if matches!(head.type_(), UnitType::Compilation | UnitType::Skeleton(_)) + && let Some(unit) = self.get_unit(head)? + && unit.match_pc(probe) + { + return unit.find_frames(probe); } } diff --git a/src/common/exception/src/elf/dwarf_inline_functions.rs b/src/common/exception/src/elf/dwarf_inline_functions.rs index 2a95b0172365a..7b6bdf2475ae5 100644 --- a/src/common/exception/src/elf/dwarf_inline_functions.rs +++ b/src/common/exception/src/elf/dwarf_inline_functions.rs @@ -80,17 +80,17 @@ impl SubroutineAttrs { } } gimli::DW_AT_abstract_origin | gimli::DW_AT_specification => { - if self.name.is_none() { - if let Ok(Some(v)) = unit.name_attr(attr.value(), 16) { - self.name = Some(v); - } + if self.name.is_none() + && let Ok(Some(v)) = unit.name_attr(attr.value(), 16) + { + self.name = Some(v); } } gimli::DW_AT_call_file => { - if let AttributeValue::FileIndex(idx) = attr.value() { - if let Ok(filename) = unit.find_file(idx) { - self.file = filename; - } + if let AttributeValue::FileIndex(idx) = attr.value() + && let Ok(filename) = unit.find_file(idx) + { + self.file = filename; } } gimli::DW_AT_call_line => { diff --git a/src/common/exception/src/elf/library_loader.rs b/src/common/exception/src/elf/library_loader.rs index fe7c9c56163e1..58da1d1ec5009 100644 --- a/src/common/exception/src/elf/library_loader.rs +++ b/src/common/exception/src/elf/library_loader.rs @@ -31,9 +31,9 @@ use object::Object; use object::ObjectSymbol; use object::ObjectSymbolTable; +use crate::elf::ElfFile; use crate::elf::library_manager::Library; use crate::elf::library_symbol::Symbol; -use crate::elf::ElfFile; #[derive(Debug)] pub struct LibraryLoader { @@ -96,14 +96,16 @@ impl LibraryLoader { let name = format!("{}", library_path.display()); let file = std::fs::File::open(library_path)?; let file_len = file.metadata()?.len(); - let ptr = libc::mmap( - ptr::null_mut(), - file_len as libc::size_t, - libc::PROT_READ, - libc::MAP_PRIVATE, - file.as_raw_fd(), - 0, - ); + let ptr = unsafe { + libc::mmap( + ptr::null_mut(), + file_len as libc::size_t, + libc::PROT_READ, + libc::MAP_PRIVATE, + file.as_raw_fd(), + 0, + ) + }; match std::ptr::eq(ptr, libc::MAP_FAILED) { true => Err(std::io::Error::other("Cannot mmap")), @@ -112,13 +114,13 @@ impl LibraryLoader { } unsafe fn load_library(&mut self, info: &dl_phdr_info) -> std::io::Result { - let library_name = match info.dlpi_name.is_null() || *info.dlpi_name == 0 { + let library_name = match info.dlpi_name.is_null() || unsafe { *info.dlpi_name } == 0 { true => match self.libraries.is_empty() { true => OsString::from("/proc/self/exe"), false => OsString::new(), }, false => { - let bytes = CStr::from_ptr(info.dlpi_name).to_bytes(); + let bytes = unsafe { CStr::from_ptr(info.dlpi_name) }.to_bytes(); OsStr::from_bytes(bytes).to_owned() } }; @@ -128,9 +130,9 @@ impl LibraryLoader { } let binary_path = std::fs::canonicalize(library_name)?.to_path_buf(); - let mut binary_library = self.mmap_library(binary_path.clone())?; + let mut binary_library = unsafe { self.mmap_library(binary_path.clone())? }; - let Some(binary_build_id) = binary_library.build_id() else { + let Some(binary_build_id) = (unsafe { binary_library.build_id() }) else { let binary_data = binary_library.data(); binary_library.address_begin = info.dlpi_addr as usize; binary_library.address_end = info.dlpi_addr as usize + binary_data.len(); @@ -142,8 +144,8 @@ impl LibraryLoader { binary_debug_path.set_extension("debug"); if std::fs::exists(&binary_debug_path)? { - let mut library = self.mmap_library(binary_debug_path)?; - if matches!(library.build_id(), Some(v) if v == binary_build_id) { + let mut library = unsafe { self.mmap_library(binary_debug_path)? }; + if matches!(unsafe { library.build_id() }, Some(v) if v == binary_build_id) { let library_data = library.data(); library.address_begin = info.dlpi_addr as usize; library.address_end = info.dlpi_addr as usize + library_data.len(); @@ -162,8 +164,8 @@ impl LibraryLoader { system_named_debug_path.set_extension("debug"); if std::fs::exists(&system_named_debug_path)? { - let mut library = self.mmap_library(system_named_debug_path)?; - if matches!(library.build_id(), Some(v) if v == binary_build_id) { + let mut library = unsafe { self.mmap_library(system_named_debug_path)? }; + if matches!(unsafe { library.build_id() }, Some(v) if v == binary_build_id) { let library_data = library.data(); library.address_begin = info.dlpi_addr as usize; library.address_end = info.dlpi_addr as usize + library_data.len(); @@ -187,9 +189,9 @@ impl LibraryLoader { system_id_debug_path.set_extension("debug"); if std::fs::exists(&system_id_debug_path)? { - let mut library = self.mmap_library(system_id_debug_path)?; + let mut library = unsafe { self.mmap_library(system_id_debug_path)? }; - if matches!(library.build_id(), Some(v) if v == binary_build_id) { + if matches!(unsafe { library.build_id() }, Some(v) if v == binary_build_id) { let library_data = library.data(); library.address_begin = info.dlpi_addr as usize; library.address_end = info.dlpi_addr as usize + library_data.len(); @@ -205,8 +207,8 @@ impl LibraryLoader { } pub unsafe fn load_libraries(&mut self, info: &dl_phdr_info) { - if let Ok(mut library) = self.load_library(info) { - self.load_symbols(&mut library); + if let Ok(mut library) = unsafe { self.load_library(info) } { + unsafe { self.load_symbols(&mut library) }; self.libraries.push(library); } } @@ -233,7 +235,9 @@ impl LibraryLoader { } unsafe extern "C" fn callback(info: *mut dl_phdr_info, _size: size_t, v: *mut c_void) -> c_int { - let loader = &mut *v.cast::(); - loader.load_libraries(&*info); + let loader = unsafe { &mut *v.cast::() }; + unsafe { + loader.load_libraries(&*info); + } 0 } diff --git a/src/common/exception/src/elf/library_manager.rs b/src/common/exception/src/elf/library_manager.rs index 2f8c84c58ca6b..cb6a0f80a142f 100644 --- a/src/common/exception/src/elf/library_manager.rs +++ b/src/common/exception/src/elf/library_manager.rs @@ -21,10 +21,10 @@ use std::sync::Arc; use object::Object; use once_cell::sync::OnceCell; +use crate::elf::ElfFile; use crate::elf::dwarf::Dwarf; use crate::elf::library_loader::LibraryLoader; use crate::elf::library_symbol::Symbol; -use crate::elf::ElfFile; use crate::exception_backtrace::PhysicalAddr; use crate::exception_backtrace::ResolvedStackFrame; use crate::exception_backtrace::StackFrame; @@ -111,10 +111,10 @@ impl LibraryManager { fn find_library_by_build_id(&self, build_id: &Arc>) -> Option<&Library> { for library in &self.libraries { - if let Some(v) = &library.build_id { - if v == build_id { - return Some(library); - } + if let Some(v) = &library.build_id + && v == build_id + { + return Some(library); } } diff --git a/src/common/exception/src/elf/mod.rs b/src/common/exception/src/elf/mod.rs index c1696452ef161..6b4f909a7d131 100644 --- a/src/common/exception/src/elf/mod.rs +++ b/src/common/exception/src/elf/mod.rs @@ -15,6 +15,8 @@ mod dwarf; mod dwarf_inline_functions; mod dwarf_subprogram; + +#[allow(clippy::collapsible_if)] mod dwarf_unit; mod library_loader; mod library_manager; diff --git a/src/common/exception/src/exception.rs b/src/common/exception/src/exception.rs index 4e22191cd31ea..cd6c57b18dd9d 100644 --- a/src/common/exception/src/exception.rs +++ b/src/common/exception/src/exception.rs @@ -21,11 +21,11 @@ use std::marker::PhantomData; use thiserror::Error; -use crate::exception_backtrace::capture; -use crate::span::pretty_print_error; -use crate::span::Span; use crate::ErrorFrame; use crate::StackTrace; +use crate::exception_backtrace::capture; +use crate::span::Span; +use crate::span::pretty_print_error; #[derive(Error)] pub struct ErrorCode { diff --git a/src/common/exception/src/exception_backtrace.rs b/src/common/exception/src/exception_backtrace.rs index 848ff7ebd2281..a1f9569fb2774 100644 --- a/src/common/exception/src/exception_backtrace.rs +++ b/src/common/exception/src/exception_backtrace.rs @@ -12,20 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::hash_map::Entry; use std::collections::HashMap; +use std::collections::hash_map::Entry; use std::fmt::Debug; use std::fmt::Formatter; use std::fmt::Write; use std::hash::Hash; use std::hash::Hasher; -use std::sync::atomic::AtomicUsize; -use std::sync::atomic::Ordering; use std::sync::Arc; use std::sync::LazyLock; use std::sync::Mutex; use std::sync::PoisonError; use std::sync::RwLock; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; // 0: not specified 1: disable 2: enable pub static USER_SET_ENABLE_BACKTRACE: AtomicUsize = AtomicUsize::new(0); diff --git a/src/common/exception/src/exception_code.rs b/src/common/exception/src/exception_code.rs index 1470be0d0597f..16c004e7213db 100644 --- a/src/common/exception/src/exception_code.rs +++ b/src/common/exception/src/exception_code.rs @@ -14,11 +14,11 @@ #![allow(non_snake_case)] -use crate::exception_backtrace::capture; use crate::ErrorCode; +use crate::exception_backtrace::capture; macro_rules! build_exceptions { - ($($(#[$meta:meta])* $body:ident($code:expr)),*$(,)*) => { + ($($(#[$meta:meta])* $body:ident($code:expr_2021)),*$(,)*) => { impl ErrorCode { $( @@ -746,17 +746,17 @@ mod tests { for line in text.lines() { let trimmed = line.trim(); - if let Some(idx1) = trimmed.find('(') { - if let Some(idx2) = trimmed[idx1..].find(')') { - let name = trimmed[..idx1].trim(); - let code_str = &trimmed[idx1 + 1..idx1 + idx2].trim(); - if !name.is_empty() - && !code_str.is_empty() - && code_str.chars().all(|c| c.is_ascii_digit()) - { - let code: u32 = code_str.parse().unwrap(); - map.entry(code).or_default().push(name.to_string()); - } + if let Some(idx1) = trimmed.find('(') + && let Some(idx2) = trimmed[idx1..].find(')') + { + let name = trimmed[..idx1].trim(); + let code_str = &trimmed[idx1 + 1..idx1 + idx2].trim(); + if !name.is_empty() + && !code_str.is_empty() + && code_str.chars().all(|c| c.is_ascii_digit()) + { + let code: u32 = code_str.parse().unwrap(); + map.entry(code).or_default().push(name.to_string()); } } } diff --git a/src/common/exception/src/exception_into.rs b/src/common/exception/src/exception_into.rs index df84a23c7ee79..4945fb7786a2e 100644 --- a/src/common/exception/src/exception_into.rs +++ b/src/common/exception/src/exception_into.rs @@ -19,12 +19,12 @@ use std::fmt::Formatter; use geozero::error::GeozeroError; -use crate::exception_backtrace::capture; -use crate::span::Span; use crate::ErrorCode; use crate::ErrorFrame; use crate::ParseError; use crate::StackTrace; +use crate::exception_backtrace::capture; +use crate::span::Span; #[derive(thiserror::Error)] enum OtherErrors { @@ -183,12 +183,19 @@ impl From for ErrorCode { impl From for ErrorCode { fn from(error: opendal::Error) -> Self { + let msg = error.message(); + let detail = error.to_string(); + let detail = detail + .strip_suffix(msg) + .and_then(|err| err.strip_suffix(" => ")) + .unwrap_or(&detail); + match error.kind() { - opendal::ErrorKind::NotFound => ErrorCode::StorageNotFound(error.to_string()), + opendal::ErrorKind::NotFound => ErrorCode::StorageNotFound(msg).add_detail(detail), opendal::ErrorKind::PermissionDenied => { - ErrorCode::StoragePermissionDenied(error.to_string()) + ErrorCode::StoragePermissionDenied(msg).add_detail(detail) } - _ => ErrorCode::StorageOther(format!("{error:?}")), + _ => ErrorCode::StorageOther(msg).add_detail(detail), } } } diff --git a/src/common/exception/src/lib.rs b/src/common/exception/src/lib.rs index 34a202c86ae59..eeae4f3f44b66 100644 --- a/src/common/exception/src/lib.rs +++ b/src/common/exception/src/lib.rs @@ -13,6 +13,7 @@ // limitations under the License. #![allow(clippy::uninlined_format_args)] +#![allow(clippy::collapsible_if)] extern crate core; @@ -25,18 +26,18 @@ mod exception_code; mod exception_flight; mod exception_into; -pub use context::display_error_stack; pub use context::ErrorFrame; pub use context::ResultExt; -pub use databend_common_ast::span; +pub use context::display_error_stack; pub use databend_common_ast::ParseError; -pub use exception::error_code_groups; +pub use databend_common_ast::span; pub use exception::ErrorCode; pub use exception::ErrorCodeResultExt; pub use exception::Result; pub use exception::ToErrorCode; -pub use exception_backtrace::set_backtrace; +pub use exception::error_code_groups; pub use exception_backtrace::StackTrace; pub use exception_backtrace::USER_SET_ENABLE_BACKTRACE; +pub use exception_backtrace::set_backtrace; pub use exception_into::SerializedError; pub use span::merge_span; diff --git a/src/common/exception/tests/it/error_code_result_ext.rs b/src/common/exception/tests/it/error_code_result_ext.rs index aea1ec3e81b9d..2690adff6a5d6 100644 --- a/src/common/exception/tests/it/error_code_result_ext.rs +++ b/src/common/exception/tests/it/error_code_result_ext.rs @@ -14,10 +14,10 @@ //! Tests for ErrorCodeResultExt trait functionality -use databend_common_exception::error_code_groups; use databend_common_exception::ErrorCode; use databend_common_exception::ErrorCodeResultExt; use databend_common_exception::Result; +use databend_common_exception::error_code_groups; fn create_unknown_database_error() -> Result { Err(ErrorCode::UnknownDatabase("test_db")) diff --git a/src/common/frozen_api/src/lib.rs b/src/common/frozen_api/src/lib.rs index 00681c1f40fe0..fd52c3cfc2a29 100644 --- a/src/common/frozen_api/src/lib.rs +++ b/src/common/frozen_api/src/lib.rs @@ -19,8 +19,8 @@ //! ## Usage //! //! ```rust -//! use databend_common_frozen_api::frozen_api; //! use databend_common_frozen_api::FrozenAPI; +//! use databend_common_frozen_api::frozen_api; //! //! // Child structs: must derive FrozenAPI //! #[derive(FrozenAPI)] @@ -48,16 +48,16 @@ //! **Rule**: If struct uses `#[frozen_api]`, it and ALL its custom field types must `#[derive(FrozenAPI)]`. use proc_macro::TokenStream; -use quote::quote; use quote::ToTokens; +use quote::quote; use sha2::Digest; use sha2::Sha256; -use syn::parse_macro_input; use syn::Data; use syn::DeriveInput; use syn::Fields; use syn::LitStr; use syn::Type; +use syn::parse_macro_input; /// Compile-time API freezing macro that prevents accidental breaking changes. #[proc_macro_attribute] @@ -269,10 +269,10 @@ fn has_custom_types_in_field(ty: &Type) -> bool { // Check generic arguments if let syn::PathArguments::AngleBracketed(args) = &segment.arguments { for arg in &args.args { - if let syn::GenericArgument::Type(inner_type) = arg { - if has_custom_types_in_field(inner_type) { - return true; - } + if let syn::GenericArgument::Type(inner_type) = arg + && has_custom_types_in_field(inner_type) + { + return true; } } } diff --git a/src/common/grpc/src/dns_resolver.rs b/src/common/grpc/src/dns_resolver.rs index 7045230935c56..8caae83188a8c 100644 --- a/src/common/grpc/src/dns_resolver.rs +++ b/src/common/grpc/src/dns_resolver.rs @@ -30,8 +30,8 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use hickory_resolver::TokioResolver; use hyper::Uri; -use hyper_util::client::legacy::connect::dns::Name; use hyper_util::client::legacy::connect::HttpConnector; +use hyper_util::client::legacy::connect::dns::Name; use log::info; use tonic::transport::Certificate; use tonic::transport::Channel; diff --git a/src/common/grpc/src/grpc_token.rs b/src/common/grpc/src/grpc_token.rs index ef400c0c70430..b05119ff75881 100644 --- a/src/common/grpc/src/grpc_token.rs +++ b/src/common/grpc/src/grpc_token.rs @@ -35,11 +35,10 @@ impl GrpcToken { pub fn try_create_token(&self, claim: GrpcClaim) -> Result { let claims = Claims::with_custom_claims(claim, Duration::from_days(3650)); - self.key - .authenticate(claims) - .map_err_to_code(ErrorCode::AuthenticateFailure, || { - "Cannot create flight token, because authenticate failure" - }) + self.key.authenticate(claims).map_err_to_code( + ErrorCode::AuthenticateFailure, + || "Cannot create flight token, because authenticate failure", + ) } pub fn try_verify_token(&self, token: String) -> Result { diff --git a/src/common/hashtable/src/container.rs b/src/common/hashtable/src/container.rs index 51619418e0509..290c97b73974d 100644 --- a/src/common/hashtable/src/container.rs +++ b/src/common/hashtable/src/container.rs @@ -12,14 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::alloc::handle_alloc_error; use std::alloc::Allocator; use std::alloc::Layout; +use std::alloc::handle_alloc_error; use std::mem::MaybeUninit; use std::ops::Deref; use std::ops::DerefMut; -use std::ptr::null_mut; use std::ptr::NonNull; +use std::ptr::null_mut; use databend_common_base::runtime::drop_guard; @@ -76,24 +76,27 @@ unsafe impl Container for HeapContainer { } unsafe fn new_zeroed(len: usize, allocator: Self::A) -> Self { - Self(Box::new_zeroed_slice_in(len, allocator).assume_init()) + unsafe { Self(Box::new_zeroed_slice_in(len, allocator).assume_init()) } } unsafe fn grow_zeroed(&mut self, new_len: usize) { - debug_assert!(self.len() <= new_len); - let old_layout = Layout::array::(self.len()).unwrap(); - let new_layout = Layout::array::(new_len).unwrap(); - let old_box = std::ptr::read(&self.0); - let (old_raw, allocator) = Box::into_raw_with_allocator(old_box); - let old_ptr = NonNull::new(old_raw).unwrap().cast(); - let grow_res = allocator.grow_zeroed(old_ptr, old_layout, new_layout); - - match grow_res { - Err(_) => handle_alloc_error(new_layout), - Ok(new_ptr) => { - let new_raw = std::ptr::slice_from_raw_parts_mut(new_ptr.cast().as_ptr(), new_len); - let new_box = Box::from_raw_in(new_raw, allocator); - std::ptr::write(self, Self(new_box)); + unsafe { + debug_assert!(self.len() <= new_len); + let old_layout = Layout::array::(self.len()).unwrap(); + let new_layout = Layout::array::(new_len).unwrap(); + let old_box = std::ptr::read(&self.0); + let (old_raw, allocator) = Box::into_raw_with_allocator(old_box); + let old_ptr = NonNull::new(old_raw).unwrap().cast(); + let grow_res = allocator.grow_zeroed(old_ptr, old_layout, new_layout); + + match grow_res { + Err(_) => handle_alloc_error(new_layout), + Ok(new_ptr) => { + let new_raw = + std::ptr::slice_from_raw_parts_mut(new_ptr.cast().as_ptr(), new_len); + let new_box = Box::from_raw_in(new_raw, allocator); + std::ptr::write(self, Self(new_box)); + } } } } @@ -183,39 +186,41 @@ unsafe impl Container for StackContaine } unsafe fn grow_zeroed(&mut self, new_len: usize) { - debug_assert!(self.len <= new_len); - if new_len <= N { - self.len = new_len; - } else if self.ptr.is_null() { - let layout = Layout::array::(new_len).unwrap(); - let allocated_bytes = self.allocator.allocate_zeroed(layout); - - match allocated_bytes { - Err(_) => handle_alloc_error(layout), - Ok(allocated_bytes) => { - self.ptr = allocated_bytes.cast().as_ptr(); - std::ptr::copy_nonoverlapping( - self.array.as_ptr() as *mut _, - self.ptr, - self.len, - ); - self.len = new_len; - } - }; - } else { - let old_layout = Layout::array::(self.len).unwrap(); - let new_layout = Layout::array::(new_len).unwrap(); - - let old_ptr = NonNull::new_unchecked(self.ptr).cast(); - let reallocated_bytes = self.allocator.grow_zeroed(old_ptr, old_layout, new_layout); - - match reallocated_bytes { - Err(_) => handle_alloc_error(new_layout), - Ok(reallocated_bytes) => { - self.ptr = reallocated_bytes.cast::().as_ptr(); - self.len = new_len; - } - }; + unsafe { + debug_assert!(self.len <= new_len); + if new_len <= N { + self.len = new_len; + } else if self.ptr.is_null() { + let layout = Layout::array::(new_len).unwrap(); + let allocated_bytes = self.allocator.allocate_zeroed(layout); + + match allocated_bytes { + Err(_) => handle_alloc_error(layout), + Ok(allocated_bytes) => { + self.ptr = allocated_bytes.cast().as_ptr(); + std::ptr::copy_nonoverlapping( + self.array.as_ptr() as *mut _, + self.ptr, + self.len, + ); + self.len = new_len; + } + }; + } else { + let old_layout = Layout::array::(self.len).unwrap(); + let new_layout = Layout::array::(new_len).unwrap(); + + let old_ptr = NonNull::new_unchecked(self.ptr).cast(); + let reallocated_bytes = self.allocator.grow_zeroed(old_ptr, old_layout, new_layout); + + match reallocated_bytes { + Err(_) => handle_alloc_error(new_layout), + Ok(reallocated_bytes) => { + self.ptr = reallocated_bytes.cast::().as_ptr(); + self.len = new_len; + } + }; + } } } } diff --git a/src/common/hashtable/src/dictionary_string_hashtable.rs b/src/common/hashtable/src/dictionary_string_hashtable.rs index cfde4dc66c2b0..98bbc91188521 100644 --- a/src/common/hashtable/src/dictionary_string_hashtable.rs +++ b/src/common/hashtable/src/dictionary_string_hashtable.rs @@ -22,17 +22,17 @@ use std::sync::Arc; use bumpalo::Bump; use databend_common_base::mem_allocator::DefaultAllocator; +use crate::DictionaryStringHashMap; +use crate::FastHash; +use crate::HashtableLike; +use crate::StringHashSet; use crate::container::Container; use crate::container::HeapContainer; use crate::short_string_hashtable::FallbackKey; -use crate::table0::Entry; use crate::table_empty::TableEmpty; +use crate::table0::Entry; use crate::traits::EntryMutRefLike; use crate::traits::EntryRefLike; -use crate::DictionaryStringHashMap; -use crate::FastHash; -use crate::HashtableLike; -use crate::StringHashSet; #[derive(Clone, Copy, Debug, Eq)] pub struct DictionaryKeys { @@ -212,24 +212,26 @@ impl DictionaryStringHashTable { key: &[NonNull<[u8]>], hash: u64, ) -> Option<&DictionaryEntry> { - assume(key.len() == self.dict_keys); - let index = (hash as usize) & (self.entries.len() - 1); - for i in (index..self.entries.len()).chain(0..index) { - assume(i < self.entries.len()); - if self.entries[i].is_zero() { - return None; - } + unsafe { + assume(key.len() == self.dict_keys); + let index = (hash as usize) & (self.entries.len() - 1); + for i in (index..self.entries.len()).chain(0..index) { + assume(i < self.entries.len()); + if self.entries[i].is_zero() { + return None; + } - let slice = std::slice::from_raw_parts( - self.entries[i].key.assume_init_ref().as_ptr(), - self.dict_keys, - ); + let slice = std::slice::from_raw_parts( + self.entries[i].key.assume_init_ref().as_ptr(), + self.dict_keys, + ); - if key == slice { - return Some(&self.entries[i]); + if key == slice { + return Some(&self.entries[i]); + } } + None } - None } /// # Safety @@ -241,24 +243,26 @@ impl DictionaryStringHashTable { key: &[NonNull<[u8]>], hash: u64, ) -> Option<&mut DictionaryEntry> { - assume(key.len() == self.dict_keys); - let index = (hash as usize) & (self.entries.len() - 1); - for i in (index..self.entries.len()).chain(0..index) { - assume(i < self.entries.len()); - if self.entries[i].is_zero() { - return None; - } + unsafe { + assume(key.len() == self.dict_keys); + let index = (hash as usize) & (self.entries.len() - 1); + for i in (index..self.entries.len()).chain(0..index) { + assume(i < self.entries.len()); + if self.entries[i].is_zero() { + return None; + } - let slice = std::slice::from_raw_parts( - self.entries[i].key.assume_init_ref().as_ptr(), - self.dict_keys, - ); + let slice = std::slice::from_raw_parts( + self.entries[i].key.assume_init_ref().as_ptr(), + self.dict_keys, + ); - if key == slice { - return Some(&mut self.entries[i]); + if key == slice { + return Some(&mut self.entries[i]); + } } + None } - None } pub fn dictionary_slot_iter(&self) -> DictionarySlotIter<'_> { @@ -362,9 +366,11 @@ impl HashtableLike for DictionaryStringHashTable { &mut self, key: &Self::Key, ) -> Result<&mut MaybeUninit, &mut Self::Value> { - match self.insert_and_entry(key) { - Ok(mut e) => Ok(&mut *(e.get_mut_ptr() as *mut MaybeUninit)), - Err(mut e) => Err(&mut *e.get_mut_ptr()), + unsafe { + match self.insert_and_entry(key) { + Ok(mut e) => Ok(&mut *(e.get_mut_ptr() as *mut MaybeUninit)), + Err(mut e) => Err(&mut *e.get_mut_ptr()), + } } } @@ -372,7 +378,7 @@ impl HashtableLike for DictionaryStringHashTable { &mut self, key: &Self::Key, ) -> Result, Self::EntryMutRef<'_>> { - self.insert_and_entry_with_hash(key, key.fast_hash()) + unsafe { self.insert_and_entry_with_hash(key, key.fast_hash()) } } unsafe fn insert_and_entry_with_hash( @@ -380,56 +386,58 @@ impl HashtableLike for DictionaryStringHashTable { key: &Self::Key, hash: u64, ) -> Result, Self::EntryMutRef<'_>> { - self.check_grow(); - - assume(key.keys.len() == self.dict_keys); - let mut dictionary_keys = Vec::with_capacity(self.dict_keys); - - for key in key.keys.as_ref() { - // println!("insert: {:?}", String::from_utf8(key.as_ref().to_vec()).unwrap()); - dictionary_keys.push(NonNull::from( - match self.dictionary_hashset.insert_and_entry(key.as_ref()) { - Ok(e) => e.key(), - Err(e) => e.key(), - }, - )); - } + unsafe { + self.check_grow(); - let index = (hash as usize) & (self.entries.len() - 1); - for i in (index..self.entries.len()).chain(0..index) { - assume(i < self.entries.len()); + assume(key.keys.len() == self.dict_keys); + let mut dictionary_keys = Vec::with_capacity(self.dict_keys); - if self.entries[i].is_zero() { - self.entries_len += 1; - - let global_keys = self.arena.alloc_slice_copy(&dictionary_keys); - // for key in global_keys.iter() { - // println!("insert: {:?}", String::from_utf8(key.as_ref().to_vec()).unwrap()); - // } - self.entries[i].hash = hash; - self.entries[i] - .key - .write(NonNull::new(global_keys.as_mut_ptr()).unwrap()); - return Ok(DictionaryMutEntryRef::create( - &mut self.entries[i], - self.dict_keys, + for key in key.keys.as_ref() { + // println!("insert: {:?}", String::from_utf8(key.as_ref().to_vec()).unwrap()); + dictionary_keys.push(NonNull::from( + match self.dictionary_hashset.insert_and_entry(key.as_ref()) { + Ok(e) => e.key(), + Err(e) => e.key(), + }, )); } - let slice = std::slice::from_raw_parts( - self.entries[i].key.assume_init_ref().as_ptr(), - self.dict_keys, - ); + let index = (hash as usize) & (self.entries.len() - 1); + for i in (index..self.entries.len()).chain(0..index) { + assume(i < self.entries.len()); + + if self.entries[i].is_zero() { + self.entries_len += 1; + + let global_keys = self.arena.alloc_slice_copy(&dictionary_keys); + // for key in global_keys.iter() { + // println!("insert: {:?}", String::from_utf8(key.as_ref().to_vec()).unwrap()); + // } + self.entries[i].hash = hash; + self.entries[i] + .key + .write(NonNull::new(global_keys.as_mut_ptr()).unwrap()); + return Ok(DictionaryMutEntryRef::create( + &mut self.entries[i], + self.dict_keys, + )); + } - if dictionary_keys.as_slice() == slice { - return Err(DictionaryMutEntryRef::create( - &mut self.entries[i], + let slice = std::slice::from_raw_parts( + self.entries[i].key.assume_init_ref().as_ptr(), self.dict_keys, - )); + ); + + if dictionary_keys.as_slice() == slice { + return Err(DictionaryMutEntryRef::create( + &mut self.entries[i], + self.dict_keys, + )); + } } - } - panic!("the hash table overflows") + panic!("the hash table overflows") + } } fn iter(&self) -> Self::Iterator<'_> { diff --git a/src/common/hashtable/src/hashjoin_string_hashtable.rs b/src/common/hashtable/src/hashjoin_string_hashtable.rs index b9fd9c0c266d2..574e6afd42b34 100644 --- a/src/common/hashtable/src/hashjoin_string_hashtable.rs +++ b/src/common/hashtable/src/hashjoin_string_hashtable.rs @@ -22,13 +22,13 @@ use databend_common_base::mem_allocator::DefaultAllocator; use databend_common_column::bitmap::Bitmap; use super::traits::HashJoinHashtableLike; +use crate::RowPtr; use crate::hashjoin_hashtable::combine_header; use crate::hashjoin_hashtable::early_filtering; use crate::hashjoin_hashtable::hash_bits; use crate::hashjoin_hashtable::new_header; use crate::hashjoin_hashtable::remove_header_tag; use crate::traits::hash_join_fast_string_hash; -use crate::RowPtr; pub const STRING_EARLY_SIZE: usize = 4; pub struct StringRawEntry { diff --git a/src/common/hashtable/src/hashtable.rs b/src/common/hashtable/src/hashtable.rs index 6591217a2447c..cb197a8372c6c 100644 --- a/src/common/hashtable/src/hashtable.rs +++ b/src/common/hashtable/src/hashtable.rs @@ -133,29 +133,33 @@ where &mut self, key: K, ) -> Result<&mut Entry, &mut Entry> { - if unlikely(K::equals_zero(&key)) { - let res = self.zero.is_some(); - if !res { - *self.zero = Some(MaybeUninit::zeroed().assume_init()); - } - let zero = self.zero.as_mut().unwrap(); - if res { - return Err(zero); - } else { - return Ok(zero); + unsafe { + if unlikely(K::equals_zero(&key)) { + let res = self.zero.is_some(); + if !res { + *self.zero = Some(MaybeUninit::zeroed().assume_init()); + } + let zero = self.zero.as_mut().unwrap(); + if res { + return Err(zero); + } else { + return Ok(zero); + } } + self.table.check_grow(); + self.table.insert(key) } - self.table.check_grow(); - self.table.insert(key) } /// # Safety /// /// The returned uninitialized value should be written immediately. #[inline(always)] pub unsafe fn insert(&mut self, key: K) -> Result<&mut MaybeUninit, &mut V> { - match self.insert_and_entry(key) { - Ok(e) => Ok(&mut e.val), - Err(e) => Err(e.val.assume_init_mut()), + unsafe { + match self.insert_and_entry(key) { + Ok(e) => Ok(&mut e.val), + Err(e) => Err(e.val.assume_init_mut()), + } } } pub fn iter(&self) -> HashtableIter<'_, K, V> { @@ -286,7 +290,7 @@ where &mut self, key: &Self::Key, ) -> Result<&mut MaybeUninit, &mut Self::Value> { - self.insert(*key) + unsafe { self.insert(*key) } } #[inline(always)] @@ -294,20 +298,22 @@ where &mut self, key: &Self::Key, ) -> Result, Self::EntryMutRef<'_>> { - if unlikely(K::equals_zero(key)) { - let res = self.zero.is_some(); - if !res { - *self.zero = Some(MaybeUninit::zeroed().assume_init()); - } - let zero = self.zero.as_mut().unwrap(); - if res { - return Err(zero); - } else { - return Ok(zero); + unsafe { + if unlikely(K::equals_zero(key)) { + let res = self.zero.is_some(); + if !res { + *self.zero = Some(MaybeUninit::zeroed().assume_init()); + } + let zero = self.zero.as_mut().unwrap(); + if res { + return Err(zero); + } else { + return Ok(zero); + } } + self.table.check_grow(); + self.table.insert(*key) } - self.table.check_grow(); - self.table.insert(*key) } #[inline(always)] @@ -316,22 +322,24 @@ where key: &Self::Key, hash: u64, ) -> Result, Self::EntryMutRef<'_>> { - if unlikely(K::equals_zero(key)) { - let res = self.zero.is_some(); - if !res { - *self.zero = Some(MaybeUninit::zeroed().assume_init()); - } - let zero = self.zero.as_mut().unwrap(); - if res { - return Err(zero); - } else { - return Ok(zero); + unsafe { + if unlikely(K::equals_zero(key)) { + let res = self.zero.is_some(); + if !res { + *self.zero = Some(MaybeUninit::zeroed().assume_init()); + } + let zero = self.zero.as_mut().unwrap(); + if res { + return Err(zero); + } else { + return Ok(zero); + } } - } - self.table.check_grow(); + self.table.check_grow(); - self.table.insert_with_hash(*key, hash) + self.table.insert_with_hash(*key, hash) + } } fn iter(&self) -> Self::Iterator<'_> { diff --git a/src/common/hashtable/src/keys_ref.rs b/src/common/hashtable/src/keys_ref.rs index d45023a15805c..def815d3efb4e 100644 --- a/src/common/hashtable/src/keys_ref.rs +++ b/src/common/hashtable/src/keys_ref.rs @@ -39,7 +39,7 @@ impl KeysRef { #[allow(clippy::missing_safety_doc)] #[inline] pub unsafe fn as_slice(&self) -> &[u8] { - std::slice::from_raw_parts(self.address as *const u8, self.length) + unsafe { std::slice::from_raw_parts(self.address as *const u8, self.length) } } } diff --git a/src/common/hashtable/src/lib.rs b/src/common/hashtable/src/lib.rs index d35597478752c..3f253413f6aa5 100644 --- a/src/common/hashtable/src/lib.rs +++ b/src/common/hashtable/src/lib.rs @@ -18,7 +18,6 @@ #![feature(allocator_api)] #![feature(arbitrary_self_types)] #![feature(ptr_metadata)] -#![feature(maybe_uninit_slice)] #![feature(trusted_len)] extern crate core; @@ -43,7 +42,6 @@ pub mod traits; mod utils; pub use table0::Entry as HashtableEntry; -pub use traits::hash_join_fast_string_hash; pub use traits::BloomHash; pub use traits::EntryMutRefLike as HashtableEntryMutRefLike; pub use traits::EntryRefLike as HashtableEntryRefLike; @@ -51,6 +49,7 @@ pub use traits::FastHash; pub use traits::HashtableLike; pub use traits::Keyable as HashtableKeyable; pub use traits::UnsizedKeyable as HashtableUnsizedKeyable; +pub use traits::hash_join_fast_string_hash; use crate::lookup_hashtable::LookupHashtable; use crate::lookup_hashtable::LookupTableIter; @@ -108,8 +107,8 @@ pub type LookupHashMapIterMut<'a, K, const CAPACITY: usize, V> = pub use hashjoin_hashtable::RawEntry; pub use hashjoin_hashtable::RowPtr; -pub use hashjoin_string_hashtable::StringRawEntry; pub use hashjoin_string_hashtable::STRING_EARLY_SIZE; +pub use hashjoin_string_hashtable::StringRawEntry; pub use keys_ref::KeysRef; pub use partitioned_hashtable::hash2bucket; pub type HashJoinHashMap = @@ -117,6 +116,6 @@ pub type HashJoinHashMap = pub type BinaryHashJoinHashMap = hashjoin_string_hashtable::HashJoinStringHashTable; pub use traits::HashJoinHashtableLike; -pub use utils::fast_memcmp; pub use utils::Interval; pub use utils::MergeIntoBlockInfoIndex; +pub use utils::fast_memcmp; diff --git a/src/common/hashtable/src/lookup_hashtable.rs b/src/common/hashtable/src/lookup_hashtable.rs index 19be8634c7983..98af1ff764a3f 100644 --- a/src/common/hashtable/src/lookup_hashtable.rs +++ b/src/common/hashtable/src/lookup_hashtable.rs @@ -19,8 +19,8 @@ use std::mem::MaybeUninit; use databend_common_base::mem_allocator::DefaultAllocator; -use crate::table0::Entry; use crate::HashtableLike; +use crate::table0::Entry; pub struct LookupHashtable< K: Sized, @@ -138,9 +138,11 @@ macro_rules! lookup_impl { &mut self, key: &$ty, ) -> Result<&mut MaybeUninit, &mut Self::Value> { - match self.insert_and_entry(key) { - Ok(e) => Ok(&mut e.val), - Err(e) => Err(e.val.assume_init_mut()), + unsafe { + match self.insert_and_entry(key) { + Ok(e) => Ok(&mut e.val), + Err(e) => Err(e.val.assume_init_mut()), + } } } @@ -165,7 +167,7 @@ macro_rules! lookup_impl { key: &$ty, _hash: u64, ) -> Result, Self::EntryMutRef<'_>> { - self.insert_and_entry(key) + unsafe { self.insert_and_entry(key) } } fn iter(&self) -> Self::Iterator<'_> { diff --git a/src/common/hashtable/src/partitioned_hashtable.rs b/src/common/hashtable/src/partitioned_hashtable.rs index 2c2dfa93d3d84..2b34c20dcf0af 100644 --- a/src/common/hashtable/src/partitioned_hashtable.rs +++ b/src/common/hashtable/src/partitioned_hashtable.rs @@ -110,12 +110,12 @@ impl, - const BUCKETS_LG2: u32, - const HIGH_BIT: bool, - > HashtableLike for PartitionedHashtable + K: ?Sized + FastHash, + V, + Impl: HashtableLike, + const BUCKETS_LG2: u32, + const HIGH_BIT: bool, +> HashtableLike for PartitionedHashtable { type Key = Impl::Key; type Value = Impl::Value; @@ -196,9 +196,11 @@ impl< &mut self, key: &Self::Key, ) -> Result<&mut MaybeUninit, &mut Self::Value> { - let hash = key.fast_hash(); - let index = hash2bucket::(hash as usize); - self.tables[index].insert(key) + unsafe { + let hash = key.fast_hash(); + let index = hash2bucket::(hash as usize); + self.tables[index].insert(key) + } } #[inline(always)] @@ -206,9 +208,11 @@ impl< &mut self, key: &Self::Key, ) -> Result, Self::EntryMutRef<'_>> { - let hash = key.fast_hash(); - let index = hash2bucket::(hash as usize); - self.tables[index].insert_and_entry_with_hash(key, hash) + unsafe { + let hash = key.fast_hash(); + let index = hash2bucket::(hash as usize); + self.tables[index].insert_and_entry_with_hash(key, hash) + } } #[inline(always)] @@ -217,8 +221,10 @@ impl< key: &Self::Key, hash: u64, ) -> Result, Self::EntryMutRef<'_>> { - let index = hash2bucket::(hash as usize); - self.tables[index].insert_and_entry_with_hash(key, hash) + unsafe { + let index = hash2bucket::(hash as usize); + self.tables[index].insert_and_entry_with_hash(key, hash) + } } fn iter(&self) -> Self::Iterator<'_> { diff --git a/src/common/hashtable/src/short_string_hashtable.rs b/src/common/hashtable/src/short_string_hashtable.rs index 1569b59c980e9..65b1bc681adea 100644 --- a/src/common/hashtable/src/short_string_hashtable.rs +++ b/src/common/hashtable/src/short_string_hashtable.rs @@ -33,11 +33,11 @@ use super::traits::HashtableLike; use super::traits::Keyable; use super::traits::UnsizedKeyable; use super::utils::read_le; -use crate::table0::Table0Iter; -use crate::table0::Table0IterMut; use crate::table_empty::TableEmpty; use crate::table_empty::TableEmptyIter; use crate::table_empty::TableEmptyIterMut; +use crate::table0::Table0Iter; +use crate::table0::Table0IterMut; pub struct ShortStringHashtable where @@ -154,26 +154,27 @@ where key: *const K, ) -> Result, ShortStringHashtableEntryMutRef<'_, K, V>> { - let key = (*key).as_bytes(); - match key.len() { - _ if key.last().copied() == Some(0) => { - self.table4.check_grow(); - self.table4 - .insert(FallbackKey::new(key)) - .map(|x| { - self.key_size += key.len(); - ShortStringHashtableEntryMutRef( - ShortStringHashtableEntryMutRefInner::Table4(x), - ) - }) - .map_err(|x| { - ShortStringHashtableEntryMutRef( - ShortStringHashtableEntryMutRefInner::Table4(x), - ) - }) - } - 0 => { - self.table0 + unsafe { + let key = (*key).as_bytes(); + match key.len() { + _ if key.last().copied() == Some(0) => { + self.table4.check_grow(); + self.table4 + .insert(FallbackKey::new(key)) + .map(|x| { + self.key_size += key.len(); + ShortStringHashtableEntryMutRef( + ShortStringHashtableEntryMutRefInner::Table4(x), + ) + }) + .map_err(|x| { + ShortStringHashtableEntryMutRef( + ShortStringHashtableEntryMutRefInner::Table4(x), + ) + }) + } + 0 => self + .table0 .insert() .map(|x| { ShortStringHashtableEntryMutRef( @@ -184,83 +185,83 @@ where ShortStringHashtableEntryMutRef( ShortStringHashtableEntryMutRefInner::Table0(x, PhantomData), ) - }) - } - 1..=8 => { - self.table1.check_grow(); - let mut t = [0u64; 1]; - t[0] = read_le(key.as_ptr(), key.len()); - let t = std::mem::transmute::<[u64; 1], InlineKey<0>>(t); - self.table1 - .insert(t) - .map(|x| { - self.key_size += key.len(); - ShortStringHashtableEntryMutRef( - ShortStringHashtableEntryMutRefInner::Table1(x), - ) - }) - .map_err(|x| { - ShortStringHashtableEntryMutRef( - ShortStringHashtableEntryMutRefInner::Table1(x), - ) - }) - } - 9..=16 => { - self.table2.check_grow(); - let mut t = [0u64; 2]; - t[0] = (key.as_ptr() as *const u64).read_unaligned(); - t[1] = read_le(key.as_ptr().offset(8), key.len() - 8); - let t = std::mem::transmute::<[u64; 2], InlineKey<1>>(t); - self.table2 - .insert(t) - .map(|x| { - self.key_size += key.len(); - ShortStringHashtableEntryMutRef( - ShortStringHashtableEntryMutRefInner::Table2(x), - ) - }) - .map_err(|x| { - ShortStringHashtableEntryMutRef( - ShortStringHashtableEntryMutRefInner::Table2(x), - ) - }) - } - 17..=24 => { - self.table3.check_grow(); - let mut t = [0u64; 3]; - t[0] = (key.as_ptr() as *const u64).read_unaligned(); - t[1] = (key.as_ptr() as *const u64).offset(1).read_unaligned(); - t[2] = read_le(key.as_ptr().offset(16), key.len() - 16); - let t = std::mem::transmute::<[u64; 3], InlineKey<2>>(t); - self.table3 - .insert(t) - .map(|x| { - self.key_size += key.len(); - ShortStringHashtableEntryMutRef( - ShortStringHashtableEntryMutRefInner::Table3(x), - ) - }) - .map_err(|x| { - ShortStringHashtableEntryMutRef( - ShortStringHashtableEntryMutRefInner::Table3(x), - ) - }) - } - _ => { - self.table4.check_grow(); - self.table4 - .insert(FallbackKey::new(key)) - .map(|x| { - self.key_size += key.len(); - ShortStringHashtableEntryMutRef( - ShortStringHashtableEntryMutRefInner::Table4(x), - ) - }) - .map_err(|x| { - ShortStringHashtableEntryMutRef( - ShortStringHashtableEntryMutRefInner::Table4(x), - ) - }) + }), + 1..=8 => { + self.table1.check_grow(); + let mut t = [0u64; 1]; + t[0] = read_le(key.as_ptr(), key.len()); + let t = std::mem::transmute::<[u64; 1], InlineKey<0>>(t); + self.table1 + .insert(t) + .map(|x| { + self.key_size += key.len(); + ShortStringHashtableEntryMutRef( + ShortStringHashtableEntryMutRefInner::Table1(x), + ) + }) + .map_err(|x| { + ShortStringHashtableEntryMutRef( + ShortStringHashtableEntryMutRefInner::Table1(x), + ) + }) + } + 9..=16 => { + self.table2.check_grow(); + let mut t = [0u64; 2]; + t[0] = (key.as_ptr() as *const u64).read_unaligned(); + t[1] = read_le(key.as_ptr().offset(8), key.len() - 8); + let t = std::mem::transmute::<[u64; 2], InlineKey<1>>(t); + self.table2 + .insert(t) + .map(|x| { + self.key_size += key.len(); + ShortStringHashtableEntryMutRef( + ShortStringHashtableEntryMutRefInner::Table2(x), + ) + }) + .map_err(|x| { + ShortStringHashtableEntryMutRef( + ShortStringHashtableEntryMutRefInner::Table2(x), + ) + }) + } + 17..=24 => { + self.table3.check_grow(); + let mut t = [0u64; 3]; + t[0] = (key.as_ptr() as *const u64).read_unaligned(); + t[1] = (key.as_ptr() as *const u64).offset(1).read_unaligned(); + t[2] = read_le(key.as_ptr().offset(16), key.len() - 16); + let t = std::mem::transmute::<[u64; 3], InlineKey<2>>(t); + self.table3 + .insert(t) + .map(|x| { + self.key_size += key.len(); + ShortStringHashtableEntryMutRef( + ShortStringHashtableEntryMutRefInner::Table3(x), + ) + }) + .map_err(|x| { + ShortStringHashtableEntryMutRef( + ShortStringHashtableEntryMutRefInner::Table3(x), + ) + }) + } + _ => { + self.table4.check_grow(); + self.table4 + .insert(FallbackKey::new(key)) + .map(|x| { + self.key_size += key.len(); + ShortStringHashtableEntryMutRef( + ShortStringHashtableEntryMutRefInner::Table4(x), + ) + }) + .map_err(|x| { + ShortStringHashtableEntryMutRef( + ShortStringHashtableEntryMutRefInner::Table4(x), + ) + }) + } } } } @@ -270,9 +271,11 @@ where /// * The lifetime of key lives longer than the hashtable. #[inline(always)] pub unsafe fn insert_borrowing(&mut self, key: &K) -> Result<&mut MaybeUninit, &mut V> { - match self.insert_and_entry_borrowing(key) { - Ok(e) => Ok(&mut *(e.get_mut_ptr() as *mut MaybeUninit)), - Err(e) => Err(&mut *e.get_mut_ptr()), + unsafe { + match self.insert_and_entry_borrowing(key) { + Ok(e) => Ok(&mut *(e.get_mut_ptr() as *mut MaybeUninit)), + Err(e) => Err(&mut *e.get_mut_ptr()), + } } } @@ -282,28 +285,29 @@ where key: &K, ) -> Result, ShortStringHashtableEntryMutRef<'_, K, V>> { - let key = key.as_bytes(); - match key.len() { - _ if key.last().copied() == Some(0) => { - self.table4.check_grow(); - match self.table4.insert(FallbackKey::new(key)) { - Ok(e) => { - // We need to save the key to avoid drop it. - let s = self.arena.alloc_slice_copy(key); - e.set_key(FallbackKey::new_with_hash(s, e.key.assume_init_ref().hash)); - - self.key_size += key.len(); - Ok(ShortStringHashtableEntryMutRef( + unsafe { + let key = key.as_bytes(); + match key.len() { + _ if key.last().copied() == Some(0) => { + self.table4.check_grow(); + match self.table4.insert(FallbackKey::new(key)) { + Ok(e) => { + // We need to save the key to avoid drop it. + let s = self.arena.alloc_slice_copy(key); + e.set_key(FallbackKey::new_with_hash(s, e.key.assume_init_ref().hash)); + + self.key_size += key.len(); + Ok(ShortStringHashtableEntryMutRef( + ShortStringHashtableEntryMutRefInner::Table4(e), + )) + } + Err(e) => Err(ShortStringHashtableEntryMutRef( ShortStringHashtableEntryMutRefInner::Table4(e), - )) + )), } - Err(e) => Err(ShortStringHashtableEntryMutRef( - ShortStringHashtableEntryMutRefInner::Table4(e), - )), } - } - 0 => { - self.table0 + 0 => self + .table0 .insert() .map(|x| { ShortStringHashtableEntryMutRef( @@ -314,85 +318,85 @@ where ShortStringHashtableEntryMutRef( ShortStringHashtableEntryMutRefInner::Table0(x, PhantomData), ) - }) - } - - 1..=8 => { - self.table1.check_grow(); - let mut t = [0u64; 1]; - t[0] = read_le(key.as_ptr(), key.len()); - let t = std::mem::transmute::<[u64; 1], InlineKey<0>>(t); - self.table1 - .insert(t) - .map(|x| { - self.key_size += key.len(); - ShortStringHashtableEntryMutRef( - ShortStringHashtableEntryMutRefInner::Table1(x), - ) - }) - .map_err(|x| { - ShortStringHashtableEntryMutRef( - ShortStringHashtableEntryMutRefInner::Table1(x), - ) - }) - } - 9..=16 => { - self.table2.check_grow(); - let mut t = [0u64; 2]; - t[0] = (key.as_ptr() as *const u64).read_unaligned(); - t[1] = read_le(key.as_ptr().offset(8), key.len() - 8); - let t = std::mem::transmute::<[u64; 2], InlineKey<1>>(t); - self.table2 - .insert(t) - .map(|x| { - self.key_size += key.len(); - ShortStringHashtableEntryMutRef( - ShortStringHashtableEntryMutRefInner::Table2(x), - ) - }) - .map_err(|x| { - ShortStringHashtableEntryMutRef( - ShortStringHashtableEntryMutRefInner::Table2(x), - ) - }) - } - 17..=24 => { - self.table3.check_grow(); - let mut t = [0u64; 3]; - t[0] = (key.as_ptr() as *const u64).read_unaligned(); - t[1] = (key.as_ptr() as *const u64).offset(1).read_unaligned(); - t[2] = read_le(key.as_ptr().offset(16), key.len() - 16); - let t = std::mem::transmute::<[u64; 3], InlineKey<2>>(t); - self.table3 - .insert(t) - .map(|x| { - self.key_size += key.len(); - ShortStringHashtableEntryMutRef( - ShortStringHashtableEntryMutRefInner::Table3(x), - ) - }) - .map_err(|x| { - ShortStringHashtableEntryMutRef( - ShortStringHashtableEntryMutRefInner::Table3(x), - ) - }) - } - _ => { - self.table4.check_grow(); - match self.table4.insert(FallbackKey::new(key)) { - Ok(e) => { - // We need to save the key to avoid drop it. - let s = self.arena.alloc_slice_copy(key); - e.set_key(FallbackKey::new_with_hash(s, e.key.assume_init_ref().hash)); - - self.key_size += key.len(); - Ok(ShortStringHashtableEntryMutRef( + }), + + 1..=8 => { + self.table1.check_grow(); + let mut t = [0u64; 1]; + t[0] = read_le(key.as_ptr(), key.len()); + let t = std::mem::transmute::<[u64; 1], InlineKey<0>>(t); + self.table1 + .insert(t) + .map(|x| { + self.key_size += key.len(); + ShortStringHashtableEntryMutRef( + ShortStringHashtableEntryMutRefInner::Table1(x), + ) + }) + .map_err(|x| { + ShortStringHashtableEntryMutRef( + ShortStringHashtableEntryMutRefInner::Table1(x), + ) + }) + } + 9..=16 => { + self.table2.check_grow(); + let mut t = [0u64; 2]; + t[0] = (key.as_ptr() as *const u64).read_unaligned(); + t[1] = read_le(key.as_ptr().offset(8), key.len() - 8); + let t = std::mem::transmute::<[u64; 2], InlineKey<1>>(t); + self.table2 + .insert(t) + .map(|x| { + self.key_size += key.len(); + ShortStringHashtableEntryMutRef( + ShortStringHashtableEntryMutRefInner::Table2(x), + ) + }) + .map_err(|x| { + ShortStringHashtableEntryMutRef( + ShortStringHashtableEntryMutRefInner::Table2(x), + ) + }) + } + 17..=24 => { + self.table3.check_grow(); + let mut t = [0u64; 3]; + t[0] = (key.as_ptr() as *const u64).read_unaligned(); + t[1] = (key.as_ptr() as *const u64).offset(1).read_unaligned(); + t[2] = read_le(key.as_ptr().offset(16), key.len() - 16); + let t = std::mem::transmute::<[u64; 3], InlineKey<2>>(t); + self.table3 + .insert(t) + .map(|x| { + self.key_size += key.len(); + ShortStringHashtableEntryMutRef( + ShortStringHashtableEntryMutRefInner::Table3(x), + ) + }) + .map_err(|x| { + ShortStringHashtableEntryMutRef( + ShortStringHashtableEntryMutRefInner::Table3(x), + ) + }) + } + _ => { + self.table4.check_grow(); + match self.table4.insert(FallbackKey::new(key)) { + Ok(e) => { + // We need to save the key to avoid drop it. + let s = self.arena.alloc_slice_copy(key); + e.set_key(FallbackKey::new_with_hash(s, e.key.assume_init_ref().hash)); + + self.key_size += key.len(); + Ok(ShortStringHashtableEntryMutRef( + ShortStringHashtableEntryMutRefInner::Table4(e), + )) + } + Err(e) => Err(ShortStringHashtableEntryMutRef( ShortStringHashtableEntryMutRefInner::Table4(e), - )) + )), } - Err(e) => Err(ShortStringHashtableEntryMutRef( - ShortStringHashtableEntryMutRefInner::Table4(e), - )), } } } @@ -405,29 +409,30 @@ where hash: u64, ) -> Result, ShortStringHashtableEntryMutRef<'_, K, V>> { - let key = key.as_bytes(); - match key.len() { - _ if key.last().copied() == Some(0) => { - self.table4.check_grow(); - match self - .table4 - .insert_with_hash(FallbackKey::new_with_hash(key, hash), hash) - { - Ok(e) => { - // We need to save the key to avoid drop it. - let s = self.arena.alloc_slice_copy(key); - e.set_key(FallbackKey::new_with_hash(s, hash)); - Ok(ShortStringHashtableEntryMutRef( + unsafe { + let key = key.as_bytes(); + match key.len() { + _ if key.last().copied() == Some(0) => { + self.table4.check_grow(); + match self + .table4 + .insert_with_hash(FallbackKey::new_with_hash(key, hash), hash) + { + Ok(e) => { + // We need to save the key to avoid drop it. + let s = self.arena.alloc_slice_copy(key); + e.set_key(FallbackKey::new_with_hash(s, hash)); + Ok(ShortStringHashtableEntryMutRef( + ShortStringHashtableEntryMutRefInner::Table4(e), + )) + } + Err(e) => Err(ShortStringHashtableEntryMutRef( ShortStringHashtableEntryMutRefInner::Table4(e), - )) + )), } - Err(e) => Err(ShortStringHashtableEntryMutRef( - ShortStringHashtableEntryMutRefInner::Table4(e), - )), } - } - 0 => { - self.table0 + 0 => self + .table0 .insert() .map(|x| { self.key_size += key.len(); @@ -439,87 +444,87 @@ where ShortStringHashtableEntryMutRef( ShortStringHashtableEntryMutRefInner::Table0(x, PhantomData), ) - }) - } - 1..=8 => { - self.table1.check_grow(); - let mut t = [0u64; 1]; - t[0] = read_le(key.as_ptr(), key.len()); - let t = std::mem::transmute::<[u64; 1], InlineKey<0>>(t); - self.table1 - .insert_with_hash(t, hash) - .map(|x| { - self.key_size += key.len(); - ShortStringHashtableEntryMutRef( - ShortStringHashtableEntryMutRefInner::Table1(x), - ) - }) - .map_err(|x| { - ShortStringHashtableEntryMutRef( - ShortStringHashtableEntryMutRefInner::Table1(x), - ) - }) - } - 9..=16 => { - self.table2.check_grow(); - let mut t = [0u64; 2]; - t[0] = (key.as_ptr() as *const u64).read_unaligned(); - t[1] = read_le(key.as_ptr().offset(8), key.len() - 8); - let t = std::mem::transmute::<[u64; 2], InlineKey<1>>(t); - self.table2 - .insert_with_hash(t, hash) - .map(|x| { - self.key_size += key.len(); - ShortStringHashtableEntryMutRef( - ShortStringHashtableEntryMutRefInner::Table2(x), - ) - }) - .map_err(|x| { - ShortStringHashtableEntryMutRef( - ShortStringHashtableEntryMutRefInner::Table2(x), - ) - }) - } - 17..=24 => { - self.table3.check_grow(); - let mut t = [0u64; 3]; - t[0] = (key.as_ptr() as *const u64).read_unaligned(); - t[1] = (key.as_ptr() as *const u64).offset(1).read_unaligned(); - t[2] = read_le(key.as_ptr().offset(16), key.len() - 16); - let t = std::mem::transmute::<[u64; 3], InlineKey<2>>(t); - self.table3 - .insert_with_hash(t, hash) - .map(|x| { - self.key_size += key.len(); - ShortStringHashtableEntryMutRef( - ShortStringHashtableEntryMutRefInner::Table3(x), - ) - }) - .map_err(|x| { - ShortStringHashtableEntryMutRef( - ShortStringHashtableEntryMutRefInner::Table3(x), - ) - }) - } - _ => { - self.table4.check_grow(); - match self - .table4 - .insert_with_hash(FallbackKey::new_with_hash(key, hash), hash) - { - Ok(e) => { - // We need to save the key to avoid drop it. - let s = self.arena.alloc_slice_copy(key); - e.set_key(FallbackKey::new_with_hash(s, hash)); - - self.key_size += key.len(); - Ok(ShortStringHashtableEntryMutRef( + }), + 1..=8 => { + self.table1.check_grow(); + let mut t = [0u64; 1]; + t[0] = read_le(key.as_ptr(), key.len()); + let t = std::mem::transmute::<[u64; 1], InlineKey<0>>(t); + self.table1 + .insert_with_hash(t, hash) + .map(|x| { + self.key_size += key.len(); + ShortStringHashtableEntryMutRef( + ShortStringHashtableEntryMutRefInner::Table1(x), + ) + }) + .map_err(|x| { + ShortStringHashtableEntryMutRef( + ShortStringHashtableEntryMutRefInner::Table1(x), + ) + }) + } + 9..=16 => { + self.table2.check_grow(); + let mut t = [0u64; 2]; + t[0] = (key.as_ptr() as *const u64).read_unaligned(); + t[1] = read_le(key.as_ptr().offset(8), key.len() - 8); + let t = std::mem::transmute::<[u64; 2], InlineKey<1>>(t); + self.table2 + .insert_with_hash(t, hash) + .map(|x| { + self.key_size += key.len(); + ShortStringHashtableEntryMutRef( + ShortStringHashtableEntryMutRefInner::Table2(x), + ) + }) + .map_err(|x| { + ShortStringHashtableEntryMutRef( + ShortStringHashtableEntryMutRefInner::Table2(x), + ) + }) + } + 17..=24 => { + self.table3.check_grow(); + let mut t = [0u64; 3]; + t[0] = (key.as_ptr() as *const u64).read_unaligned(); + t[1] = (key.as_ptr() as *const u64).offset(1).read_unaligned(); + t[2] = read_le(key.as_ptr().offset(16), key.len() - 16); + let t = std::mem::transmute::<[u64; 3], InlineKey<2>>(t); + self.table3 + .insert_with_hash(t, hash) + .map(|x| { + self.key_size += key.len(); + ShortStringHashtableEntryMutRef( + ShortStringHashtableEntryMutRefInner::Table3(x), + ) + }) + .map_err(|x| { + ShortStringHashtableEntryMutRef( + ShortStringHashtableEntryMutRefInner::Table3(x), + ) + }) + } + _ => { + self.table4.check_grow(); + match self + .table4 + .insert_with_hash(FallbackKey::new_with_hash(key, hash), hash) + { + Ok(e) => { + // We need to save the key to avoid drop it. + let s = self.arena.alloc_slice_copy(key); + e.set_key(FallbackKey::new_with_hash(s, hash)); + + self.key_size += key.len(); + Ok(ShortStringHashtableEntryMutRef( + ShortStringHashtableEntryMutRefInner::Table4(e), + )) + } + Err(e) => Err(ShortStringHashtableEntryMutRef( ShortStringHashtableEntryMutRefInner::Table4(e), - )) + )), } - Err(e) => Err(ShortStringHashtableEntryMutRef( - ShortStringHashtableEntryMutRefInner::Table4(e), - )), } } } @@ -1194,9 +1199,11 @@ where A: Allocator + Clone + Default &mut self, key: &Self::Key, ) -> Result<&mut MaybeUninit, &mut Self::Value> { - match self.insert_and_entry(key) { - Ok(e) => Ok(&mut *(e.get_mut_ptr() as *mut MaybeUninit)), - Err(e) => Err(&mut *e.get_mut_ptr()), + unsafe { + match self.insert_and_entry(key) { + Ok(e) => Ok(&mut *(e.get_mut_ptr() as *mut MaybeUninit)), + Err(e) => Err(&mut *e.get_mut_ptr()), + } } } @@ -1226,7 +1233,7 @@ where A: Allocator + Clone + Default &mut self, key_ref: &Self::Key, ) -> Result, Self::EntryMutRef<'_>> { - self.insert_and_entry(key_ref) + unsafe { self.insert_and_entry(key_ref) } } unsafe fn insert_and_entry_with_hash( @@ -1234,6 +1241,6 @@ where A: Allocator + Clone + Default key_ref: &Self::Key, hash: u64, ) -> Result, Self::EntryMutRef<'_>> { - self.insert_and_entry_with_hash(key_ref, hash) + unsafe { self.insert_and_entry_with_hash(key_ref, hash) } } } diff --git a/src/common/hashtable/src/stack_hashtable.rs b/src/common/hashtable/src/stack_hashtable.rs index 5be8ebbb649b0..3530c81d221f3 100644 --- a/src/common/hashtable/src/stack_hashtable.rs +++ b/src/common/hashtable/src/stack_hashtable.rs @@ -135,30 +135,34 @@ where &mut self, key: K, ) -> Result<&mut Entry, &mut Entry> { - if unlikely(K::equals_zero(&key)) { - let res = self.zero.is_some(); - if !res { - *self.zero = Some(MaybeUninit::zeroed().assume_init()); - } - let zero = self.zero.as_mut().unwrap(); - if res { - return Err(zero); - } else { - return Ok(zero); + unsafe { + if unlikely(K::equals_zero(&key)) { + let res = self.zero.is_some(); + if !res { + *self.zero = Some(MaybeUninit::zeroed().assume_init()); + } + let zero = self.zero.as_mut().unwrap(); + if res { + return Err(zero); + } else { + return Ok(zero); + } } - } - self.table.check_grow(); + self.table.check_grow(); - self.table.insert(key) + self.table.insert(key) + } } /// # Safety /// /// The returned uninitialized value should be written immediately. #[inline(always)] pub unsafe fn insert(&mut self, key: K) -> Result<&mut MaybeUninit, &mut V> { - match self.insert_and_entry(key) { - Ok(e) => Ok(&mut e.val), - Err(e) => Err(e.val.assume_init_mut()), + unsafe { + match self.insert_and_entry(key) { + Ok(e) => Ok(&mut e.val), + Err(e) => Err(e.val.assume_init_mut()), + } } } pub fn iter(&self) -> StackHashtableIter<'_, K, V> { diff --git a/src/common/hashtable/src/string_hashtable.rs b/src/common/hashtable/src/string_hashtable.rs index 3e0235c98e5e6..f7b6d0ea7e41f 100644 --- a/src/common/hashtable/src/string_hashtable.rs +++ b/src/common/hashtable/src/string_hashtable.rs @@ -29,11 +29,11 @@ use super::traits::EntryRefLike; use super::traits::HashtableLike; use super::traits::UnsizedKeyable; use crate::short_string_hashtable::FallbackKey; -use crate::table0::Table0Iter; -use crate::table0::Table0IterMut; use crate::table_empty::TableEmpty; use crate::table_empty::TableEmptyIter; use crate::table_empty::TableEmptyIterMut; +use crate::table0::Table0Iter; +use crate::table0::Table0IterMut; /// Simple unsized hashtable is used for storing unsized keys in arena. It can be worked with HashMethodSerializer. /// Different from `ShortStringHashTable`, it doesn't use adaptive sub hashtable to store key values via key size. @@ -136,34 +136,36 @@ where &mut self, key: *const K, ) -> Result, StringHashtableEntryMutRef<'_, K, V>> { - let key = (*key).as_bytes(); - match key.len() { - 0 => self - .table_empty - .insert() - .map(|x| { - StringHashtableEntryMutRef(StringHashtableEntryMutRefInner::TableEmpty( - x, - PhantomData, - )) - }) - .map_err(|x| { - StringHashtableEntryMutRef(StringHashtableEntryMutRefInner::TableEmpty( - x, - PhantomData, - )) - }), - _ => { - self.table.check_grow(); - self.table - .insert(FallbackKey::new(key)) + unsafe { + let key = (*key).as_bytes(); + match key.len() { + 0 => self + .table_empty + .insert() .map(|x| { - self.key_size += key.len(); - StringHashtableEntryMutRef(StringHashtableEntryMutRefInner::Table(x)) + StringHashtableEntryMutRef(StringHashtableEntryMutRefInner::TableEmpty( + x, + PhantomData, + )) }) .map_err(|x| { - StringHashtableEntryMutRef(StringHashtableEntryMutRefInner::Table(x)) - }) + StringHashtableEntryMutRef(StringHashtableEntryMutRefInner::TableEmpty( + x, + PhantomData, + )) + }), + _ => { + self.table.check_grow(); + self.table + .insert(FallbackKey::new(key)) + .map(|x| { + self.key_size += key.len(); + StringHashtableEntryMutRef(StringHashtableEntryMutRefInner::Table(x)) + }) + .map_err(|x| { + StringHashtableEntryMutRef(StringHashtableEntryMutRefInner::Table(x)) + }) + } } } } @@ -173,9 +175,11 @@ where /// * The lifetime of key lives longer than the hashtable. #[inline(always)] pub unsafe fn insert_borrowing(&mut self, key: &K) -> Result<&mut MaybeUninit, &mut V> { - match self.insert_and_entry_borrowing(key) { - Ok(e) => Ok(&mut *(e.get_mut_ptr() as *mut MaybeUninit)), - Err(e) => Err(&mut *e.get_mut_ptr()), + unsafe { + match self.insert_and_entry_borrowing(key) { + Ok(e) => Ok(&mut *(e.get_mut_ptr() as *mut MaybeUninit)), + Err(e) => Err(&mut *e.get_mut_ptr()), + } } } @@ -184,40 +188,42 @@ where &mut self, key: &K, ) -> Result, StringHashtableEntryMutRef<'_, K, V>> { - let key = key.as_bytes(); - match key.len() { - 0 => self - .table_empty - .insert() - .map(|x| { - StringHashtableEntryMutRef(StringHashtableEntryMutRefInner::TableEmpty( - x, - PhantomData, - )) - }) - .map_err(|x| { - StringHashtableEntryMutRef(StringHashtableEntryMutRefInner::TableEmpty( - x, - PhantomData, - )) - }), - - _ => { - self.table.check_grow(); - match self.table.insert(FallbackKey::new(key)) { - Ok(e) => { - // We need to save the key to avoid drop it. - let s = self.arena.alloc_slice_copy(key); - e.set_key(FallbackKey::new_with_hash(s, e.key.assume_init_ref().hash)); - - self.key_size += key.len(); - Ok(StringHashtableEntryMutRef( - StringHashtableEntryMutRefInner::Table(e), + unsafe { + let key = key.as_bytes(); + match key.len() { + 0 => self + .table_empty + .insert() + .map(|x| { + StringHashtableEntryMutRef(StringHashtableEntryMutRefInner::TableEmpty( + x, + PhantomData, )) + }) + .map_err(|x| { + StringHashtableEntryMutRef(StringHashtableEntryMutRefInner::TableEmpty( + x, + PhantomData, + )) + }), + + _ => { + self.table.check_grow(); + match self.table.insert(FallbackKey::new(key)) { + Ok(e) => { + // We need to save the key to avoid drop it. + let s = self.arena.alloc_slice_copy(key); + e.set_key(FallbackKey::new_with_hash(s, e.key.assume_init_ref().hash)); + + self.key_size += key.len(); + Ok(StringHashtableEntryMutRef( + StringHashtableEntryMutRefInner::Table(e), + )) + } + Err(e) => Err(StringHashtableEntryMutRef( + StringHashtableEntryMutRefInner::Table(e), + )), } - Err(e) => Err(StringHashtableEntryMutRef( - StringHashtableEntryMutRefInner::Table(e), - )), } } } @@ -229,60 +235,64 @@ where key: &K, hash: u64, ) -> Result, StringHashtableEntryMutRef<'_, K, V>> { - let key = key.as_bytes(); - match key.len() { - 0 => self - .table_empty - .insert() - .map(|x| { - StringHashtableEntryMutRef(StringHashtableEntryMutRefInner::TableEmpty( - x, - PhantomData, - )) - }) - .map_err(|x| { - StringHashtableEntryMutRef(StringHashtableEntryMutRefInner::TableEmpty( - x, - PhantomData, - )) - }), - _ => { - self.table.check_grow(); - match self - .table - .insert_with_hash(FallbackKey::new_with_hash(key, hash), hash) - { - Ok(e) => { - // We need to save the key to avoid drop it. - let s = self.arena.alloc_slice_copy(key); - e.set_key(FallbackKey::new_with_hash(s, hash)); - - self.key_size += key.len(); - Ok(StringHashtableEntryMutRef( - StringHashtableEntryMutRefInner::Table(e), + unsafe { + let key = key.as_bytes(); + match key.len() { + 0 => self + .table_empty + .insert() + .map(|x| { + StringHashtableEntryMutRef(StringHashtableEntryMutRefInner::TableEmpty( + x, + PhantomData, )) + }) + .map_err(|x| { + StringHashtableEntryMutRef(StringHashtableEntryMutRefInner::TableEmpty( + x, + PhantomData, + )) + }), + _ => { + self.table.check_grow(); + match self + .table + .insert_with_hash(FallbackKey::new_with_hash(key, hash), hash) + { + Ok(e) => { + // We need to save the key to avoid drop it. + let s = self.arena.alloc_slice_copy(key); + e.set_key(FallbackKey::new_with_hash(s, hash)); + + self.key_size += key.len(); + Ok(StringHashtableEntryMutRef( + StringHashtableEntryMutRefInner::Table(e), + )) + } + Err(e) => Err(StringHashtableEntryMutRef( + StringHashtableEntryMutRefInner::Table(e), + )), } - Err(e) => Err(StringHashtableEntryMutRef( - StringHashtableEntryMutRefInner::Table(e), - )), } } } } pub unsafe fn get_slot_index(&self, key: &K) -> Option { - let key = (*key).as_bytes(); + unsafe { + let key = (*key).as_bytes(); + + if key.is_empty() { + return match self.table_empty.has_zero { + true => Some(0), + false => None, + }; + } - if key.is_empty() { - return match self.table_empty.has_zero { - true => Some(0), - false => None, - }; + self.table + .get_slot_index(&FallbackKey::new(key)) + .map(|x| 1 + x) } - - self.table - .get_slot_index(&FallbackKey::new(key)) - .map(|x| 1 + x) } } @@ -628,9 +638,11 @@ where A: Allocator + Clone + Default &mut self, key: &Self::Key, ) -> Result<&mut MaybeUninit, &mut Self::Value> { - match self.insert_and_entry(key) { - Ok(e) => Ok(&mut *(e.get_mut_ptr() as *mut MaybeUninit)), - Err(e) => Err(&mut *e.get_mut_ptr()), + unsafe { + match self.insert_and_entry(key) { + Ok(e) => Ok(&mut *(e.get_mut_ptr() as *mut MaybeUninit)), + Err(e) => Err(&mut *e.get_mut_ptr()), + } } } @@ -652,7 +664,7 @@ where A: Allocator + Clone + Default &mut self, key_ref: &Self::Key, ) -> Result, Self::EntryMutRef<'_>> { - self.insert_and_entry(key_ref) + unsafe { self.insert_and_entry(key_ref) } } unsafe fn insert_and_entry_with_hash( @@ -660,6 +672,6 @@ where A: Allocator + Clone + Default key_ref: &Self::Key, hash: u64, ) -> Result, Self::EntryMutRef<'_>> { - self.insert_and_entry_with_hash(key_ref, hash) + unsafe { self.insert_and_entry_with_hash(key_ref, hash) } } } diff --git a/src/common/hashtable/src/table0.rs b/src/common/hashtable/src/table0.rs index d9959ec2553b1..2316ae02edb8f 100644 --- a/src/common/hashtable/src/table0.rs +++ b/src/common/hashtable/src/table0.rs @@ -116,7 +116,7 @@ where /// `key` doesn't equal to zero. #[inline(always)] pub unsafe fn get(&self, key: &K) -> Option<&Entry> { - self.get_with_hash(key, key.hash()) + unsafe { self.get_with_hash(key, key.hash()) } } /// # Safety /// @@ -124,26 +124,28 @@ where /// Provided hash is correct. #[inline(always)] pub unsafe fn get_with_hash(&self, key: &K, hash: u64) -> Option<&Entry> { - assume(!K::equals_zero(key)); - let index = (hash as usize) & (self.entries.len() - 1); - for i in (index..self.entries.len()).chain(0..index) { - assume(i < self.entries.len()); - if self.entries[i].is_zero() { - return None; - } + unsafe { + assume(!K::equals_zero(key)); + let index = (hash as usize) & (self.entries.len() - 1); + for i in (index..self.entries.len()).chain(0..index) { + assume(i < self.entries.len()); + if self.entries[i].is_zero() { + return None; + } - if self.entries[i].key.assume_init_ref() == key { - return Some(&self.entries[i]); + if self.entries[i].key.assume_init_ref() == key { + return Some(&self.entries[i]); + } } + None } - None } /// # Safety /// /// `key` doesn't equal to zero. #[inline(always)] pub unsafe fn get_mut(&mut self, key: &K) -> Option<&mut Entry> { - self.get_with_hash_mut(key, key.hash()) + unsafe { self.get_with_hash_mut(key, key.hash()) } } /// # Safety /// @@ -151,34 +153,38 @@ where /// Provided hash is correct. #[inline(always)] pub unsafe fn get_with_hash_mut(&mut self, key: &K, hash: u64) -> Option<&mut Entry> { - assume(!K::equals_zero(key)); - let index = (hash as usize) & (self.entries.len() - 1); - for i in (index..self.entries.len()).chain(0..index) { - assume(i < self.entries.len()); - if self.entries[i].is_zero() { - return None; - } - if self.entries[i].key.assume_init_ref() == key { - return Some(&mut self.entries[i]); + unsafe { + assume(!K::equals_zero(key)); + let index = (hash as usize) & (self.entries.len() - 1); + for i in (index..self.entries.len()).chain(0..index) { + assume(i < self.entries.len()); + if self.entries[i].is_zero() { + return None; + } + if self.entries[i].key.assume_init_ref() == key { + return Some(&mut self.entries[i]); + } } + None } - None } pub unsafe fn get_slot_index(&self, key: &K) -> Option { - assume(!K::equals_zero(key)); + unsafe { + assume(!K::equals_zero(key)); - let index = (key.hash() as usize) & (self.entries.len() - 1); - for i in (index..self.entries.len()).chain(0..index) { - assume(i < self.entries.len()); - if self.entries[i].is_zero() { - return None; - } - if self.entries[i].key.assume_init_ref() == key { - return Some(i); + let index = (key.hash() as usize) & (self.entries.len() - 1); + for i in (index..self.entries.len()).chain(0..index) { + assume(i < self.entries.len()); + if self.entries[i].is_zero() { + return None; + } + if self.entries[i].key.assume_init_ref() == key { + return Some(i); + } } + None } - None } /// # Safety @@ -191,7 +197,7 @@ where /// Panics if the hash table overflows. #[inline(always)] pub unsafe fn insert(&mut self, key: K) -> Result<&mut Entry, &mut Entry> { - self.insert_with_hash(key, key.hash()) + unsafe { self.insert_with_hash(key, key.hash()) } } /// # Safety /// @@ -207,20 +213,22 @@ where key: K, hash: u64, ) -> Result<&mut Entry, &mut Entry> { - assume(!K::equals_zero(&key)); - let index = (hash as usize) & (self.entries.len() - 1); - for i in (index..self.entries.len()).chain(0..index) { - assume(i < self.entries.len()); - if self.entries[i].is_zero() { - self.len += 1; - self.entries[i].key.write(key); - return Ok(&mut self.entries[i]); - } - if self.entries[i].key.assume_init_ref() == &key { - return Err(&mut self.entries[i]); + unsafe { + assume(!K::equals_zero(&key)); + let index = (hash as usize) & (self.entries.len() - 1); + for i in (index..self.entries.len()).chain(0..index) { + assume(i < self.entries.len()); + if self.entries[i].is_zero() { + self.len += 1; + self.entries[i].key.write(key); + return Ok(&mut self.entries[i]); + } + if self.entries[i].key.assume_init_ref() == &key { + return Err(&mut self.entries[i]); + } } + panic!("the hash table overflows") } - panic!("the hash table overflows") } pub fn iter(&self) -> Table0Iter<'_, K, V> { Table0Iter { @@ -324,16 +332,18 @@ where A: Allocator + Clone, { pub unsafe fn set_merge(&mut self, other: &Self) { - while (self.len() + other.len()) * 2 > self.capacity() { - if (self.entries.len() >> 22) == 0 { - self.grow(2); - } else { - self.grow(1); + unsafe { + while (self.len() + other.len()) * 2 > self.capacity() { + if (self.entries.len() >> 22) == 0 { + self.grow(2); + } else { + self.grow(1); + } + } + for entry in other.iter() { + let key = entry.key.assume_init(); + let _ = self.insert(key); } - } - for entry in other.iter() { - let key = entry.key.assume_init(); - let _ = self.insert(key); } } } diff --git a/src/common/hashtable/src/traits.rs b/src/common/hashtable/src/traits.rs index 4979e29b1593a..2be4be395370b 100644 --- a/src/common/hashtable/src/traits.rs +++ b/src/common/hashtable/src/traits.rs @@ -20,8 +20,8 @@ use std::num::NonZeroU64; use databend_common_base::base::OrderedFloat; use databend_common_column::bitmap::Bitmap; -use ethnum::i256; use ethnum::U256; +use ethnum::i256; use crate::RowPtr; @@ -165,7 +165,7 @@ unsafe impl UnsizedKeyable for str { } unsafe fn from_bytes(bytes: &[u8]) -> &Self { - std::str::from_utf8_unchecked(bytes) + unsafe { std::str::from_utf8_unchecked(bytes) } } } diff --git a/src/common/hashtable/src/utils.rs b/src/common/hashtable/src/utils.rs index 1576568eab973..2af3b15d6cb1c 100644 --- a/src/common/hashtable/src/utils.rs +++ b/src/common/hashtable/src/utils.rs @@ -92,12 +92,14 @@ impl Drop for ZeroEntry { #[inline(always)] pub unsafe fn read_le(data: *const u8, len: usize) -> u64 { - assume(0 < len && len <= 8); - let s = 64 - 8 * len as isize; - if data as usize & 2048 == 0 { - (data as *const u64).read_unaligned() & (u64::MAX >> s) - } else { - (data.offset(len as isize - 8) as *const u64).read_unaligned() >> s + unsafe { + assume(0 < len && len <= 8); + let s = 64 - 8 * len as isize; + if data as usize & 2048 == 0 { + (data as *const u64).read_unaligned() & (u64::MAX >> s) + } else { + (data.offset(len as isize - 8) as *const u64).read_unaligned() >> s + } } } @@ -410,7 +412,7 @@ impl MergeIntoBlockInfoIndex { let mut new_chunk = true; while chunk_idx < chunks_offsets.len() && partial_idx < partial_unmodified.len() { // here is '<', not '<=', chunks_offsets[chunk_idx] is the count of chunks[chunk_idx] - if partial_unmodified[partial_idx].0 .1 < chunks_offsets[chunk_idx] { + if partial_unmodified[partial_idx].0.1 < chunks_offsets[chunk_idx] { if new_chunk { res.push((Vec::new(), chunk_idx as u64)); offset = res.len() - 1; @@ -629,13 +631,13 @@ fn test_chunk_offsets_skip_chunk() { let res = block_info_index.chunk_offsets(&partial_unmodified, &chunks_offsets); assert_eq!(res.len(), 2); assert_eq!(res[0].0.len(), 1); - assert_eq!(res[0].0[0].0 .0, 8); - assert_eq!(res[0].0[0].0 .1, 10); + assert_eq!(res[0].0[0].0.0, 8); + assert_eq!(res[0].0[0].0.1, 10); assert_eq!(res[1].0.len(), 2); - assert_eq!(res[1].0[0].0 .0, 40); - assert_eq!(res[1].0[0].0 .1, 46); + assert_eq!(res[1].0[0].0.0, 40); + assert_eq!(res[1].0[0].0.1, 46); - assert_eq!(res[1].0[1].0 .0, 51); - assert_eq!(res[1].0[1].0 .1, 55); + assert_eq!(res[1].0[1].0.0, 51); + assert_eq!(res[1].0[1].0.1, 55); } diff --git a/src/common/hashtable/tests/it/main.rs b/src/common/hashtable/tests/it/main.rs index fa6653ccd4ad0..5b953a5b5ae47 100644 --- a/src/common/hashtable/tests/it/main.rs +++ b/src/common/hashtable/tests/it/main.rs @@ -15,12 +15,11 @@ #![allow(clippy::arc_with_non_send_sync)] use std::ptr::NonNull; +use std::sync::Arc; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; -use std::sync::Arc; use bumpalo::Bump; -use databend_common_hashtable::fast_memcmp; use databend_common_hashtable::DictionaryKeys; use databend_common_hashtable::DictionaryStringHashMap; use databend_common_hashtable::HashMap; @@ -28,8 +27,9 @@ use databend_common_hashtable::HashtableEntryMutRefLike; use databend_common_hashtable::HashtableLike; use databend_common_hashtable::ShortStringHashMap; use databend_common_hashtable::StackHashMap; -use rand::distributions::Alphanumeric; +use databend_common_hashtable::fast_memcmp; use rand::Rng; +use rand::distributions::Alphanumeric; macro_rules! simple_test { ($t: tt) => { @@ -198,9 +198,11 @@ fn test_dictionary_hash_map() { NonNull::from(index1_str.as_bytes()), NonNull::from(index2_str.as_bytes()), ]; - assert!(hashtable - .insert_and_entry(&DictionaryKeys::create(&keys)) - .is_err()); + assert!( + hashtable + .insert_and_entry(&DictionaryKeys::create(&keys)) + .is_err() + ); } } } diff --git a/src/common/http/src/debug/home.rs b/src/common/http/src/debug/home.rs index 089015be8f91e..ded36781a66b2 100644 --- a/src/common/http/src/debug/home.rs +++ b/src/common/http/src/debug/home.rs @@ -14,8 +14,8 @@ use std::num::NonZeroI32; -use poem::web::Html; use poem::IntoResponse; +use poem::web::Html; #[derive(serde::Serialize, serde::Deserialize, Debug)] pub struct PProfRequest { diff --git a/src/common/http/src/debug/jeprof.rs b/src/common/http/src/debug/jeprof.rs index 406ee241adc04..6d9d11d4297bf 100644 --- a/src/common/http/src/debug/jeprof.rs +++ b/src/common/http/src/debug/jeprof.rs @@ -15,10 +15,10 @@ use databend_common_base::mem_allocator::dump_profile; use databend_common_exception::ErrorCode; use http::StatusCode; +use poem::Error; use poem::error::InternalServerError; use poem::web::IntoResponse; use poem::web::Query; -use poem::Error; use tempfile::Builder; #[poem::handler] diff --git a/src/common/http/src/debug/pprof.rs b/src/common/http/src/debug/pprof.rs index 5dd30a5662170..5c7af27eaac83 100644 --- a/src/common/http/src/debug/pprof.rs +++ b/src/common/http/src/debug/pprof.rs @@ -12,12 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_base::base::tokio::time::Duration; use databend_common_base::base::Profiling; +use databend_common_base::base::tokio::time::Duration; use log::debug; +use poem::IntoResponse; use poem::error::InternalServerError; use poem::web::Query; -use poem::IntoResponse; use crate::debug::PProfRequest; diff --git a/src/common/http/src/debug/stack.rs b/src/common/http/src/debug/stack.rs index 96f3b3620e237..d3bfbc08d7b1e 100644 --- a/src/common/http/src/debug/stack.rs +++ b/src/common/http/src/debug/stack.rs @@ -13,8 +13,8 @@ // limitations under the License. use databend_common_base::dump_backtrace; -use poem::web::Query; use poem::IntoResponse; +use poem::web::Query; #[derive(serde::Serialize, serde::Deserialize, Debug)] pub struct DumpStackRequest { diff --git a/src/common/http/src/health.rs b/src/common/http/src/health.rs index e5f26ac881eca..c0bc4c2d0358a 100644 --- a/src/common/http/src/health.rs +++ b/src/common/http/src/health.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use poem::web::Json; use poem::IntoResponse; +use poem::web::Json; #[derive(serde::Serialize)] pub struct HealthCheckResponse { diff --git a/src/common/http/src/http_shutdown_handlers.rs b/src/common/http/src/http_shutdown_handlers.rs index 138dcdba44e2f..e02c6b4ef6030 100644 --- a/src/common/http/src/http_shutdown_handlers.rs +++ b/src/common/http/src/http_shutdown_handlers.rs @@ -20,17 +20,17 @@ use databend_common_base::base::tokio; use databend_common_base::base::tokio::sync::broadcast; use databend_common_base::base::tokio::sync::oneshot; use databend_common_base::base::tokio::task::JoinHandle; -use futures::future::Either; use futures::FutureExt; +use futures::future::Either; use log::error; use log::info; +use poem::Endpoint; use poem::listener::Acceptor; use poem::listener::AcceptorExt; use poem::listener::IntoTlsConfigStream; use poem::listener::Listener; use poem::listener::OpensslTlsConfig; use poem::listener::TcpListener; -use poem::Endpoint; use crate::HttpError; @@ -148,19 +148,19 @@ impl HttpShutdownHandler { if let Some(abort_handle) = self.abort_handle.take() { let _ = abort_handle.send(()); } - if let Some(join_handle) = self.join_handle.take() { - if let Err(error) = join_handle.await { - error!( - "Unexpected error during shutdown Http Server {}. cause {}", - self.service_name, error - ); - } + if let Some(join_handle) = self.join_handle.take() + && let Err(error) = join_handle.await + { + error!( + "Unexpected error during shutdown Http Server {}. cause {}", + self.service_name, error + ); } - if let Some(join_handle) = self.join_handle.take() { - if let Err(_err) = tokio::time::timeout(Duration::from_secs(5), join_handle).await { - error!("Timeout during shutdown Http Server {}", self.service_name); - } + if let Some(join_handle) = self.join_handle.take() + && let Err(_err) = tokio::time::timeout(Duration::from_secs(5), join_handle).await + { + error!("Timeout during shutdown Http Server {}", self.service_name); } } else if let Some(join_handle) = self.join_handle.take() { join_handle.abort(); diff --git a/src/common/http/tests/it/health.rs b/src/common/http/tests/it/health.rs index 429bb6120e4f4..b81e4e7ec749a 100644 --- a/src/common/http/tests/it/health.rs +++ b/src/common/http/tests/it/health.rs @@ -17,10 +17,10 @@ use databend_common_http::health_handler; use http::Method; use http::StatusCode; use http::Uri; -use poem::get; use poem::Endpoint; use poem::Request; use poem::Route; +use poem::get; use pretty_assertions::assert_eq; #[tokio::test(flavor = "multi_thread")] diff --git a/src/common/io/src/bitmap.rs b/src/common/io/src/bitmap.rs index fbb3f456ffe2e..289ced1821be0 100644 --- a/src/common/io/src/bitmap.rs +++ b/src/common/io/src/bitmap.rs @@ -24,8 +24,8 @@ use std::ptr; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use roaring::treemap::Iter; use roaring::RoaringTreemap; +use roaring::treemap::Iter; use smallvec::SmallVec; // https://github.com/ClickHouse/ClickHouse/blob/516a6ed6f8bd8c5f6eed3a10e9037580b2fb6152/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h#L914 @@ -185,15 +185,15 @@ impl HybridBitmap { } fn try_demote(&mut self) { - if let HybridBitmap::Large(tree) = self { - if (tree.len() as usize) <= LARGE_THRESHOLD { - let data = mem::take(tree); - let mut set = SmallBitmap::with_capacity(data.len() as usize); - for value in data.into_iter() { - set.push(value); - } - *self = HybridBitmap::Small(set); + if let HybridBitmap::Large(tree) = self + && (tree.len() as usize) <= LARGE_THRESHOLD + { + let data = mem::take(tree); + let mut set = SmallBitmap::with_capacity(data.len() as usize); + for value in data.into_iter() { + set.push(value); } + *self = HybridBitmap::Small(set); } } } diff --git a/src/common/io/src/cursor_ext/cursor_read_bytes_ext.rs b/src/common/io/src/cursor_ext/cursor_read_bytes_ext.rs index 7f4603575c396..20b8baac57743 100644 --- a/src/common/io/src/cursor_ext/cursor_read_bytes_ext.rs +++ b/src/common/io/src/cursor_ext/cursor_read_bytes_ext.rs @@ -79,11 +79,7 @@ where T: AsRef<[u8]> fn peek_byte(&self) -> Option { let buf = Cursor::split(self).1; - if buf.is_empty() { - None - } else { - Some(buf[0]) - } + if buf.is_empty() { None } else { Some(buf[0]) } } fn eof(&mut self) -> bool { diff --git a/src/common/io/src/cursor_ext/cursor_read_datetime_ext.rs b/src/common/io/src/cursor_ext/cursor_read_datetime_ext.rs index a64f295e69a07..b63dff9779086 100644 --- a/src/common/io/src/cursor_ext/cursor_read_datetime_ext.rs +++ b/src/common/io/src/cursor_ext/cursor_read_datetime_ext.rs @@ -27,12 +27,12 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_exception::ToErrorCode; use databend_common_timezone::fast_utc_from_local; +use jiff::Timestamp; +use jiff::Zoned; use jiff::civil::Date; use jiff::civil::Time; use jiff::tz::Offset; use jiff::tz::TimeZone; -use jiff::Timestamp; -use jiff::Zoned; use crate::cursor_ext::cursor_read_bytes_ext::ReadBytesExt; use crate::datetime::parse_standard_timestamp as parse_iso_timestamp; @@ -186,8 +186,10 @@ where T: AsRef<[u8]> let n = self.keep_read(buf, |f| f.is_ascii_digit()); match n { 2 => { - let hour_offset: i32 = - lexical_core::FromLexical::from_lexical(buf.as_slice()).map_err_to_code(ErrorCode::BadBytes, || "hour offset parse error".to_string())?; + let hour_offset: i32 = lexical_core::FromLexical::from_lexical(buf.as_slice()) + .map_err_to_code(ErrorCode::BadBytes, || { + "hour offset parse error".to_string() + })?; if (0..15).contains(&hour_offset) { buf.clear(); if self.ignore_byte(b':') { @@ -198,9 +200,18 @@ where T: AsRef<[u8]> )); } let minute_offset: i32 = - lexical_core::FromLexical::from_lexical(buf.as_slice()).map_err_to_code(ErrorCode::BadBytes, || "minute offset parse error".to_string())?; + lexical_core::FromLexical::from_lexical(buf.as_slice()) + .map_err_to_code(ErrorCode::BadBytes, || { + "minute offset parse error".to_string() + })?; // max utc: 14:00, min utc: 00:00 - get_hour_minute_offset(dt, west_tz, &calc_offset, hour_offset, minute_offset) + get_hour_minute_offset( + dt, + west_tz, + &calc_offset, + hour_offset, + minute_offset, + ) } else { get_hour_minute_offset(dt, west_tz, &calc_offset, hour_offset, 0) } @@ -213,11 +224,15 @@ where T: AsRef<[u8]> } 4 => { let hour_offset = &buf.as_slice()[..2]; - let hour_offset: i32 = - lexical_core::FromLexical::from_lexical(hour_offset).map_err_to_code(ErrorCode::BadBytes, || "hour offset parse error".to_string())?; + let hour_offset: i32 = lexical_core::FromLexical::from_lexical(hour_offset) + .map_err_to_code(ErrorCode::BadBytes, || { + "hour offset parse error".to_string() + })?; let minute_offset = &buf.as_slice()[2..]; - let minute_offset: i32 = - lexical_core::FromLexical::from_lexical(minute_offset).map_err_to_code(ErrorCode::BadBytes, || "minute offset parse error".to_string())?; + let minute_offset: i32 = lexical_core::FromLexical::from_lexical(minute_offset) + .map_err_to_code(ErrorCode::BadBytes, || { + "minute offset parse error".to_string() + })?; buf.clear(); // max utc: 14:00, min utc: 00:00 if (0..15).contains(&hour_offset) { @@ -378,15 +393,18 @@ where T: AsRef<[u8]> // only date part if need_date { Ok(DateTimeResType::Date(d)) - } else if let Some(zoned) = fast_local_to_zoned(tz, &d, 0, 0, 0, 0) { - Ok(DateTimeResType::Datetime(zoned)) } else { - let zoned = tz - .to_zoned(d.to_datetime(Time::midnight())) - .map_err_to_code(ErrorCode::BadBytes, || { - format!("Failed to parse date {} as timestamp.", d) - })?; - Ok(DateTimeResType::Datetime(zoned)) + match fast_local_to_zoned(tz, &d, 0, 0, 0, 0) { + Some(zoned) => Ok(DateTimeResType::Datetime(zoned)), + _ => { + let zoned = tz + .to_zoned(d.to_datetime(Time::midnight())) + .map_err_to_code(ErrorCode::BadBytes, || { + format!("Failed to parse date {} as timestamp.", d) + })?; + Ok(DateTimeResType::Datetime(zoned)) + } + } } } } @@ -454,17 +472,17 @@ pub fn unwrap_local_time( Ok(t2) } LocalResult::None => { - if enable_dst_hour_fix { - if let Some(res2) = naive_datetime.checked_add_signed(Duration::seconds(3600)) { - return match tz.from_local_datetime(&res2) { - MappedLocalTime::Single(t) => Ok(t), - MappedLocalTime::Ambiguous(t1, _) => Ok(t1), - MappedLocalTime::None => Err(ErrorCode::BadArguments(format!( - "Local Time Error: The local time {}, {} can not map to a single unique result with timezone {}", - naive_datetime, res2, tz - ))), - }; - } + if enable_dst_hour_fix + && let Some(res2) = naive_datetime.checked_add_signed(Duration::seconds(3600)) + { + return match tz.from_local_datetime(&res2) { + MappedLocalTime::Single(t) => Ok(t), + MappedLocalTime::Ambiguous(t1, _) => Ok(t1), + MappedLocalTime::None => Err(ErrorCode::BadArguments(format!( + "Local Time Error: The local time {}, {} can not map to a single unique result with timezone {}", + naive_datetime, res2, tz + ))), + }; } Err(ErrorCode::BadArguments(format!( "The time {} can not map to a single unique result with timezone {}", diff --git a/src/common/io/src/cursor_ext/cursor_read_string_ext.rs b/src/common/io/src/cursor_ext/cursor_read_string_ext.rs index 568a0e716665d..3e053d2736327 100644 --- a/src/common/io/src/cursor_ext/cursor_read_string_ext.rs +++ b/src/common/io/src/cursor_ext/cursor_read_string_ext.rs @@ -188,10 +188,10 @@ where T: AsRef<[u8]> // Check that the pre-calculated position is correct. fn check_pos(curr_pos: usize, positions: &mut VecDeque) -> Result<()> { - if let Some(pos) = positions.pop_front() { - if curr_pos == pos { - return Ok(()); - } + if let Some(pos) = positions.pop_front() + && curr_pos == pos + { + return Ok(()); } Err(std::io::Error::new( ErrorKind::InvalidData, diff --git a/src/common/io/src/cursor_ext/mod.rs b/src/common/io/src/cursor_ext/mod.rs index d6d1465b365ad..d6278f60a9dac 100644 --- a/src/common/io/src/cursor_ext/mod.rs +++ b/src/common/io/src/cursor_ext/mod.rs @@ -20,10 +20,10 @@ mod cursor_read_string_ext; pub use cursor_checkpoint_ext::ReadCheckPointExt; pub use cursor_read_bytes_ext::ReadBytesExt; -pub use cursor_read_datetime_ext::unwrap_local_time; pub use cursor_read_datetime_ext::BufferReadDateTimeExt; pub use cursor_read_datetime_ext::DateTimeResType; +pub use cursor_read_datetime_ext::unwrap_local_time; +pub use cursor_read_number_ext::ReadNumberExt; pub use cursor_read_number_ext::collect_number; pub use cursor_read_number_ext::read_num_text_exact; -pub use cursor_read_number_ext::ReadNumberExt; pub use cursor_read_string_ext::BufferReadStringExt; diff --git a/src/common/io/src/datetime.rs b/src/common/io/src/datetime.rs index ce13bccef98a1..57aa84b398c22 100644 --- a/src/common/io/src/datetime.rs +++ b/src/common/io/src/datetime.rs @@ -139,7 +139,7 @@ pub fn parse_standard_timestamp(input: &[u8]) -> Option> None => { return Some(Err(ErrorCode::BadBytes( "Invalid timezone offset: hour part must be digits".to_string(), - ))) + ))); } }; idx += 2; @@ -156,7 +156,7 @@ pub fn parse_standard_timestamp(input: &[u8]) -> Option> None => { return Some(Err(ErrorCode::BadBytes( "Invalid timezone offset minute part".to_string(), - ))) + ))); } }; idx += 2; @@ -169,7 +169,7 @@ pub fn parse_standard_timestamp(input: &[u8]) -> Option> None => { return Some(Err(ErrorCode::BadBytes( "Invalid timezone offset minute part".to_string(), - ))) + ))); } }; idx += 2; diff --git a/src/common/io/src/geography.rs b/src/common/io/src/geography.rs index ef436ad44b92c..3d939e584f651 100644 --- a/src/common/io/src/geography.rs +++ b/src/common/io/src/geography.rs @@ -33,13 +33,13 @@ pub fn geography_from_ewkt_bytes(ewkt: &[u8]) -> Result> { pub fn geography_from_ewkt(input: &str) -> Result> { let input = input.trim(); let (geo, parsed_srid) = ewkt_str_to_geo(input)?; - if let Some(parsed_srid) = parsed_srid { - if parsed_srid != 4326 { - return Err(ErrorCode::GeometryError(format!( - "SRIDs other than 4326 are not supported. Got SRID: {}", - parsed_srid - ))); - } + if let Some(parsed_srid) = parsed_srid + && parsed_srid != 4326 + { + return Err(ErrorCode::GeometryError(format!( + "SRIDs other than 4326 are not supported. Got SRID: {}", + parsed_srid + ))); } geo.coords_iter().try_for_each(|c| check_point(c.x, c.y))?; geo.to_ewkb(geozero::CoordDimensions::xy(), parsed_srid) diff --git a/src/common/io/src/geometry.rs b/src/common/io/src/geometry.rs index c22747386b479..bd4d649929fb6 100644 --- a/src/common/io/src/geometry.rs +++ b/src/common/io/src/geometry.rs @@ -18,9 +18,6 @@ use std::str::FromStr; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use geo::Geometry; -use geozero::geo_types::GeoWriter; -use geozero::geojson::GeoJson; -use geozero::wkb::Ewkb; use geozero::CoordDimensions; use geozero::GeomProcessor; use geozero::GeozeroGeometry; @@ -28,6 +25,9 @@ use geozero::ToGeo; use geozero::ToJson; use geozero::ToWkb; use geozero::ToWkt; +use geozero::geo_types::GeoWriter; +use geozero::geojson::GeoJson; +use geozero::wkb::Ewkb; use serde::Deserialize; use serde::Serialize; use wkt::TryFromWkt; @@ -165,13 +165,13 @@ pub(crate) fn ewkt_str_to_geo(input: &str) -> Result<(Geometry, Option)> { if input.starts_with(['s']) || input.starts_with(['S']) { if let Some((srid_input, wkt_input)) = input.split_once(';') { let srid_input = srid_input.to_uppercase(); - if let Some(srid_str) = srid_input.strip_prefix("SRID") { - if let Some(srid_str) = srid_str.trim().strip_prefix("=") { - let parsed_srid = srid_str.trim().parse::()?; - let geo = Geometry::try_from_wkt_str(wkt_input) - .map_err(|e| ErrorCode::GeometryError(e.to_string()))?; - return Ok((geo, Some(parsed_srid))); - } + if let Some(srid_str) = srid_input.strip_prefix("SRID") + && let Some(srid_str) = srid_str.trim().strip_prefix("=") + { + let parsed_srid = srid_str.trim().parse::()?; + let geo = Geometry::try_from_wkt_str(wkt_input) + .map_err(|e| ErrorCode::GeometryError(e.to_string()))?; + return Ok((geo, Some(parsed_srid))); } } Err(ErrorCode::GeometryError("invalid srid")) diff --git a/src/common/io/src/lib.rs b/src/common/io/src/lib.rs index b4036513b3466..4c7b97442e6a4 100644 --- a/src/common/io/src/lib.rs +++ b/src/common/io/src/lib.rs @@ -19,7 +19,6 @@ #![feature(can_vector)] #![feature(read_buf)] #![feature(slice_internals)] -#![feature(maybe_uninit_slice)] #![feature(cursor_split)] #![feature(buf_read_has_data_left)] @@ -47,13 +46,16 @@ mod stat_buffer; pub mod interval; pub mod wkb; +pub use bitmap::HybridBitmap; pub use bitmap::deserialize_bitmap; pub use bitmap::parse_bitmap; -pub use bitmap::HybridBitmap; pub use decimal::display_decimal_128; pub use decimal::display_decimal_256; pub use escape::escape_string; pub use escape::escape_string_with_quote; +pub use geometry::Axis; +pub use geometry::Extremum; +pub use geometry::GeometryDataType; pub use geometry::ewkb_to_geo; pub use geometry::geo_to_ewkb; pub use geometry::geo_to_ewkt; @@ -65,7 +67,4 @@ pub use geometry::geometry_from_ewkt; pub use geometry::geometry_type_name; pub use geometry::parse_bytes_to_ewkb; pub use geometry::read_srid; -pub use geometry::Axis; -pub use geometry::Extremum; -pub use geometry::GeometryDataType; pub use interval::Interval; diff --git a/src/common/io/src/number.rs b/src/common/io/src/number.rs index 76e7622152ea4..e12039b943275 100644 --- a/src/common/io/src/number.rs +++ b/src/common/io/src/number.rs @@ -16,8 +16,8 @@ use std::str::FromStr; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use enumflags2::bitflags; use enumflags2::BitFlags; +use enumflags2::bitflags; // Template Patterns for Numeric Formatting // https://github.com/postgres/postgres/blob/master/src/backend/utils/adt/formatting.c @@ -830,7 +830,7 @@ fn num_processor(nodes: &[FormatNode], desc: NumDesc, num_part: NumPart) -> Resu // } else { // TODO: Why is this not the same as the postgres implementation? np.sign_wrote = true; // needn't sign - // } + // } } else { if np.sign && np.desc.flag.contains(NumFlag::FillMode) { np.desc.flag.remove(NumFlag::Bracket) @@ -859,16 +859,16 @@ fn num_processor(nodes: &[FormatNode], desc: NumDesc, num_part: NumPart) -> Resu // those digits. But don't advance last_relevant beyond the last // character of the np.number string, which is a hazard if the // number got shortened due to precision limitations. - if let Some(last_relevant) = np.last_relevant { - if np.desc.zero_end > np.out_pre_spaces { - // note that np.number cannot be zero-length here - let last_zero_pos = np.number.len() - 1; - let last_zero_pos = last_zero_pos.min(np.desc.zero_end - np.out_pre_spaces); - - if last_relevant.1 < last_zero_pos { - let ch = np.number[last_zero_pos]; - np.last_relevant = Some((ch, last_zero_pos)) - } + if let Some(last_relevant) = np.last_relevant + && np.desc.zero_end > np.out_pre_spaces + { + // note that np.number cannot be zero-length here + let last_zero_pos = np.number.len() - 1; + let last_zero_pos = last_zero_pos.min(np.desc.zero_end - np.out_pre_spaces); + + if last_relevant.1 < last_zero_pos { + let ch = np.number[last_zero_pos]; + np.last_relevant = Some((ch, last_zero_pos)) } } } diff --git a/src/common/io/src/prelude.rs b/src/common/io/src/prelude.rs index a8f5aaeb2fe66..400b5a62c8139 100644 --- a/src/common/io/src/prelude.rs +++ b/src/common/io/src/prelude.rs @@ -16,8 +16,8 @@ pub use bytes::BufMut; pub use bytes::BytesMut; pub use crate::binary_read::BinaryRead; -pub use crate::binary_write::put_uvarint; pub use crate::binary_write::BinaryWrite; +pub use crate::binary_write::put_uvarint; pub use crate::bincode_serialization::*; pub use crate::borsh_serialization::*; pub use crate::format_settings::FormatSettings; diff --git a/src/common/io/tests/it/main.rs b/src/common/io/tests/it/main.rs index 9deb879a3a64c..882540240ae01 100644 --- a/src/common/io/tests/it/main.rs +++ b/src/common/io/tests/it/main.rs @@ -1,4 +1,3 @@ -#![feature(cursor_split)] // Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,6 +13,7 @@ // limitations under the License. #![allow(clippy::uninlined_format_args)] #![feature(assert_matches)] +#![feature(cursor_split)] extern crate core; diff --git a/src/common/license/src/display_jwt_claims.rs b/src/common/license/src/display_jwt_claims.rs index a9440f8418f01..f219e40c638ef 100644 --- a/src/common/license/src/display_jwt_claims.rs +++ b/src/common/license/src/display_jwt_claims.rs @@ -15,9 +15,9 @@ use std::fmt; use std::time::Duration; -use display_more::display_unix_epoch::DisplayUnixTimeStamp; use display_more::DisplayOptionExt; use display_more::DisplayUnixTimeStampExt; +use display_more::display_unix_epoch::DisplayUnixTimeStamp; use jwt_simple::claims::JWTClaims; use jwt_simple::prelude::coarsetime; @@ -52,13 +52,13 @@ where T: fmt::Display /// Add `display_jwt_claims` method to `JWTClaims` if `T` is Display. pub trait DisplayJWTClaimsExt { - fn display_jwt_claims(&self) -> DisplayJWTClaims; + fn display_jwt_claims(&self) -> DisplayJWTClaims<'_, T>; } impl DisplayJWTClaimsExt for JWTClaims where T: fmt::Display { - fn display_jwt_claims(&self) -> DisplayJWTClaims { + fn display_jwt_claims(&self) -> DisplayJWTClaims<'_, T> { DisplayJWTClaims { claims: self } } } diff --git a/src/common/metrics/src/count.rs b/src/common/metrics/src/count.rs index 24ee2b6eeab8b..63a50761c3875 100644 --- a/src/common/metrics/src/count.rs +++ b/src/common/metrics/src/count.rs @@ -116,9 +116,9 @@ where C: Count #[cfg(test)] mod tests { + use std::sync::Arc; use std::sync::atomic::AtomicI64; use std::sync::atomic::Ordering; - use std::sync::Arc; use crate::count::Count; use crate::count::WithCount; diff --git a/src/common/metrics/src/metrics/auth.rs b/src/common/metrics/src/metrics/auth.rs index e551b37071a5a..f8a0028b1f8a3 100644 --- a/src/common/metrics/src/metrics/auth.rs +++ b/src/common/metrics/src/metrics/auth.rs @@ -15,10 +15,10 @@ use std::sync::LazyLock; use std::time::Duration; -use databend_common_base::runtime::metrics::register_counter_family; -use databend_common_base::runtime::metrics::register_histogram_family_in_milliseconds; use databend_common_base::runtime::metrics::FamilyCounter; use databend_common_base::runtime::metrics::FamilyHistogram; +use databend_common_base::runtime::metrics::register_counter_family; +use databend_common_base::runtime::metrics::register_histogram_family_in_milliseconds; use crate::VecLabels; diff --git a/src/common/metrics/src/metrics/cache.rs b/src/common/metrics/src/metrics/cache.rs index d85595c1cb335..1deb3ee360374 100644 --- a/src/common/metrics/src/metrics/cache.rs +++ b/src/common/metrics/src/metrics/cache.rs @@ -14,10 +14,10 @@ use std::sync::LazyLock; -use databend_common_base::runtime::metrics::register_counter_family; -use databend_common_base::runtime::metrics::register_histogram_family_in_milliseconds; use databend_common_base::runtime::metrics::FamilyCounter; use databend_common_base::runtime::metrics::FamilyHistogram; +use databend_common_base::runtime::metrics::register_counter_family; +use databend_common_base::runtime::metrics::register_histogram_family_in_milliseconds; use prometheus_client::encoding::EncodeLabelSet; #[derive(Clone, Debug, EncodeLabelSet, Hash, PartialEq, Eq)] diff --git a/src/common/metrics/src/metrics/cluster.rs b/src/common/metrics/src/metrics/cluster.rs index e9d2691ddac42..b36e481f9fd18 100644 --- a/src/common/metrics/src/metrics/cluster.rs +++ b/src/common/metrics/src/metrics/cluster.rs @@ -14,10 +14,10 @@ use std::sync::LazyLock; -use databend_common_base::runtime::metrics::register_counter_family; -use databend_common_base::runtime::metrics::register_gauge_family; use databend_common_base::runtime::metrics::FamilyCounter; use databend_common_base::runtime::metrics::FamilyGauge; +use databend_common_base::runtime::metrics::register_counter_family; +use databend_common_base::runtime::metrics::register_gauge_family; static CLUSTER_CLUSTER_HEARTBEAT_COUNT: LazyLock>> = LazyLock::new(|| register_counter_family("cluster_heartbeat_count")); diff --git a/src/common/metrics/src/metrics/external_server.rs b/src/common/metrics/src/metrics/external_server.rs index 703f3e57c01fb..206cd9341ccdd 100644 --- a/src/common/metrics/src/metrics/external_server.rs +++ b/src/common/metrics/src/metrics/external_server.rs @@ -15,11 +15,11 @@ use std::sync::LazyLock; use std::time::Duration; +use databend_common_base::runtime::metrics::FamilyCounter; +use databend_common_base::runtime::metrics::FamilyHistogram; use databend_common_base::runtime::metrics::register_counter_family; use databend_common_base::runtime::metrics::register_histogram_family_in_milliseconds; use databend_common_base::runtime::metrics::register_histogram_family_in_rows; -use databend_common_base::runtime::metrics::FamilyCounter; -use databend_common_base::runtime::metrics::FamilyHistogram; use crate::VecLabels; diff --git a/src/common/metrics/src/metrics/http.rs b/src/common/metrics/src/metrics/http.rs index 28353aedb6a88..fe7faf97295d4 100644 --- a/src/common/metrics/src/metrics/http.rs +++ b/src/common/metrics/src/metrics/http.rs @@ -15,12 +15,12 @@ use std::sync::LazyLock; use std::time::Duration; -use databend_common_base::runtime::metrics::register_counter; -use databend_common_base::runtime::metrics::register_counter_family; -use databend_common_base::runtime::metrics::register_histogram_family_in_seconds; use databend_common_base::runtime::metrics::Counter; use databend_common_base::runtime::metrics::FamilyCounter; use databend_common_base::runtime::metrics::FamilyHistogram; +use databend_common_base::runtime::metrics::register_counter; +use databend_common_base::runtime::metrics::register_counter_family; +use databend_common_base::runtime::metrics::register_histogram_family_in_seconds; use crate::VecLabels; diff --git a/src/common/metrics/src/metrics/interpreter.rs b/src/common/metrics/src/metrics/interpreter.rs index 0b168bf75c37e..3d931fe1bb638 100644 --- a/src/common/metrics/src/metrics/interpreter.rs +++ b/src/common/metrics/src/metrics/interpreter.rs @@ -14,10 +14,10 @@ use std::sync::LazyLock; -use databend_common_base::runtime::metrics::register_counter_family; -use databend_common_base::runtime::metrics::register_histogram_family_in_milliseconds; use databend_common_base::runtime::metrics::FamilyCounter; use databend_common_base::runtime::metrics::FamilyHistogram; +use databend_common_base::runtime::metrics::register_counter_family; +use databend_common_base::runtime::metrics::register_histogram_family_in_milliseconds; use crate::VecLabels; diff --git a/src/common/metrics/src/metrics/lock.rs b/src/common/metrics/src/metrics/lock.rs index d325d6cbe40fc..b13bc34e331da 100644 --- a/src/common/metrics/src/metrics/lock.rs +++ b/src/common/metrics/src/metrics/lock.rs @@ -14,10 +14,10 @@ use std::sync::LazyLock; -use databend_common_base::runtime::metrics::register_counter; -use databend_common_base::runtime::metrics::register_counter_family; use databend_common_base::runtime::metrics::Counter; use databend_common_base::runtime::metrics::FamilyCounter; +use databend_common_base::runtime::metrics::register_counter; +use databend_common_base::runtime::metrics::register_counter_family; use crate::VecLabels; diff --git a/src/common/metrics/src/metrics/mysql.rs b/src/common/metrics/src/metrics/mysql.rs index 5187f75adc704..9242a919c7c7f 100644 --- a/src/common/metrics/src/metrics/mysql.rs +++ b/src/common/metrics/src/metrics/mysql.rs @@ -15,8 +15,8 @@ use std::sync::LazyLock; use std::time::Duration; -use databend_common_base::runtime::metrics::register_histogram_in_milliseconds; use databend_common_base::runtime::metrics::Histogram; +use databend_common_base::runtime::metrics::register_histogram_in_milliseconds; pub static MYSQL_PROCESSOR_REQUEST_DURATION: LazyLock = LazyLock::new(|| register_histogram_in_milliseconds("mysql_process_request_duration_ms")); diff --git a/src/common/metrics/src/metrics/session.rs b/src/common/metrics/src/metrics/session.rs index e3e27786cfd7a..a8c531ed4f803 100644 --- a/src/common/metrics/src/metrics/session.rs +++ b/src/common/metrics/src/metrics/session.rs @@ -15,12 +15,12 @@ use std::sync::LazyLock; use std::time::Duration; -use databend_common_base::runtime::metrics::register_counter; -use databend_common_base::runtime::metrics::register_gauge; -use databend_common_base::runtime::metrics::register_histogram_in_milliseconds; use databend_common_base::runtime::metrics::Counter; use databend_common_base::runtime::metrics::Gauge; use databend_common_base::runtime::metrics::Histogram; +use databend_common_base::runtime::metrics::register_counter; +use databend_common_base::runtime::metrics::register_gauge; +use databend_common_base::runtime::metrics::register_histogram_in_milliseconds; pub static SESSION_CONNECT_NUMBERS: LazyLock = LazyLock::new(|| register_counter("session_connect_numbers")); diff --git a/src/common/metrics/src/metrics/storage.rs b/src/common/metrics/src/metrics/storage.rs index 9b86215fccda9..5b8f2b155c1ef 100644 --- a/src/common/metrics/src/metrics/storage.rs +++ b/src/common/metrics/src/metrics/storage.rs @@ -15,14 +15,14 @@ use std::sync::LazyLock; use std::time::Duration; -use databend_common_base::runtime::metrics::register_counter; -use databend_common_base::runtime::metrics::register_counter_family; -use databend_common_base::runtime::metrics::register_gauge; -use databend_common_base::runtime::metrics::register_histogram_in_milliseconds; use databend_common_base::runtime::metrics::Counter; use databend_common_base::runtime::metrics::FamilyCounter; use databend_common_base::runtime::metrics::Gauge; use databend_common_base::runtime::metrics::Histogram; +use databend_common_base::runtime::metrics::register_counter; +use databend_common_base::runtime::metrics::register_counter_family; +use databend_common_base::runtime::metrics::register_gauge; +use databend_common_base::runtime::metrics::register_histogram_in_milliseconds; use prometheus_client::encoding::EncodeLabelSet; // Common metrics. diff --git a/src/common/metrics/src/metrics/system.rs b/src/common/metrics/src/metrics/system.rs index 2bfd6dab586c6..3a960ee4cca2a 100644 --- a/src/common/metrics/src/metrics/system.rs +++ b/src/common/metrics/src/metrics/system.rs @@ -14,8 +14,8 @@ use std::sync::LazyLock; -use databend_common_base::runtime::metrics::register_gauge_family; use databend_common_base::runtime::metrics::FamilyGauge; +use databend_common_base::runtime::metrics::register_gauge_family; pub static SYSTEM_VERSION_GAUGE: LazyLock>> = LazyLock::new(|| register_gauge_family("system_version")); diff --git a/src/common/native/src/compression/binary/dict.rs b/src/common/native/src/compression/binary/dict.rs index d4fa8e4135150..58a4e7782ba8c 100644 --- a/src/common/native/src/compression/binary/dict.rs +++ b/src/common/native/src/compression/binary/dict.rs @@ -21,16 +21,15 @@ use databend_common_column::types::Index; use super::BinaryCompression; use super::BinaryStats; +use crate::compression::Compression; use crate::compression::get_bits_needed; -use crate::compression::integer::compress_integer; -use crate::compression::integer::decompress_integer; use crate::compression::integer::Dict; use crate::compression::integer::DictEncoder; +use crate::compression::integer::compress_integer; +use crate::compression::integer::decompress_integer; use crate::compression::is_valid; -use crate::compression::Compression; use crate::error::Error; use crate::error::Result; -use crate::general_err; use crate::util::AsBytes; use crate::write::WriteOptions; diff --git a/src/common/native/src/compression/binary/freq.rs b/src/common/native/src/compression/binary/freq.rs index b408a1a39d8cc..1b9135b35c3d1 100644 --- a/src/common/native/src/compression/binary/freq.rs +++ b/src/common/native/src/compression/binary/freq.rs @@ -22,11 +22,10 @@ use roaring::RoaringBitmap; use super::BinaryCompression; use super::BinaryStats; -use crate::compression::integer::Freq; use crate::compression::Compression; +use crate::compression::integer::Freq; use crate::error::Error; use crate::error::Result; -use crate::general_err; use crate::write::WriteOptions; impl BinaryCompression for Freq { diff --git a/src/common/native/src/compression/binary/mod.rs b/src/common/native/src/compression/binary/mod.rs index afa799b685199..b089857b19f00 100644 --- a/src/common/native/src/compression/binary/mod.rs +++ b/src/common/native/src/compression/binary/mod.rs @@ -24,15 +24,15 @@ use databend_common_column::bitmap::Bitmap; use databend_common_column::types::Index; use databend_common_expression::types::Buffer; +use super::Compression; use super::basic::CommonCompression; use super::integer::Dict; use super::integer::Freq; use super::integer::OneValue; -use super::Compression; use crate::error::Error; use crate::error::Result; -use crate::read::read_basic::read_compress_header; use crate::read::NativeReadBuf; +use crate::read::read_basic::read_compress_header; use crate::write::WriteOptions; pub fn compress_binary( diff --git a/src/common/native/src/compression/binary/one_value.rs b/src/common/native/src/compression/binary/one_value.rs index 3a087a9c7c807..511a21b99170e 100644 --- a/src/common/native/src/compression/binary/one_value.rs +++ b/src/common/native/src/compression/binary/one_value.rs @@ -20,11 +20,10 @@ use databend_common_column::binary::BinaryColumn; use super::BinaryCompression; use super::BinaryStats; -use crate::compression::integer::OneValue; use crate::compression::Compression; +use crate::compression::integer::OneValue; use crate::error::Error; use crate::error::Result; -use crate::general_err; use crate::write::WriteOptions; impl BinaryCompression for OneValue { diff --git a/src/common/native/src/compression/boolean/mod.rs b/src/common/native/src/compression/boolean/mod.rs index 62b2e49bb0afe..c78b7ff00bb7d 100644 --- a/src/common/native/src/compression/boolean/mod.rs +++ b/src/common/native/src/compression/boolean/mod.rs @@ -17,17 +17,17 @@ mod rle; use databend_common_column::bitmap::MutableBitmap; use databend_common_expression::types::Bitmap; -use rand::thread_rng; use rand::Rng; +use rand::thread_rng; +use super::Compression; use super::basic::CommonCompression; use super::integer::OneValue; use super::integer::Rle; -use super::Compression; use crate::error::Error; use crate::error::Result; -use crate::read::read_basic::read_compress_header; use crate::read::NativeReadBuf; +use crate::read::read_basic::read_compress_header; use crate::write::WriteOptions; pub fn compress_boolean( diff --git a/src/common/native/src/compression/boolean/one_value.rs b/src/common/native/src/compression/boolean/one_value.rs index 210a4abbd264b..b4e5c03f705f4 100644 --- a/src/common/native/src/compression/boolean/one_value.rs +++ b/src/common/native/src/compression/boolean/one_value.rs @@ -16,11 +16,10 @@ use databend_common_column::bitmap::Bitmap; use databend_common_column::bitmap::MutableBitmap; use super::BooleanCompression; -use crate::compression::integer::OneValue; use crate::compression::Compression; +use crate::compression::integer::OneValue; use crate::error::Error; use crate::error::Result; -use crate::general_err; impl BooleanCompression for OneValue { fn to_compression(&self) -> Compression { diff --git a/src/common/native/src/compression/boolean/rle.rs b/src/common/native/src/compression/boolean/rle.rs index 6417384b617d1..2506f3fa026c7 100644 --- a/src/common/native/src/compression/boolean/rle.rs +++ b/src/common/native/src/compression/boolean/rle.rs @@ -17,13 +17,13 @@ use byteorder::ReadBytesExt; use databend_common_column::bitmap::Bitmap; use databend_common_column::bitmap::MutableBitmap; -use super::compress_sample_ratio; use super::BooleanCompression; use super::BooleanStats; -use crate::compression::integer::Rle; +use super::compress_sample_ratio; use crate::compression::Compression; use crate::compression::SAMPLE_COUNT; use crate::compression::SAMPLE_SIZE; +use crate::compression::integer::Rle; use crate::error::Result; impl BooleanCompression for Rle { diff --git a/src/common/native/src/compression/double/dict.rs b/src/common/native/src/compression/double/dict.rs index c580fc2f3aa59..81404931d0290 100644 --- a/src/common/native/src/compression/double/dict.rs +++ b/src/common/native/src/compression/double/dict.rs @@ -16,19 +16,18 @@ use byteorder::LittleEndian; use byteorder::ReadBytesExt; use databend_common_column::buffer::Buffer; -use super::traits::DoubleType; use super::DoubleCompression; use super::DoubleStats; +use super::traits::DoubleType; +use crate::compression::Compression; use crate::compression::get_bits_needed; -use crate::compression::integer::compress_integer; -use crate::compression::integer::decompress_integer; use crate::compression::integer::Dict; use crate::compression::integer::DictEncoder; use crate::compression::integer::RawNative; -use crate::compression::Compression; +use crate::compression::integer::compress_integer; +use crate::compression::integer::decompress_integer; use crate::error::Error; use crate::error::Result; -use crate::general_err; use crate::write::WriteOptions; impl DoubleCompression for Dict { diff --git a/src/common/native/src/compression/double/freq.rs b/src/common/native/src/compression/double/freq.rs index 237bb31c7200c..901bd8e152f1a 100644 --- a/src/common/native/src/compression/double/freq.rs +++ b/src/common/native/src/compression/double/freq.rs @@ -20,13 +20,13 @@ use byteorder::ReadBytesExt; use databend_common_column::buffer::Buffer; use roaring::RoaringBitmap; -use super::compress_double; use super::DoubleCompression; use super::DoubleStats; use super::DoubleType; +use super::compress_double; +use crate::compression::Compression; use crate::compression::double::decompress_double; use crate::compression::integer::Freq; -use crate::compression::Compression; use crate::error::Result; use crate::write::WriteOptions; diff --git a/src/common/native/src/compression/double/mod.rs b/src/common/native/src/compression/double/mod.rs index 18583f75e2673..520641e8570c0 100644 --- a/src/common/native/src/compression/double/mod.rs +++ b/src/common/native/src/compression/double/mod.rs @@ -24,22 +24,22 @@ use std::collections::HashMap; use databend_common_column::bitmap::Bitmap; use databend_common_column::bitmap::MutableBitmap; use databend_common_column::buffer::Buffer; -use rand::thread_rng; use rand::Rng; +use rand::thread_rng; pub use self::one_value::OneValue; use self::patas::Patas; pub use self::traits::DoubleType; +use super::Compression; use super::basic::CommonCompression; use super::integer::Dict; use super::integer::Freq; use super::integer::Rle; use super::is_valid; -use super::Compression; use crate::error::Error; use crate::error::Result; -use crate::read::read_basic::read_compress_header; use crate::read::NativeReadBuf; +use crate::read::read_basic::read_compress_header; use crate::write::WriteOptions; pub fn compress_double( diff --git a/src/common/native/src/compression/double/patas.rs b/src/common/native/src/compression/double/patas.rs index 5692e5122ff65..d736865f43215 100644 --- a/src/common/native/src/compression/double/patas.rs +++ b/src/common/native/src/compression/double/patas.rs @@ -23,10 +23,10 @@ use databend_common_column::types::NativeType; use ringbuffer::AllocRingBuffer; use ringbuffer::RingBuffer; -use super::compress_sample_ratio; use super::DoubleCompression; use super::DoubleStats; use super::DoubleType; +use super::compress_sample_ratio; use crate::compression::Compression; use crate::compression::SAMPLE_COUNT; use crate::compression::SAMPLE_SIZE; diff --git a/src/common/native/src/compression/double/rle.rs b/src/common/native/src/compression/double/rle.rs index eaa966e75af4e..cd0d042f97369 100644 --- a/src/common/native/src/compression/double/rle.rs +++ b/src/common/native/src/compression/double/rle.rs @@ -20,15 +20,15 @@ use byteorder::ReadBytesExt; use databend_common_column::buffer::Buffer; use databend_common_expression::types::Bitmap; -use super::compress_sample_ratio; use super::DoubleCompression; use super::DoubleStats; use super::DoubleType; -use crate::compression::integer::Rle; -use crate::compression::is_valid; +use super::compress_sample_ratio; use crate::compression::Compression; use crate::compression::SAMPLE_COUNT; use crate::compression::SAMPLE_SIZE; +use crate::compression::integer::Rle; +use crate::compression::is_valid; use crate::error::Result; use crate::write::WriteOptions; diff --git a/src/common/native/src/compression/integer/bp.rs b/src/common/native/src/compression/integer/bp.rs index f8bc9ced4f62f..5d0e1b4b8fd4c 100644 --- a/src/common/native/src/compression/integer/bp.rs +++ b/src/common/native/src/compression/integer/bp.rs @@ -20,10 +20,10 @@ use bitpacking::BitPacker4x; use byteorder::ReadBytesExt; use databend_common_column::buffer::Buffer; -use super::compress_sample_ratio; use super::IntegerCompression; use super::IntegerStats; use super::IntegerType; +use super::compress_sample_ratio; use crate::compression::Compression; use crate::compression::SAMPLE_COUNT; use crate::compression::SAMPLE_SIZE; diff --git a/src/common/native/src/compression/integer/delta_bp.rs b/src/common/native/src/compression/integer/delta_bp.rs index c6f15e8741418..b421d093fb59e 100644 --- a/src/common/native/src/compression/integer/delta_bp.rs +++ b/src/common/native/src/compression/integer/delta_bp.rs @@ -20,10 +20,10 @@ use bitpacking::BitPacker4x; use byteorder::ReadBytesExt; use databend_common_column::buffer::Buffer; -use super::compress_sample_ratio; use super::IntegerCompression; use super::IntegerStats; use super::IntegerType; +use super::compress_sample_ratio; use crate::compression::Compression; use crate::compression::SAMPLE_COUNT; use crate::compression::SAMPLE_SIZE; diff --git a/src/common/native/src/compression/integer/dict.rs b/src/common/native/src/compression/integer/dict.rs index 10d842e96891e..7356c318dfe95 100644 --- a/src/common/native/src/compression/integer/dict.rs +++ b/src/common/native/src/compression/integer/dict.rs @@ -19,11 +19,11 @@ use byteorder::ReadBytesExt; use databend_common_column::buffer::Buffer; use databend_common_column::types::NativeType; -use super::compress_integer; -use super::decompress_integer; use super::IntegerCompression; use super::IntegerStats; use super::IntegerType; +use super::compress_integer; +use super::decompress_integer; use crate::error::Error; use crate::error::Result; @@ -168,12 +168,11 @@ where T: AsBytes + PartialEq + Clone } } -use hashbrown_v0_14::hash_map::RawEntryMut; use hashbrown_v0_14::HashMap; +use hashbrown_v0_14::hash_map::RawEntryMut; -use crate::compression::get_bits_needed; use crate::compression::Compression; -use crate::general_err; +use crate::compression::get_bits_needed; use crate::util::AsBytes; use crate::write::WriteOptions; diff --git a/src/common/native/src/compression/integer/freq.rs b/src/common/native/src/compression/integer/freq.rs index 66b7d865dcdaa..8a4d5396fc69d 100644 --- a/src/common/native/src/compression/integer/freq.rs +++ b/src/common/native/src/compression/integer/freq.rs @@ -21,11 +21,11 @@ use byteorder::ReadBytesExt; use databend_common_column::buffer::Buffer; use roaring::RoaringBitmap; -use super::compress_integer; -use super::decompress_integer; use super::IntegerCompression; use super::IntegerStats; use super::IntegerType; +use super::compress_integer; +use super::decompress_integer; use crate::compression::Compression; use crate::error::Result; use crate::write::WriteOptions; diff --git a/src/common/native/src/compression/integer/mod.rs b/src/common/native/src/compression/integer/mod.rs index 1d8ecefd30d2d..eb414b4ed75e7 100644 --- a/src/common/native/src/compression/integer/mod.rs +++ b/src/common/native/src/compression/integer/mod.rs @@ -25,8 +25,8 @@ use std::collections::HashMap; use databend_common_column::bitmap::Bitmap; use databend_common_column::bitmap::MutableBitmap; use databend_common_column::buffer::Buffer; -use rand::thread_rng; use rand::Rng; +use rand::thread_rng; use self::bp::Bitpacking; use self::delta_bp::DeltaBitpacking; @@ -37,12 +37,12 @@ pub use self::freq::Freq; pub use self::one_value::OneValue; pub use self::rle::Rle; pub use self::traits::IntegerType; -use super::basic::CommonCompression; use super::Compression; +use super::basic::CommonCompression; use crate::error::Error; use crate::error::Result; -use crate::read::read_basic::read_compress_header; use crate::read::NativeReadBuf; +use crate::read::read_basic::read_compress_header; use crate::write::WriteOptions; pub fn compress_integer( diff --git a/src/common/native/src/compression/integer/rle.rs b/src/common/native/src/compression/integer/rle.rs index ca21bc1c7b019..839487b7944fd 100644 --- a/src/common/native/src/compression/integer/rle.rs +++ b/src/common/native/src/compression/integer/rle.rs @@ -20,14 +20,14 @@ use byteorder::ReadBytesExt; use databend_common_column::buffer::Buffer; use databend_common_expression::types::Bitmap; -use super::compress_sample_ratio; use super::IntegerCompression; use super::IntegerStats; use super::IntegerType; -use crate::compression::is_valid; +use super::compress_sample_ratio; use crate::compression::Compression; use crate::compression::SAMPLE_COUNT; use crate::compression::SAMPLE_SIZE; +use crate::compression::is_valid; use crate::error::Result; use crate::write::WriteOptions; diff --git a/src/common/native/src/compression/integer/traits.rs b/src/common/native/src/compression/integer/traits.rs index 4b63c464f87d7..038837616a9fb 100644 --- a/src/common/native/src/compression/integer/traits.rs +++ b/src/common/native/src/compression/integer/traits.rs @@ -15,9 +15,9 @@ use std::cmp::Ordering; use std::hash::Hash; +use databend_common_column::types::NativeType; use databend_common_column::types::i256; use databend_common_column::types::months_days_micros; -use databend_common_column::types::NativeType; pub trait IntegerType: NativeType + PartialOrd + Hash + Eq { fn compare_i64(&self, i: i64) -> Ordering; diff --git a/src/common/native/src/lib.rs b/src/common/native/src/lib.rs index 38debadb17dfa..59242bb9480f4 100644 --- a/src/common/native/src/lib.rs +++ b/src/common/native/src/lib.rs @@ -13,6 +13,9 @@ // limitations under the License. #![allow(clippy::useless_transmute)] +#![allow(clippy::derivable_impls)] +#![allow(clippy::collapsible_if)] +#![allow(clippy::manual_is_multiple_of)] #[macro_use] mod error; diff --git a/src/common/native/src/nested.rs b/src/common/native/src/nested.rs index 3be2e3b926cbb..71d859c94e17c 100644 --- a/src/common/native/src/nested.rs +++ b/src/common/native/src/nested.rs @@ -14,14 +14,14 @@ use std::ops::Range; +use databend_common_expression::Column; +use databend_common_expression::TableDataType; use databend_common_expression::types::AnyType; use databend_common_expression::types::ArrayColumn; use databend_common_expression::types::Bitmap; use databend_common_expression::types::Buffer; use databend_common_expression::types::NumberColumn; use databend_common_expression::types::VectorColumn; -use databend_common_expression::Column; -use databend_common_expression::TableDataType; use crate::error::Result; diff --git a/src/common/native/src/read/array/binary.rs b/src/common/native/src/read/array/binary.rs index 083972b8ee0c1..e63e1e1fe3467 100644 --- a/src/common/native/src/read/array/binary.rs +++ b/src/common/native/src/read/array/binary.rs @@ -15,21 +15,21 @@ use std::io::Cursor; use databend_common_column::binary::BinaryColumn; +use databend_common_expression::Column; +use databend_common_expression::TableDataType; use databend_common_expression::types::Bitmap; use databend_common_expression::types::Buffer; use databend_common_expression::types::GeographyColumn; -use databend_common_expression::Column; -use databend_common_expression::TableDataType; +use crate::PageMeta; use crate::compression::binary::decompress_binary; use crate::error::Result; use crate::nested::InitNested; use crate::nested::NestedState; -use crate::read::read_basic::*; use crate::read::BufReader; use crate::read::NativeReadBuf; use crate::read::PageIterator; -use crate::PageMeta; +use crate::read::read_basic::*; #[derive(Debug)] pub struct BinaryNestedIter diff --git a/src/common/native/src/read/array/boolean.rs b/src/common/native/src/read/array/boolean.rs index b5edb6bdf9b20..eb2b32f0bf9d7 100644 --- a/src/common/native/src/read/array/boolean.rs +++ b/src/common/native/src/read/array/boolean.rs @@ -19,15 +19,15 @@ use databend_common_column::bitmap::MutableBitmap; use databend_common_expression::Column; use databend_common_expression::TableDataType; +use crate::PageMeta; use crate::compression::boolean::decompress_boolean; use crate::error::Result; use crate::nested::InitNested; use crate::nested::NestedState; -use crate::read::read_basic::*; use crate::read::BufReader; use crate::read::NativeReadBuf; use crate::read::PageIterator; -use crate::PageMeta; +use crate::read::read_basic::*; #[derive(Debug)] pub struct BooleanNestedIter diff --git a/src/common/native/src/read/array/decimal.rs b/src/common/native/src/read/array/decimal.rs index 6850916db449e..18e835c527d3f 100644 --- a/src/common/native/src/read/array/decimal.rs +++ b/src/common/native/src/read/array/decimal.rs @@ -17,21 +17,21 @@ use std::io::Cursor; use std::marker::PhantomData; use databend_common_column::buffer::Buffer; -use databend_common_expression::types::Decimal; -use databend_common_expression::types::DecimalSize; use databend_common_expression::Column; use databend_common_expression::TableDataType; +use databend_common_expression::types::Decimal; +use databend_common_expression::types::DecimalSize; -use crate::compression::integer::decompress_integer; +use crate::PageMeta; use crate::compression::integer::IntegerType; +use crate::compression::integer::decompress_integer; use crate::error::Result; use crate::nested::InitNested; use crate::nested::NestedState; -use crate::read::read_basic::*; use crate::read::BufReader; use crate::read::NativeReadBuf; use crate::read::PageIterator; -use crate::PageMeta; +use crate::read::read_basic::*; #[derive(Debug)] pub struct DecimalNestedIter diff --git a/src/common/native/src/read/array/double.rs b/src/common/native/src/read/array/double.rs index 741a25ea87815..20bba33ca0345 100644 --- a/src/common/native/src/read/array/double.rs +++ b/src/common/native/src/read/array/double.rs @@ -17,22 +17,22 @@ use std::io::Cursor; use std::marker::PhantomData; use databend_common_column::buffer::Buffer; +use databend_common_expression::Column; +use databend_common_expression::TableDataType; use databend_common_expression::types::ArgType; use databend_common_expression::types::Number; use databend_common_expression::types::NumberType; -use databend_common_expression::Column; -use databend_common_expression::TableDataType; -use crate::compression::double::decompress_double; +use crate::PageMeta; use crate::compression::double::DoubleType; +use crate::compression::double::decompress_double; use crate::error::Result; use crate::nested::InitNested; use crate::nested::NestedState; -use crate::read::read_basic::*; use crate::read::BufReader; use crate::read::NativeReadBuf; use crate::read::PageIterator; -use crate::PageMeta; +use crate::read::read_basic::*; #[derive(Debug)] pub struct DoubleNestedIter diff --git a/src/common/native/src/read/array/fixed_list.rs b/src/common/native/src/read/array/fixed_list.rs index 6b8aa0009fc30..93d63eae6f8fc 100644 --- a/src/common/native/src/read/array/fixed_list.rs +++ b/src/common/native/src/read/array/fixed_list.rs @@ -16,8 +16,8 @@ use databend_common_expression::Column; use databend_common_expression::TableDataType; use crate::error::Result; -use crate::nested::create_fixed_list; use crate::nested::NestedState; +use crate::nested::create_fixed_list; use crate::read::deserialize::DynIter; /// An iterator adapter over [`DynIter`] assumed to be encoded as List columns diff --git a/src/common/native/src/read/array/integer.rs b/src/common/native/src/read/array/integer.rs index f0ad2e0d86e6a..99a275e76c86e 100644 --- a/src/common/native/src/read/array/integer.rs +++ b/src/common/native/src/read/array/integer.rs @@ -17,21 +17,21 @@ use std::io::Cursor; use std::marker::PhantomData; use databend_common_column::buffer::Buffer; -use databend_common_expression::types::ArgType; -use databend_common_expression::types::Number; use databend_common_expression::Column; use databend_common_expression::TableDataType; +use databend_common_expression::types::ArgType; +use databend_common_expression::types::Number; -use crate::compression::integer::decompress_integer; +use crate::PageMeta; use crate::compression::integer::IntegerType; +use crate::compression::integer::decompress_integer; use crate::error::Result; use crate::nested::InitNested; use crate::nested::NestedState; -use crate::read::read_basic::*; use crate::read::BufReader; use crate::read::NativeReadBuf; use crate::read::PageIterator; -use crate::PageMeta; +use crate::read::read_basic::*; #[derive(Debug)] pub struct IntegerNestedIter diff --git a/src/common/native/src/read/array/interval.rs b/src/common/native/src/read/array/interval.rs index 98343d63756cc..9b06d589a6a38 100644 --- a/src/common/native/src/read/array/interval.rs +++ b/src/common/native/src/read/array/interval.rs @@ -16,20 +16,20 @@ use std::io::Cursor; use databend_common_column::buffer::Buffer; use databend_common_column::types::months_days_micros; -use databend_common_expression::types::ArgType; -use databend_common_expression::types::IntervalType; use databend_common_expression::Column; use databend_common_expression::TableDataType; +use databend_common_expression::types::ArgType; +use databend_common_expression::types::IntervalType; +use crate::PageMeta; use crate::compression::integer::decompress_integer; use crate::error::Result; use crate::nested::InitNested; use crate::nested::NestedState; -use crate::read::read_basic::*; use crate::read::BufReader; use crate::read::NativeReadBuf; use crate::read::PageIterator; -use crate::PageMeta; +use crate::read::read_basic::*; #[derive(Debug)] pub struct IntervalNestedIter diff --git a/src/common/native/src/read/array/list.rs b/src/common/native/src/read/array/list.rs index 3421d2473eee1..27e9eaf6211f6 100644 --- a/src/common/native/src/read/array/list.rs +++ b/src/common/native/src/read/array/list.rs @@ -16,8 +16,8 @@ use databend_common_expression::Column; use databend_common_expression::TableDataType; use crate::error::Result; -use crate::nested::create_list; use crate::nested::NestedState; +use crate::nested::create_list; use crate::read::deserialize::DynIter; /// An iterator adapter over [`DynIter`] assumed to be encoded as List columns diff --git a/src/common/native/src/read/array/map.rs b/src/common/native/src/read/array/map.rs index 4f0d4a986a58b..4d0dd249c18a6 100644 --- a/src/common/native/src/read/array/map.rs +++ b/src/common/native/src/read/array/map.rs @@ -16,8 +16,8 @@ use databend_common_expression::Column; use databend_common_expression::TableDataType; use crate::error::Result; -use crate::nested::create_map; use crate::nested::NestedState; +use crate::nested::create_map; use crate::read::deserialize::DynIter; /// An iterator adapter over [`DynIter`] assumed to be encoded as Map columns diff --git a/src/common/native/src/read/array/struct_.rs b/src/common/native/src/read/array/struct_.rs index 37b12f3c0204f..1c4c8956acbd6 100644 --- a/src/common/native/src/read/array/struct_.rs +++ b/src/common/native/src/read/array/struct_.rs @@ -16,8 +16,8 @@ use databend_common_expression::Column; use databend_common_expression::TableDataType; use crate::error::Result; -use crate::nested::create_struct; use crate::nested::NestedState; +use crate::nested::create_struct; use crate::read::deserialize::NestedIters; type StructValues = Vec>>; diff --git a/src/common/native/src/read/array/timestamp_tz.rs b/src/common/native/src/read/array/timestamp_tz.rs index 953fbb0d65547..6c1efaa36ed80 100644 --- a/src/common/native/src/read/array/timestamp_tz.rs +++ b/src/common/native/src/read/array/timestamp_tz.rs @@ -18,18 +18,18 @@ use std::io::Cursor; use databend_common_column::buffer::Buffer; use databend_common_column::error::Result; use databend_common_column::types::timestamp_tz; -use databend_common_expression::types::timestamp_tz::TimestampTzType; -use databend_common_expression::types::ArgType; use databend_common_expression::Column; use databend_common_expression::TableDataType; +use databend_common_expression::types::ArgType; +use databend_common_expression::types::timestamp_tz::TimestampTzType; +use crate::PageMeta; use crate::compression::integer::decompress_integer; use crate::nested::InitNested; use crate::nested::NestedState; -use crate::read::read_basic::read_nested; use crate::read::NativeReadBuf; use crate::read::PageIterator; -use crate::PageMeta; +use crate::read::read_basic::read_nested; #[derive(Debug)] pub struct TimestampTzNestedIter diff --git a/src/common/native/src/read/array/view.rs b/src/common/native/src/read/array/view.rs index 8a8b7a84f8b82..16bd32e59efbf 100644 --- a/src/common/native/src/read/array/view.rs +++ b/src/common/native/src/read/array/view.rs @@ -18,21 +18,21 @@ use byteorder::LittleEndian; use byteorder::ReadBytesExt; use databend_common_column::binview::Utf8ViewColumn; use databend_common_column::binview::View; -use databend_common_expression::types::Bitmap; -use databend_common_expression::types::Buffer; use databend_common_expression::Column; use databend_common_expression::TableDataType; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::Buffer; +use crate::CommonCompression; +use crate::PageMeta; use crate::compression::integer::decompress_integer; use crate::error::Result; use crate::nested::InitNested; use crate::nested::NestedState; -use crate::read::read_basic::*; use crate::read::BufReader; use crate::read::NativeReadBuf; use crate::read::PageIterator; -use crate::CommonCompression; -use crate::PageMeta; +use crate::read::read_basic::*; #[derive(Debug)] pub struct ViewColNestedIter diff --git a/src/common/native/src/read/batch_read.rs b/src/common/native/src/read/batch_read.rs index 9c0d0fd7d7604..18c51222251fc 100644 --- a/src/common/native/src/read/batch_read.rs +++ b/src/common/native/src/read/batch_read.rs @@ -12,24 +12,24 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_expression::Column; +use databend_common_expression::TableDataType; use databend_common_expression::types::DateType; use databend_common_expression::types::DecimalDataType; use databend_common_expression::types::NumberType; use databend_common_expression::types::TimestampType; -use databend_common_expression::Column; -use databend_common_expression::TableDataType; -use super::array::*; use super::NativeReadBuf; +use super::array::*; +use crate::PageMeta; use crate::error::Result; +use crate::nested::InitNested; +use crate::nested::NestedState; use crate::nested::create_fixed_list; use crate::nested::create_list; use crate::nested::create_map; use crate::nested::create_struct; -use crate::nested::InitNested; -use crate::nested::NestedState; use crate::util::n_columns; -use crate::PageMeta; fn read_nested_column( mut readers: Vec, diff --git a/src/common/native/src/read/deserialize.rs b/src/common/native/src/read/deserialize.rs index 70191b9370e6f..1fe0ac46bfcf9 100644 --- a/src/common/native/src/read/deserialize.rs +++ b/src/common/native/src/read/deserialize.rs @@ -12,16 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_expression::Column; +use databend_common_expression::TableDataType; +use databend_common_expression::TableField; use databend_common_expression::types::DateType; use databend_common_expression::types::DecimalDataType; use databend_common_expression::types::NumberType; use databend_common_expression::types::TimestampType; -use databend_common_expression::Column; -use databend_common_expression::TableDataType; -use databend_common_expression::TableField; -use super::array::*; use super::PageIterator; +use super::array::*; use crate::error::Result; use crate::nested::InitNested; use crate::nested::NestedState; diff --git a/src/common/native/src/read/mod.rs b/src/common/native/src/read/mod.rs index dd9d9cea6e840..ecb20467af9c1 100644 --- a/src/common/native/src/read/mod.rs +++ b/src/common/native/src/read/mod.rs @@ -19,15 +19,15 @@ use batch_read::batch_read_column; use databend_common_expression::Column; use databend_common_expression::TableDataType; use databend_common_expression::TableField; -pub use deserialize::column_iters; pub use deserialize::ColumnIter; +pub use deserialize::column_iters; use crate::error::Result; pub(crate) mod read_basic; use std::io::BufReader; -use super::nested::InitNested; use super::PageMeta; +use super::nested::InitNested; pub mod reader; pub trait NativeReadBuf: std::io::BufRead { diff --git a/src/common/native/src/read/reader.rs b/src/common/native/src/read/reader.rs index e407e26f90038..33bfa612fedeb 100644 --- a/src/common/native/src/read/reader.rs +++ b/src/common/native/src/read/reader.rs @@ -19,14 +19,14 @@ use std::io::SeekFrom; use databend_common_expression::TableSchema; use opendal::Reader; -use super::read_basic::read_u32; -use super::read_basic::read_u64; use super::NativeReadBuf; use super::PageIterator; -use crate::error::Error; -use crate::error::Result; +use super::read_basic::read_u32; +use super::read_basic::read_u64; use crate::ColumnMeta; use crate::PageMeta; +use crate::error::Error; +use crate::error::Result; const DEFAULT_FOOTER_SIZE: u64 = 64 * 1024; diff --git a/src/common/native/src/stat.rs b/src/common/native/src/stat.rs index c8f24adace5ae..91c0a484ac5b9 100644 --- a/src/common/native/src/stat.rs +++ b/src/common/native/src/stat.rs @@ -14,14 +14,14 @@ use std::io::BufRead; -use databend_common_expression::types::NumberDataType; use databend_common_expression::TableDataType; use databend_common_expression::TableField; +use databend_common_expression::types::NumberDataType; +use crate::CommonCompression; use crate::compression::Compression; use crate::error::Result; use crate::read::PageIterator; -use crate::CommonCompression; #[derive(Debug)] pub struct ColumnInfo { @@ -187,15 +187,16 @@ mod test { use std::io::BufRead; use databend_common_column::binary::BinaryColumn; - use databend_common_expression::infer_schema_type; - use databend_common_expression::types::Int64Type; use databend_common_expression::Column; use databend_common_expression::FromData; use databend_common_expression::TableField; use databend_common_expression::TableSchema; + use databend_common_expression::infer_schema_type; + use databend_common_expression::types::Int64Type; - use super::stat_simple; use super::ColumnInfo; + use super::stat_simple; + use crate::CommonCompression; use crate::read::reader::NativeReader; use crate::stat::PageBody; use crate::util::env::remove_all_env; @@ -203,7 +204,6 @@ mod test { use crate::util::env::set_freq_env; use crate::write::NativeWriter; use crate::write::WriteOptions; - use crate::CommonCompression; const PAGE_SIZE: usize = 2048; const PAGE_PER_COLUMN: usize = 10; diff --git a/src/common/native/src/util/env.rs b/src/common/native/src/util/env.rs index d9594c9bce509..e56ce4d287c24 100644 --- a/src/common/native/src/util/env.rs +++ b/src/common/native/src/util/env.rs @@ -48,7 +48,7 @@ pub fn check_patas_env() -> bool { } pub fn remove_env(env: &str) { - env::remove_var(env); + unsafe { env::remove_var(env) }; } pub fn remove_all_env() { @@ -60,21 +60,21 @@ pub fn remove_all_env() { } pub fn set_dict_env() { - env::set_var(DICT_ENV, "1"); + unsafe { env::set_var(DICT_ENV, "1") }; } pub fn set_rle_env() { - env::set_var(RLE_ENV, "1"); + unsafe { env::set_var(RLE_ENV, "1") }; } pub fn set_freq_env() { - env::set_var(FREQ_ENV, "1"); + unsafe { env::set_var(FREQ_ENV, "1") }; } pub fn set_bitpack_env() { - env::set_var(BITPACK_ENV, "1"); + unsafe { env::set_var(BITPACK_ENV, "1") }; } pub fn set_patas_env() { - env::set_var(PATAS_ENV, "1"); + unsafe { env::set_var(PATAS_ENV, "1") }; } diff --git a/src/common/native/src/util/mod.rs b/src/common/native/src/util/mod.rs index c8e6de31bef42..de1b53fc156aa 100644 --- a/src/common/native/src/util/mod.rs +++ b/src/common/native/src/util/mod.rs @@ -38,9 +38,9 @@ macro_rules! with_match_integer_double_type { $body_primitive }; } - use databend_common_expression::types::NumberDataType::*; use databend_common_expression::types::F32; use databend_common_expression::types::F64; + use databend_common_expression::types::NumberDataType::*; match $key_type { Int8 => __with_ty__! { i8 }, diff --git a/src/common/native/src/write/common.rs b/src/common/native/src/write/common.rs index 0237a0acd736f..8c7b07c5581c0 100644 --- a/src/common/native/src/write/common.rs +++ b/src/common/native/src/write/common.rs @@ -16,17 +16,17 @@ use std::io::Write; use databend_common_expression::Column; -use super::write; use super::NativeWriter; +use super::write; +use crate::ColumnMeta; +use crate::EOF_MARKER; +use crate::PageMeta; use crate::compression::CommonCompression; use crate::compression::Compression; use crate::error::Result; use crate::nested::slice_nest_column; use crate::nested::to_leaves; use crate::nested::to_nested; -use crate::ColumnMeta; -use crate::PageMeta; -use crate::EOF_MARKER; /// Options declaring the behaviour of writing to IPC #[derive(Debug, Clone, PartialEq, Default)] diff --git a/src/common/native/src/write/primitive.rs b/src/common/native/src/write/primitive.rs index 34049118865ef..5e2eb41e3ab34 100644 --- a/src/common/native/src/write/primitive.rs +++ b/src/common/native/src/write/primitive.rs @@ -16,9 +16,9 @@ use std::io::Write; use databend_common_column::bitmap::Bitmap; use databend_common_column::buffer::Buffer; -use databend_common_column::types::i256; use databend_common_column::types::NativeType; use databend_common_column::types::PrimitiveType; +use databend_common_column::types::i256; use databend_common_expression::types::F32; use databend_common_expression::types::F64; diff --git a/src/common/native/src/write/serialize.rs b/src/common/native/src/write/serialize.rs index b1905d1fcb947..59b5e48d8de6a 100644 --- a/src/common/native/src/write/serialize.rs +++ b/src/common/native/src/write/serialize.rs @@ -17,6 +17,7 @@ use std::io::Write; use databend_common_column::bitmap::Bitmap; use databend_common_column::buffer::Buffer; use databend_common_column::types::i256; +use databend_common_expression::Column; use databend_common_expression::types::AnyType; use databend_common_expression::types::DataType; use databend_common_expression::types::DecimalColumn; @@ -26,10 +27,9 @@ use databend_common_expression::types::NumberColumn; use databend_common_expression::types::ValueType; use databend_common_expression::visitor::ValueVisitor; use databend_common_expression::with_number_mapped_type; -use databend_common_expression::Column; -use super::boolean::write_bitmap; use super::WriteOptions; +use super::boolean::write_bitmap; use crate::error::Error; use crate::error::Result; use crate::nested::Nested; diff --git a/src/common/native/src/write/writer.rs b/src/common/native/src/write/writer.rs index d56b82ac371bc..837313acb3c29 100644 --- a/src/common/native/src/write/writer.rs +++ b/src/common/native/src/write/writer.rs @@ -17,13 +17,13 @@ use std::io::Write; use databend_common_expression::Column; use databend_common_expression::TableSchema; -use super::common::write_eof; use super::common::WriteOptions; -use crate::error::Error; -use crate::error::Result; +use super::common::write_eof; use crate::ColumnMeta; use crate::STRAWBOAT_MAGIC; use crate::STRAWBOAT_VERSION; +use crate::error::Error; +use crate::error::Result; #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) enum State { diff --git a/src/common/native/tests/it/native/io.rs b/src/common/native/tests/it/native/io.rs index 61f5e61226cf7..ce0dc11e60bc5 100644 --- a/src/common/native/tests/it/native/io.rs +++ b/src/common/native/tests/it/native/io.rs @@ -15,21 +15,21 @@ use std::io::BufRead; use std::io::BufReader; -use databend_common_expression::infer_schema_type; -use databend_common_expression::types::*; use databend_common_expression::Column; use databend_common_expression::FromData; use databend_common_expression::TableField; use databend_common_expression::TableSchema; +use databend_common_expression::infer_schema_type; +use databend_common_expression::types::*; +use databend_common_native::ColumnMeta; +use databend_common_native::CommonCompression; +use databend_common_native::PageMeta; use databend_common_native::n_columns; use databend_common_native::read::batch_read::batch_read_column; use databend_common_native::read::deserialize::column_iters; use databend_common_native::read::reader::NativeReader; use databend_common_native::write::NativeWriter; use databend_common_native::write::WriteOptions; -use databend_common_native::ColumnMeta; -use databend_common_native::CommonCompression; -use databend_common_native::PageMeta; pub const WRITE_PAGE: usize = 2048; pub const SMALL_WRITE_PAGE: usize = 2; diff --git a/src/common/native/tests/it/native/read_meta.rs b/src/common/native/tests/it/native/read_meta.rs index c3565d69ab5be..b63c657a140de 100644 --- a/src/common/native/tests/it/native/read_meta.rs +++ b/src/common/native/tests/it/native/read_meta.rs @@ -12,17 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_expression::infer_schema_type; use databend_common_expression::TableField; use databend_common_expression::TableSchema; +use databend_common_expression::infer_schema_type; +use databend_common_native::ColumnMeta; +use databend_common_native::CommonCompression; use databend_common_native::read::reader::read_meta; use databend_common_native::write::NativeWriter; use databend_common_native::write::WriteOptions; -use databend_common_native::ColumnMeta; -use databend_common_native::CommonCompression; -use super::io::new_test_column; use super::io::WRITE_PAGE; +use super::io::new_test_column; fn write_data(dest: &mut Vec) -> Vec { let chunk = new_test_column(); diff --git a/src/common/storage/Cargo.toml b/src/common/storage/Cargo.toml index 4f06485f6796b..8f4a9854e01d3 100644 --- a/src/common/storage/Cargo.toml +++ b/src/common/storage/Cargo.toml @@ -34,6 +34,8 @@ iceberg = { workspace = true } log = { workspace = true } lru = { workspace = true } opendal = { workspace = true } +opendal-layer-immutable-index = { workspace = true } +opendal-layer-observe-metrics-common = { workspace = true } parquet = { workspace = true } prometheus-client = { workspace = true } regex = { workspace = true } diff --git a/src/common/storage/src/column_node.rs b/src/common/storage/src/column_node.rs index 46619584ebacd..26427e4538e6b 100644 --- a/src/common/storage/src/column_node.rs +++ b/src/common/storage/src/column_node.rs @@ -137,7 +137,7 @@ impl ColumnNodes { let column_node = &column_nodes[path[0]]; if path.len() > 1 { return match &column_node.children { - Some(ref children) => Self::traverse_path(children, &path[1..]), + Some(children) => Self::traverse_path(children, &path[1..]), None => Err(ErrorCode::Internal(format!( "Cannot get column_node by path: {:?}", path @@ -156,9 +156,7 @@ impl ColumnNodes { let is_nested = is_nested || column_node.children.is_some(); if path.len() > 1 { return match &column_node.children { - Some(ref children) => { - Self::traverse_path_nested_aware(children, &path[1..], is_nested) - } + Some(children) => Self::traverse_path_nested_aware(children, &path[1..], is_nested), None => Err(ErrorCode::Internal(format!( "Cannot get column_node by path: {:?}", path diff --git a/src/common/storage/src/config.rs b/src/common/storage/src/config.rs index 7a0c832fc3bd7..1043bfcd6fb7e 100644 --- a/src/common/storage/src/config.rs +++ b/src/common/storage/src/config.rs @@ -18,8 +18,8 @@ use std::sync::Arc; use databend_common_auth::RefreshableToken; use databend_common_auth::TokenFile; -use databend_common_base::base::tokio::sync::RwLock; use databend_common_base::base::GlobalInstance; +use databend_common_base::base::tokio::sync::RwLock; use databend_common_meta_app::storage::StorageParams; use serde::Deserialize; use serde::Serialize; diff --git a/src/common/storage/src/copy.rs b/src/common/storage/src/copy.rs index a3464e33ed4fe..86ae35404c14f 100644 --- a/src/common/storage/src/copy.rs +++ b/src/common/storage/src/copy.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use dashmap::mapref::entry::Entry; use dashmap::DashMap; +use dashmap::mapref::entry::Entry; use databend_common_ast::ast::OnErrorMode; use databend_common_exception::ErrorCode; use serde::Deserialize; diff --git a/src/common/storage/src/http_client.rs b/src/common/storage/src/http_client.rs index fc712aef06559..05d4debaef2cb 100644 --- a/src/common/storage/src/http_client.rs +++ b/src/common/storage/src/http_client.rs @@ -22,11 +22,11 @@ use databend_common_metrics::storage::metrics_inc_storage_http_requests_count; use futures::TryStreamExt; use http::Request; use http::Response; -use opendal::raw::parse_content_encoding; -use opendal::raw::parse_content_length; +use opendal::Buffer; use opendal::raw::HttpBody; use opendal::raw::HttpFetch; -use opendal::Buffer; +use opendal::raw::parse_content_encoding; +use opendal::raw::parse_content_length; use url::Url; pub struct StorageHttpClient { diff --git a/src/common/storage/src/lib.rs b/src/common/storage/src/lib.rs index 62e7462f35235..d04d0dd66e9db 100644 --- a/src/common/storage/src/lib.rs +++ b/src/common/storage/src/lib.rs @@ -1,4 +1,3 @@ -#![feature(box_patterns)] // Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,6 +29,9 @@ //! - Intermediate data generated by query could be stored by temporary operator. #![allow(clippy::uninlined_format_args)] +#![allow(clippy::collapsible_if)] +#![allow(clippy::let_unit_value)] +#![feature(box_patterns)] mod config; pub use config::ShareTableConfig; @@ -39,10 +41,11 @@ mod http_client; pub use http_client::StorageHttpClient; mod operator; -pub use operator::check_operator; -pub use operator::init_operator; pub use operator::DataOperator; pub use operator::OperatorRegistry; +pub use operator::Scheme; +pub use operator::check_operator; +pub use operator::init_operator; mod operator_cache; @@ -61,12 +64,12 @@ pub use parquet::read_metadata_async; pub use parquet::read_parquet_schema_async_rs; mod stage; -pub use stage::init_stage_operator; +pub use stage::STDIN_FD; pub use stage::StageFileInfo; pub use stage::StageFileInfoStream; pub use stage::StageFileStatus; pub use stage::StageFilesInfo; -pub use stage::STDIN_FD; +pub use stage::init_stage_operator; mod copy; mod histogram; @@ -80,9 +83,9 @@ pub use copy::CopyStatus; pub use copy::FileParseError; pub use copy::FileParseErrorAtLine; pub use copy::FileStatus; +pub use histogram::DEFAULT_HISTOGRAM_BUCKETS; pub use histogram::Histogram; pub use histogram::HistogramBucket; -pub use histogram::DEFAULT_HISTOGRAM_BUCKETS; pub use merge::MutationStatus; pub use meta_hll::MetaHLL; pub use meta_hll::MetaHLL12; diff --git a/src/common/storage/src/metrics.rs b/src/common/storage/src/metrics.rs index 894c83ff1b968..6c858cf955a33 100644 --- a/src/common/storage/src/metrics.rs +++ b/src/common/storage/src/metrics.rs @@ -12,13 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::Arc; use std::sync::atomic::AtomicU64; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; -use std::sync::Arc; use std::time::Instant; -use opendal::raw::oio; +use opendal::Buffer; +use opendal::Metadata; +use opendal::Result; use opendal::raw::Access; use opendal::raw::Layer; use opendal::raw::LayeredAccess; @@ -29,9 +31,7 @@ use opendal::raw::RpDelete; use opendal::raw::RpList; use opendal::raw::RpRead; use opendal::raw::RpWrite; -use opendal::Buffer; -use opendal::Metadata; -use opendal::Result; +use opendal::raw::oio; /// StorageMetrics represents the metrics of storage (all bytes metrics are compressed size). #[derive(Debug, Default)] diff --git a/src/common/storage/src/metrics_layer.rs b/src/common/storage/src/metrics_layer.rs index 90cd024e9a6e0..78d4e3831583a 100644 --- a/src/common/storage/src/metrics_layer.rs +++ b/src/common/storage/src/metrics_layer.rs @@ -17,15 +17,15 @@ use std::fmt::Debug; use std::sync::LazyLock; use std::time::Duration; -use databend_common_base::runtime::metrics::register_counter_family; -use databend_common_base::runtime::metrics::register_gauge_family; -use databend_common_base::runtime::metrics::register_histogram_family; use databend_common_base::runtime::metrics::FamilyCounter; use databend_common_base::runtime::metrics::FamilyGauge; use databend_common_base::runtime::metrics::FamilyHistogram; -use opendal::layers::observe; +use databend_common_base::runtime::metrics::register_counter_family; +use databend_common_base::runtime::metrics::register_gauge_family; +use databend_common_base::runtime::metrics::register_histogram_family; use opendal::raw::Access; use opendal::raw::Layer; +use opendal_layer_observe_metrics_common as observe; use prometheus_client::encoding::EncodeLabel; use prometheus_client::encoding::EncodeLabelSet; use prometheus_client::encoding::LabelSetEncoder; diff --git a/src/common/storage/src/operator.rs b/src/common/storage/src/operator.rs index d7c99b8c0a3f8..5f9834696fb8c 100644 --- a/src/common/storage/src/operator.rs +++ b/src/common/storage/src/operator.rs @@ -16,17 +16,17 @@ use std::env; use std::io::Error; use std::io::ErrorKind; use std::io::Result; +use std::str::FromStr; use std::sync::LazyLock; use std::time::Duration; use anyhow::anyhow; use databend_common_base::base::GlobalInstance; -use databend_common_base::runtime::metrics::register_counter_family; -use databend_common_base::runtime::metrics::FamilyCounter; use databend_common_base::runtime::GlobalIORuntime; use databend_common_base::runtime::TrySpawn; +use databend_common_base::runtime::metrics::FamilyCounter; +use databend_common_base::runtime::metrics::register_counter_family; use databend_common_exception::ErrorCode; -use databend_common_meta_app::storage::set_s3_storage_class; use databend_common_meta_app::storage::S3StorageClass; use databend_common_meta_app::storage::StorageAzblobConfig; use databend_common_meta_app::storage::StorageCosConfig; @@ -44,28 +44,29 @@ use databend_common_meta_app::storage::StorageOssConfig; use databend_common_meta_app::storage::StorageParams; use databend_common_meta_app::storage::StorageS3Config; use databend_common_meta_app::storage::StorageWebhdfsConfig; +use databend_common_meta_app::storage::set_s3_storage_class; use databend_enterprise_storage_encryption::get_storage_encryption_handler; use log::warn; +use opendal::Builder; +use opendal::Operator; use opendal::layers::AsyncBacktraceLayer; use opendal::layers::ConcurrentLimitLayer; use opendal::layers::FastraceLayer; use opendal::layers::HttpClientLayer; -use opendal::layers::ImmutableIndexLayer; use opendal::layers::LoggingLayer; use opendal::layers::RetryInterceptor; use opendal::layers::RetryLayer; use opendal::layers::TimeoutLayer; use opendal::raw::HttpClient; use opendal::services; -use opendal::Builder; -use opendal::Operator; +use opendal_layer_immutable_index::ImmutableIndexLayer; +use crate::StorageConfig; +use crate::StorageHttpClient; use crate::http_client::get_storage_http_client; use crate::metrics_layer::METRICS_LAYER; use crate::operator_cache::get_operator_cache; use crate::runtime_layer::RuntimeLayer; -use crate::StorageConfig; -use crate::StorageHttpClient; static METRIC_OPENDAL_RETRIES_COUNT: LazyLock>> = LazyLock::new(|| register_counter_family("opendal_retries_count")); @@ -403,8 +404,8 @@ fn init_s3_operator(cfg: &StorageS3Config) -> Result { .session_token(&cfg.security_token) .role_arn(&cfg.role_arn) .external_id(&cfg.external_id) - // It's safe to allow anonymous since opendal will perform the check first. - .allow_anonymous() + // Don't enable it otherwise we will get Permission in stat unknown files + // .allow_anonymous() // Root. .root(&cfg.root); @@ -654,3 +655,76 @@ impl OperatorRegistry for iceberg::io::FileIO { Ok((file_io.get_operator().clone(), &location[pos..])) } } + +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +pub enum Scheme { + Azblob, + Gcs, + Hdfs, + Ipfs, + S3, + Oss, + Obs, + Cos, + Http, + Fs, + Webhdfs, + Huggingface, + Custom(&'static str), +} + +impl Scheme { + /// Convert self into static str. + pub fn into_static(self) -> &'static str { + self.into() + } +} + +impl From for &'static str { + fn from(v: Scheme) -> Self { + match v { + Scheme::Azblob => "azblob", + Scheme::Gcs => "gcs", + Scheme::Hdfs => "hdfs", + Scheme::Ipfs => "ipfs", + Scheme::S3 => "s3", + Scheme::Oss => "oss", + Scheme::Obs => "obs", + Scheme::Cos => "cos", + Scheme::Http => "http", + Scheme::Fs => "fs", + Scheme::Webhdfs => "webhdfs", + Scheme::Huggingface => "huggingface", + Scheme::Custom(s) => s, + } + } +} + +impl FromStr for Scheme { + type Err = Error; + + fn from_str(s: &str) -> Result { + let s = s.to_lowercase(); + match s.as_str() { + "azblob" => Ok(Scheme::Azblob), + "gcs" => Ok(Scheme::Gcs), + "hdfs" => Ok(Scheme::Hdfs), + "ipfs" => Ok(Scheme::Ipfs), + "s3" | "s3a" => Ok(Scheme::S3), + "oss" => Ok(Scheme::Oss), + "obs" => Ok(Scheme::Obs), + "cos" => Ok(Scheme::Cos), + "http" | "https" => Ok(Scheme::Http), + "fs" => Ok(Scheme::Fs), + "webhdfs" => Ok(Scheme::Webhdfs), + "huggingface" | "hf" => Ok(Scheme::Huggingface), + _ => Ok(Scheme::Custom(Box::leak(s.into_boxed_str()))), + } + } +} + +impl std::fmt::Display for Scheme { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.into_static()) + } +} diff --git a/src/common/storage/src/operator_cache.rs b/src/common/storage/src/operator_cache.rs index 95a52ab96a569..33c9b35cfe0b2 100644 --- a/src/common/storage/src/operator_cache.rs +++ b/src/common/storage/src/operator_cache.rs @@ -17,10 +17,10 @@ use std::sync::Arc; use std::sync::LazyLock; use std::sync::Mutex; -use databend_common_base::runtime::metrics::register_counter; -use databend_common_base::runtime::metrics::register_gauge; use databend_common_base::runtime::metrics::Counter; use databend_common_base::runtime::metrics::Gauge; +use databend_common_base::runtime::metrics::register_counter; +use databend_common_base::runtime::metrics::register_gauge; use databend_common_exception::Result as DatabendResult; use databend_common_meta_app::storage::StorageParams; use log::debug; diff --git a/src/common/storage/src/parquet.rs b/src/common/storage/src/parquet.rs index f07c3fe2c7e1c..dcf640c839dab 100644 --- a/src/common/storage/src/parquet.rs +++ b/src/common/storage/src/parquet.rs @@ -18,8 +18,8 @@ use std::sync::Arc; use arrow_schema::Schema as ArrowSchema; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::converts::arrow::EXTENSION_KEY; use databend_common_expression::FieldIndex; +use databend_common_expression::converts::arrow::EXTENSION_KEY; use opendal::Operator; use parquet::arrow::parquet_to_arrow_schema; use parquet::errors::ParquetError; @@ -213,14 +213,14 @@ pub fn traverse_parquet_schema_tree( #[cfg(test)] mod tests { - use databend_common_expression::types::NumberDataType; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; + use databend_common_expression::types::NumberDataType; use parquet::arrow::ArrowSchemaConverter; - use crate::parquet::build_parquet_schema_tree; use crate::parquet::ParquetSchemaTreeNode; + use crate::parquet::build_parquet_schema_tree; #[test] fn test_build_parquet_schema_tree() { diff --git a/src/common/storage/src/runtime_layer.rs b/src/common/storage/src/runtime_layer.rs index 2ce25bafeb78d..64b4b025a074d 100644 --- a/src/common/storage/src/runtime_layer.rs +++ b/src/common/storage/src/runtime_layer.rs @@ -19,7 +19,9 @@ use std::sync::Arc; use databend_common_base::runtime::Runtime; use databend_common_base::runtime::ThreadTracker; use databend_common_base::runtime::TrySpawn; -use opendal::raw::oio; +use opendal::Buffer; +use opendal::Metadata; +use opendal::Result; use opendal::raw::Access; use opendal::raw::Layer; use opendal::raw::LayeredAccess; @@ -37,9 +39,7 @@ use opendal::raw::RpPresign; use opendal::raw::RpRead; use opendal::raw::RpStat; use opendal::raw::RpWrite; -use opendal::Buffer; -use opendal::Metadata; -use opendal::Result; +use opendal::raw::oio; /// # TODO /// @@ -307,26 +307,18 @@ impl oio::List for RuntimeIO { } impl oio::Delete for RuntimeIO { - fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> { - self.inner.as_mut().unwrap().delete(path, args) + async fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> { + self.inner.as_mut().unwrap().delete(path, args).await } - async fn flush(&mut self) -> Result { + async fn close(&mut self) -> Result<()> { let mut r = self.inner.take().expect("deleter must be valid"); let runtime = self.runtime.clone(); - let (r, res) = runtime - .try_spawn( - async move { - let res = r.flush().await; - (r, res) - }, - Some(self.spawn_task_name.clone()), - ) - .expect("spawn must success") + let _ = runtime + .spawn(async move { r.close().await }) .await - .expect("join must success"); - self.inner = Some(r); - res + .expect("join must success")?; + Ok(()) } } diff --git a/src/common/storage/src/stage.rs b/src/common/storage/src/stage.rs index dc99d215532fe..6249648de87b9 100644 --- a/src/common/storage/src/stage.rs +++ b/src/common/storage/src/stage.rs @@ -24,17 +24,17 @@ use databend_common_exception::Result; use databend_common_meta_app::principal::StageInfo; use databend_common_meta_app::principal::StageType; use databend_common_meta_app::principal::UserIdentity; -use futures::stream; use futures::Stream; use futures::StreamExt; use futures::TryStreamExt; +use futures::stream; use opendal::EntryMode; use opendal::Metadata; use opendal::Operator; use regex::Regex; -use crate::init_operator; use crate::DataOperator; +use crate::init_operator; #[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq, Eq)] pub enum StageFileStatus { @@ -59,7 +59,10 @@ impl StageFileInfo { path, size: meta.content_length(), md5: meta.content_md5().map(str::to_string), - last_modified: meta.last_modified(), + last_modified: meta.last_modified().map(|m| { + let ns = m.into_inner().as_nanosecond(); + DateTime::from_timestamp_nanos(ns as i64) + }), etag: meta.etag().map(str::to_string), status: StageFileStatus::NeedCopy, creator: None, diff --git a/src/common/storage/src/statistics.rs b/src/common/storage/src/statistics.rs index 814d95dbdcb79..e367529783d54 100644 --- a/src/common/storage/src/statistics.rs +++ b/src/common/storage/src/statistics.rs @@ -18,8 +18,8 @@ use std::fmt::Formatter; use databend_common_base::base::OrderedFloat; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::number::NumberScalar; use databend_common_expression::Scalar; +use databend_common_expression::types::number::NumberScalar; pub type F64 = OrderedFloat; diff --git a/src/common/storage/tests/it/column_node.rs b/src/common/storage/tests/it/column_node.rs index 8567c6d8f6dfd..0877c22acd541 100644 --- a/src/common/storage/tests/it/column_node.rs +++ b/src/common/storage/tests/it/column_node.rs @@ -13,11 +13,11 @@ // limitations under the License. use databend_common_exception::Result; -use databend_common_expression::create_test_complex_schema; -use databend_common_expression::types::NumberDataType; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; +use databend_common_expression::create_test_complex_schema; +use databend_common_expression::types::NumberDataType; use databend_common_storage::ColumnNode; use databend_common_storage::ColumnNodes; diff --git a/src/common/timezone/src/lib.rs b/src/common/timezone/src/lib.rs index ea59e0f36ad4b..86b68f460fee5 100644 --- a/src/common/timezone/src/lib.rs +++ b/src/common/timezone/src/lib.rs @@ -16,12 +16,12 @@ use std::sync::Arc; use std::sync::LazyLock; use std::sync::RwLock; -use jiff::civil::date; +use jiff::SignedDuration; use jiff::civil::Date; use jiff::civil::Time; use jiff::civil::Weekday; +use jiff::civil::date; use jiff::tz::TimeZone; -use jiff::SignedDuration; const LUT_MIN_YEAR: i32 = 1900; const LUT_MAX_YEAR: i32 = 2299; @@ -132,18 +132,17 @@ impl DayEntry { let mut offset = self.offset_at_start; if let (Some(trans), Some(transition_elapsed)) = (self.transition_utc, self.transition_elapsed) + && seconds >= trans { - if seconds >= trans { - if transition_elapsed == 0 { - if self.offset_change > 0 { - // Gap at the top of the day: skip the missing span but retain - // the post-transition offset that `offset_at_start` already has. - local_seconds += self.offset_change as i64; - } - } else { + if transition_elapsed == 0 { + if self.offset_change > 0 { + // Gap at the top of the day: skip the missing span but retain + // the post-transition offset that `offset_at_start` already has. local_seconds += self.offset_change as i64; - offset += self.offset_change; } + } else { + local_seconds += self.offset_change as i64; + offset += self.offset_change; } } debug_assert!(local_seconds >= 0); diff --git a/src/common/tracing/src/config.rs b/src/common/tracing/src/config.rs index 4429c46f9f3c7..df1588b0c62fd 100644 --- a/src/common/tracing/src/config.rs +++ b/src/common/tracing/src/config.rs @@ -298,18 +298,13 @@ impl FromStr for LogFormat { } } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] pub enum OTLPProtocol { Http, + #[default] Grpc, } -impl Default for OTLPProtocol { - fn default() -> Self { - Self::Grpc - } -} - impl Display for OTLPProtocol { fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { match self { @@ -422,7 +417,8 @@ impl Display for HistoryConfig { .iter() .map(|f| format!("{}({} hours)", f.table_name.clone(), f.retention)) .join(", "), - self.storage_params.as_ref() + self.storage_params + .as_ref() .map_or("None".to_string(), |p| p.to_string()) ) } diff --git a/src/common/tracing/src/crash_hook.rs b/src/common/tracing/src/crash_hook.rs index 077cf6ba013c4..805e211efcf92 100644 --- a/src/common/tracing/src/crash_hook.rs +++ b/src/common/tracing/src/crash_hook.rs @@ -27,8 +27,8 @@ use std::time::Duration; use databend_common_base::runtime::Thread; use databend_common_base::runtime::ThreadTracker; use databend_common_exception::StackTrace; -use jiff::tz::TimeZone; use jiff::Zoned; +use jiff::tz::TimeZone; const BUFFER_SIZE: usize = { size_of::() // sig @@ -200,13 +200,13 @@ impl JmpBuffer { } #[cfg(test)] -extern "C" { +unsafe extern "C" { // https://man7.org/linux/man-pages/man3/sigsetjmp.3p.html #[cfg_attr(target_env = "gnu", link_name = "__sigsetjmp")] - pub fn sigsetjmp(jb: *mut JmpBuffer, save_mask: i32) -> i32; + pub unsafe fn sigsetjmp(jb: *mut JmpBuffer, save_mask: i32) -> i32; // https://man7.org/linux/man-pages/man3/siglongjmp.3p.html - pub fn siglongjmp(jb: *mut JmpBuffer, val: i32) -> !; + pub unsafe fn siglongjmp(jb: *mut JmpBuffer, val: i32) -> !; } #[cfg(test)] @@ -267,78 +267,84 @@ fn signal_message(sig: i32, si_code: i32, si_addr: usize) -> String { } unsafe extern "C" fn signal_handler(sig: i32, info: *mut libc::siginfo_t, uc: *mut libc::c_void) { - let lock = CRASH_HANDLER_LOCK.lock(); - let mut guard = lock.unwrap_or_else(PoisonError::into_inner); + unsafe { + let lock = CRASH_HANDLER_LOCK.lock(); + let mut guard = lock.unwrap_or_else(PoisonError::into_inner); - if let Some(crash_handler) = guard.as_mut() { - crash_handler.recv_signal(sig, info, uc); - } + if let Some(crash_handler) = guard.as_mut() { + crash_handler.recv_signal(sig, info, uc); + } - #[cfg(test)] - { - drop(guard); - siglongjmp(addr_of_mut!(TEST_JMP_BUFFER), 1); - } + #[cfg(test)] + { + drop(guard); + siglongjmp(addr_of_mut!(TEST_JMP_BUFFER), 1); + } - #[allow(unreachable_code)] - if sig != libc::SIGTRAP { - match libc::SIG_ERR == libc::signal(sig, libc::SIG_DFL) { - true => std::process::exit(1), - false => match libc::raise(sig) { - 0 => {} - _ => std::process::exit(1), - }, + #[allow(unreachable_code)] + if sig != libc::SIGTRAP { + match libc::SIG_ERR == libc::signal(sig, libc::SIG_DFL) { + true => std::process::exit(1), + false => match libc::raise(sig) { + 0 => {} + _ => std::process::exit(1), + }, + } } } } pub unsafe fn add_signal_handler(signals: Vec) { - let mut sa = std::mem::zeroed::(); + unsafe { + let mut sa = std::mem::zeroed::(); - sa.sa_flags = libc::SA_ONSTACK | libc::SA_SIGINFO; - sa.sa_sigaction = signal_handler as usize; + sa.sa_flags = libc::SA_ONSTACK | libc::SA_SIGINFO; + sa.sa_sigaction = signal_handler as *const () as usize; - libc::sigemptyset(&mut sa.sa_mask); + libc::sigemptyset(&mut sa.sa_mask); - for signal in &signals { - libc::sigaddset(&mut sa.sa_mask, *signal); - } + for signal in &signals { + libc::sigaddset(&mut sa.sa_mask, *signal); + } - for signal in &signals { - libc::sigaction(*signal, &sa, std::ptr::null_mut()); + for signal in &signals { + libc::sigaction(*signal, &sa, std::ptr::null_mut()); + } } } // https://man7.org/linux/man-pages/man2/sigaltstack.2.html pub unsafe fn add_signal_stack(stack_bytes: usize) { - let page_size = libc::sysconf(libc::_SC_PAGESIZE) as usize; - let alloc_size = page_size + stack_bytes; - - let stack_memory_arena = libc::mmap( - std::ptr::null_mut(), - alloc_size, - libc::PROT_NONE, - libc::MAP_PRIVATE | libc::MAP_ANON, - -1, - 0, - ); - - if std::ptr::eq(stack_memory_arena, libc::MAP_FAILED) { - return; - } + unsafe { + let page_size = libc::sysconf(libc::_SC_PAGESIZE) as usize; + let alloc_size = page_size + stack_bytes; + + let stack_memory_arena = libc::mmap( + std::ptr::null_mut(), + alloc_size, + libc::PROT_NONE, + libc::MAP_PRIVATE | libc::MAP_ANON, + -1, + 0, + ); + + if std::ptr::eq(stack_memory_arena, libc::MAP_FAILED) { + return; + } - let stack_ptr = (stack_memory_arena as usize + page_size) as *mut libc::c_void; + let stack_ptr = (stack_memory_arena as usize + page_size) as *mut libc::c_void; - if libc::mprotect(stack_ptr, stack_bytes, libc::PROT_READ | libc::PROT_WRITE) != 0 { - libc::munmap(stack_ptr, alloc_size); - return; - } + if libc::mprotect(stack_ptr, stack_bytes, libc::PROT_READ | libc::PROT_WRITE) != 0 { + libc::munmap(stack_ptr, alloc_size); + return; + } - let mut new_signal_stack = std::mem::zeroed::(); - new_signal_stack.ss_sp = stack_memory_arena; - new_signal_stack.ss_size = stack_bytes; - if libc::sigaltstack(&new_signal_stack, std::ptr::null_mut()) != 0 { - libc::munmap(stack_ptr, alloc_size); + let mut new_signal_stack = std::mem::zeroed::(); + new_signal_stack.ss_sp = stack_memory_arena; + new_signal_stack.ss_size = stack_bytes; + if libc::sigaltstack(&new_signal_stack, std::ptr::null_mut()) != 0 { + libc::munmap(stack_ptr, alloc_size); + } } } @@ -433,31 +439,32 @@ pub struct SignalListener; impl SignalListener { pub fn spawn(mut file: File, crash_version: String) { - Thread::named_spawn(Some(String::from("SignalListener")), move || loop { - let mut buffer = [0_u8; BUFFER_SIZE]; + Thread::named_spawn(Some(String::from("SignalListener")), move || { + loop { + let mut buffer = [0_u8; BUFFER_SIZE]; - if file.read_exact(&mut buffer).is_ok() { - let pos = 0; - let (sig, pos) = read_i32(&buffer, pos); - let (si_code, pos) = read_i32(&buffer, pos); - let (si_addr, pos) = read_u64(&buffer, pos); - let (crash_query_id, pos) = read_string(&buffer, pos); - - let (frames_len, mut pos) = read_u64(&buffer, pos); - let mut frames = Vec::with_capacity(50); - - for _ in 0..frames_len { - let (ip, new_pos) = read_u64(&buffer, pos); - frames.push(ip); - pos = new_pos; - } + if file.read_exact(&mut buffer).is_ok() { + let pos = 0; + let (sig, pos) = read_i32(&buffer, pos); + let (si_code, pos) = read_i32(&buffer, pos); + let (si_addr, pos) = read_u64(&buffer, pos); + let (crash_query_id, pos) = read_string(&buffer, pos); - let id = std::process::id(); - let signal_mess = signal_message(sig, si_code, si_addr as usize); - let stack_trace = StackTrace::from_ips(&frames); + let (frames_len, mut pos) = read_u64(&buffer, pos); + let mut frames = Vec::with_capacity(50); - eprintln!( - "{:#^80}\n\ + for _ in 0..frames_len { + let (ip, new_pos) = read_u64(&buffer, pos); + frames.push(ip); + pos = new_pos; + } + + let id = std::process::id(); + let signal_mess = signal_message(sig, si_code, si_addr as usize); + let stack_trace = StackTrace::from_ips(&frames); + + eprintln!( + "{:#^80}\n\ PID: {}\n\ Version: {}\n\ Timestamp(UTC): {}\n\ @@ -465,17 +472,17 @@ impl SignalListener { QueryId: {:?}\n\ Signal Message: {}\n\ Backtrace:\n{:?}", - " Crash fault info ", - id, - crash_version, - Zoned::now().with_time_zone(TimeZone::UTC), - Zoned::now(), - crash_query_id, - signal_mess, - stack_trace - ); - log::error!( - "{:#^80}\n\ + " Crash fault info ", + id, + crash_version, + Zoned::now().with_time_zone(TimeZone::UTC), + Zoned::now(), + crash_query_id, + signal_mess, + stack_trace + ); + log::error!( + "{:#^80}\n\ PID: {}\n\ Version: {}\n\ Timestamp(UTC): {}\n\ @@ -483,15 +490,16 @@ impl SignalListener { QueryId: {:?}\n\ Signal Message: {}\n\ Backtrace:\n{:?}", - " Crash fault info ", - id, - crash_version, - Zoned::now().with_time_zone(TimeZone::UTC), - Zoned::now(), - crash_query_id, - signal_mess, - stack_trace - ); + " Crash fault info ", + id, + crash_version, + Zoned::now().with_time_zone(TimeZone::UTC), + Zoned::now(), + crash_query_id, + signal_mess, + stack_trace + ); + } } }); } @@ -504,13 +512,13 @@ mod tests { use databend_common_base::runtime::ThreadTracker; + use crate::crash_hook::BUFFER_SIZE; + use crate::crash_hook::TEST_JMP_BUFFER; use crate::crash_hook::pipe_file; use crate::crash_hook::read_i32; use crate::crash_hook::read_string; use crate::crash_hook::read_u64; use crate::crash_hook::sigsetjmp; - use crate::crash_hook::BUFFER_SIZE; - use crate::crash_hook::TEST_JMP_BUFFER; use crate::set_crash_hook; #[test] diff --git a/src/common/tracing/src/filter.rs b/src/common/tracing/src/filter.rs index 7808d84b03871..941777e888345 100644 --- a/src/common/tracing/src/filter.rs +++ b/src/common/tracing/src/filter.rs @@ -20,10 +20,10 @@ pub struct ThreadTrackerFilter; impl logforth::Filter for ThreadTrackerFilter { fn enabled(&self, metadata: &log::Metadata) -> FilterResult { - if let Some(settings) = ThreadTracker::capture_log_settings() { - if metadata.level() > settings.level { - return FilterResult::Reject; - } + if let Some(settings) = ThreadTracker::capture_log_settings() + && metadata.level() > settings.level + { + return FilterResult::Reject; } FilterResult::Neutral diff --git a/src/common/tracing/src/init.rs b/src/common/tracing/src/init.rs index 8749fe710ab79..cd3f45a787a16 100644 --- a/src/common/tracing/src/init.rs +++ b/src/common/tracing/src/init.rs @@ -14,35 +14,35 @@ use std::borrow::Cow; use std::collections::BTreeMap; +use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; -use std::sync::Arc; +use databend_common_base::base::GlobalInstance; use databend_common_base::base::tokio; use databend_common_base::base::tokio::sync::RwLock; -use databend_common_base::base::GlobalInstance; use databend_common_base::runtime::Thread; use fastrace::prelude::*; use log::LevelFilter; -use logforth::filter::env_filter::EnvFilterBuilder; use logforth::filter::EnvFilter; +use logforth::filter::env_filter::EnvFilterBuilder; use opendal::Operator; use opentelemetry_otlp::Compression; use opentelemetry_otlp::WithExportConfig; use opentelemetry_otlp::WithTonicConfig; +use crate::Config; use crate::config::LogFormat; use crate::config::OTLPProtocol; use crate::filter::ThreadTrackerFilter; -use crate::loggers::new_rolling_file_appender; use crate::loggers::IdenticalLayout; use crate::loggers::JsonLayout; use crate::loggers::TextLayout; +use crate::loggers::new_rolling_file_appender; use crate::predefined_tables::table_to_target; use crate::query_log_collector::QueryLogCollector; use crate::remote_log::RemoteLog; use crate::structlog::StructLogReporter; -use crate::Config; const HEADER_TRACE_PARENT: &str = "traceparent"; diff --git a/src/common/tracing/src/lib.rs b/src/common/tracing/src/lib.rs index 48b4634315a48..6d6846599e183 100644 --- a/src/common/tracing/src/lib.rs +++ b/src/common/tracing/src/lib.rs @@ -30,9 +30,10 @@ mod structlog; mod predefined_tables; mod query_log_collector; -pub use crash_hook::pipe_file; pub use crash_hook::SignalListener; +pub use crash_hook::pipe_file; +pub use crate::config::CONFIG_DEFAULT_LOG_LEVEL; pub use crate::config::Config; pub use crate::config::FileConfig; pub use crate::config::HistoryConfig; @@ -46,23 +47,22 @@ pub use crate::config::QueryLogConfig; pub use crate::config::StderrConfig; pub use crate::config::StructLogConfig; pub use crate::config::TracingConfig; -pub use crate::config::CONFIG_DEFAULT_LOG_LEVEL; pub use crate::crash_hook::set_crash_hook; +pub use crate::init::GlobalLogger; pub use crate::init::init_logging; pub use crate::init::inject_span_to_tonic_request; pub use crate::init::start_trace_for_remote_request; -pub use crate::init::GlobalLogger; pub use crate::panic_hook::log_panic; pub use crate::panic_hook::set_panic_hook; +pub use crate::predefined_tables::HistoryTable; pub use crate::predefined_tables::get_all_history_table_names; pub use crate::predefined_tables::init_history_tables; -pub use crate::predefined_tables::HistoryTable; -pub use crate::remote_log::convert_to_batch; pub use crate::remote_log::LogBuffer as RemoteLogBuffer; pub use crate::remote_log::LogMessage; pub use crate::remote_log::RemoteLog; pub use crate::remote_log::RemoteLogElement; pub use crate::remote_log::RemoteLogGuard; +pub use crate::remote_log::convert_to_batch; pub use crate::structlog::DummyReporter; pub use crate::structlog::StructLogReporter; pub fn closure_name() -> &'static str { diff --git a/src/common/tracing/src/loggers.rs b/src/common/tracing/src/loggers.rs index 856366ab3641e..ba790ce634a1a 100644 --- a/src/common/tracing/src/loggers.rs +++ b/src/common/tracing/src/loggers.rs @@ -17,22 +17,22 @@ use std::path::Path; use databend_common_base::runtime::LimitMemGuard; use databend_common_base::runtime::ThreadTracker; -use jiff::fmt::temporal::DateTimePrinter; -use jiff::tz::TimeZone; use jiff::Timestamp; use jiff::Zoned; +use jiff::fmt::temporal::DateTimePrinter; +use jiff::tz::TimeZone; +use log::Record; use log::kv::Key; use log::kv::Source; use log::kv::Value; use log::kv::VisitSource; -use log::Record; -use logforth::append::rolling_file::RollingFileWriter; -use logforth::append::rolling_file::Rotation; -use logforth::append::RollingFile; -use logforth::non_blocking::NonBlockingBuilder; use logforth::Append; use logforth::Diagnostic; use logforth::Layout; +use logforth::append::RollingFile; +use logforth::append::rolling_file::RollingFileWriter; +use logforth::append::rolling_file::Rotation; +use logforth::non_blocking::NonBlockingBuilder; use serde_json::Map; use serde_json::Value as JsonValue; @@ -244,8 +244,12 @@ mod tests { .file(Some("test_file.rs")) .line(Some(42)) .build(), - expected_text_output: format!("{timestamp} INFO test::module: test_file.rs:42 test message"), - expected_json_output: format!(r#"{{"timestamp":"{timestamp}","level":"INFO","fields":{{"message":"test message"}}}}"#), + expected_text_output: format!( + "{timestamp} INFO test::module: test_file.rs:42 test message" + ), + expected_json_output: format!( + r#"{{"timestamp":"{timestamp}","level":"INFO","fields":{{"message":"test message"}}}}"# + ), }, TestCase { name: "empty message", @@ -259,7 +263,9 @@ mod tests { .line(Some(42)) .build(), expected_text_output: format!("{timestamp} INFO test::module: test_file.rs:42 "), - expected_json_output: format!(r#"{{"timestamp":"{timestamp}","level":"INFO","fields":{{"message":""}}}}"#), + expected_json_output: format!( + r#"{{"timestamp":"{timestamp}","level":"INFO","fields":{{"message":""}}}}"# + ), }, TestCase { name: "error level", @@ -272,8 +278,12 @@ mod tests { .file(Some("test_file.rs")) .line(Some(42)) .build(), - expected_text_output: format!("{timestamp} ERROR test::module: test_file.rs:42 error occurred"), - expected_json_output: format!(r#"{{"timestamp":"{timestamp}","level":"ERROR","fields":{{"message":"error occurred"}}}}"#), + expected_text_output: format!( + "{timestamp} ERROR test::module: test_file.rs:42 error occurred" + ), + expected_json_output: format!( + r#"{{"timestamp":"{timestamp}","level":"ERROR","fields":{{"message":"error occurred"}}}}"# + ), }, TestCase { name: "warn level", @@ -286,8 +296,12 @@ mod tests { .file(Some("test_file.rs")) .line(Some(42)) .build(), - expected_text_output: format!("{timestamp} WARN test::module: test_file.rs:42 warning message"), - expected_json_output: format!(r#"{{"timestamp":"{timestamp}","level":"WARN","fields":{{"message":"warning message"}}}}"#), + expected_text_output: format!( + "{timestamp} WARN test::module: test_file.rs:42 warning message" + ), + expected_json_output: format!( + r#"{{"timestamp":"{timestamp}","level":"WARN","fields":{{"message":"warning message"}}}}"# + ), }, TestCase { name: "debug level", @@ -300,8 +314,12 @@ mod tests { .file(Some("test_file.rs")) .line(Some(42)) .build(), - expected_text_output: format!("{timestamp} DEBUG test::module: test_file.rs:42 debug info"), - expected_json_output: format!(r#"{{"timestamp":"{timestamp}","level":"DEBUG","fields":{{"message":"debug info"}}}}"#), + expected_text_output: format!( + "{timestamp} DEBUG test::module: test_file.rs:42 debug info" + ), + expected_json_output: format!( + r#"{{"timestamp":"{timestamp}","level":"DEBUG","fields":{{"message":"debug info"}}}}"# + ), }, TestCase { name: "trace level", @@ -314,8 +332,12 @@ mod tests { .file(Some("test_file.rs")) .line(Some(42)) .build(), - expected_text_output: format!("{timestamp} TRACE test::module: test_file.rs:42 trace data"), - expected_json_output: format!(r#"{{"timestamp":"{timestamp}","level":"TRACE","fields":{{"message":"trace data"}}}}"#), + expected_text_output: format!( + "{timestamp} TRACE test::module: test_file.rs:42 trace data" + ), + expected_json_output: format!( + r#"{{"timestamp":"{timestamp}","level":"TRACE","fields":{{"message":"trace data"}}}}"# + ), }, TestCase { name: "message with key-values", @@ -332,8 +354,12 @@ mod tests { ("action", &"login" as _), ]) .build(), - expected_text_output: format!("{timestamp} INFO test::module: test_file.rs:42 user action user_id=123 action=login"), - expected_json_output: format!(r#"{{"timestamp":"{timestamp}","level":"INFO","fields":{{"message":"user action","user_id":123,"action":"login"}}}}"#), + expected_text_output: format!( + "{timestamp} INFO test::module: test_file.rs:42 user action user_id=123 action=login" + ), + expected_json_output: format!( + r#"{{"timestamp":"{timestamp}","level":"INFO","fields":{{"message":"user action","user_id":123,"action":"login"}}}}"# + ), }, TestCase { name: "error with context", @@ -351,8 +377,12 @@ mod tests { ("host", &"localhost" as _), ]) .build(), - expected_text_output: format!("{timestamp} ERROR test::module: test_file.rs:42 database connection failed error_code=500 retry_count=3 host=localhost"), - expected_json_output: format!(r#"{{"timestamp":"{timestamp}","level":"ERROR","fields":{{"message":"database connection failed","error_code":500,"retry_count":3,"host":"localhost"}}}}"#), + expected_text_output: format!( + "{timestamp} ERROR test::module: test_file.rs:42 database connection failed error_code=500 retry_count=3 host=localhost" + ), + expected_json_output: format!( + r#"{{"timestamp":"{timestamp}","level":"ERROR","fields":{{"message":"database connection failed","error_code":500,"retry_count":3,"host":"localhost"}}}}"# + ), }, ] } diff --git a/src/common/tracing/src/panic_hook.rs b/src/common/tracing/src/panic_hook.rs index 1e25c83c10221..e9dbd8f0b8dbb 100644 --- a/src/common/tracing/src/panic_hook.rs +++ b/src/common/tracing/src/panic_hook.rs @@ -13,8 +13,8 @@ // limitations under the License. use std::panic::PanicHookInfo; -use std::sync::atomic::Ordering; use std::sync::Arc; +use std::sync::atomic::Ordering; use backtrace::Backtrace; use backtrace::BacktraceFrame; diff --git a/src/common/tracing/src/predefined_tables/mod.rs b/src/common/tracing/src/predefined_tables/mod.rs index dc7448e42c9fb..5112abdfe5a2a 100644 --- a/src/common/tracing/src/predefined_tables/mod.rs +++ b/src/common/tracing/src/predefined_tables/mod.rs @@ -14,7 +14,7 @@ mod history_tables; +pub use history_tables::HistoryTable; pub use history_tables::get_all_history_table_names; pub use history_tables::init_history_tables; pub use history_tables::table_to_target; -pub use history_tables::HistoryTable; diff --git a/src/common/tracing/src/query_log_collector.rs b/src/common/tracing/src/query_log_collector.rs index 68805fe03b40d..8e4ab22808c27 100644 --- a/src/common/tracing/src/query_log_collector.rs +++ b/src/common/tracing/src/query_log_collector.rs @@ -21,8 +21,8 @@ use log::Record; use logforth::Append; use logforth::Diagnostic; -use crate::loggers::format_timestamp; use crate::loggers::KvWriter; +use crate::loggers::format_timestamp; #[derive(Debug)] pub struct QueryLogCollector {} diff --git a/src/common/tracing/src/remote_log.rs b/src/common/tracing/src/remote_log.rs index 92ebd989d795f..0a3216549bccb 100644 --- a/src/common/tracing/src/remote_log.rs +++ b/src/common/tracing/src/remote_log.rs @@ -15,29 +15,29 @@ use std::collections::BTreeMap; use std::fmt::Debug; use std::path::Path; +use std::sync::Arc; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; use std::sync::mpsc; use std::sync::mpsc::channel; -use std::sync::Arc; use std::time::Duration; +use arrow_array::RecordBatch; use arrow_array::builder::StringBuilder; use arrow_array::builder::TimestampMicrosecondBuilder; -use arrow_array::RecordBatch; use arrow_schema::DataType; use arrow_schema::Field; use arrow_schema::Schema; use arrow_schema::TimeUnit; -use async_channel::bounded; use async_channel::Receiver; use async_channel::Sender; +use async_channel::bounded; use concurrent_queue::ConcurrentQueue; use databend_common_base::base::uuid; -use databend_common_base::runtime::spawn; use databend_common_base::runtime::Runtime; use databend_common_base::runtime::ThreadTracker; use databend_common_base::runtime::TrySpawn; +use databend_common_base::runtime::spawn; use databend_common_exception::Result; use jiff::Timestamp; use log::Record; @@ -51,9 +51,9 @@ use parquet::file::properties::EnabledStatistics; use parquet::file::properties::WriterProperties; use serde_json::Map; -use crate::loggers::collect_kvs; use crate::Config; use crate::GlobalLogger; +use crate::loggers::collect_kvs; /// An appender that sends log records to persistent storage #[derive(Debug)] diff --git a/src/common/tracing/tests/it/logger.rs b/src/common/tracing/tests/it/logger.rs index abc17c27307de..daf887513ed27 100644 --- a/src/common/tracing/tests/it/logger.rs +++ b/src/common/tracing/tests/it/logger.rs @@ -14,8 +14,8 @@ use std::collections::BTreeMap; -use databend_common_tracing::init_logging; use databend_common_tracing::Config; +use databend_common_tracing::init_logging; /// Test rolling files logger /// generate 30 log files, then check if there are 2 files diff --git a/src/common/tracing/tests/it/remote_log.rs b/src/common/tracing/tests/it/remote_log.rs index 532bdd467ed2e..f064a106bc0c5 100644 --- a/src/common/tracing/tests/it/remote_log.rs +++ b/src/common/tracing/tests/it/remote_log.rs @@ -17,10 +17,9 @@ use std::sync::Arc; use std::time::Duration; use async_channel::bounded; -use databend_common_base::base::tokio; use databend_common_base::base::GlobalInstance; +use databend_common_base::base::tokio; use databend_common_exception::Result; -use databend_common_tracing::convert_to_batch; use databend_common_tracing::Config; use databend_common_tracing::GlobalLogger; use databend_common_tracing::LogMessage; @@ -28,10 +27,11 @@ use databend_common_tracing::RemoteLog; use databend_common_tracing::RemoteLogBuffer; use databend_common_tracing::RemoteLogElement; use databend_common_tracing::RemoteLogGuard; +use databend_common_tracing::convert_to_batch; use log::Level; use log::Record; -use opendal::services; use opendal::Operator; +use opendal::services; fn setup() -> Result<(RemoteLog, Box)> { let mut labels = BTreeMap::new(); diff --git a/src/meta/api/src/auto_increment_api_test_suite.rs b/src/meta/api/src/auto_increment_api_test_suite.rs index 841e8d387a4c3..faff31cda4884 100644 --- a/src/meta/api/src/auto_increment_api_test_suite.rs +++ b/src/meta/api/src/auto_increment_api_test_suite.rs @@ -15,13 +15,12 @@ use std::sync::Arc; use chrono::Utc; -use databend_common_expression::types::NumberDataType; use databend_common_expression::AutoIncrementExpr; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; +use databend_common_expression::types::NumberDataType; use databend_common_meta_app::principal::AutoIncrementKey; -use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_app::schema::AutoIncrementStorageIdent; use databend_common_meta_app::schema::CreateDatabaseReq; use databend_common_meta_app::schema::CreateOption; @@ -32,6 +31,7 @@ use databend_common_meta_app::schema::GetAutoIncrementNextValueReq; use databend_common_meta_app::schema::ListDroppedTableReq; use databend_common_meta_app::schema::TableMeta; use databend_common_meta_app::schema::TableNameIdent; +use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::Key; @@ -40,7 +40,6 @@ use databend_common_meta_types::MetaError; use fastrace::func_name; use log::info; -use crate::kv_pb_api::KVPbApi; use crate::AutoIncrementApi; use crate::DatabaseApi; use crate::DatamaskApi; @@ -48,6 +47,7 @@ use crate::GarbageCollectionApi; use crate::RowAccessPolicyApi; use crate::SchemaApi; use crate::TableApi; +use crate::kv_pb_api::KVPbApi; /// Test suite of `AutoIncrementApi`. /// @@ -88,16 +88,15 @@ impl AutoIncrementApiTestSuite { util.create_db().await?; let schema = || { - Arc::new(TableSchema::new(vec![TableField::new( - "number", - TableDataType::Number(NumberDataType::UInt64), - ) - .with_auto_increment_expr(Some(AutoIncrementExpr { - column_id: 0, - start: 0, - step: 1, - is_ordered: true, - }))])) + Arc::new(TableSchema::new(vec![ + TableField::new("number", TableDataType::Number(NumberDataType::UInt64)) + .with_auto_increment_expr(Some(AutoIncrementExpr { + column_id: 0, + start: 0, + step: 1, + is_ordered: true, + })), + ])) }; let options = || maplit::btreemap! {"opt‐1".into() => "val-1".into()}; diff --git a/src/meta/api/src/auto_increment_nextval_impl.rs b/src/meta/api/src/auto_increment_nextval_impl.rs index 731587d780991..19b3311eb1cd9 100644 --- a/src/meta/api/src/auto_increment_nextval_impl.rs +++ b/src/meta/api/src/auto_increment_nextval_impl.rs @@ -18,11 +18,11 @@ use databend_common_meta_app::schema::AutoIncrementStorageIdent; use databend_common_meta_app::tenant::ToTenant; use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::Key; -use databend_common_meta_types::anyerror::func_name; -use databend_common_meta_types::protobuf::FetchIncreaseU64Response; use databend_common_meta_types::MetaError; use databend_common_meta_types::TxnOp; use databend_common_meta_types::TxnRequest; +use databend_common_meta_types::anyerror::func_name; +use databend_common_meta_types::protobuf::FetchIncreaseU64Response; use log::debug; use crate::errors::AutoIncrementError; diff --git a/src/meta/api/src/catalog_api.rs b/src/meta/api/src/catalog_api.rs index 849934b95e07d..962264ba9a94a 100644 --- a/src/meta/api/src/catalog_api.rs +++ b/src/meta/api/src/catalog_api.rs @@ -14,15 +14,15 @@ use std::sync::Arc; +use databend_common_meta_app::KeyWithTenant; use databend_common_meta_app::app_error::AppError; -use databend_common_meta_app::schema::catalog_id_ident::CatalogId; -use databend_common_meta_app::schema::catalog_name_ident::CatalogNameIdentRaw; use databend_common_meta_app::schema::CatalogIdToNameIdent; use databend_common_meta_app::schema::CatalogInfo; use databend_common_meta_app::schema::CatalogMeta; use databend_common_meta_app::schema::CatalogNameIdent; use databend_common_meta_app::schema::ListCatalogReq; -use databend_common_meta_app::KeyWithTenant; +use databend_common_meta_app::schema::catalog_id_ident::CatalogId; +use databend_common_meta_app::schema::catalog_name_ident::CatalogNameIdentRaw; use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::DirName; use databend_common_meta_kvapi::kvapi::Key; diff --git a/src/meta/api/src/data_mask_api.rs b/src/meta/api/src/data_mask_api.rs index cf2d3d7113e14..2926ab636983f 100644 --- a/src/meta/api/src/data_mask_api.rs +++ b/src/meta/api/src/data_mask_api.rs @@ -12,12 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_meta_app::data_mask::data_mask_name_ident; use databend_common_meta_app::data_mask::CreateDatamaskReply; use databend_common_meta_app::data_mask::CreateDatamaskReq; use databend_common_meta_app::data_mask::DataMaskId; use databend_common_meta_app::data_mask::DataMaskNameIdent; use databend_common_meta_app::data_mask::DatamaskMeta; +use databend_common_meta_app::data_mask::data_mask_name_ident; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_app::tenant_key::errors::ExistError; use databend_common_meta_types::MetaError; diff --git a/src/meta/api/src/data_mask_api_impl.rs b/src/meta/api/src/data_mask_api_impl.rs index 63ac3cdad0da3..40ed61b7a8a19 100644 --- a/src/meta/api/src/data_mask_api_impl.rs +++ b/src/meta/api/src/data_mask_api_impl.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_meta_app::data_mask::data_mask_name_ident; +use databend_common_meta_app::KeyWithTenant; use databend_common_meta_app::data_mask::CreateDatamaskReply; use databend_common_meta_app::data_mask::CreateDatamaskReq; use databend_common_meta_app::data_mask::DataMaskId; @@ -25,11 +25,11 @@ use databend_common_meta_app::data_mask::MaskPolicyIdTableId; use databend_common_meta_app::data_mask::MaskPolicyTableIdIdent; use databend_common_meta_app::data_mask::MaskPolicyTableIdListIdent; use databend_common_meta_app::data_mask::MaskpolicyTableIdList; +use databend_common_meta_app::data_mask::data_mask_name_ident; use databend_common_meta_app::id_generator::IdGenerator; use databend_common_meta_app::row_access_policy::RowAccessPolicyNameIdent; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_app::tenant_key::errors::ExistError; -use databend_common_meta_app::KeyWithTenant; use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::DirName; use databend_common_meta_types::MetaError; diff --git a/src/meta/api/src/database_api.rs b/src/meta/api/src/database_api.rs index a1354967b2b81..447f2d481ef26 100644 --- a/src/meta/api/src/database_api.rs +++ b/src/meta/api/src/database_api.rs @@ -17,6 +17,7 @@ use std::fmt::Display; use std::sync::Arc; use chrono::Utc; +use databend_common_meta_app::KeyWithTenant; use databend_common_meta_app::app_error::AppError; use databend_common_meta_app::app_error::CreateDatabaseWithDropTime; use databend_common_meta_app::app_error::DatabaseAlreadyExists; @@ -25,8 +26,6 @@ use databend_common_meta_app::app_error::UndropDbWithNoDropTime; use databend_common_meta_app::app_error::UnknownDatabase; use databend_common_meta_app::app_error::UnknownDatabaseId; use databend_common_meta_app::id_generator::IdGenerator; -use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; -use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdentRaw; use databend_common_meta_app::schema::CreateDatabaseReply; use databend_common_meta_app::schema::CreateDatabaseReq; use databend_common_meta_app::schema::CreateOption; @@ -43,7 +42,8 @@ use databend_common_meta_app::schema::RenameDatabaseReply; use databend_common_meta_app::schema::RenameDatabaseReq; use databend_common_meta_app::schema::UndropDatabaseReply; use databend_common_meta_app::schema::UndropDatabaseReq; -use databend_common_meta_app::KeyWithTenant; +use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; +use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdentRaw; use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::DirName; use databend_common_meta_types::ConditionResult::Eq; diff --git a/src/meta/api/src/database_util.rs b/src/meta/api/src/database_util.rs index f2b4922de2599..db79c6cdac1e6 100644 --- a/src/meta/api/src/database_util.rs +++ b/src/meta/api/src/database_util.rs @@ -15,16 +15,16 @@ use std::fmt::Display; use chrono::Utc; +use databend_common_meta_app::KeyWithTenant; use databend_common_meta_app::app_error::AppError; use databend_common_meta_app::app_error::DropDbWithDropTime; use databend_common_meta_app::principal::OwnershipObject; use databend_common_meta_app::principal::TenantOwnershipObjectIdent; -use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_app::schema::DatabaseId; use databend_common_meta_app::schema::DatabaseIdHistoryIdent; use databend_common_meta_app::schema::DatabaseMeta; use databend_common_meta_app::schema::DbIdList; -use databend_common_meta_app::KeyWithTenant; +use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_kvapi::kvapi; use databend_common_meta_types::ConditionResult::Eq; use databend_common_meta_types::MetaError; diff --git a/src/meta/api/src/dictionary_api.rs b/src/meta/api/src/dictionary_api.rs index 3243f2cb3b40c..90e7b6d1a1615 100644 --- a/src/meta/api/src/dictionary_api.rs +++ b/src/meta/api/src/dictionary_api.rs @@ -13,9 +13,6 @@ // limitations under the License. use databend_common_meta_app::app_error::AppError; -use databend_common_meta_app::schema::dictionary_id_ident::DictionaryId; -use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; -use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameRsc; use databend_common_meta_app::schema::CreateDictionaryReply; use databend_common_meta_app::schema::CreateDictionaryReq; use databend_common_meta_app::schema::DictionaryIdentity; @@ -24,6 +21,9 @@ use databend_common_meta_app::schema::ListDictionaryReq; use databend_common_meta_app::schema::RenameDictionaryReq; use databend_common_meta_app::schema::UpdateDictionaryReply; use databend_common_meta_app::schema::UpdateDictionaryReq; +use databend_common_meta_app::schema::dictionary_id_ident::DictionaryId; +use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; +use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameRsc; use databend_common_meta_app::tenant_key::errors::ExistError; use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::DirName; diff --git a/src/meta/api/src/error_util.rs b/src/meta/api/src/error_util.rs index 1771fa5acb373..fe9fc39110fb2 100644 --- a/src/meta/api/src/error_util.rs +++ b/src/meta/api/src/error_util.rs @@ -22,8 +22,8 @@ use databend_common_meta_app::app_error::TableAlreadyExists; use databend_common_meta_app::app_error::UnknownDatabase; use databend_common_meta_app::app_error::UnknownDatabaseId; use databend_common_meta_app::app_error::UnknownTable; -use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_app::schema::TableNameIdent; +use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use crate::kv_app_error::KVAppError; diff --git a/src/meta/api/src/garbage_collection_api.rs b/src/meta/api/src/garbage_collection_api.rs index e4264fc5697b1..61fa20129c986 100644 --- a/src/meta/api/src/garbage_collection_api.rs +++ b/src/meta/api/src/garbage_collection_api.rs @@ -25,10 +25,6 @@ use databend_common_meta_app::app_error::MarkDatabaseMetaAsGCInProgressFailed; use databend_common_meta_app::principal::AutoIncrementKey; use databend_common_meta_app::principal::OwnershipObject; use databend_common_meta_app::principal::TenantOwnershipObjectIdent; -use databend_common_meta_app::schema::index_id_ident::IndexIdIdent; -use databend_common_meta_app::schema::index_id_to_name_ident::IndexIdToNameIdent; -use databend_common_meta_app::schema::table_niv::TableNIV; -use databend_common_meta_app::schema::vacuum_watermark_ident::VacuumWatermarkIdent; use databend_common_meta_app::schema::AutoIncrementStorageIdent; use databend_common_meta_app::schema::DBIdTableName; use databend_common_meta_app::schema::DatabaseId; @@ -43,15 +39,19 @@ use databend_common_meta_app::schema::TableId; use databend_common_meta_app::schema::TableIdHistoryIdent; use databend_common_meta_app::schema::TableIdToName; use databend_common_meta_app::schema::VacuumWatermark; +use databend_common_meta_app::schema::index_id_ident::IndexIdIdent; +use databend_common_meta_app::schema::index_id_to_name_ident::IndexIdToNameIdent; +use databend_common_meta_app::schema::table_niv::TableNIV; +use databend_common_meta_app::schema::vacuum_watermark_ident::VacuumWatermarkIdent; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::DirName; use databend_common_meta_kvapi::kvapi::Key; -use databend_common_meta_types::txn_op::Request; -use databend_common_meta_types::txn_op_response::Response; use databend_common_meta_types::MetaError; use databend_common_meta_types::SeqV; use databend_common_meta_types::TxnRequest; +use databend_common_meta_types::txn_op::Request; +use databend_common_meta_types::txn_op_response::Response; use display_more::DisplaySliceExt; use fastrace::func_name; use futures::StreamExt; diff --git a/src/meta/api/src/index_api.rs b/src/meta/api/src/index_api.rs index 90a5d289e7842..fb4700b0b7ed3 100644 --- a/src/meta/api/src/index_api.rs +++ b/src/meta/api/src/index_api.rs @@ -15,19 +15,11 @@ use std::collections::HashMap; use databend_common_base::base::uuid::Uuid; +use databend_common_meta_app::KeyWithTenant; use databend_common_meta_app::app_error::AppError; use databend_common_meta_app::app_error::DuplicatedIndexColumnId; use databend_common_meta_app::app_error::IndexColumnIdNotFound; use databend_common_meta_app::app_error::UnknownTableId; -use databend_common_meta_app::schema::index_id_ident::IndexId; -use databend_common_meta_app::schema::index_id_ident::IndexIdIdent; -use databend_common_meta_app::schema::index_id_to_name_ident::IndexIdToNameIdent; -use databend_common_meta_app::schema::index_name_ident::IndexName; -use databend_common_meta_app::schema::index_name_ident::IndexNameIdent; -use databend_common_meta_app::schema::marked_deleted_index_id::MarkedDeletedIndexId; -use databend_common_meta_app::schema::marked_deleted_index_ident::MarkedDeletedIndexIdIdent; -use databend_common_meta_app::schema::marked_deleted_table_index_id::MarkedDeletedTableIndexId; -use databend_common_meta_app::schema::marked_deleted_table_index_ident::MarkedDeletedTableIndexIdIdent; use databend_common_meta_app::schema::CreateIndexReply; use databend_common_meta_app::schema::CreateIndexReq; use databend_common_meta_app::schema::CreateOption; @@ -41,9 +33,17 @@ use databend_common_meta_app::schema::IndexNameIdentRaw; use databend_common_meta_app::schema::ListIndexesReq; use databend_common_meta_app::schema::TableId; use databend_common_meta_app::schema::TableIndex; +use databend_common_meta_app::schema::index_id_ident::IndexId; +use databend_common_meta_app::schema::index_id_ident::IndexIdIdent; +use databend_common_meta_app::schema::index_id_to_name_ident::IndexIdToNameIdent; +use databend_common_meta_app::schema::index_name_ident::IndexName; +use databend_common_meta_app::schema::index_name_ident::IndexNameIdent; +use databend_common_meta_app::schema::marked_deleted_index_id::MarkedDeletedIndexId; +use databend_common_meta_app::schema::marked_deleted_index_ident::MarkedDeletedIndexIdIdent; +use databend_common_meta_app::schema::marked_deleted_table_index_id::MarkedDeletedTableIndexId; +use databend_common_meta_app::schema::marked_deleted_table_index_ident::MarkedDeletedTableIndexIdIdent; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_app::tenant_key::errors::UnknownError; -use databend_common_meta_app::KeyWithTenant; use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::DirName; use databend_common_meta_kvapi::kvapi::Key; diff --git a/src/meta/api/src/kv_pb_api/codec.rs b/src/meta/api/src/kv_pb_api/codec.rs index 7d515dfbe8551..9a76d4ee428b7 100644 --- a/src/meta/api/src/kv_pb_api/codec.rs +++ b/src/meta/api/src/kv_pb_api/codec.rs @@ -14,10 +14,10 @@ use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::NonEmptyItem; -use databend_common_meta_types::protobuf::StreamItem; use databend_common_meta_types::Change; use databend_common_meta_types::Operation; use databend_common_meta_types::SeqV; +use databend_common_meta_types::protobuf::StreamItem; use databend_common_proto_conv::FromToProto; use crate::kv_pb_api::errors::NoneValue; @@ -84,12 +84,12 @@ where { let buf = &seqv.data; let x: Result<_, PbDecodeError> = try { - let p: T::PB = prost::Message::decode(buf.as_ref())?; - let v: T = FromToProto::from_pb(p)?; + let p: T::PB = prost::Message::decode(buf.as_ref()).map_err(PbDecodeError::from)?; + let v: T = FromToProto::from_pb(p).map_err(PbDecodeError::from)?; SeqV::new_with_meta(seqv.seq, seqv.meta, v) }; - x.map_err(|e| e.with_context(context())) + x.map_err(|e: PbDecodeError| e.with_context(context())) } /// Decode key and protobuf encoded value from `StreamItem`. diff --git a/src/meta/api/src/kv_pb_api/errors/read_error.rs b/src/meta/api/src/kv_pb_api/errors/read_error.rs index 654745de5cfb7..3439de10f1896 100644 --- a/src/meta/api/src/kv_pb_api/errors/read_error.rs +++ b/src/meta/api/src/kv_pb_api/errors/read_error.rs @@ -13,9 +13,9 @@ // limitations under the License. use databend_common_meta_kvapi::kvapi; -use databend_common_meta_types::anyerror::AnyError; use databend_common_meta_types::InvalidReply; use databend_common_meta_types::MetaError; +use databend_common_meta_types::anyerror::AnyError; use crate::kv_pb_api::errors::PbDecodeError; diff --git a/src/meta/api/src/kv_pb_api/mod.rs b/src/meta/api/src/kv_pb_api/mod.rs index ba24d7096a3fe..f577b60be5a0e 100644 --- a/src/meta/api/src/kv_pb_api/mod.rs +++ b/src/meta/api/src/kv_pb_api/mod.rs @@ -21,25 +21,25 @@ mod upsert_pb; use std::future::Future; +use databend_common_meta_app::KeyWithTenant; use databend_common_meta_app::data_id::DataId; use databend_common_meta_app::tenant_key::resource::TenantResource; -use databend_common_meta_app::KeyWithTenant; use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::DirName; use databend_common_meta_kvapi::kvapi::KVApi; use databend_common_meta_kvapi::kvapi::KvApiExt; use databend_common_meta_kvapi::kvapi::NonEmptyItem; -use databend_common_meta_types::protobuf::StreamItem; use databend_common_meta_types::Change; use databend_common_meta_types::SeqV; use databend_common_meta_types::UpsertKV; +use databend_common_meta_types::protobuf::StreamItem; use databend_common_proto_conv::FromToProto; +use futures::TryStreamExt; use futures::future::FutureExt; use futures::future::TryFutureExt; use futures::stream; use futures::stream::BoxStream; use futures::stream::StreamExt; -use futures::TryStreamExt; use itertools::Itertools; use seq_marked::SeqValue; @@ -524,12 +524,12 @@ mod tests { use databend_common_meta_kvapi::kvapi::KVApi; use databend_common_meta_kvapi::kvapi::KVStream; use databend_common_meta_kvapi::kvapi::UpsertKVReply; - use databend_common_meta_types::protobuf::StreamItem; use databend_common_meta_types::MetaError; use databend_common_meta_types::SeqV; use databend_common_meta_types::TxnReply; use databend_common_meta_types::TxnRequest; use databend_common_meta_types::UpsertKV; + use databend_common_meta_types::protobuf::StreamItem; use databend_common_proto_conv::FromToProto; use futures::StreamExt; use futures::TryStreamExt; @@ -810,7 +810,8 @@ mod tests { let want = vec![ "InvalidReply: source:(PbDecodeError: failed to decode Protobuf message: invalid varint; when:(decode value of __fd_catalog_by_id/1))", "InvalidReply: source:(PbDecodeError: failed to decode Protobuf message: invalid varint; when:(decode value of __fd_catalog_by_id/2))", - "InvalidReply: source:(PbDecodeError: failed to decode Protobuf message: invalid varint; when:(decode value of __fd_catalog_by_id/3))"]; + "InvalidReply: source:(PbDecodeError: failed to decode Protobuf message: invalid varint; when:(decode value of __fd_catalog_by_id/3))", + ]; assert_eq!(errors, want); diff --git a/src/meta/api/src/lib.rs b/src/meta/api/src/lib.rs index 8fc901d875c0d..30347f72a75fe 100644 --- a/src/meta/api/src/lib.rs +++ b/src/meta/api/src/lib.rs @@ -15,6 +15,8 @@ #![allow(clippy::uninlined_format_args)] #![allow(clippy::diverging_sub_expression)] #![allow(clippy::type_complexity)] +#![allow(clippy::collapsible_if)] +#![allow(clippy::unnecessary_unwrap)] #![feature(try_blocks)] extern crate databend_common_meta_types; diff --git a/src/meta/api/src/name_id_value_api.rs b/src/meta/api/src/name_id_value_api.rs index 536afd9f51e8d..8cd0e0b1013aa 100644 --- a/src/meta/api/src/name_id_value_api.rs +++ b/src/meta/api/src/name_id_value_api.rs @@ -14,12 +14,12 @@ use std::future::Future; +use databend_common_meta_app::KeyWithTenant; use databend_common_meta_app::data_id::DataId; use databend_common_meta_app::id_generator::IdGenerator; use databend_common_meta_app::primitive::Id; use databend_common_meta_app::tenant_key::ident::TIdent; use databend_common_meta_app::tenant_key::resource::TenantResource; -use databend_common_meta_app::KeyWithTenant; use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::DirName; use databend_common_meta_kvapi::kvapi::KVApi; @@ -401,19 +401,19 @@ mod tests { use std::collections::BTreeMap; use async_trait::async_trait; - use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_app::schema::DatabaseId; use databend_common_meta_app::schema::DatabaseMeta; + use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_kvapi::kvapi::KVApi; use databend_common_meta_kvapi::kvapi::KVStream; use databend_common_meta_kvapi::kvapi::UpsertKVReply; - use databend_common_meta_types::protobuf::StreamItem; use databend_common_meta_types::MetaError; use databend_common_meta_types::SeqV; use databend_common_meta_types::TxnReply; use databend_common_meta_types::TxnRequest; use databend_common_meta_types::UpsertKV; + use databend_common_meta_types::protobuf::StreamItem; use databend_common_proto_conv::FromToProto; use futures::StreamExt; use prost::Message; diff --git a/src/meta/api/src/reply.rs b/src/meta/api/src/reply.rs index 8e81dff965e38..d9796bb6da1af 100644 --- a/src/meta/api/src/reply.rs +++ b/src/meta/api/src/reply.rs @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_meta_types::protobuf::RaftReply; use databend_common_meta_types::InvalidReply; use databend_common_meta_types::MetaAPIError; use databend_common_meta_types::TxnOpResponse; use databend_common_meta_types::TxnReply; +use databend_common_meta_types::protobuf::RaftReply; use serde::de::DeserializeOwned; pub fn reply_to_api_result(msg: RaftReply) -> Result @@ -45,9 +45,9 @@ mod tests { i: i32, } - use databend_common_meta_types::protobuf::RaftReply; use databend_common_meta_types::MetaAPIError; use databend_common_meta_types::MetaNetworkError; + use databend_common_meta_types::protobuf::RaftReply; use crate::reply::reply_to_api_result; @@ -62,9 +62,11 @@ mod tests { let res: Result = reply_to_api_result(msg); match res { Err(MetaAPIError::NetworkError(MetaNetworkError::InvalidReply(inv_reply))) => { - assert!(inv_reply - .to_string() - .starts_with("InvalidReply: can not decode RaftReply.data")); + assert!( + inv_reply + .to_string() + .starts_with("InvalidReply: can not decode RaftReply.data") + ); } _ => { unreachable!("expect InvalidReply") @@ -80,9 +82,11 @@ mod tests { let res: Result = reply_to_api_result(msg); match res { Err(MetaAPIError::NetworkError(MetaNetworkError::InvalidReply(inv_reply))) => { - assert!(inv_reply - .to_string() - .starts_with("InvalidReply: can not decode RaftReply.error")); + assert!( + inv_reply + .to_string() + .starts_with("InvalidReply: can not decode RaftReply.error") + ); } _ => { unreachable!("expect InvalidReply") diff --git a/src/meta/api/src/row_access_policy_api.rs b/src/meta/api/src/row_access_policy_api.rs index d534f53510b86..2ac9628053400 100644 --- a/src/meta/api/src/row_access_policy_api.rs +++ b/src/meta/api/src/row_access_policy_api.rs @@ -12,12 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_meta_app::row_access_policy::row_access_policy_name_ident; use databend_common_meta_app::row_access_policy::CreateRowAccessPolicyReply; use databend_common_meta_app::row_access_policy::CreateRowAccessPolicyReq; use databend_common_meta_app::row_access_policy::RowAccessPolicyId; use databend_common_meta_app::row_access_policy::RowAccessPolicyMeta; use databend_common_meta_app::row_access_policy::RowAccessPolicyNameIdent; +use databend_common_meta_app::row_access_policy::row_access_policy_name_ident; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_app::tenant_key::errors::ExistError; use databend_common_meta_types::MetaError; diff --git a/src/meta/api/src/row_access_policy_api_impl.rs b/src/meta/api/src/row_access_policy_api_impl.rs index 3453f63134930..a257acd43c1f3 100644 --- a/src/meta/api/src/row_access_policy_api_impl.rs +++ b/src/meta/api/src/row_access_policy_api_impl.rs @@ -12,10 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_meta_app::KeyWithTenant; use databend_common_meta_app::data_mask::DataMaskNameIdent; use databend_common_meta_app::id_generator::IdGenerator; -use databend_common_meta_app::row_access_policy::row_access_policy_name_ident; -use databend_common_meta_app::row_access_policy::row_access_policy_table_id_ident::RowAccessPolicyIdTableId; use databend_common_meta_app::row_access_policy::CreateRowAccessPolicyReply; use databend_common_meta_app::row_access_policy::CreateRowAccessPolicyReq; use databend_common_meta_app::row_access_policy::RowAccessPolicyId; @@ -25,9 +24,10 @@ use databend_common_meta_app::row_access_policy::RowAccessPolicyMeta; use databend_common_meta_app::row_access_policy::RowAccessPolicyNameIdent; use databend_common_meta_app::row_access_policy::RowAccessPolicyNameIdentRaw; use databend_common_meta_app::row_access_policy::RowAccessPolicyTableIdIdent; +use databend_common_meta_app::row_access_policy::row_access_policy_name_ident; +use databend_common_meta_app::row_access_policy::row_access_policy_table_id_ident::RowAccessPolicyIdTableId; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_app::tenant_key::errors::ExistError; -use databend_common_meta_app::KeyWithTenant; use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::DirName; use databend_common_meta_types::MetaError; diff --git a/src/meta/api/src/schema_api.rs b/src/meta/api/src/schema_api.rs index aed26b1ea3a73..7de4e8572c73b 100644 --- a/src/meta/api/src/schema_api.rs +++ b/src/meta/api/src/schema_api.rs @@ -16,6 +16,7 @@ use std::collections::HashSet; use std::fmt::Display; use std::time::Duration; +use ConditionResult::Eq; use chrono::DateTime; use chrono::Utc; use databend_common_meta_app::app_error::AppError; @@ -32,16 +33,11 @@ use databend_common_meta_app::data_mask::MaskPolicyTableId; use databend_common_meta_app::data_mask::MaskPolicyTableIdIdent; use databend_common_meta_app::principal::OwnershipObject; use databend_common_meta_app::principal::TenantOwnershipObjectIdent; -use databend_common_meta_app::row_access_policy::row_access_policy_table_id_ident::RowAccessPolicyIdTableId; use databend_common_meta_app::row_access_policy::RowAccessPolicyId; use databend_common_meta_app::row_access_policy::RowAccessPolicyIdIdent; use databend_common_meta_app::row_access_policy::RowAccessPolicyTableId; use databend_common_meta_app::row_access_policy::RowAccessPolicyTableIdIdent; -use databend_common_meta_app::schema::marked_deleted_index_id::MarkedDeletedIndexId; -use databend_common_meta_app::schema::marked_deleted_index_ident::MarkedDeletedIndexIdIdent; -use databend_common_meta_app::schema::marked_deleted_table_index_id::MarkedDeletedTableIndexId; -use databend_common_meta_app::schema::marked_deleted_table_index_ident::MarkedDeletedTableIndexIdIdent; -use databend_common_meta_app::schema::vacuum_watermark_ident::VacuumWatermarkIdent; +use databend_common_meta_app::row_access_policy::row_access_policy_table_id_ident::RowAccessPolicyIdTableId; use databend_common_meta_app::schema::DBIdTableName; use databend_common_meta_app::schema::DatabaseId; use databend_common_meta_app::schema::DatabaseMeta; @@ -56,6 +52,11 @@ use databend_common_meta_app::schema::TableMeta; use databend_common_meta_app::schema::TableNameIdent; use databend_common_meta_app::schema::UndropTableByIdReq; use databend_common_meta_app::schema::UndropTableReq; +use databend_common_meta_app::schema::marked_deleted_index_id::MarkedDeletedIndexId; +use databend_common_meta_app::schema::marked_deleted_index_ident::MarkedDeletedIndexIdIdent; +use databend_common_meta_app::schema::marked_deleted_table_index_id::MarkedDeletedTableIndexId; +use databend_common_meta_app::schema::marked_deleted_table_index_ident::MarkedDeletedTableIndexIdIdent; +use databend_common_meta_app::schema::vacuum_watermark_ident::VacuumWatermarkIdent; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::Key; @@ -69,7 +70,6 @@ use fastrace::func_name; use log::debug; use log::error; use log::warn; -use ConditionResult::Eq; use crate::catalog_api::CatalogApi; use crate::data_retention_util::is_drop_time_retainable; diff --git a/src/meta/api/src/schema_api_test_suite.rs b/src/meta/api/src/schema_api_test_suite.rs index e084ff468c2e9..3c18786fc1140 100644 --- a/src/meta/api/src/schema_api_test_suite.rs +++ b/src/meta/api/src/schema_api_test_suite.rs @@ -26,31 +26,22 @@ use chrono::Utc; use databend_common_base::runtime::Runtime; use databend_common_base::runtime::TrySpawn; use databend_common_exception::ErrorCode; -use databend_common_expression::types::NumberDataType; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; +use databend_common_expression::types::NumberDataType; +use databend_common_meta_app::KeyWithTenant; use databend_common_meta_app::app_error::AppError; use databend_common_meta_app::data_mask::CreateDatamaskReq; use databend_common_meta_app::data_mask::DataMaskNameIdent; use databend_common_meta_app::data_mask::DatamaskMeta; use databend_common_meta_app::data_mask::MaskPolicyIdTableId; use databend_common_meta_app::data_mask::MaskPolicyTableIdIdent; -use databend_common_meta_app::row_access_policy::row_access_policy_table_id_ident::RowAccessPolicyIdTableId; use databend_common_meta_app::row_access_policy::CreateRowAccessPolicyReq; use databend_common_meta_app::row_access_policy::RowAccessPolicyMeta; use databend_common_meta_app::row_access_policy::RowAccessPolicyNameIdent; use databend_common_meta_app::row_access_policy::RowAccessPolicyTableIdIdent; -use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; -use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdentRaw; -use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; -use databend_common_meta_app::schema::index_id_ident::IndexId; -use databend_common_meta_app::schema::index_id_ident::IndexIdIdent; -use databend_common_meta_app::schema::index_id_to_name_ident::IndexIdToNameIdent; -use databend_common_meta_app::schema::least_visible_time_ident::LeastVisibleTimeIdent; -use databend_common_meta_app::schema::sequence_storage::SequenceStorageIdent; -use databend_common_meta_app::schema::table_niv::TableNIV; -use databend_common_meta_app::schema::vacuum_watermark_ident::VacuumWatermarkIdent; +use databend_common_meta_app::row_access_policy::row_access_policy_table_id_ident::RowAccessPolicyIdTableId; use databend_common_meta_app::schema::CatalogMeta; use databend_common_meta_app::schema::CatalogNameIdent; use databend_common_meta_app::schema::CatalogOption; @@ -131,9 +122,18 @@ use databend_common_meta_app::schema::UpdateMultiTableMetaReq; use databend_common_meta_app::schema::UpdateTableMetaReq; use databend_common_meta_app::schema::UpsertTableCopiedFileReq; use databend_common_meta_app::schema::UpsertTableOptionReq; +use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; +use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdentRaw; +use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; +use databend_common_meta_app::schema::index_id_ident::IndexId; +use databend_common_meta_app::schema::index_id_ident::IndexIdIdent; +use databend_common_meta_app::schema::index_id_to_name_ident::IndexIdToNameIdent; +use databend_common_meta_app::schema::least_visible_time_ident::LeastVisibleTimeIdent; +use databend_common_meta_app::schema::sequence_storage::SequenceStorageIdent; +use databend_common_meta_app::schema::table_niv::TableNIV; +use databend_common_meta_app::schema::vacuum_watermark_ident::VacuumWatermarkIdent; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_app::tenant::ToTenant; -use databend_common_meta_app::KeyWithTenant; use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::Key; use databend_common_meta_kvapi::kvapi::KvApiExt; @@ -144,6 +144,12 @@ use fastrace::func_name; use log::debug; use log::info; +use crate::DEFAULT_MGET_SIZE; +use crate::DatamaskApi; +use crate::RowAccessPolicyApi; +use crate::SchemaApi; +use crate::SequenceApi; +use crate::TableApi; use crate::deserialize_struct; use crate::kv_app_error::KVAppError; use crate::kv_pb_api::KVPbApi; @@ -152,12 +158,6 @@ use crate::serialize_struct; use crate::testing::get_kv_data; use crate::testing::get_kv_u64_data; use crate::util::IdempotentKVTxnSender; -use crate::DatamaskApi; -use crate::RowAccessPolicyApi; -use crate::SchemaApi; -use crate::SequenceApi; -use crate::TableApi; -use crate::DEFAULT_MGET_SIZE; /// Test suite of `SchemaApi`. /// @@ -3697,10 +3697,12 @@ impl SchemaApiTestSuite { }; mt.set_table_column_mask_policy(set_req).await?; table_info = util.get_table().await?; - assert!(table_info - .meta - .column_mask_policy_columns_ids - .contains_key(&number_column_id)); + assert!( + table_info + .meta + .column_mask_policy_columns_ids + .contains_key(&number_column_id) + ); // Drop the table (this deletes the table-policy reference), then drop the policy. util.drop_table_by_id().await?; diff --git a/src/meta/api/src/security_api.rs b/src/meta/api/src/security_api.rs index 3a40a482da9c3..d08eed32feaab 100644 --- a/src/meta/api/src/security_api.rs +++ b/src/meta/api/src/security_api.rs @@ -18,9 +18,9 @@ use databend_common_meta_app::app_error::UnknownTableId; use databend_common_meta_app::data_mask::MaskPolicyIdTableId; use databend_common_meta_app::data_mask::MaskPolicyTableId; use databend_common_meta_app::data_mask::MaskPolicyTableIdIdent; -use databend_common_meta_app::row_access_policy::row_access_policy_table_id_ident::RowAccessPolicyIdTableId; use databend_common_meta_app::row_access_policy::RowAccessPolicyTableId; use databend_common_meta_app::row_access_policy::RowAccessPolicyTableIdIdent; +use databend_common_meta_app::row_access_policy::row_access_policy_table_id_ident::RowAccessPolicyIdTableId; use databend_common_meta_app::schema::SecurityPolicyColumnMap; use databend_common_meta_app::schema::SetSecurityPolicyAction; use databend_common_meta_app::schema::SetTableColumnMaskPolicyReply; diff --git a/src/meta/api/src/sequence_api_impl.rs b/src/meta/api/src/sequence_api_impl.rs index 5415358d17d24..2336fe72ce55c 100644 --- a/src/meta/api/src/sequence_api_impl.rs +++ b/src/meta/api/src/sequence_api_impl.rs @@ -19,8 +19,6 @@ use databend_common_meta_app::app_error::SequenceError; use databend_common_meta_app::app_error::UnsupportedSequenceStorageVersion; use databend_common_meta_app::app_error::WrongSequenceCount; use databend_common_meta_app::primitive::Id; -use databend_common_meta_app::schema::sequence_storage::SequenceStorageIdent; -use databend_common_meta_app::schema::sequence_storage::SequenceStorageValue; use databend_common_meta_app::schema::CreateOption; use databend_common_meta_app::schema::CreateSequenceReply; use databend_common_meta_app::schema::CreateSequenceReq; @@ -30,6 +28,8 @@ use databend_common_meta_app::schema::GetSequenceNextValueReply; use databend_common_meta_app::schema::GetSequenceNextValueReq; use databend_common_meta_app::schema::SequenceIdent; use databend_common_meta_app::schema::SequenceMeta; +use databend_common_meta_app::schema::sequence_storage::SequenceStorageIdent; +use databend_common_meta_app::schema::sequence_storage::SequenceStorageValue; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::DirName; @@ -41,6 +41,7 @@ use fastrace::func_name; use futures::TryStreamExt; use log::debug; +use crate::SequenceApi; use crate::kv_app_error::KVAppError; use crate::kv_pb_api::KVPbApi; use crate::sequence_nextval_impl::NextVal; @@ -50,7 +51,6 @@ use crate::txn_condition_util::txn_cond_seq; use crate::txn_core_util::send_txn; use crate::txn_op_builder_util::txn_put_pb; use crate::txn_op_del; -use crate::SequenceApi; #[async_trait::async_trait] #[tonic::async_trait] diff --git a/src/meta/api/src/sequence_nextval_impl.rs b/src/meta/api/src/sequence_nextval_impl.rs index 6eaeadfbd0e6e..3c00ffff6f983 100644 --- a/src/meta/api/src/sequence_nextval_impl.rs +++ b/src/meta/api/src/sequence_nextval_impl.rs @@ -15,9 +15,9 @@ use databend_common_meta_app::app_error::AppError; use databend_common_meta_app::app_error::OutOfSequenceRange; use databend_common_meta_app::app_error::SequenceError; -use databend_common_meta_app::schema::sequence_storage::SequenceStorageIdent; use databend_common_meta_app::schema::SequenceIdent; use databend_common_meta_app::schema::SequenceMeta; +use databend_common_meta_app::schema::sequence_storage::SequenceStorageIdent; use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::Key; use databend_common_meta_types::MetaError; @@ -133,13 +133,13 @@ where KV: kvapi::KVApi + ?Sized if got_delta < delta { return Err(KVAppError::AppError(AppError::SequenceError( - SequenceError::OutOfSequenceRange(OutOfSequenceRange::new( - self.ident.name(), - format!( - "{sequence_meta}: count: {count}; expected delta: {delta}, but got: {resp}", - ), - )), - ))); + SequenceError::OutOfSequenceRange(OutOfSequenceRange::new( + self.ident.name(), + format!( + "{sequence_meta}: count: {count}; expected delta: {delta}, but got: {resp}", + ), + )), + ))); } Ok(Ok((resp.before, resp.after))) diff --git a/src/meta/api/src/table_api.rs b/src/meta/api/src/table_api.rs index 0ab4c34abf88d..1d0e356ae79ef 100644 --- a/src/meta/api/src/table_api.rs +++ b/src/meta/api/src/table_api.rs @@ -20,6 +20,7 @@ use std::sync::Arc; use chrono::DateTime; use chrono::Utc; +use databend_common_meta_app::KeyWithTenant; use databend_common_meta_app::app_error::AppError; use databend_common_meta_app::app_error::CommitTableMetaError; use databend_common_meta_app::app_error::CreateAsDropTableWithoutDropTime; @@ -42,9 +43,6 @@ use databend_common_meta_app::app_error::ViewAlreadyExists; use databend_common_meta_app::id_generator::IdGenerator; use databend_common_meta_app::primitive::Id; use databend_common_meta_app::principal::AutoIncrementKey; -use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; -use databend_common_meta_app::schema::least_visible_time_ident::LeastVisibleTimeIdent; -use databend_common_meta_app::schema::table_niv::TableNIV; use databend_common_meta_app::schema::AutoIncrementStorageIdent; use databend_common_meta_app::schema::AutoIncrementStorageValue; use databend_common_meta_app::schema::CommitTableMetaReply; @@ -89,14 +87,13 @@ use databend_common_meta_app::schema::UpdateMultiTableMetaResult; use databend_common_meta_app::schema::UpdateTableMetaReply; use databend_common_meta_app::schema::UpsertTableOptionReply; use databend_common_meta_app::schema::UpsertTableOptionReq; -use databend_common_meta_app::KeyWithTenant; +use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; +use databend_common_meta_app::schema::least_visible_time_ident::LeastVisibleTimeIdent; +use databend_common_meta_app::schema::table_niv::TableNIV; use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::DirName; use databend_common_meta_kvapi::kvapi::Key; use databend_common_meta_kvapi::kvapi::KvApiExt; -use databend_common_meta_types::protobuf as pb; -use databend_common_meta_types::txn_op::Request; -use databend_common_meta_types::txn_op_response::Response; use databend_common_meta_types::ConditionResult::Eq; use databend_common_meta_types::MatchSeqExt; use databend_common_meta_types::MetaError; @@ -106,20 +103,24 @@ use databend_common_meta_types::TxnGetRequest; use databend_common_meta_types::TxnGetResponse; use databend_common_meta_types::TxnOp; use databend_common_meta_types::TxnRequest; +use databend_common_meta_types::protobuf as pb; +use databend_common_meta_types::txn_op::Request; +use databend_common_meta_types::txn_op_response::Response; use fastrace::func_name; use log::debug; use log::error; use log::info; use seq_marked::SeqValue; +use crate::DEFAULT_MGET_SIZE; use crate::assert_table_exist; use crate::database_api::DatabaseApi; use crate::database_util::get_db_or_err; use crate::deserialize_struct; use crate::error_util::table_has_to_not_exist; use crate::fetch_id; -use crate::garbage_collection_api::get_history_tables_for_gc; use crate::garbage_collection_api::ORPHAN_POSTFIX; +use crate::garbage_collection_api::get_history_tables_for_gc; use crate::get_u64_value; use crate::kv_app_error::KVAppError; use crate::kv_fetch_util::deserialize_id_get_response; @@ -146,7 +147,6 @@ use crate::txn_op_put; use crate::txn_put_pb; use crate::util::IdempotentKVTxnResponse; use crate::util::IdempotentKVTxnSender; -use crate::DEFAULT_MGET_SIZE; /// TableApi defines APIs for table lifecycle and metadata management. /// @@ -1143,7 +1143,10 @@ where let succ = match txn_response { IdempotentKVTxnResponse::Success(_) => true, IdempotentKVTxnResponse::AlreadyCommitted => { - info!( "Transaction ID {} exists, the corresponding commit_table_meta transaction has been executed successfully", txn_sender.get_txn_id() ); + info!( + "Transaction ID {} exists, the corresponding commit_table_meta transaction has been executed successfully", + txn_sender.get_txn_id() + ); true } IdempotentKVTxnResponse::Failed(_) => false, diff --git a/src/meta/api/src/testing.rs b/src/meta/api/src/testing.rs index 63a205e34ae0f..78b6b549e3246 100644 --- a/src/meta/api/src/testing.rs +++ b/src/meta/api/src/testing.rs @@ -16,11 +16,11 @@ use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::KvApiExt; -use databend_common_meta_types::anyerror::AnyError; use databend_common_meta_types::MetaAPIError; use databend_common_meta_types::MetaDataError; use databend_common_meta_types::MetaDataReadError; use databend_common_meta_types::MetaError; +use databend_common_meta_types::anyerror::AnyError; use databend_common_proto_conv::FromToProto; use crate::kv_app_error::KVAppError; diff --git a/src/meta/api/src/txn_condition_util.rs b/src/meta/api/src/txn_condition_util.rs index d324b21518119..af15926581302 100644 --- a/src/meta/api/src/txn_condition_util.rs +++ b/src/meta/api/src/txn_condition_util.rs @@ -16,9 +16,9 @@ use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::DirName; -use databend_common_meta_types::txn_condition::Target; use databend_common_meta_types::ConditionResult; use databend_common_meta_types::TxnCondition; +use databend_common_meta_types::txn_condition::Target; /// Build a TxnCondition that compares the seq of a record. pub fn txn_cond_eq_seq(key: &impl kvapi::Key, seq: u64) -> TxnCondition { diff --git a/src/meta/api/src/util.rs b/src/meta/api/src/util.rs index 6f2bc509c0289..d43ccfce91e5c 100644 --- a/src/meta/api/src/util.rs +++ b/src/meta/api/src/util.rs @@ -16,14 +16,14 @@ use std::time::Duration; use databend_common_base::base::uuid::Uuid; use databend_common_meta_kvapi::kvapi; -use databend_common_meta_types::anyerror::AnyError; -use databend_common_meta_types::txn_op_response::Response; use databend_common_meta_types::InvalidReply; use databend_common_meta_types::MetaError; use databend_common_meta_types::TxnCondition; use databend_common_meta_types::TxnOp; use databend_common_meta_types::TxnOpResponse; use databend_common_meta_types::TxnRequest; +use databend_common_meta_types::anyerror::AnyError; +use databend_common_meta_types::txn_op_response::Response; use log::info; use crate::txn_core_util::send_txn; diff --git a/src/meta/app-storage/src/lib.rs b/src/meta/app-storage/src/lib.rs index 29db4a9fd1dd8..2d7af9e9374e5 100644 --- a/src/meta/app-storage/src/lib.rs +++ b/src/meta/app-storage/src/lib.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![feature(no_sanitize)] mod storage_params; pub use storage_params::*; diff --git a/src/meta/app-storage/src/storage_params.rs b/src/meta/app-storage/src/storage_params.rs index 9fe4b03682bcb..3a8cdea0ed5ce 100644 --- a/src/meta/app-storage/src/storage_params.rs +++ b/src/meta/app-storage/src/storage_params.rs @@ -638,7 +638,6 @@ pub struct StorageMokaConfig { } impl Default for StorageMokaConfig { - #[no_sanitize(address)] fn default() -> Self { Self { // Use 1G as default. diff --git a/src/meta/app-types/src/non_empty.rs b/src/meta/app-types/src/non_empty.rs index d1da9809ecf10..adca0931bae9c 100644 --- a/src/meta/app-types/src/non_empty.rs +++ b/src/meta/app-types/src/non_empty.rs @@ -59,7 +59,7 @@ impl NonEmptyString { &self.non_empty } - pub fn as_non_empty_str(&self) -> NonEmptyStr { + pub fn as_non_empty_str(&self) -> NonEmptyStr<'_> { NonEmptyStr::new(&self.non_empty).unwrap() } } diff --git a/src/meta/app/src/app_error.rs b/src/meta/app/src/app_error.rs index 704f61c0da96c..dcebdd2abfafe 100644 --- a/src/meta/app/src/app_error.rs +++ b/src/meta/app/src/app_error.rs @@ -20,14 +20,14 @@ use databend_common_exception::ErrorCode; use databend_common_meta_types::MatchSeq; use crate::data_mask::data_mask_name_ident; -use crate::principal::procedure_name_ident; use crate::principal::ProcedureIdentity; +use crate::principal::procedure_name_ident; use crate::row_access_policy::row_access_policy_name_ident; +use crate::schema::DictionaryIdentity; +use crate::schema::SequenceRsc; use crate::schema::catalog_name_ident; use crate::schema::dictionary_name_ident; use crate::schema::index_name_ident; -use crate::schema::DictionaryIdentity; -use crate::schema::SequenceRsc; use crate::tenant_key::errors::ExistError; use crate::tenant_key::errors::UnknownError; use crate::tenant_key::ident::TIdent; @@ -264,7 +264,9 @@ impl UndropTableHasNoHistory { } #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] -#[error("Cannot undrop table '{table_name}': table was dropped at {drop_time} before vacuum started at {retention}. Data may have been cleaned up.")] +#[error( + "Cannot undrop table '{table_name}': table was dropped at {drop_time} before vacuum started at {retention}. Data may have been cleaned up." +)] pub struct UndropTableRetentionGuard { table_name: String, drop_time: DateTime, @@ -583,7 +585,9 @@ impl UnknownTableId { } #[derive(thiserror::Error, Debug, Clone, PartialEq, Eq)] -#[error("VirtualColumnIdOutBound: the virtual column id `{column_id}` is outside the range `{lower}` to `{upper}`")] +#[error( + "VirtualColumnIdOutBound: the virtual column id `{column_id}` is outside the range `{lower}` to `{upper}`" +)] pub struct VirtualColumnIdOutBound { column_id: u32, lower: u32, @@ -1316,8 +1320,8 @@ impl AppErrorMessage for DuplicatedUpsertFiles {} impl AppErrorMessage for CommitTableMetaError { fn message(&self) -> String { format!( - "Create table '{}' failed, possibly because a table with the same name already exists", - self.table_name + "Create table '{}' failed, possibly because a table with the same name already exists. Context: {}", + self.table_name, self.context ) } } diff --git a/src/meta/app/src/data_id.rs b/src/meta/app/src/data_id.rs index 46c8c76075d51..bdb39b25f497b 100644 --- a/src/meta/app/src/data_id.rs +++ b/src/meta/app/src/data_id.rs @@ -143,11 +143,11 @@ impl KeyCodec for DataId { /// so that the upper level code can be simplified to using just one trait `prost::Message` to access meta-service data. mod prost_message_impl { + use prost::DecodeError; use prost::bytes::Buf; use prost::bytes::BufMut; use prost::encoding::DecodeContext; use prost::encoding::WireType; - use prost::DecodeError; use crate::data_id::DataId; use crate::tenant_key::resource::TenantResource; diff --git a/src/meta/app/src/data_mask/data_mask_name_ident.rs b/src/meta/app/src/data_mask/data_mask_name_ident.rs index 619a274b2edc3..d2baa0e170912 100644 --- a/src/meta/app/src/data_mask/data_mask_name_ident.rs +++ b/src/meta/app/src/data_mask/data_mask_name_ident.rs @@ -37,10 +37,10 @@ mod kvapi_impl { use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::Key; + use crate::KeyWithTenant; use crate::data_mask::DataMaskId; use crate::data_mask::DataMaskNameIdent; use crate::tenant_key::resource::TenantResource; - use crate::KeyWithTenant; pub struct Resource; impl TenantResource for Resource { diff --git a/src/meta/app/src/data_mask/mask_policy_policy_table_id_ident.rs b/src/meta/app/src/data_mask/mask_policy_policy_table_id_ident.rs index b926ecf3ac777..f39674144ecac 100644 --- a/src/meta/app/src/data_mask/mask_policy_policy_table_id_ident.rs +++ b/src/meta/app/src/data_mask/mask_policy_policy_table_id_ident.rs @@ -54,8 +54,8 @@ mod kvapi_impl { use databend_common_meta_kvapi::kvapi; - use crate::data_mask::mask_policy_policy_table_id_ident::MaskPolicyTableIdIdent; use crate::data_mask::MaskPolicyTableId; + use crate::data_mask::mask_policy_policy_table_id_ident::MaskPolicyTableIdIdent; use crate::tenant_key::resource::TenantResource; pub struct Resource; diff --git a/src/meta/app/src/lib.rs b/src/meta/app/src/lib.rs index 87474d345656c..c8774d7f03e9f 100644 --- a/src/meta/app/src/lib.rs +++ b/src/meta/app/src/lib.rs @@ -20,7 +20,8 @@ #![allow(clippy::uninlined_format_args)] #![allow(non_local_definitions)] -#![feature(no_sanitize)] +#![allow(clippy::collapsible_if)] +#![allow(clippy::derivable_impls)] pub mod app_error; pub mod data_id; diff --git a/src/meta/app/src/primitive.rs b/src/meta/app/src/primitive.rs index eb7d14a346e15..9454a1469cd73 100644 --- a/src/meta/app/src/primitive.rs +++ b/src/meta/app/src/primitive.rs @@ -114,11 +114,11 @@ mod prost_message_impl { use std::ops::Deref; use std::ops::DerefMut; + use prost::DecodeError; use prost::bytes::Buf; use prost::bytes::BufMut; use prost::encoding::DecodeContext; use prost::encoding::WireType; - use prost::DecodeError; use crate::primitive::Id; diff --git a/src/meta/app/src/principal/connection_ident.rs b/src/meta/app/src/principal/connection_ident.rs index 4fed6bb5bf42e..c0b3ec0312848 100644 --- a/src/meta/app/src/principal/connection_ident.rs +++ b/src/meta/app/src/principal/connection_ident.rs @@ -24,8 +24,8 @@ mod kvapi_impl { use databend_common_exception::ErrorCode; use databend_common_meta_kvapi::kvapi; - use crate::principal::connection_ident::ConnectionIdent; use crate::principal::UserDefinedConnection; + use crate::principal::connection_ident::ConnectionIdent; use crate::tenant_key::errors::ExistError; use crate::tenant_key::errors::UnknownError; use crate::tenant_key::resource::TenantResource; diff --git a/src/meta/app/src/principal/file_format.rs b/src/meta/app/src/principal/file_format.rs index 19e829c44e2ee..91111b39a95c7 100644 --- a/src/meta/app/src/principal/file_format.rs +++ b/src/meta/app/src/principal/file_format.rs @@ -24,9 +24,9 @@ use databend_common_ast::ast::FileFormatOptions; use databend_common_ast::ast::FileFormatValue; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_io::GeometryDataType; use databend_common_io::constants::NULL_BYTES_ESCAPE; use databend_common_io::escape_string; -use databend_common_io::GeometryDataType; use paste::paste; use serde::Deserialize; use serde::Serialize; diff --git a/src/meta/app/src/principal/mod.rs b/src/meta/app/src/principal/mod.rs index 8572fd34e9524..55cb77c84ec9b 100644 --- a/src/meta/app/src/principal/mod.rs +++ b/src/meta/app/src/principal/mod.rs @@ -123,14 +123,14 @@ pub use user_defined_function::UDAFScript; pub use user_defined_function::UDFDefinition; pub use user_defined_function::UDFScript; pub use user_defined_function::UDFServer; +pub use user_defined_function::UDTF; pub use user_defined_function::UDTFServer; pub use user_defined_function::UserDefinedFunction; -pub use user_defined_function::UDTF; pub use user_grant::GrantEntry; pub use user_grant::GrantObject; -pub use user_grant::UserGrantSet; pub use user_grant::SENSITIVE_SYSTEM_RESOURCE; pub use user_grant::SYSTEM_TABLES_ALLOW_LIST; +pub use user_grant::UserGrantSet; pub use user_identity::UserIdentity; pub use user_info::UserInfo; pub use user_info::UserOption; diff --git a/src/meta/app/src/principal/ownership_object.rs b/src/meta/app/src/principal/ownership_object.rs index 0ac3c66780d75..95999eaff1819 100644 --- a/src/meta/app/src/principal/ownership_object.rs +++ b/src/meta/app/src/principal/ownership_object.rs @@ -94,13 +94,13 @@ impl fmt::Display for OwnershipObject { fn fmt(&self, f: &mut fmt::Formatter) -> std::result::Result<(), fmt::Error> { match self { OwnershipObject::Database { - ref catalog_name, - ref db_id, + catalog_name, + db_id, } => write!(f, "'{}'.'{}'.*", catalog_name, db_id), OwnershipObject::Table { - ref catalog_name, - ref db_id, - ref table_id, + catalog_name, + db_id, + table_id, } => { write!(f, "'{}'.'{}'.'{}'", catalog_name, db_id, table_id) } diff --git a/src/meta/app/src/principal/procedure.rs b/src/meta/app/src/principal/procedure.rs index 8bfdb4ea614e5..79049a246fba8 100644 --- a/src/meta/app/src/principal/procedure.rs +++ b/src/meta/app/src/principal/procedure.rs @@ -22,12 +22,12 @@ use chrono::Utc; use databend_common_expression::types::DataType; use display_more::DisplaySliceExt; +use crate::KeyWithTenant; +use crate::principal::ProcedureIdentity; use crate::principal::procedure_id_ident::ProcedureIdIdent; use crate::principal::procedure_name_ident::ProcedureNameIdent; -use crate::principal::ProcedureIdentity; use crate::tenant::Tenant; use crate::tenant::ToTenant; -use crate::KeyWithTenant; #[derive(Clone, Debug, PartialEq)] pub struct ProcedureInfo { diff --git a/src/meta/app/src/principal/procedure_id_ident.rs b/src/meta/app/src/principal/procedure_id_ident.rs index 83ed4f563999d..7be1498591c9e 100644 --- a/src/meta/app/src/principal/procedure_id_ident.rs +++ b/src/meta/app/src/principal/procedure_id_ident.rs @@ -44,8 +44,8 @@ mod kvapi_impl { use databend_common_meta_kvapi::kvapi; - use crate::principal::procedure_id_ident::ProcedureIdIdent; use crate::principal::ProcedureMeta; + use crate::principal::procedure_id_ident::ProcedureIdIdent; use crate::tenant_key::resource::TenantResource; pub struct Resource; diff --git a/src/meta/app/src/principal/procedure_id_to_name.rs b/src/meta/app/src/principal/procedure_id_to_name.rs index 52a1257314196..89d5d9012544f 100644 --- a/src/meta/app/src/principal/procedure_id_to_name.rs +++ b/src/meta/app/src/principal/procedure_id_to_name.rs @@ -38,8 +38,8 @@ mod kvapi_impl { use databend_common_meta_kvapi::kvapi; - use crate::principal::procedure_id_to_name::ProcedureIdToNameIdent; use crate::principal::ProcedureIdentity; + use crate::principal::procedure_id_to_name::ProcedureIdToNameIdent; use crate::tenant_key::resource::TenantResource; // TODO(TIdent): parent should return Some(ProcedureIdIdent::new(self.procedure_id).to_string_key()) diff --git a/src/meta/app/src/principal/procedure_name_ident.rs b/src/meta/app/src/principal/procedure_name_ident.rs index d5b900b1b1c28..1275478698bff 100644 --- a/src/meta/app/src/principal/procedure_name_ident.rs +++ b/src/meta/app/src/principal/procedure_name_ident.rs @@ -38,10 +38,10 @@ mod kvapi_impl { use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::Key; - use crate::principal::procedure_id_ident::ProcedureId; + use crate::KeyWithTenant; use crate::principal::ProcedureNameIdent; + use crate::principal::procedure_id_ident::ProcedureId; use crate::tenant_key::resource::TenantResource; - use crate::KeyWithTenant; pub struct ProcedureName; impl TenantResource for ProcedureName { diff --git a/src/meta/app/src/principal/stage_file_ident.rs b/src/meta/app/src/principal/stage_file_ident.rs index 6af7091fe7687..21ec950c3835b 100644 --- a/src/meta/app/src/principal/stage_file_ident.rs +++ b/src/meta/app/src/principal/stage_file_ident.rs @@ -70,8 +70,8 @@ mod kvapi_impl { mod tests { use databend_common_meta_kvapi::kvapi::Key; - use crate::principal::stage_file_ident::StageFileIdent; use crate::principal::StageIdent; + use crate::principal::stage_file_ident::StageFileIdent; use crate::tenant::Tenant; #[test] diff --git a/src/meta/app/src/principal/tenant_ownership_object_ident.rs b/src/meta/app/src/principal/tenant_ownership_object_ident.rs index c085514351d23..2af9ae5cf8bde 100644 --- a/src/meta/app/src/principal/tenant_ownership_object_ident.rs +++ b/src/meta/app/src/principal/tenant_ownership_object_ident.rs @@ -108,10 +108,10 @@ mod tests { use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::Key; + use crate::KeyWithTenant; use crate::principal::OwnershipObject; use crate::principal::TenantOwnershipObjectIdent; use crate::tenant::Tenant; - use crate::KeyWithTenant; #[test] fn test_tenant_ownership_object_tenant_prefix() { diff --git a/src/meta/app/src/principal/user_defined_file_format_ident.rs b/src/meta/app/src/principal/user_defined_file_format_ident.rs index 77af563d1a180..e5094dae7707e 100644 --- a/src/meta/app/src/principal/user_defined_file_format_ident.rs +++ b/src/meta/app/src/principal/user_defined_file_format_ident.rs @@ -24,8 +24,8 @@ mod kvapi_impl { use databend_common_exception::ErrorCode; use databend_common_meta_kvapi::kvapi; - use crate::principal::user_defined_file_format_ident::UserDefinedFileFormatIdent; use crate::principal::UserDefinedFileFormat; + use crate::principal::user_defined_file_format_ident::UserDefinedFileFormatIdent; use crate::tenant_key::errors::ExistError; use crate::tenant_key::errors::UnknownError; use crate::tenant_key::resource::TenantResource; diff --git a/src/meta/app/src/principal/user_defined_function.rs b/src/meta/app/src/principal/user_defined_function.rs index a9ca10dc4136d..fb61fab7f2896 100644 --- a/src/meta/app/src/principal/user_defined_function.rs +++ b/src/meta/app/src/principal/user_defined_function.rs @@ -18,8 +18,8 @@ use std::fmt::Formatter; use chrono::DateTime; use chrono::Utc; -use databend_common_expression::types::DataType; use databend_common_expression::DataField; +use databend_common_expression::types::DataType; #[derive(Clone, Debug, Eq, PartialEq)] pub struct LambdaUDF { @@ -361,7 +361,10 @@ impl Display for UDFDefinition { } write!(f, "{} {}", item.name(), item.data_type())?; } - write!(f, " }} RETURNS {return_type} LANGUAGE {language} IMPORTS = {imports:?} PACKAGES = {packages:?} RUNTIME_VERSION = {runtime_version} AS $${code}$$")?; + write!( + f, + " }} RETURNS {return_type} LANGUAGE {language} IMPORTS = {imports:?} PACKAGES = {packages:?} RUNTIME_VERSION = {runtime_version} AS $${code}$$" + )?; } UDFDefinition::UDTF(UDTF { arg_types, diff --git a/src/meta/app/src/principal/user_grant.rs b/src/meta/app/src/principal/user_grant.rs index 63298b2a70b6b..c216d9a6eb3cd 100644 --- a/src/meta/app/src/principal/user_grant.rs +++ b/src/meta/app/src/principal/user_grant.rs @@ -165,12 +165,12 @@ impl fmt::Display for GrantObject { fn fmt(&self, f: &mut fmt::Formatter) -> std::result::Result<(), fmt::Error> { match self { GrantObject::Global => write!(f, "*.*"), - GrantObject::Database(ref cat, ref db) => write!(f, "'{}'.'{}'.*", cat, db), - GrantObject::DatabaseById(ref cat, ref db) => write!(f, "'{}'.'{}'.*", cat, db), - GrantObject::Table(ref cat, ref db, ref table) => { + GrantObject::Database(cat, db) => write!(f, "'{}'.'{}'.*", cat, db), + GrantObject::DatabaseById(cat, db) => write!(f, "'{}'.'{}'.*", cat, db), + GrantObject::Table(cat, db, table) => { write!(f, "'{}'.'{}'.'{}'", cat, db, table) } - GrantObject::TableById(ref cat, ref db, ref table) => { + GrantObject::TableById(cat, db, table) => { write!(f, "'{}'.'{}'.'{}'", cat, db, table) } GrantObject::UDF(udf) => write!(f, "UDF {udf}"), diff --git a/src/meta/app/src/principal/user_info.rs b/src/meta/app/src/principal/user_info.rs index 2774bb014bf7e..48f0488e9a947 100644 --- a/src/meta/app/src/principal/user_info.rs +++ b/src/meta/app/src/principal/user_info.rs @@ -20,16 +20,16 @@ use chrono::Utc; use databend_common_ast::ast::UserOptionItem; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use enumflags2::bitflags; use enumflags2::BitFlags; +use enumflags2::bitflags; use serde::Deserialize; use serde::Serialize; use crate::principal::AuthInfo; +use crate::principal::BUILTIN_ROLE_ACCOUNT_ADMIN; use crate::principal::UserGrantSet; use crate::principal::UserIdentity; use crate::principal::UserQuota; -use crate::principal::BUILTIN_ROLE_ACCOUNT_ADMIN; #[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq, Default)] #[serde(default)] diff --git a/src/meta/app/src/principal/user_privilege.rs b/src/meta/app/src/principal/user_privilege.rs index 5ef3a2abaf5a3..6a45ee32a1130 100644 --- a/src/meta/app/src/principal/user_privilege.rs +++ b/src/meta/app/src/principal/user_privilege.rs @@ -16,9 +16,9 @@ use std::fmt; use std::fmt::Display; use std::ops; +use enumflags2::BitFlags; use enumflags2::bitflags; use enumflags2::make_bitflags; -use enumflags2::BitFlags; // Note: // 1. If add new privilege type, need add forward test diff --git a/src/meta/app/src/row_access_policy/row_access_policy_id_to_name_ident.rs b/src/meta/app/src/row_access_policy/row_access_policy_id_to_name_ident.rs index aa1568ec17a4c..80266d48e5d42 100644 --- a/src/meta/app/src/row_access_policy/row_access_policy_id_to_name_ident.rs +++ b/src/meta/app/src/row_access_policy/row_access_policy_id_to_name_ident.rs @@ -39,8 +39,8 @@ mod kvapi_impl { use databend_common_meta_kvapi::kvapi; - use crate::row_access_policy::row_access_policy_id_to_name_ident::RowAccessPolicyIdToNameIdent; use crate::row_access_policy::RowAccessPolicyNameIdentRaw; + use crate::row_access_policy::row_access_policy_id_to_name_ident::RowAccessPolicyIdToNameIdent; use crate::tenant_key::resource::TenantResource; pub struct Resource; diff --git a/src/meta/app/src/row_access_policy/row_access_policy_name_ident.rs b/src/meta/app/src/row_access_policy/row_access_policy_name_ident.rs index eeedc3dde4754..a4f9ed20c4378 100644 --- a/src/meta/app/src/row_access_policy/row_access_policy_name_ident.rs +++ b/src/meta/app/src/row_access_policy/row_access_policy_name_ident.rs @@ -37,10 +37,10 @@ mod kvapi_impl { use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::Key; + use crate::KeyWithTenant; use crate::row_access_policy::RowAccessPolicyId; use crate::row_access_policy::RowAccessPolicyNameIdent; use crate::tenant_key::resource::TenantResource; - use crate::KeyWithTenant; pub struct Resource; impl TenantResource for Resource { diff --git a/src/meta/app/src/row_access_policy/row_access_policy_table_id_ident.rs b/src/meta/app/src/row_access_policy/row_access_policy_table_id_ident.rs index 53d7268ff986d..5cd6c2e7e6200 100644 --- a/src/meta/app/src/row_access_policy/row_access_policy_table_id_ident.rs +++ b/src/meta/app/src/row_access_policy/row_access_policy_table_id_ident.rs @@ -78,8 +78,8 @@ mod kvapi_impl { mod tests { use databend_common_meta_kvapi::kvapi::Key; - use crate::row_access_policy::row_access_policy_table_id_ident::RowAccessPolicyIdTableId; use crate::row_access_policy::RowAccessPolicyTableIdIdent; + use crate::row_access_policy::row_access_policy_table_id_ident::RowAccessPolicyIdTableId; use crate::tenant::Tenant; #[test] diff --git a/src/meta/app/src/schema/catalog.rs b/src/meta/app/src/schema/catalog.rs index 37d9e6def181b..4628cd3386576 100644 --- a/src/meta/app/src/schema/catalog.rs +++ b/src/meta/app/src/schema/catalog.rs @@ -18,13 +18,13 @@ use std::fmt::Display; use chrono::DateTime; use chrono::Utc; -use crate::schema::catalog::catalog_info::CatalogId; -use crate::schema::catalog_id_ident; +use crate::KeyWithTenant; use crate::schema::CatalogIdIdent; use crate::schema::CatalogNameIdent; +use crate::schema::catalog::catalog_info::CatalogId; +use crate::schema::catalog_id_ident; use crate::storage::StorageParams; use crate::tenant::Tenant; -use crate::KeyWithTenant; #[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] pub enum CatalogType { diff --git a/src/meta/app/src/schema/catalog_id_to_name_ident.rs b/src/meta/app/src/schema/catalog_id_to_name_ident.rs index b9a0da3d4b8b9..b7b1278c30ddc 100644 --- a/src/meta/app/src/schema/catalog_id_to_name_ident.rs +++ b/src/meta/app/src/schema/catalog_id_to_name_ident.rs @@ -38,8 +38,8 @@ mod kvapi_impl { use databend_common_meta_kvapi::kvapi; - use crate::schema::catalog_name_ident::CatalogNameIdentRaw; use crate::schema::CatalogIdToNameIdent; + use crate::schema::catalog_name_ident::CatalogNameIdentRaw; use crate::tenant_key::resource::TenantResource; // TODO(TIdent): parent should return Some(CatalogIdIdent::new(self.catalog_id).to_string_key()) diff --git a/src/meta/app/src/schema/catalog_name_ident.rs b/src/meta/app/src/schema/catalog_name_ident.rs index 00a2c1c76c25a..fc4aab24361a9 100644 --- a/src/meta/app/src/schema/catalog_name_ident.rs +++ b/src/meta/app/src/schema/catalog_name_ident.rs @@ -28,10 +28,10 @@ mod kvapi_impl { use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::Key; - use crate::schema::catalog_id_ident::CatalogId; + use crate::KeyWithTenant; use crate::schema::CatalogNameIdent; + use crate::schema::catalog_id_ident::CatalogId; use crate::tenant_key::resource::TenantResource; - use crate::KeyWithTenant; pub struct CatalogNameRsc; impl TenantResource for CatalogNameRsc { diff --git a/src/meta/app/src/schema/database.rs b/src/meta/app/src/schema/database.rs index 3cf187b37e731..99f890347cded 100644 --- a/src/meta/app/src/schema/database.rs +++ b/src/meta/app/src/schema/database.rs @@ -23,11 +23,11 @@ use chrono::Utc; use databend_common_meta_types::SeqV; use super::CreateOption; +use crate::KeyWithTenant; use crate::schema::database_id::DatabaseId; use crate::schema::database_name_ident::DatabaseNameIdent; use crate::tenant::Tenant; use crate::tenant::ToTenant; -use crate::KeyWithTenant; #[derive(Clone, Debug, Eq, PartialEq)] pub struct DatabaseInfo { @@ -328,9 +328,9 @@ impl ListDatabaseReq { mod kvapi_key_impl { use databend_common_meta_kvapi::kvapi; - use crate::schema::database_name_ident::DatabaseNameIdentRaw; use crate::schema::DatabaseId; use crate::schema::DatabaseIdToName; + use crate::schema::database_name_ident::DatabaseNameIdentRaw; impl kvapi::KeyCodec for DatabaseIdToName { fn encode_key(&self, b: kvapi::KeyBuilder) -> kvapi::KeyBuilder { diff --git a/src/meta/app/src/schema/database_name_ident.rs b/src/meta/app/src/schema/database_name_ident.rs index a89a8401ec78f..a8708b8f04818 100644 --- a/src/meta/app/src/schema/database_name_ident.rs +++ b/src/meta/app/src/schema/database_name_ident.rs @@ -38,8 +38,8 @@ mod kvapi_impl { use databend_common_meta_kvapi::kvapi::Key; use crate::primitive::Id; - use crate::schema::database_name_ident::DatabaseNameIdent; use crate::schema::DatabaseId; + use crate::schema::database_name_ident::DatabaseNameIdent; use crate::tenant_key::resource::TenantResource; pub struct DatabaseNameRsc; diff --git a/src/meta/app/src/schema/dictionary.rs b/src/meta/app/src/schema/dictionary.rs index b52bbd279c3b5..2d5a682de4b30 100644 --- a/src/meta/app/src/schema/dictionary.rs +++ b/src/meta/app/src/schema/dictionary.rs @@ -25,10 +25,10 @@ use databend_common_exception::Result; use databend_common_expression::TableSchema; use super::dictionary_name_ident::DictionaryNameIdent; +use crate::KeyWithTenant; use crate::schema::DictionaryIdentity; use crate::tenant::Tenant; use crate::tenant::ToTenant; -use crate::KeyWithTenant; /// Represents the metadata of a dictionary within the system. #[derive(serde::Serialize, serde::Deserialize, Clone, Debug, Eq, PartialEq)] diff --git a/src/meta/app/src/schema/dictionary_id_ident.rs b/src/meta/app/src/schema/dictionary_id_ident.rs index 8621af66552f9..ed2ae51d70eb8 100644 --- a/src/meta/app/src/schema/dictionary_id_ident.rs +++ b/src/meta/app/src/schema/dictionary_id_ident.rs @@ -36,8 +36,8 @@ mod kvapi_impl { use databend_common_meta_kvapi::kvapi; - use crate::schema::dictionary_id_ident::DictionaryIdIdent; use crate::schema::DictionaryMeta; + use crate::schema::dictionary_id_ident::DictionaryIdIdent; use crate::tenant_key::resource::TenantResource; pub struct DictionaryIdRsc; diff --git a/src/meta/app/src/schema/dictionary_name_ident.rs b/src/meta/app/src/schema/dictionary_name_ident.rs index 31ff9702fad40..03402078ccad6 100644 --- a/src/meta/app/src/schema/dictionary_name_ident.rs +++ b/src/meta/app/src/schema/dictionary_name_ident.rs @@ -13,9 +13,9 @@ // limitations under the License. use super::DictionaryIdentity; +use crate::KeyWithTenant; use crate::tenant::ToTenant; use crate::tenant_key::ident::TIdent; -use crate::KeyWithTenant; /// A dictionary identity belonging to a tenant. pub type DictionaryNameIdent = TIdent; diff --git a/src/meta/app/src/schema/index.rs b/src/meta/app/src/schema/index.rs index 64727cea633b6..868ef95adddd0 100644 --- a/src/meta/app/src/schema/index.rs +++ b/src/meta/app/src/schema/index.rs @@ -22,10 +22,10 @@ use chrono::Utc; use databend_common_meta_types::MetaId; use super::CreateOption; +use crate::KeyWithTenant; use crate::schema::IndexNameIdent; use crate::tenant::Tenant; use crate::tenant::ToTenant; -use crate::KeyWithTenant; #[derive( serde::Serialize, diff --git a/src/meta/app/src/schema/index_id_ident.rs b/src/meta/app/src/schema/index_id_ident.rs index 9f58edb47b424..1146d29d86590 100644 --- a/src/meta/app/src/schema/index_id_ident.rs +++ b/src/meta/app/src/schema/index_id_ident.rs @@ -27,8 +27,8 @@ mod kvapi_impl { use databend_common_meta_kvapi::kvapi; - use crate::schema::index_id_ident::IndexIdIdent; use crate::schema::IndexMeta; + use crate::schema::index_id_ident::IndexIdIdent; use crate::tenant_key::resource::TenantResource; pub struct IndexIdResource; diff --git a/src/meta/app/src/schema/index_id_to_name_ident.rs b/src/meta/app/src/schema/index_id_to_name_ident.rs index 6d2a8094f05d1..06264e9dfd6f6 100644 --- a/src/meta/app/src/schema/index_id_to_name_ident.rs +++ b/src/meta/app/src/schema/index_id_to_name_ident.rs @@ -25,8 +25,8 @@ mod kvapi_impl { use databend_common_meta_kvapi::kvapi; - use crate::schema::index_id_to_name_ident::IndexIdToNameIdent; use crate::schema::IndexNameIdentRaw; + use crate::schema::index_id_to_name_ident::IndexIdToNameIdent; use crate::tenant_key::resource::TenantResource; pub struct IndexIdToName; diff --git a/src/meta/app/src/schema/index_name_ident.rs b/src/meta/app/src/schema/index_name_ident.rs index 28070e1b9d994..8a6b4e8ddde45 100644 --- a/src/meta/app/src/schema/index_name_ident.rs +++ b/src/meta/app/src/schema/index_name_ident.rs @@ -39,10 +39,10 @@ mod kvapi_impl { use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::Key; - use crate::schema::index_id_ident::IndexId; + use crate::KeyWithTenant; use crate::schema::IndexNameIdent; + use crate::schema::index_id_ident::IndexId; use crate::tenant_key::resource::TenantResource; - use crate::KeyWithTenant; pub struct IndexName; impl TenantResource for IndexName { diff --git a/src/meta/app/src/schema/least_visible_time_ident.rs b/src/meta/app/src/schema/least_visible_time_ident.rs index 91952723d02a6..db9ead09ad867 100644 --- a/src/meta/app/src/schema/least_visible_time_ident.rs +++ b/src/meta/app/src/schema/least_visible_time_ident.rs @@ -27,8 +27,8 @@ impl LeastVisibleTimeIdent { mod kvapi_impl { use databend_common_meta_kvapi::kvapi; - use crate::schema::least_visible_time_ident::LeastVisibleTimeIdent; use crate::schema::LeastVisibleTime; + use crate::schema::least_visible_time_ident::LeastVisibleTimeIdent; use crate::tenant_key::resource::TenantResource; pub struct LeastVisibleTimeRsc; diff --git a/src/meta/app/src/schema/marked_deleted_index_ident.rs b/src/meta/app/src/schema/marked_deleted_index_ident.rs index e3192cdfaf17d..3b644aad09fa1 100644 --- a/src/meta/app/src/schema/marked_deleted_index_ident.rs +++ b/src/meta/app/src/schema/marked_deleted_index_ident.rs @@ -24,8 +24,8 @@ mod kvapi_impl { use databend_common_meta_kvapi::kvapi; - use crate::schema::marked_deleted_index_ident::MarkedDeletedIndexIdIdent; use crate::schema::MarkedDeletedIndexMeta; + use crate::schema::marked_deleted_index_ident::MarkedDeletedIndexIdIdent; use crate::tenant_key::resource::TenantResource; /// The meta-service key for storing id of dropped but not vacuumed index diff --git a/src/meta/app/src/schema/table/ops.rs b/src/meta/app/src/schema/table/ops.rs index 21374b1af172f..3beb43d2beabe 100644 --- a/src/meta/app/src/schema/table/ops.rs +++ b/src/meta/app/src/schema/table/ops.rs @@ -100,10 +100,10 @@ impl TableMeta { mod tests { use std::sync::Arc; - use databend_common_expression::types::NumberDataType; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; + use databend_common_expression::types::NumberDataType; use super::*; diff --git a/src/meta/app/src/tenant_key/ident.rs b/src/meta/app/src/tenant_key/ident.rs index c0570ea1231c2..9f8f894999563 100644 --- a/src/meta/app/src/tenant_key/ident.rs +++ b/src/meta/app/src/tenant_key/ident.rs @@ -19,13 +19,13 @@ use std::fmt::Display; use std::hash::Hash; use std::hash::Hasher; +use crate::KeyWithTenant; use crate::tenant::Tenant; use crate::tenant::ToTenant; use crate::tenant_key::errors::ExistError; use crate::tenant_key::errors::UnknownError; use crate::tenant_key::raw::TIdentRaw; use crate::tenant_key::resource::TenantResource; -use crate::KeyWithTenant; /// `[T]enant[Ident]` is a common meta-service key structure in form of `//`. pub struct TIdent { @@ -199,10 +199,10 @@ mod kvapi_key_impl { use databend_common_meta_kvapi::kvapi::KeyCodec; use databend_common_meta_kvapi::kvapi::KeyError; + use crate::KeyWithTenant; use crate::tenant::Tenant; use crate::tenant_key::ident::TIdent; use crate::tenant_key::resource::TenantResource; - use crate::KeyWithTenant; impl kvapi::KeyCodec for TIdent where diff --git a/src/meta/app/src/tenant_key/raw.rs b/src/meta/app/src/tenant_key/raw.rs index 5dfe1fa70e28b..cee38d79ebe12 100644 --- a/src/meta/app/src/tenant_key/raw.rs +++ b/src/meta/app/src/tenant_key/raw.rs @@ -19,10 +19,10 @@ use std::fmt::Debug; use std::hash::Hash; use std::hash::Hasher; +use crate::KeyWithTenant; use crate::tenant::Tenant; use crate::tenant_key::ident::TIdent; use crate::tenant_key::resource::TenantResource; -use crate::KeyWithTenant; /// The storage of [`TIdent`] as raw value, without per-tenant config. /// diff --git a/src/meta/binaries/meta/ee_main.rs b/src/meta/binaries/meta/ee_main.rs index 873254b72797c..bc6797db1c07d 100644 --- a/src/meta/binaries/meta/ee_main.rs +++ b/src/meta/binaries/meta/ee_main.rs @@ -13,7 +13,7 @@ // limitations under the License. #![feature(stmt_expr_attributes)] -#![allow(clippy::uninlined_format_args)] +#![allow(clippy::collapsible_if, clippy::uninlined_format_args)] mod entry; mod kvapi; diff --git a/src/meta/binaries/meta/entry.rs b/src/meta/binaries/meta/entry.rs index 9200c76f65e77..e565e008aa5f2 100644 --- a/src/meta/binaries/meta/entry.rs +++ b/src/meta/binaries/meta/entry.rs @@ -22,17 +22,17 @@ use databend_common_base::base::StopHandle; use databend_common_base::base::Stoppable; use databend_common_base::runtime::GlobalIORuntime; use databend_common_grpc::RpcClientConf; -use databend_common_meta_raft_store::ondisk::OnDisk; use databend_common_meta_raft_store::ondisk::DATA_VERSION; +use databend_common_meta_raft_store::ondisk::OnDisk; use databend_common_meta_sled_store::openraft::MessageSummary; use databend_common_meta_store::MetaStoreProvider; -use databend_common_meta_types::node::Node; use databend_common_meta_types::Cmd; use databend_common_meta_types::LogEntry; use databend_common_meta_types::MetaAPIError; +use databend_common_meta_types::node::Node; use databend_common_storage::init_operator; -use databend_common_tracing::set_panic_hook; use databend_common_tracing::GlobalLogger; +use databend_common_tracing::set_panic_hook; use databend_common_version::BUILD_INFO; use databend_common_version::DATABEND_COMMIT_VERSION; use databend_common_version::DATABEND_GIT_SEMVER; @@ -46,13 +46,13 @@ use databend_meta::meta_node::meta_handle::MetaHandle; use databend_meta::meta_node::meta_worker::MetaWorker; use databend_meta::meta_service::MetaNode; use databend_meta::metrics::server_metrics; +use databend_meta::version::MIN_METACLI_SEMVER; use databend_meta::version::raft_client_requires; use databend_meta::version::raft_server_provides; -use databend_meta::version::MIN_METACLI_SEMVER; use log::info; use log::warn; -use tokio::time::sleep; use tokio::time::Instant; +use tokio::time::sleep; use crate::kvapi::KvApiCommand; diff --git a/src/meta/binaries/meta/oss_main.rs b/src/meta/binaries/meta/oss_main.rs index 01f88eb369497..2aa599ec22af9 100644 --- a/src/meta/binaries/meta/oss_main.rs +++ b/src/meta/binaries/meta/oss_main.rs @@ -13,6 +13,7 @@ // limitations under the License. #![feature(stmt_expr_attributes)] +#![allow(clippy::collapsible_if)] mod entry; mod kvapi; diff --git a/src/meta/binaries/metabench/main.rs b/src/meta/binaries/metabench/main.rs index ca5894af5a355..364de3cebfb60 100644 --- a/src/meta/binaries/metabench/main.rs +++ b/src/meta/binaries/metabench/main.rs @@ -12,14 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(clippy::uninlined_format_args)] +#![allow( + clippy::collapsible_if, + clippy::manual_is_multiple_of, + clippy::uninlined_format_args +)] use std::collections::BTreeMap; use std::fmt::Debug; use std::fmt::Display; +use std::sync::Arc; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; -use std::sync::Arc; use std::time::Duration; use std::time::Instant; @@ -27,11 +31,10 @@ use chrono::Utc; use clap::Parser; use databend_common_base::base::tokio; use databend_common_base::runtime; -use databend_common_meta_api::serialize_struct; -use databend_common_meta_api::txn_op_put; use databend_common_meta_api::DatabaseApi; use databend_common_meta_api::TableApi; -use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; +use databend_common_meta_api::serialize_struct; +use databend_common_meta_api::txn_op_put; use databend_common_meta_app::schema::CreateDatabaseReq; use databend_common_meta_app::schema::CreateOption; use databend_common_meta_app::schema::CreateTableReq; @@ -41,20 +44,21 @@ use databend_common_meta_app::schema::TableCopiedFileInfo; use databend_common_meta_app::schema::TableCopiedFileNameIdent; use databend_common_meta_app::schema::TableNameIdent; use databend_common_meta_app::schema::UpsertTableOptionReq; +use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_app::tenant::Tenant; -use databend_common_meta_client::required; use databend_common_meta_client::ClientHandle; use databend_common_meta_client::MetaGrpcClient; +use databend_common_meta_client::required; use databend_common_meta_kvapi::kvapi::KVApi; use databend_common_meta_semaphore::Semaphore; use databend_common_meta_types::MatchSeq; use databend_common_meta_types::Operation; use databend_common_meta_types::TxnRequest; use databend_common_meta_types::UpsertKV; -use databend_common_tracing::init_logging; use databend_common_tracing::FileConfig; use databend_common_tracing::LogFormat; use databend_common_tracing::StderrConfig; +use databend_common_tracing::init_logging; use databend_common_version::BUILD_INFO; use databend_common_version::METASRV_COMMIT_VERSION; use futures::TryStreamExt; diff --git a/src/meta/binaries/metactl/main.rs b/src/meta/binaries/metactl/main.rs index 1a812ff122c6e..efffc90631735 100644 --- a/src/meta/binaries/metactl/main.rs +++ b/src/meta/binaries/metactl/main.rs @@ -23,9 +23,9 @@ use clap::CommandFactory; use clap::Parser; use clap::Subcommand; use databend_common_base::base::tokio; -use databend_common_meta_client::errors::CreationError; use databend_common_meta_client::ClientHandle; use databend_common_meta_client::MetaGrpcClient; +use databend_common_meta_client::errors::CreationError; use databend_common_meta_control::admin::MetaAdminClient; use databend_common_meta_control::args::BenchArgs; use databend_common_meta_control::args::DumpRaftLogWalArgs; @@ -51,12 +51,12 @@ use databend_common_meta_control::keys_layout_from_grpc; use databend_common_meta_control::lua_support; use databend_common_meta_kvapi::kvapi::KVApi; use databend_common_meta_kvapi::kvapi::KvApiExt; -use databend_common_meta_types::protobuf::WatchRequest; use databend_common_meta_types::UpsertKV; -use databend_common_tracing::init_logging; +use databend_common_meta_types::protobuf::WatchRequest; use databend_common_tracing::Config as LogConfig; use databend_common_tracing::FileConfig; use databend_common_tracing::LogFormat; +use databend_common_tracing::init_logging; use databend_common_version::BUILD_INFO; use databend_common_version::METASRV_COMMIT_VERSION; use display_more::DisplayOptionExt; diff --git a/src/meta/binaries/metaverifier/main.rs b/src/meta/binaries/metaverifier/main.rs index 106997a4a3e29..6e6edb4e30a77 100644 --- a/src/meta/binaries/metaverifier/main.rs +++ b/src/meta/binaries/metaverifier/main.rs @@ -18,8 +18,8 @@ use std::collections::BTreeMap; use std::collections::HashSet; use std::fmt::Debug; use std::fs; -use std::sync::mpsc; use std::sync::Arc; +use std::sync::mpsc; use std::time::Duration; use std::time::Instant; @@ -35,15 +35,15 @@ use databend_common_meta_kvapi::kvapi::KvApiExt; use databend_common_meta_types::MatchSeq; use databend_common_meta_types::Operation; use databend_common_meta_types::UpsertKV; -use databend_common_tracing::init_logging; use databend_common_tracing::FileConfig; use databend_common_tracing::LogFormat; use databend_common_tracing::StderrConfig; +use databend_common_tracing::init_logging; use databend_common_version::BUILD_INFO; use databend_common_version::METASRV_COMMIT_VERSION; -use rand::rngs::StdRng; use rand::Rng; use rand::SeedableRng; +use rand::rngs::StdRng; use serde::Deserialize; use serde::Serialize; diff --git a/src/meta/cache/src/meta_client_source.rs b/src/meta/cache/src/meta_client_source.rs index 79807b8bbbb4e..b1ba83e50c44d 100644 --- a/src/meta/cache/src/meta_client_source.rs +++ b/src/meta/cache/src/meta_client_source.rs @@ -17,20 +17,20 @@ use std::sync::Arc; use databend_common_meta_client::ClientHandle; -use databend_common_meta_types::protobuf::WatchRequest; -use databend_common_meta_types::protobuf::WatchResponse; use databend_common_meta_types::MetaClientError; use databend_common_meta_types::SeqV; +use databend_common_meta_types::protobuf::WatchRequest; +use databend_common_meta_types::protobuf::WatchResponse; use futures::StreamExt; use log::debug; use log::warn; +use sub_cache::Source; use sub_cache::errors::ConnectionClosed; use sub_cache::errors::SubscribeError; use sub_cache::errors::Unsupported; use sub_cache::event_stream::Change; use sub_cache::event_stream::Event; use sub_cache::event_stream::EventStream; -use sub_cache::Source; use tonic::Status; pub struct MetaClientSource { diff --git a/src/meta/client/src/channel_manager.rs b/src/meta/client/src/channel_manager.rs index 89937ba22912c..416126f823470 100644 --- a/src/meta/client/src/channel_manager.rs +++ b/src/meta/client/src/channel_manager.rs @@ -21,23 +21,23 @@ use databend_common_base::containers::ItemManager; use databend_common_grpc::ConnectionFactory; use databend_common_grpc::GrpcConnectionError; use databend_common_grpc::RpcClientTlsConfig; -use databend_common_meta_types::protobuf::meta_service_client::MetaServiceClient; use databend_common_meta_types::ConnectionError; use databend_common_meta_types::GrpcConfig; use databend_common_meta_types::MetaClientError; use databend_common_meta_types::MetaNetworkError; +use databend_common_meta_types::protobuf::meta_service_client::MetaServiceClient; use log::info; use once_cell::sync::OnceCell; use parking_lot::Mutex; use tonic::async_trait; use tonic::transport::Channel; +use crate::FeatureSpec; +use crate::MetaGrpcClient; use crate::endpoints::Endpoints; use crate::established_client::EstablishedClient; use crate::grpc_client::AuthInterceptor; use crate::grpc_client::RealClient; -use crate::FeatureSpec; -use crate::MetaGrpcClient; #[derive(Debug)] pub struct MetaChannelManager { diff --git a/src/meta/client/src/client_handle.rs b/src/meta/client/src/client_handle.rs index 9ce3484b0f01a..2a871ec527fe7 100644 --- a/src/meta/client/src/client_handle.rs +++ b/src/meta/client/src/client_handle.rs @@ -15,9 +15,9 @@ use std::fmt::Debug; use std::fmt::Display; use std::fmt::Formatter; +use std::sync::Arc; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; -use std::sync::Arc; use anyerror::AnyError; use databend_common_base::base::tokio::sync::mpsc::UnboundedSender; @@ -27,17 +27,17 @@ use databend_common_base::runtime::ThreadTracker; use databend_common_base::runtime::UnlimitedFuture; use databend_common_meta_kvapi::kvapi::ListKVReq; use databend_common_meta_kvapi::kvapi::UpsertKVReply; +use databend_common_meta_types::ConnectionError; +use databend_common_meta_types::MetaClientError; +use databend_common_meta_types::MetaError; +use databend_common_meta_types::TxnRequest; +use databend_common_meta_types::UpsertKV; use databend_common_meta_types::protobuf::ClientInfo; use databend_common_meta_types::protobuf::ClusterStatus; use databend_common_meta_types::protobuf::MemberListReply; use databend_common_meta_types::protobuf::StreamItem; use databend_common_meta_types::protobuf::WatchRequest; use databend_common_meta_types::protobuf::WatchResponse; -use databend_common_meta_types::ConnectionError; -use databend_common_meta_types::MetaClientError; -use databend_common_meta_types::MetaError; -use databend_common_meta_types::TxnRequest; -use databend_common_meta_types::UpsertKV; use databend_common_metrics::count::Count; use fastrace::Span; use log::debug; @@ -45,14 +45,14 @@ use log::error; use log::info; use tonic::codegen::BoxStream; -use crate::established_client::EstablishedClient; -use crate::grpc_metrics; -use crate::message; -use crate::message::Response; use crate::ClientWorkerRequest; use crate::InitFlag; use crate::RequestFor; use crate::Streamed; +use crate::established_client::EstablishedClient; +use crate::grpc_metrics; +use crate::message; +use crate::message::Response; /// A handle to access meta-client worker. /// The worker will be actually running in a dedicated runtime: `MetaGrpcClient.rt`. diff --git a/src/meta/client/src/established_client.rs b/src/meta/client/src/established_client.rs index 25c044eb8cb4a..78c379f92602b 100644 --- a/src/meta/client/src/established_client.rs +++ b/src/meta/client/src/established_client.rs @@ -13,13 +13,16 @@ // limitations under the License. use std::fmt; +use std::sync::Arc; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; -use std::sync::Arc; use chrono::Utc; +use databend_common_meta_types::GrpcHelper; +use databend_common_meta_types::MetaHandshakeError; +use databend_common_meta_types::TxnReply; +use databend_common_meta_types::TxnRequest; use databend_common_meta_types::protobuf as pb; -use databend_common_meta_types::protobuf::meta_service_client::MetaServiceClient; use databend_common_meta_types::protobuf::ClientInfo; use databend_common_meta_types::protobuf::ClusterStatus; use databend_common_meta_types::protobuf::Empty; @@ -33,28 +36,25 @@ use databend_common_meta_types::protobuf::RaftRequest; use databend_common_meta_types::protobuf::StreamItem; use databend_common_meta_types::protobuf::WatchRequest; use databend_common_meta_types::protobuf::WatchResponse; -use databend_common_meta_types::GrpcHelper; -use databend_common_meta_types::MetaHandshakeError; -use databend_common_meta_types::TxnReply; -use databend_common_meta_types::TxnRequest; +use databend_common_meta_types::protobuf::meta_service_client::MetaServiceClient; use display_more::DisplayOptionExt; use log::debug; use log::error; use log::info; use log::warn; use parking_lot::Mutex; +use tonic::Response; +use tonic::Status; use tonic::codec::Streaming; use tonic::codegen::InterceptedService; use tonic::transport::Channel; -use tonic::Response; -use tonic::Status; -use crate::endpoints::rotate_failing_endpoint; +use crate::FeatureSpec; use crate::endpoints::Endpoints; +use crate::endpoints::rotate_failing_endpoint; use crate::grpc_client::AuthInterceptor; use crate::grpc_client::RealClient; use crate::required::Features; -use crate::FeatureSpec; /// Update the client state according to the result of an RPC. trait HandleRPCResult { diff --git a/src/meta/client/src/grpc_action.rs b/src/meta/client/src/grpc_action.rs index 8570287496111..6e7e819388eed 100644 --- a/src/meta/client/src/grpc_action.rs +++ b/src/meta/client/src/grpc_action.rs @@ -23,6 +23,11 @@ use databend_common_meta_kvapi::kvapi::ListKVReq; use databend_common_meta_kvapi::kvapi::MGetKVReply; use databend_common_meta_kvapi::kvapi::MGetKVReq; use databend_common_meta_kvapi::kvapi::UpsertKVReply; +use databend_common_meta_types::GrpcHelper; +use databend_common_meta_types::InvalidArgument; +use databend_common_meta_types::TxnReply; +use databend_common_meta_types::TxnRequest; +use databend_common_meta_types::UpsertKV; use databend_common_meta_types::protobuf::ClientInfo; use databend_common_meta_types::protobuf::ClusterStatus; use databend_common_meta_types::protobuf::MemberListReply; @@ -30,15 +35,11 @@ use databend_common_meta_types::protobuf::RaftRequest; use databend_common_meta_types::protobuf::StreamItem; use databend_common_meta_types::protobuf::WatchRequest; use databend_common_meta_types::protobuf::WatchResponse; -use databend_common_meta_types::GrpcHelper; -use databend_common_meta_types::InvalidArgument; -use databend_common_meta_types::TxnReply; -use databend_common_meta_types::TxnRequest; -use databend_common_meta_types::UpsertKV; use log::debug; -use tonic::codegen::BoxStream; use tonic::Request; +use tonic::codegen::BoxStream; +use crate::InitFlag; use crate::established_client::EstablishedClient; use crate::message::ExportReq; use crate::message::GetClientInfo; @@ -47,7 +48,6 @@ use crate::message::GetEndpoints; use crate::message::GetMemberList; use crate::message::MakeEstablishedClient; use crate::message::Streamed; -use crate::InitFlag; /// Bind a request type to its corresponding response type. pub trait RequestFor: Clone + fmt::Debug { diff --git a/src/meta/client/src/grpc_client.rs b/src/meta/client/src/grpc_client.rs index d62ba7d7df97d..5c9c180a5a1f9 100644 --- a/src/meta/client/src/grpc_client.rs +++ b/src/meta/client/src/grpc_client.rs @@ -21,13 +21,13 @@ use std::time::Duration; use std::time::Instant; use arrow_flight::BasicAuth; +use databend_common_base::base::BuildInfoRef; use databend_common_base::base::tokio::select; use databend_common_base::base::tokio::sync::mpsc; use databend_common_base::base::tokio::sync::mpsc::UnboundedReceiver; use databend_common_base::base::tokio::sync::oneshot; use databend_common_base::base::tokio::sync::oneshot::Sender as OneSend; use databend_common_base::base::tokio::time::sleep; -use databend_common_base::base::BuildInfoRef; use databend_common_base::containers::Pool; use databend_common_base::future::TimedFutureExt; use databend_common_base::runtime::Runtime; @@ -36,9 +36,15 @@ use databend_common_base::runtime::TrySpawn; use databend_common_base::runtime::UnlimitedFuture; use databend_common_grpc::RpcClientConf; use databend_common_grpc::RpcClientTlsConfig; +use databend_common_meta_types::ConnectionError; +use databend_common_meta_types::MetaClientError; +use databend_common_meta_types::MetaError; +use databend_common_meta_types::MetaHandshakeError; +use databend_common_meta_types::MetaNetworkError; +use databend_common_meta_types::TxnReply; +use databend_common_meta_types::TxnRequest; use databend_common_meta_types::anyerror::AnyError; use databend_common_meta_types::protobuf as pb; -use databend_common_meta_types::protobuf::meta_service_client::MetaServiceClient; use databend_common_meta_types::protobuf::ClientInfo; use databend_common_meta_types::protobuf::ClusterStatus; use databend_common_meta_types::protobuf::Empty; @@ -49,17 +55,11 @@ use databend_common_meta_types::protobuf::MemberListRequest; use databend_common_meta_types::protobuf::RaftRequest; use databend_common_meta_types::protobuf::WatchRequest; use databend_common_meta_types::protobuf::WatchResponse; -use databend_common_meta_types::ConnectionError; -use databend_common_meta_types::MetaClientError; -use databend_common_meta_types::MetaError; -use databend_common_meta_types::MetaHandshakeError; -use databend_common_meta_types::MetaNetworkError; -use databend_common_meta_types::TxnReply; -use databend_common_meta_types::TxnRequest; +use databend_common_meta_types::protobuf::meta_service_client::MetaServiceClient; +use fastrace::Span; use fastrace::func_name; use fastrace::func_path; use fastrace::future::FutureExt as MTFutureExt; -use fastrace::Span; use futures::stream::StreamExt; use log::debug; use log::error; @@ -69,36 +69,36 @@ use once_cell::sync::OnceCell; use parking_lot::Mutex; use prost::Message; use semver::Version; +use tonic::Code; +use tonic::Request; +use tonic::Status; +use tonic::Streaming; use tonic::codegen::BoxStream; use tonic::codegen::InterceptedService; use tonic::metadata::MetadataValue; use tonic::service::Interceptor; use tonic::transport::Channel; -use tonic::Code; -use tonic::Request; -use tonic::Status; -use tonic::Streaming; -use crate::endpoints::rotate_failing_endpoint; +use crate::ClientHandle; +use crate::ClientWorkerRequest; +use crate::FeatureSpec; +use crate::MetaChannelManager; +use crate::MetaGrpcReadReq; use crate::endpoints::Endpoints; +use crate::endpoints::rotate_failing_endpoint; use crate::errors::CreationError; use crate::established_client::EstablishedClient; use crate::from_digit_ver; use crate::grpc_metrics; use crate::message; use crate::message::Response; +use crate::required::Features; use crate::required::features; use crate::required::std; use crate::required::supported_features; -use crate::required::Features; use crate::rpc_handler::ResponseAction; use crate::rpc_handler::RpcHandler; use crate::to_digit_ver; -use crate::ClientHandle; -use crate::ClientWorkerRequest; -use crate::FeatureSpec; -use crate::MetaChannelManager; -use crate::MetaGrpcReadReq; const RPC_RETRIES: usize = 4; const AUTH_TOKEN_KEY: &str = "auth-token-bin"; diff --git a/src/meta/client/src/grpc_metrics.rs b/src/meta/client/src/grpc_metrics.rs index 89600ec48a537..6b33290313adf 100644 --- a/src/meta/client/src/grpc_metrics.rs +++ b/src/meta/client/src/grpc_metrics.rs @@ -15,12 +15,12 @@ use std::error::Error; use std::sync::LazyLock; -use databend_common_base::runtime::metrics::register_counter_family; -use databend_common_base::runtime::metrics::register_gauge; -use databend_common_base::runtime::metrics::register_histogram_family_in_milliseconds; use databend_common_base::runtime::metrics::FamilyCounter; use databend_common_base::runtime::metrics::FamilyHistogram; use databend_common_base::runtime::metrics::Gauge; +use databend_common_base::runtime::metrics::register_counter_family; +use databend_common_base::runtime::metrics::register_gauge; +use databend_common_base::runtime::metrics::register_histogram_family_in_milliseconds; use databend_common_meta_types::MetaError; use databend_common_metrics::VecLabels; diff --git a/src/meta/client/src/lib.rs b/src/meta/client/src/lib.rs index 24ffae65e292b..59f4b83264fe6 100644 --- a/src/meta/client/src/lib.rs +++ b/src/meta/client/src/lib.rs @@ -13,6 +13,7 @@ // limitations under the License. #![allow(clippy::uninlined_format_args)] +#![allow(clippy::collapsible_if)] extern crate core; diff --git a/src/meta/client/src/message.rs b/src/meta/client/src/message.rs index 0bd8295f5ec68..3011555d6a059 100644 --- a/src/meta/client/src/message.rs +++ b/src/meta/client/src/message.rs @@ -19,6 +19,10 @@ use databend_common_base::base::tokio::sync::oneshot::Sender; use databend_common_base::runtime::TrackingPayload; use databend_common_meta_kvapi::kvapi::ListKVReq; use databend_common_meta_kvapi::kvapi::MGetKVReq; +use databend_common_meta_types::MetaClientError; +use databend_common_meta_types::MetaError; +use databend_common_meta_types::TxnReply; +use databend_common_meta_types::TxnRequest; use databend_common_meta_types::protobuf::ClientInfo; use databend_common_meta_types::protobuf::ClusterStatus; use databend_common_meta_types::protobuf::ExportedChunk; @@ -26,10 +30,6 @@ use databend_common_meta_types::protobuf::MemberListReply; use databend_common_meta_types::protobuf::StreamItem; use databend_common_meta_types::protobuf::WatchRequest; use databend_common_meta_types::protobuf::WatchResponse; -use databend_common_meta_types::MetaClientError; -use databend_common_meta_types::MetaError; -use databend_common_meta_types::TxnReply; -use databend_common_meta_types::TxnRequest; use fastrace::Span; use tonic::codegen::BoxStream; @@ -203,7 +203,7 @@ impl Response { .map(|x| x as &(dyn std::error::Error + 'static)) } - let e = match self { + match self { Response::StreamMGet(res) => to_err(res), Response::StreamList(res) => to_err(res), Response::Txn(res) => to_err(res), @@ -215,9 +215,7 @@ impl Response { Response::GetClusterStatus(res) => to_err(res), Response::GetMemberList(res) => to_err(res), Response::GetClientInfo(res) => to_err(res), - }; - - e + } } } diff --git a/src/meta/client/src/rpc_handler.rs b/src/meta/client/src/rpc_handler.rs index 7e38646be0899..ed03bbc66a9cf 100644 --- a/src/meta/client/src/rpc_handler.rs +++ b/src/meta/client/src/rpc_handler.rs @@ -29,9 +29,9 @@ use tonic::Code; use tonic::Response; use tonic::Status; -use crate::established_client::EstablishedClient; use crate::FeatureSpec; use crate::MetaGrpcClient; +use crate::established_client::EstablishedClient; /// Represents the action to take after processing an RPC response. /// @@ -183,11 +183,7 @@ impl<'a> RpcHandler<'a> { warn!( "MetaGrpcClient::{} retryable error: elapsed: {:?}; error: {:?}; with {}; request: {:?}", - self.required_feature.0, - elapsed, - status, - client_display, - request + self.required_feature.0, elapsed, status, client_display, request ); self.rpc_failures.push((client_display, status)); @@ -198,11 +194,7 @@ impl<'a> RpcHandler<'a> { } else { warn!( "MetaGrpcClient::{} non-retryable error: elapsed: {:?}; error: {:?}; with {}; request: {:?}", - self.required_feature.0, - elapsed, - status, - established_client, - request + self.required_feature.0, elapsed, status, established_client, request ); Err(status) diff --git a/src/meta/client/tests/it/grpc_client.rs b/src/meta/client/tests/it/grpc_client.rs index 820d8813cf268..4b1198dc958f1 100644 --- a/src/meta/client/tests/it/grpc_client.rs +++ b/src/meta/client/tests/it/grpc_client.rs @@ -18,15 +18,15 @@ use std::time::Duration; use databend_common_base::base::tokio; use databend_common_grpc::ConnectionFactory; use databend_common_meta_client::ClientHandle; +use databend_common_meta_client::MIN_METASRV_SEMVER; use databend_common_meta_client::MetaChannelManager; use databend_common_meta_client::MetaGrpcClient; use databend_common_meta_client::Streamed; -use databend_common_meta_client::MIN_METASRV_SEMVER; use databend_common_meta_kvapi::kvapi::KVApi; use databend_common_meta_kvapi::kvapi::MGetKVReq; -use databend_common_meta_types::protobuf::StreamItem; use databend_common_meta_types::MetaError; use databend_common_meta_types::UpsertKV; +use databend_common_meta_types::protobuf::StreamItem; use futures::StreamExt; use log::info; use tonic::codegen::BoxStream; @@ -78,8 +78,7 @@ async fn test_grpc_client_handshake_timeout() { MetaGrpcClient::handshake(&mut client, &MIN_METASRV_SEMVER, &[], "root", "xxx").await; let got = res.unwrap_err(); - let expect = - "HandshakeError with databend-meta: Connection Failure; cause: tonic::status::Status: status: Cancelled, message: \"Timeout expired\", details: [], metadata: MetadataMap { headers: {} }; source: transport error; source: Timeout expired"; + let expect = "HandshakeError with databend-meta: Connection Failure; cause: tonic::status::Status: status: Cancelled, message: \"Timeout expired\", details: [], metadata: MetadataMap { headers: {} }; source: transport error; source: Timeout expired"; assert_eq!(got.to_string(), expect); } diff --git a/src/meta/client/tests/it/grpc_server.rs b/src/meta/client/tests/it/grpc_server.rs index 0652967af6341..59398c1849625 100644 --- a/src/meta/client/tests/it/grpc_server.rs +++ b/src/meta/client/tests/it/grpc_server.rs @@ -19,10 +19,8 @@ use std::time::Duration; use databend_common_base::base::tokio; use databend_common_base::base::tokio::sync::oneshot; use databend_common_base::base::tokio::task::JoinHandle; -use databend_common_meta_client::to_digit_ver; use databend_common_meta_client::MIN_METASRV_SEMVER; -use databend_common_meta_types::protobuf::meta_service_server::MetaService; -use databend_common_meta_types::protobuf::meta_service_server::MetaServiceServer; +use databend_common_meta_client::to_digit_ver; use databend_common_meta_types::protobuf::ClientInfo; use databend_common_meta_types::protobuf::ClusterStatus; use databend_common_meta_types::protobuf::Empty; @@ -39,14 +37,16 @@ use databend_common_meta_types::protobuf::TxnReply; use databend_common_meta_types::protobuf::TxnRequest; use databend_common_meta_types::protobuf::WatchRequest; use databend_common_meta_types::protobuf::WatchResponse; +use databend_common_meta_types::protobuf::meta_service_server::MetaService; +use databend_common_meta_types::protobuf::meta_service_server::MetaServiceServer; use futures::Stream; use rand::Rng; -use tonic::codegen::BoxStream; -use tonic::transport::Server; use tonic::Request; use tonic::Response; use tonic::Status; use tonic::Streaming; +use tonic::codegen::BoxStream; +use tonic::transport::Server; /// A service that times out a kv_api() call, without impl other API. pub struct GrpcServiceForTestImpl {} diff --git a/src/meta/control/src/export_from_grpc.rs b/src/meta/control/src/export_from_grpc.rs index 3c69478446842..a4527a6d4a66d 100644 --- a/src/meta/control/src/export_from_grpc.rs +++ b/src/meta/control/src/export_from_grpc.rs @@ -20,8 +20,8 @@ use std::net::ToSocketAddrs; use anyhow::anyhow; use databend_common_base::base::BuildInfoRef; -use databend_common_meta_client::required; use databend_common_meta_client::MetaGrpcClient; +use databend_common_meta_client::required; use databend_common_meta_raft_store::key_spaces::RaftStoreEntry; use databend_common_meta_types::protobuf; use tokio::net::TcpSocket; diff --git a/src/meta/control/src/import.rs b/src/meta/control/src/import.rs index ef7517edb2d35..99474a79aa2d0 100644 --- a/src/meta/control/src/import.rs +++ b/src/meta/control/src/import.rs @@ -14,8 +14,8 @@ use std::collections::BTreeMap; use std::collections::BTreeSet; -use std::fs::remove_dir_all; use std::fs::File; +use std::fs::remove_dir_all; use std::io; use std::io::BufRead; use std::io::BufReader; @@ -27,19 +27,19 @@ use databend_common_meta_raft_store::config::RaftConfig; use databend_common_meta_raft_store::ondisk::DataVersion; use databend_common_meta_raft_store::raft_log_v004; use databend_common_meta_sled_store::init_get_sled_db; -use databend_common_meta_sled_store::openraft::storage::RaftLogStorageExt; use databend_common_meta_sled_store::openraft::RaftSnapshotBuilder; +use databend_common_meta_sled_store::openraft::storage::RaftLogStorageExt; +use databend_common_meta_types::Cmd; +use databend_common_meta_types::Endpoint; +use databend_common_meta_types::LogEntry; use databend_common_meta_types::node::Node; -use databend_common_meta_types::raft_types::new_log_id; use databend_common_meta_types::raft_types::Entry; use databend_common_meta_types::raft_types::EntryPayload; use databend_common_meta_types::raft_types::LogId; use databend_common_meta_types::raft_types::Membership; use databend_common_meta_types::raft_types::NodeId; use databend_common_meta_types::raft_types::StoredMembership; -use databend_common_meta_types::Cmd; -use databend_common_meta_types::Endpoint; -use databend_common_meta_types::LogEntry; +use databend_common_meta_types::raft_types::new_log_id; use databend_meta::store::RaftStore; use display_more::display_option::DisplayOptionExt; use raft_log::api::raft_log_writer::RaftLogWriter; diff --git a/src/meta/control/src/import_v004.rs b/src/meta/control/src/import_v004.rs index 192c705fd1612..eb0fcc8e6cd00 100644 --- a/src/meta/control/src/import_v004.rs +++ b/src/meta/control/src/import_v004.rs @@ -24,9 +24,9 @@ use databend_common_meta_raft_store::ondisk::Header; use databend_common_meta_raft_store::ondisk::OnDisk; use databend_common_meta_raft_store::raft_log_v004; use databend_common_meta_raft_store::raft_log_v004::RaftLogV004; -use databend_common_meta_raft_store::sm_v003::adapter::SMEntryV002ToV004; use databend_common_meta_raft_store::sm_v003::SnapshotStoreV004; use databend_common_meta_raft_store::sm_v003::WriteEntry; +use databend_common_meta_raft_store::sm_v003::adapter::SMEntryV002ToV004; use databend_common_meta_raft_store::state_machine::MetaSnapshotId; use databend_common_meta_types::raft_types::LogId; use databend_common_meta_types::sys_data::SysData; diff --git a/src/meta/control/src/lib.rs b/src/meta/control/src/lib.rs index cf2b49b24e66c..506e93b4ec168 100644 --- a/src/meta/control/src/lib.rs +++ b/src/meta/control/src/lib.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(clippy::uninlined_format_args)] +#![allow(clippy::unnecessary_unwrap, clippy::uninlined_format_args)] pub mod admin; pub mod args; diff --git a/src/meta/control/src/lua_support.rs b/src/meta/control/src/lua_support.rs index 249047cbf41a5..454b704f515d0 100644 --- a/src/meta/control/src/lua_support.rs +++ b/src/meta/control/src/lua_support.rs @@ -18,9 +18,9 @@ use std::sync::Arc; use std::time::Duration; use databend_common_base::base::BuildInfoRef; -use databend_common_meta_client::errors::CreationError; use databend_common_meta_client::ClientHandle; use databend_common_meta_client::MetaGrpcClient; +use databend_common_meta_client::errors::CreationError; use databend_common_meta_kvapi::kvapi::KVApi; use databend_common_meta_kvapi::kvapi::KvApiExt; use databend_common_meta_types::UpsertKV; diff --git a/src/meta/control/src/reading.rs b/src/meta/control/src/reading.rs index f0c406f3615d0..e491c94b7ea4e 100644 --- a/src/meta/control/src/reading.rs +++ b/src/meta/control/src/reading.rs @@ -19,8 +19,8 @@ use std::io::Lines; use std::iter::Peekable; use databend_common_meta_raft_store::key_spaces::RaftStoreEntry; -use databend_common_meta_raft_store::ondisk::DataVersion; use databend_common_meta_raft_store::ondisk::DATA_VERSION; +use databend_common_meta_raft_store::ondisk::DataVersion; use databend_common_meta_raft_store::ondisk::TREE_HEADER; /// Import from lines of exported data and Return the max log id that is found. diff --git a/src/meta/kvapi-test-suite/src/kvapi_test_suite.rs b/src/meta/kvapi-test-suite/src/kvapi_test_suite.rs index 8a466e06f4566..e9c858c854d22 100644 --- a/src/meta/kvapi-test-suite/src/kvapi_test_suite.rs +++ b/src/meta/kvapi-test-suite/src/kvapi_test_suite.rs @@ -17,15 +17,6 @@ use std::time::SystemTime; use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::KvApiExt; -use databend_common_meta_types::normalize_meta::NormalizeMeta; -use databend_common_meta_types::protobuf as pb; -use databend_common_meta_types::protobuf::BooleanExpression; -use databend_common_meta_types::protobuf::FetchIncreaseU64Response; -use databend_common_meta_types::protobuf::KvMeta; -use databend_common_meta_types::txn_condition; -use databend_common_meta_types::txn_op; -use databend_common_meta_types::txn_op_response; -use databend_common_meta_types::txn_op_response::Response; use databend_common_meta_types::ConditionResult; use databend_common_meta_types::MatchSeq; use databend_common_meta_types::MetaSpec; @@ -45,6 +36,15 @@ use databend_common_meta_types::TxnReply; use databend_common_meta_types::TxnRequest; use databend_common_meta_types::UpsertKV; use databend_common_meta_types::With; +use databend_common_meta_types::normalize_meta::NormalizeMeta; +use databend_common_meta_types::protobuf as pb; +use databend_common_meta_types::protobuf::BooleanExpression; +use databend_common_meta_types::protobuf::FetchIncreaseU64Response; +use databend_common_meta_types::protobuf::KvMeta; +use databend_common_meta_types::txn_condition; +use databend_common_meta_types::txn_op; +use databend_common_meta_types::txn_op_response; +use databend_common_meta_types::txn_op_response::Response; use display_more::DisplayOptionExt; use display_more::DisplaySliceExt; use fastrace::func_name; diff --git a/src/meta/kvapi/src/kvapi/api.rs b/src/meta/kvapi/src/kvapi/api.rs index c18909e29c4e8..169ff0bab587e 100644 --- a/src/meta/kvapi/src/kvapi/api.rs +++ b/src/meta/kvapi/src/kvapi/api.rs @@ -15,12 +15,12 @@ use std::ops::Deref; use async_trait::async_trait; -use databend_common_meta_types::errors; -use databend_common_meta_types::protobuf::StreamItem; use databend_common_meta_types::Change; use databend_common_meta_types::TxnReply; use databend_common_meta_types::TxnRequest; use databend_common_meta_types::UpsertKV; +use databend_common_meta_types::errors; +use databend_common_meta_types::protobuf::StreamItem; use futures_util::stream::BoxStream; use crate::kvapi; diff --git a/src/meta/kvapi/src/kvapi/dir_name.rs b/src/meta/kvapi/src/kvapi/dir_name.rs index d49350b71b0f0..083d59aa27bc2 100644 --- a/src/meta/kvapi/src/kvapi/dir_name.rs +++ b/src/meta/kvapi/src/kvapi/dir_name.rs @@ -96,9 +96,9 @@ impl Key for DirName { #[cfg(test)] mod tests { + use crate::kvapi::Key; use crate::kvapi::dir_name::DirName; use crate::kvapi::testing::FooKey; - use crate::kvapi::Key; #[test] fn test_dir_name_from_key() { diff --git a/src/meta/kvapi/src/kvapi/key.rs b/src/meta/kvapi/src/kvapi/key.rs index 75d213b895822..e90a082db47c3 100644 --- a/src/meta/kvapi/src/kvapi/key.rs +++ b/src/meta/kvapi/src/kvapi/key.rs @@ -83,9 +83,9 @@ where Self: Sized #[cfg(test)] mod tests { - use crate::kvapi::testing::FooKey; use crate::kvapi::DirName; use crate::kvapi::Key; + use crate::kvapi::testing::FooKey; #[test] fn test_with_key_space() { diff --git a/src/meta/kvapi/src/kvapi/key_parser.rs b/src/meta/kvapi/src/kvapi/key_parser.rs index fb1dfbc7eb0ab..f04e12a6ad4e5 100644 --- a/src/meta/kvapi/src/kvapi/key_parser.rs +++ b/src/meta/kvapi/src/kvapi/key_parser.rs @@ -18,10 +18,10 @@ use std::string::FromUtf8Error; use databend_common_meta_app_types::non_empty::NonEmptyString; +use crate::kvapi::KeyError; use crate::kvapi::helper::decode_id; use crate::kvapi::helper::unescape; use crate::kvapi::helper::unescape_specified; -use crate::kvapi::KeyError; /// A helper for parsing a string key into structured key. pub struct KeyParser<'s> { @@ -179,8 +179,8 @@ impl<'s> KeyParser<'s> { #[cfg(test)] mod tests { - use crate::kvapi::key_parser::KeyParser; use crate::kvapi::KeyError; + use crate::kvapi::key_parser::KeyParser; #[test] fn test_key_parser_new_prefixed() -> anyhow::Result<()> { diff --git a/src/meta/kvapi/src/kvapi/kv_api_ext.rs b/src/meta/kvapi/src/kvapi/kv_api_ext.rs index 733904b7ca272..841ede5b6a9b1 100644 --- a/src/meta/kvapi/src/kvapi/kv_api_ext.rs +++ b/src/meta/kvapi/src/kvapi/kv_api_ext.rs @@ -13,8 +13,8 @@ // limitations under the License. use async_trait::async_trait; -use databend_common_meta_types::errors; use databend_common_meta_types::SeqV; +use databend_common_meta_types::errors; use futures_util::StreamExt; use futures_util::TryStreamExt; use log::debug; diff --git a/src/meta/kvapi/src/kvapi/testing.rs b/src/meta/kvapi/src/kvapi/testing.rs index 2b04244d7a1e1..9abe8f171f8af 100644 --- a/src/meta/kvapi/src/kvapi/testing.rs +++ b/src/meta/kvapi/src/kvapi/testing.rs @@ -19,6 +19,7 @@ use crate::kvapi::KeyError; use crate::kvapi::KeyParser; use crate::kvapi::Value; +#[allow(dead_code)] #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct FooKey { pub(crate) a: u64, @@ -41,6 +42,7 @@ impl KeyCodec for FooKey { } } +#[allow(dead_code)] #[derive(Debug)] pub(crate) struct FooValue; diff --git a/src/meta/process/src/examples.rs b/src/meta/process/src/examples.rs index 45bd5196bae47..796152309608e 100644 --- a/src/meta/process/src/examples.rs +++ b/src/meta/process/src/examples.rs @@ -15,8 +15,8 @@ use std::collections::BTreeMap; use clap::Parser; -use databend_common_tracing::init_logging; use databend_common_tracing::Config as LogConfig; +use databend_common_tracing::init_logging; use serde::Deserialize; use serde::Serialize; diff --git a/src/meta/process/src/kv_processor.rs b/src/meta/process/src/kv_processor.rs index d15f1cbdcd7cb..63be6976b6477 100644 --- a/src/meta/process/src/kv_processor.rs +++ b/src/meta/process/src/kv_processor.rs @@ -14,9 +14,6 @@ use anyhow::Error; use databend_common_meta_raft_store::key_spaces::RaftStoreEntry; -use databend_common_meta_types::raft_types::Entry; -use databend_common_meta_types::txn_condition::Target; -use databend_common_meta_types::txn_op::Request; use databend_common_meta_types::Cmd; use databend_common_meta_types::LogEntry; use databend_common_meta_types::Operation; @@ -26,6 +23,9 @@ use databend_common_meta_types::TxnOp; use databend_common_meta_types::TxnPutRequest; use databend_common_meta_types::TxnRequest; use databend_common_meta_types::UpsertKV; +use databend_common_meta_types::raft_types::Entry; +use databend_common_meta_types::txn_condition::Target; +use databend_common_meta_types::txn_op::Request; use openraft::EntryPayload; use crate::process::Process; diff --git a/src/meta/proto-conv/src/catalog_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/catalog_from_to_protobuf_impl.rs index c5a0e3e02c6fa..242cd1f1e7da6 100644 --- a/src/meta/proto-conv/src/catalog_from_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/catalog_from_to_protobuf_impl.rs @@ -21,11 +21,11 @@ use databend_common_meta_app::schema as mt; use databend_common_meta_app::storage::StorageParams; use databend_common_protos::pb; -use crate::reader_check_msg; use crate::FromToProto; use crate::Incompatible; use crate::MIN_READER_VER; use crate::VER; +use crate::reader_check_msg; impl FromToProto for mt::CatalogMeta { type PB = pb::CatalogMeta; diff --git a/src/meta/proto-conv/src/client_session_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/client_session_from_to_protobuf_impl.rs index 73f1a147a9fa9..7259dcc01b071 100644 --- a/src/meta/proto-conv/src/client_session_from_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/client_session_from_to_protobuf_impl.rs @@ -18,11 +18,11 @@ use databend_common_meta_app::principal::client_session as mt; use databend_common_protos::pb; -use crate::reader_check_msg; use crate::FromToProto; use crate::Incompatible; use crate::MIN_READER_VER; use crate::VER; +use crate::reader_check_msg; impl FromToProto for mt::ClientSession { type PB = pb::ClientSession; diff --git a/src/meta/proto-conv/src/config_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/config_from_to_protobuf_impl.rs index 4f8935169e99f..64a66cdc10406 100644 --- a/src/meta/proto-conv/src/config_from_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/config_from_to_protobuf_impl.rs @@ -25,11 +25,11 @@ use databend_common_meta_app::storage::StorageS3Config; use databend_common_meta_app::storage::StorageWebhdfsConfig; use databend_common_protos::pb; -use crate::reader_check_msg; use crate::FromToProto; use crate::Incompatible; use crate::MIN_READER_VER; use crate::VER; +use crate::reader_check_msg; impl FromToProto for mt::storage::StorageParams { type PB = pb::StorageConfig; diff --git a/src/meta/proto-conv/src/connection_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/connection_from_to_protobuf_impl.rs index 1e0d8c72e3e0d..794c8cca79003 100644 --- a/src/meta/proto-conv/src/connection_from_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/connection_from_to_protobuf_impl.rs @@ -15,11 +15,11 @@ use databend_common_meta_app::principal as mt; use databend_common_protos::pb; -use crate::reader_check_msg; use crate::FromToProto; use crate::Incompatible; use crate::MIN_READER_VER; use crate::VER; +use crate::reader_check_msg; impl FromToProto for mt::UserDefinedConnection { type PB = pb::UserDefinedConnection; diff --git a/src/meta/proto-conv/src/data_mask_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/data_mask_from_to_protobuf_impl.rs index ca305981bcb07..848a543233c00 100644 --- a/src/meta/proto-conv/src/data_mask_from_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/data_mask_from_to_protobuf_impl.rs @@ -22,11 +22,11 @@ use chrono::Utc; use databend_common_meta_app::data_mask as mt; use databend_common_protos::pb; -use crate::reader_check_msg; use crate::FromToProto; use crate::Incompatible; use crate::MIN_READER_VER; use crate::VER; +use crate::reader_check_msg; impl FromToProto for mt::DatamaskMeta { type PB = pb::DatamaskMeta; diff --git a/src/meta/proto-conv/src/database_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/database_from_to_protobuf_impl.rs index 0cdb2ac58fd67..c7e308fe55114 100644 --- a/src/meta/proto-conv/src/database_from_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/database_from_to_protobuf_impl.rs @@ -20,11 +20,11 @@ use chrono::Utc; use databend_common_meta_app::schema as mt; use databend_common_protos::pb; -use crate::reader_check_msg; use crate::FromToProto; use crate::Incompatible; use crate::MIN_READER_VER; use crate::VER; +use crate::reader_check_msg; impl FromToProto for mt::DatabaseMeta { type PB = pb::DatabaseMeta; diff --git a/src/meta/proto-conv/src/dictionary_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/dictionary_from_to_protobuf_impl.rs index 1086c73f806ff..1472f76e670bd 100644 --- a/src/meta/proto-conv/src/dictionary_from_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/dictionary_from_to_protobuf_impl.rs @@ -20,11 +20,11 @@ use databend_common_expression as ex; use databend_common_meta_app::schema as mt; use databend_common_protos::pb; -use crate::reader_check_msg; use crate::FromToProto; use crate::Incompatible; use crate::MIN_READER_VER; use crate::VER; +use crate::reader_check_msg; impl FromToProto for mt::DictionaryMeta { type PB = pb::DictionaryMeta; diff --git a/src/meta/proto-conv/src/file_format_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/file_format_from_to_protobuf_impl.rs index ecd21493f138e..64ad06b77b192 100644 --- a/src/meta/proto-conv/src/file_format_from_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/file_format_from_to_protobuf_impl.rs @@ -25,12 +25,12 @@ use databend_common_meta_app::principal::StageFileCompression; use databend_common_protos::pb; use num::FromPrimitive; -use crate::reader_check_msg; use crate::FromToProto; use crate::FromToProtoEnum; use crate::Incompatible; use crate::MIN_READER_VER; use crate::VER; +use crate::reader_check_msg; impl FromToProtoEnum for mt::principal::StageFileFormatType { type PBEnum = pb::StageFileFormatType; diff --git a/src/meta/proto-conv/src/index_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/index_from_to_protobuf_impl.rs index 6b9a8b424ba70..95224ffd27b99 100644 --- a/src/meta/proto-conv/src/index_from_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/index_from_to_protobuf_impl.rs @@ -21,11 +21,11 @@ use databend_common_meta_app::schema as mt; use databend_common_protos::pb; use num::FromPrimitive; -use crate::reader_check_msg; use crate::FromToProto; use crate::Incompatible; use crate::MIN_READER_VER; use crate::VER; +use crate::reader_check_msg; impl FromToProto for mt::IndexMeta { type PB = pb::IndexMeta; diff --git a/src/meta/proto-conv/src/least_visible_time_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/least_visible_time_from_to_protobuf_impl.rs index d22c60838cc51..7e9febe0af834 100644 --- a/src/meta/proto-conv/src/least_visible_time_from_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/least_visible_time_from_to_protobuf_impl.rs @@ -20,11 +20,11 @@ use chrono::Utc; use databend_common_meta_app::schema as mt; use databend_common_protos::pb; -use crate::reader_check_msg; use crate::FromToProto; use crate::Incompatible; use crate::MIN_READER_VER; use crate::VER; +use crate::reader_check_msg; impl FromToProto for mt::LeastVisibleTime { type PB = pb::LeastVisibleTime; diff --git a/src/meta/proto-conv/src/lib.rs b/src/meta/proto-conv/src/lib.rs index b03adb66a7a9c..ccb2560fa6a5f 100644 --- a/src/meta/proto-conv/src/lib.rs +++ b/src/meta/proto-conv/src/lib.rs @@ -96,8 +96,8 @@ mod vacuum_watermark_from_to_protobuf_impl; pub use from_to_protobuf::FromToProto; pub use from_to_protobuf::FromToProtoEnum; pub use from_to_protobuf::Incompatible; -pub use util::missing; -pub use util::reader_check_msg; pub use util::MIN_MSG_VER; pub use util::MIN_READER_VER; pub use util::VER; +pub use util::missing; +pub use util::reader_check_msg; diff --git a/src/meta/proto-conv/src/lock_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/lock_from_to_protobuf_impl.rs index 6b271d99e5f72..7eb66a016a0ff 100644 --- a/src/meta/proto-conv/src/lock_from_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/lock_from_to_protobuf_impl.rs @@ -23,11 +23,11 @@ use databend_common_meta_app_types::non_empty::NonEmptyString; use databend_common_protos::pb; use num::FromPrimitive; -use crate::reader_check_msg; use crate::FromToProto; use crate::Incompatible; use crate::MIN_READER_VER; use crate::VER; +use crate::reader_check_msg; impl FromToProto for mt::LockKey { type PB = pb::LockKey; diff --git a/src/meta/proto-conv/src/owner_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/owner_from_to_protobuf_impl.rs index 837a844fcb5d6..04b03a95c9957 100644 --- a/src/meta/proto-conv/src/owner_from_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/owner_from_to_protobuf_impl.rs @@ -17,11 +17,11 @@ use chrono::Utc; use databend_common_meta_app::schema as mt; use databend_common_protos::pb; -use crate::reader_check_msg; use crate::FromToProto; use crate::Incompatible; use crate::MIN_READER_VER; use crate::VER; +use crate::reader_check_msg; impl FromToProto for mt::Ownership { type PB = pb::Ownership; diff --git a/src/meta/proto-conv/src/ownership_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/ownership_from_to_protobuf_impl.rs index dad92e22ef1c6..204718f96a160 100644 --- a/src/meta/proto-conv/src/ownership_from_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/ownership_from_to_protobuf_impl.rs @@ -18,11 +18,11 @@ use databend_common_meta_app as mt; use databend_common_protos::pb; -use crate::reader_check_msg; use crate::FromToProto; use crate::Incompatible; use crate::MIN_READER_VER; use crate::VER; +use crate::reader_check_msg; impl FromToProto for mt::principal::OwnershipInfo { type PB = pb::OwnershipInfo; diff --git a/src/meta/proto-conv/src/procedure_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/procedure_from_to_protobuf_impl.rs index 3039546e4b8e0..2d7a9da2b2969 100644 --- a/src/meta/proto-conv/src/procedure_from_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/procedure_from_to_protobuf_impl.rs @@ -17,17 +17,17 @@ use chrono::DateTime; use chrono::Utc; +use databend_common_expression::TableDataType; use databend_common_expression::infer_schema_type; use databend_common_expression::types::DataType; -use databend_common_expression::TableDataType; use databend_common_meta_app as mt; use databend_common_protos::pb; -use crate::reader_check_msg; use crate::FromToProto; use crate::Incompatible; use crate::MIN_READER_VER; use crate::VER; +use crate::reader_check_msg; impl FromToProto for mt::principal::ProcedureIdentity { type PB = pb::ProcedureIdentity; diff --git a/src/meta/proto-conv/src/role_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/role_from_to_protobuf_impl.rs index 0f605e51182e5..78a63b78da3c7 100644 --- a/src/meta/proto-conv/src/role_from_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/role_from_to_protobuf_impl.rs @@ -22,11 +22,11 @@ use chrono::Utc; use databend_common_meta_app as mt; use databend_common_protos::pb; -use crate::reader_check_msg; use crate::FromToProto; use crate::Incompatible; use crate::MIN_READER_VER; use crate::VER; +use crate::reader_check_msg; impl FromToProto for mt::principal::RoleInfo { type PB = pb::RoleInfo; diff --git a/src/meta/proto-conv/src/row_access_policy_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/row_access_policy_from_to_protobuf_impl.rs index 98f5588e767c5..193510c4c696b 100644 --- a/src/meta/proto-conv/src/row_access_policy_from_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/row_access_policy_from_to_protobuf_impl.rs @@ -22,11 +22,11 @@ use chrono::Utc; use databend_common_meta_app::row_access_policy as mt; use databend_common_protos::pb; -use crate::reader_check_msg; use crate::FromToProto; use crate::Incompatible; use crate::MIN_READER_VER; use crate::VER; +use crate::reader_check_msg; impl FromToProto for mt::RowAccessPolicyMeta { type PB = pb::RowAccessPolicyMeta; diff --git a/src/meta/proto-conv/src/row_access_policy_table_id_form_to_protobuf_impl.rs b/src/meta/proto-conv/src/row_access_policy_table_id_form_to_protobuf_impl.rs index 3348d9da590e2..c4773f0e1c6c5 100644 --- a/src/meta/proto-conv/src/row_access_policy_table_id_form_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/row_access_policy_table_id_form_to_protobuf_impl.rs @@ -18,11 +18,11 @@ use databend_common_meta_app::row_access_policy as mt; use databend_common_protos::pb; -use crate::reader_check_msg; use crate::FromToProto; use crate::Incompatible; use crate::MIN_READER_VER; use crate::VER; +use crate::reader_check_msg; impl FromToProto for mt::RowAccessPolicyTableId { type PB = pb::RowAccessPolicyTableId; diff --git a/src/meta/proto-conv/src/schema_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/schema_from_to_protobuf_impl.rs index 44c8e362deff8..c125096f71b7d 100644 --- a/src/meta/proto-conv/src/schema_from_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/schema_from_to_protobuf_impl.rs @@ -16,21 +16,21 @@ //! Everytime update anything in this file, update the `VER` and let the tests pass. use databend_common_expression as ex; -use databend_common_expression::types::NumberDataType; use databend_common_expression::TableDataType; use databend_common_expression::VariantDataType; +use databend_common_expression::types::NumberDataType; use databend_common_protos::pb; use databend_common_protos::pb::data_type::Dt; use databend_common_protos::pb::data_type::Dt24; use databend_common_protos::pb::number::Num; use databend_common_protos::pb::variant_data_type; -use crate::reader_check_msg; use crate::FromToProto; use crate::FromToProtoEnum; use crate::Incompatible; use crate::MIN_READER_VER; use crate::VER; +use crate::reader_check_msg; impl FromToProto for ex::TableSchema { type PB = pb::DataSchema; diff --git a/src/meta/proto-conv/src/sequence_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/sequence_from_to_protobuf_impl.rs index f698612f6578f..eb2e531c30061 100644 --- a/src/meta/proto-conv/src/sequence_from_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/sequence_from_to_protobuf_impl.rs @@ -20,11 +20,11 @@ use chrono::Utc; use databend_common_meta_app::schema::SequenceMeta; use databend_common_protos::pb; -use crate::reader_check_msg; use crate::FromToProto; use crate::Incompatible; use crate::MIN_READER_VER; use crate::VER; +use crate::reader_check_msg; impl FromToProto for SequenceMeta { type PB = pb::SequenceMeta; diff --git a/src/meta/proto-conv/src/stage_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/stage_from_to_protobuf_impl.rs index e1c4eb0bdcec7..fea02a658ea69 100644 --- a/src/meta/proto-conv/src/stage_from_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/stage_from_to_protobuf_impl.rs @@ -24,12 +24,12 @@ use databend_common_protos::pb; use mt::principal::FileFormatOptionsReader; use num::FromPrimitive; -use crate::reader_check_msg; use crate::FromToProto; use crate::FromToProtoEnum; use crate::Incompatible; use crate::MIN_READER_VER; use crate::VER; +use crate::reader_check_msg; impl FromToProto for mt::principal::StageParams { type PB = pb::stage_info::StageParams; diff --git a/src/meta/proto-conv/src/table_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/table_from_to_protobuf_impl.rs index 0ff39f0904e7a..3b425405f25db 100644 --- a/src/meta/proto-conv/src/table_from_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/table_from_to_protobuf_impl.rs @@ -31,11 +31,11 @@ use databend_common_meta_app_types::non_empty::NonEmptyString; use databend_common_protos::pb; use num::FromPrimitive; -use crate::reader_check_msg; use crate::FromToProto; use crate::Incompatible; use crate::MIN_READER_VER; use crate::VER; +use crate::reader_check_msg; impl FromToProto for mt::TableCopiedFileInfo { type PB = pb::TableCopiedFileInfo; diff --git a/src/meta/proto-conv/src/task_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/task_from_to_protobuf_impl.rs index 381aa50a8712f..e654b745a9d6e 100644 --- a/src/meta/proto-conv/src/task_from_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/task_from_to_protobuf_impl.rs @@ -20,11 +20,11 @@ use databend_common_protos::pb; use databend_common_protos::pb::task_message::DeleteTask; use databend_common_protos::pb::task_message::Message; -use crate::reader_check_msg; use crate::FromToProto; use crate::Incompatible; use crate::MIN_READER_VER; use crate::VER; +use crate::reader_check_msg; impl FromToProto for mt::Task { type PB = pb::Task; diff --git a/src/meta/proto-conv/src/tenant_quota_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/tenant_quota_from_to_protobuf_impl.rs index 61b671993473d..42582b29979bf 100644 --- a/src/meta/proto-conv/src/tenant_quota_from_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/tenant_quota_from_to_protobuf_impl.rs @@ -18,11 +18,11 @@ use databend_common_meta_app::tenant; use databend_common_protos::pb; -use crate::reader_check_msg; use crate::FromToProto; use crate::Incompatible; use crate::MIN_READER_VER; use crate::VER; +use crate::reader_check_msg; impl FromToProto for tenant::TenantQuota { type PB = pb::TenantQuota; diff --git a/src/meta/proto-conv/src/tident_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/tident_from_to_protobuf_impl.rs index 30913aae2a636..9fd6303a08517 100644 --- a/src/meta/proto-conv/src/tident_from_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/tident_from_to_protobuf_impl.rs @@ -19,11 +19,11 @@ use databend_common_meta_app::tenant_key::raw; use databend_common_meta_app::tenant_key::resource::TenantResource; use databend_common_protos::pb; -use crate::reader_check_msg; use crate::FromToProto; use crate::Incompatible; use crate::MIN_READER_VER; use crate::VER; +use crate::reader_check_msg; impl FromToProto for raw::TIdentRaw where R: TenantResource diff --git a/src/meta/proto-conv/src/token_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/token_from_to_protobuf_impl.rs index b14119145fa49..09ed52a759031 100644 --- a/src/meta/proto-conv/src/token_from_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/token_from_to_protobuf_impl.rs @@ -19,11 +19,11 @@ use databend_common_meta_app::principal::user_token as mt; use databend_common_protos::pb; use num::FromPrimitive; -use crate::reader_check_msg; use crate::FromToProto; use crate::Incompatible; use crate::MIN_READER_VER; use crate::VER; +use crate::reader_check_msg; impl FromToProto for mt::QueryTokenInfo { type PB = pb::TokenInfo; diff --git a/src/meta/proto-conv/src/udf_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/udf_from_to_protobuf_impl.rs index ad73f73882072..d7758f86c6557 100644 --- a/src/meta/proto-conv/src/udf_from_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/udf_from_to_protobuf_impl.rs @@ -14,19 +14,19 @@ use chrono::DateTime; use chrono::Utc; -use databend_common_expression::infer_schema_type; -use databend_common_expression::types::DataType; use databend_common_expression::TableDataType; use databend_common_expression::TableField; +use databend_common_expression::infer_schema_type; +use databend_common_expression::types::DataType; use databend_common_meta_app::principal as mt; use databend_common_protos::pb; use databend_common_protos::pb::UdtfArg; -use crate::reader_check_msg; use crate::FromToProto; use crate::Incompatible; use crate::MIN_READER_VER; use crate::VER; +use crate::reader_check_msg; impl FromToProto for mt::LambdaUDF { type PB = pb::LambdaUdf; diff --git a/src/meta/proto-conv/src/user_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/user_from_to_protobuf_impl.rs index 4458aa0709743..5dee941288b1b 100644 --- a/src/meta/proto-conv/src/user_from_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/user_from_to_protobuf_impl.rs @@ -25,11 +25,11 @@ use databend_common_protos::pb; use enumflags2::BitFlags; use num::FromPrimitive; -use crate::reader_check_msg; use crate::FromToProto; use crate::Incompatible; use crate::MIN_READER_VER; use crate::VER; +use crate::reader_check_msg; impl FromToProto for mt::principal::AuthInfo { type PB = pb::AuthInfo; @@ -160,7 +160,10 @@ impl FromToProto for mt::principal::GrantObject { reader_check_msg(p.ver, p.min_reader_ver)?; let Some(object) = p.object else { - return Err(Incompatible::new(format!("Incompatible GrantObject type: Data contains an unrecognized variant for {} version", p.ver))); + return Err(Incompatible::new(format!( + "Incompatible GrantObject type: Data contains an unrecognized variant for {} version", + p.ver + ))); }; match object { diff --git a/src/meta/proto-conv/src/util.rs b/src/meta/proto-conv/src/util.rs index 3deca046806b6..7e6ee9a862422 100644 --- a/src/meta/proto-conv/src/util.rs +++ b/src/meta/proto-conv/src/util.rs @@ -214,12 +214,10 @@ pub const MIN_MSG_VER: u64 = 1; pub fn reader_check_msg(msg_ver: u64, msg_min_reader_ver: u64) -> Result<(), Incompatible> { // The reader version must be big enough if VER < msg_min_reader_ver { - return Err(Incompatible::new( - format!( - "executable ver={} is smaller than the min reader version({}) that can read this message", - VER, msg_min_reader_ver - ), - )); + return Err(Incompatible::new(format!( + "executable ver={} is smaller than the min reader version({}) that can read this message", + VER, msg_min_reader_ver + ))); } // The message version must be big enough diff --git a/src/meta/proto-conv/src/vacuum_watermark_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/vacuum_watermark_from_to_protobuf_impl.rs index 79da592a256af..4bfe79c016a64 100644 --- a/src/meta/proto-conv/src/vacuum_watermark_from_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/vacuum_watermark_from_to_protobuf_impl.rs @@ -19,11 +19,11 @@ use chrono::Utc; use databend_common_meta_app::schema as mt; use databend_common_protos::pb; -use crate::reader_check_msg; use crate::FromToProto; use crate::Incompatible; use crate::MIN_READER_VER; use crate::VER; +use crate::reader_check_msg; impl FromToProto for mt::VacuumWatermark { type PB = pb::VacuumWatermark; diff --git a/src/meta/proto-conv/tests/it/proto_conv.rs b/src/meta/proto-conv/tests/it/proto_conv.rs index 4a56759c2c70c..f5df6aedebdbb 100644 --- a/src/meta/proto-conv/tests/it/proto_conv.rs +++ b/src/meta/proto-conv/tests/it/proto_conv.rs @@ -16,17 +16,17 @@ use std::collections::BTreeMap; use std::sync::Arc; use std::vec; -use ce::types::decimal::DecimalSize; use ce::types::DecimalDataType; use ce::types::NumberDataType; +use ce::types::decimal::DecimalSize; use chrono::DateTime; use chrono::TimeZone; use chrono::Utc; use databend_common_expression as ce; -use databend_common_expression::types::DataType; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; +use databend_common_expression::types::DataType; use databend_common_meta_app::schema as mt; use databend_common_meta_app::schema::CatalogOption; use databend_common_meta_app::schema::IcebergCatalogOption; @@ -369,13 +369,11 @@ fn test_incompatible() -> anyhow::Result<()> { let res = mt::DatabaseMeta::from_pb(p); assert_eq!( - Incompatible::new( - format!( - "executable ver={} is smaller than the min reader version({}) that can read this message", - VER, - VER + 1 - ) - ), + Incompatible::new(format!( + "executable ver={} is smaller than the min reader version({}) that can read this message", + VER, + VER + 1 + )), res.unwrap_err() ); diff --git a/src/meta/proto-conv/tests/it/user_proto_conv.rs b/src/meta/proto-conv/tests/it/user_proto_conv.rs index 56f96455bcddf..0868a3eb1cb6a 100644 --- a/src/meta/proto-conv/tests/it/user_proto_conv.rs +++ b/src/meta/proto-conv/tests/it/user_proto_conv.rs @@ -471,13 +471,11 @@ fn test_user_incompatible() -> anyhow::Result<()> { let res = mt::principal::UserInfo::from_pb(p); assert_eq!( - Incompatible::new( - format!( - "executable ver={} is smaller than the min reader version({}) that can read this message", - VER, - VER + 1 - ) - ), + Incompatible::new(format!( + "executable ver={} is smaller than the min reader version({}) that can read this message", + VER, + VER + 1 + )), res.unwrap_err() ); } @@ -490,13 +488,11 @@ fn test_user_incompatible() -> anyhow::Result<()> { let res = mt::principal::StageFile::from_pb(p); assert_eq!( - Incompatible::new( - format!( - "executable ver={} is smaller than the min reader version({}) that can read this message", - VER, - VER + 1 - ) - ), + Incompatible::new(format!( + "executable ver={} is smaller than the min reader version({}) that can read this message", + VER, + VER + 1 + )), res.unwrap_err() ) } @@ -509,13 +505,11 @@ fn test_user_incompatible() -> anyhow::Result<()> { let res = mt::principal::StageInfo::from_pb(p); assert_eq!( - Incompatible::new( - format!( - "executable ver={} is smaller than the min reader version({}) that can read this message", - VER, - VER + 1 - ) - ), + Incompatible::new(format!( + "executable ver={} is smaller than the min reader version({}) that can read this message", + VER, + VER + 1 + )), res.unwrap_err() ) } @@ -528,13 +522,11 @@ fn test_user_incompatible() -> anyhow::Result<()> { let res = mt::principal::StageInfo::from_pb(p); assert_eq!( - Incompatible::new( - format!( - "executable ver={} is smaller than the min reader version({}) that can read this message", - VER, - VER + 1 - ) - ), + Incompatible::new(format!( + "executable ver={} is smaller than the min reader version({}) that can read this message", + VER, + VER + 1 + )), res.unwrap_err() ); } @@ -547,13 +539,11 @@ fn test_user_incompatible() -> anyhow::Result<()> { let res = mt::principal::StageInfo::from_pb(p); assert_eq!( - Incompatible::new( - format!( - "executable ver={} is smaller than the min reader version({}) that can read this message", - VER, - VER + 1 - ) - ), + Incompatible::new(format!( + "executable ver={} is smaller than the min reader version({}) that can read this message", + VER, + VER + 1 + )), res.unwrap_err() ); } @@ -566,13 +556,11 @@ fn test_user_incompatible() -> anyhow::Result<()> { let res = mt::principal::StageInfo::from_pb(p); assert_eq!( - Incompatible::new( - format!( - "executable ver={} is smaller than the min reader version({}) that can read this message", - VER, - VER + 1 - ) - ), + Incompatible::new(format!( + "executable ver={} is smaller than the min reader version({}) that can read this message", + VER, + VER + 1 + )), res.unwrap_err() ); } @@ -585,13 +573,11 @@ fn test_user_incompatible() -> anyhow::Result<()> { let res = mt::principal::StageInfo::from_pb(p); assert_eq!( - Incompatible::new( - format!( - "executable ver={} is smaller than the min reader version({}) that can read this message", - VER, - VER + 1 - ) - ), + Incompatible::new(format!( + "executable ver={} is smaller than the min reader version({}) that can read this message", + VER, + VER + 1 + )), res.unwrap_err() ) } @@ -604,13 +590,11 @@ fn test_user_incompatible() -> anyhow::Result<()> { let res = mt::principal::StageInfo::from_pb(p); assert_eq!( - Incompatible::new( - format!( - "executable ver={} is smaller than the min reader version({}) that can read this message", - VER, - VER + 1 - ) - ), + Incompatible::new(format!( + "executable ver={} is smaller than the min reader version({}) that can read this message", + VER, + VER + 1 + )), res.unwrap_err() ) } diff --git a/src/meta/proto-conv/tests/it/v026_schema.rs b/src/meta/proto-conv/tests/it/v026_schema.rs index fa71840a6fd41..c85c2b40eb74a 100644 --- a/src/meta/proto-conv/tests/it/v026_schema.rs +++ b/src/meta/proto-conv/tests/it/v026_schema.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_expression::types::NumberDataType; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; +use databend_common_expression::types::NumberDataType; use fastrace::func_name; use crate::common; diff --git a/src/meta/proto-conv/tests/it/v027_schema.rs b/src/meta/proto-conv/tests/it/v027_schema.rs index 0c5682a1862e6..351e7109e6bc7 100644 --- a/src/meta/proto-conv/tests/it/v027_schema.rs +++ b/src/meta/proto-conv/tests/it/v027_schema.rs @@ -12,12 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_expression::types::decimal::DecimalSize; -use databend_common_expression::types::DecimalDataType; -use databend_common_expression::types::NumberDataType; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; +use databend_common_expression::types::DecimalDataType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::decimal::DecimalSize; use fastrace::func_name; use crate::common; diff --git a/src/meta/proto-conv/tests/it/v028_schema.rs b/src/meta/proto-conv/tests/it/v028_schema.rs index 946faae6c2393..275c133210841 100644 --- a/src/meta/proto-conv/tests/it/v028_schema.rs +++ b/src/meta/proto-conv/tests/it/v028_schema.rs @@ -12,12 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_expression::types::decimal::DecimalSize; -use databend_common_expression::types::DecimalDataType; -use databend_common_expression::types::NumberDataType; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; +use databend_common_expression::types::DecimalDataType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::decimal::DecimalSize; use fastrace::func_name; use crate::common; diff --git a/src/meta/proto-conv/tests/it/v029_schema.rs b/src/meta/proto-conv/tests/it/v029_schema.rs index ac4039dfe29f1..3fa796eec0290 100644 --- a/src/meta/proto-conv/tests/it/v029_schema.rs +++ b/src/meta/proto-conv/tests/it/v029_schema.rs @@ -12,12 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_expression::types::decimal::DecimalSize; -use databend_common_expression::types::DecimalDataType; -use databend_common_expression::types::NumberDataType; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; +use databend_common_expression::types::DecimalDataType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::decimal::DecimalSize; use fastrace::func_name; use crate::common; diff --git a/src/meta/proto-conv/tests/it/v034_schema.rs b/src/meta/proto-conv/tests/it/v034_schema.rs index 7c8f3e8c5087a..b36f88e0ed502 100644 --- a/src/meta/proto-conv/tests/it/v034_schema.rs +++ b/src/meta/proto-conv/tests/it/v034_schema.rs @@ -12,12 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_expression::types::decimal::DecimalSize; -use databend_common_expression::types::DecimalDataType; -use databend_common_expression::types::NumberDataType; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; +use databend_common_expression::types::DecimalDataType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::decimal::DecimalSize; use fastrace::func_name; use crate::common; diff --git a/src/meta/proto-conv/tests/it/v044_table_meta.rs b/src/meta/proto-conv/tests/it/v044_table_meta.rs index 13e37821b6f24..ed23bfc11ebe1 100644 --- a/src/meta/proto-conv/tests/it/v044_table_meta.rs +++ b/src/meta/proto-conv/tests/it/v044_table_meta.rs @@ -18,8 +18,8 @@ use std::sync::Arc; use chrono::TimeZone; use chrono::Utc; use databend_common_expression as ce; -use databend_common_expression::types::NumberDataType; use databend_common_expression::ComputedExpr; +use databend_common_expression::types::NumberDataType; use databend_common_meta_app::schema as mt; use fastrace::func_name; use maplit::btreemap; diff --git a/src/meta/proto-conv/tests/it/v055_table_meta.rs b/src/meta/proto-conv/tests/it/v055_table_meta.rs index e85d29b373c51..1b10d68b9a3bf 100644 --- a/src/meta/proto-conv/tests/it/v055_table_meta.rs +++ b/src/meta/proto-conv/tests/it/v055_table_meta.rs @@ -18,8 +18,8 @@ use std::sync::Arc; use chrono::TimeZone; use chrono::Utc; use databend_common_expression as ce; -use databend_common_expression::types::NumberDataType; use databend_common_expression::ComputedExpr; +use databend_common_expression::types::NumberDataType; use databend_common_meta_app::schema as mt; use fastrace::func_name; use maplit::btreemap; diff --git a/src/meta/proto-conv/tests/it/v074_table_db_meta.rs b/src/meta/proto-conv/tests/it/v074_table_db_meta.rs index 7f421ffd91d1c..7f0a2c5b4739d 100644 --- a/src/meta/proto-conv/tests/it/v074_table_db_meta.rs +++ b/src/meta/proto-conv/tests/it/v074_table_db_meta.rs @@ -18,8 +18,8 @@ use std::sync::Arc; use chrono::TimeZone; use chrono::Utc; use databend_common_expression as ce; -use databend_common_expression::types::NumberDataType; use databend_common_expression::ComputedExpr; +use databend_common_expression::types::NumberDataType; use databend_common_meta_app::schema as mt; use fastrace::func_name; use maplit::btreemap; diff --git a/src/meta/proto-conv/tests/it/v080_geometry_datatype.rs b/src/meta/proto-conv/tests/it/v080_geometry_datatype.rs index 41b0e1a0946c7..d0849cb1fcd76 100644 --- a/src/meta/proto-conv/tests/it/v080_geometry_datatype.rs +++ b/src/meta/proto-conv/tests/it/v080_geometry_datatype.rs @@ -18,12 +18,12 @@ use std::sync::Arc; use chrono::TimeZone; use chrono::Utc; use databend_common_expression as ce; -use databend_common_expression::types::decimal::DecimalSize; -use databend_common_expression::types::DecimalDataType; -use databend_common_expression::types::NumberDataType; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; +use databend_common_expression::types::DecimalDataType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::decimal::DecimalSize; use databend_common_meta_app::schema as mt; use fastrace::func_name; use maplit::btreemap; diff --git a/src/meta/proto-conv/tests/it/v082_table_index.rs b/src/meta/proto-conv/tests/it/v082_table_index.rs index 83736d464f7e1..c1c7687d4172c 100644 --- a/src/meta/proto-conv/tests/it/v082_table_index.rs +++ b/src/meta/proto-conv/tests/it/v082_table_index.rs @@ -18,8 +18,8 @@ use std::sync::Arc; use chrono::TimeZone; use chrono::Utc; use databend_common_expression as ce; -use databend_common_expression::types::NumberDataType; use databend_common_expression::ComputedExpr; +use databend_common_expression::types::NumberDataType; use databend_common_meta_app::schema as mt; use databend_common_meta_app::schema::TableIndexType; use fastrace::func_name; diff --git a/src/meta/proto-conv/tests/it/v085_table_index.rs b/src/meta/proto-conv/tests/it/v085_table_index.rs index f5b73f21ec99d..c3df168938b33 100644 --- a/src/meta/proto-conv/tests/it/v085_table_index.rs +++ b/src/meta/proto-conv/tests/it/v085_table_index.rs @@ -18,8 +18,8 @@ use std::sync::Arc; use chrono::TimeZone; use chrono::Utc; use databend_common_expression as ce; -use databend_common_expression::types::NumberDataType; use databend_common_expression::ComputedExpr; +use databend_common_expression::types::NumberDataType; use databend_common_meta_app::schema as mt; use databend_common_meta_app::schema::TableIndexType; use fastrace::func_name; diff --git a/src/meta/proto-conv/tests/it/v086_table_index.rs b/src/meta/proto-conv/tests/it/v086_table_index.rs index 819703a23668a..9ced9d7f94079 100644 --- a/src/meta/proto-conv/tests/it/v086_table_index.rs +++ b/src/meta/proto-conv/tests/it/v086_table_index.rs @@ -18,8 +18,8 @@ use std::sync::Arc; use chrono::TimeZone; use chrono::Utc; use databend_common_expression as ce; -use databend_common_expression::types::NumberDataType; use databend_common_expression::ComputedExpr; +use databend_common_expression::types::NumberDataType; use databend_common_meta_app::schema as mt; use databend_common_meta_app::schema::TableIndexType; use fastrace::func_name; diff --git a/src/meta/proto-conv/tests/it/v094_table_meta.rs b/src/meta/proto-conv/tests/it/v094_table_meta.rs index 8b1c44ac82a4b..8dda9828236a1 100644 --- a/src/meta/proto-conv/tests/it/v094_table_meta.rs +++ b/src/meta/proto-conv/tests/it/v094_table_meta.rs @@ -18,8 +18,8 @@ use std::sync::Arc; use chrono::TimeZone; use chrono::Utc; use databend_common_expression as ce; -use databend_common_expression::types::NumberDataType; use databend_common_expression::ComputedExpr; +use databend_common_expression::types::NumberDataType; use databend_common_meta_app::schema as mt; use fastrace::func_name; use maplit::btreemap; diff --git a/src/meta/proto-conv/tests/it/v107_geography_datatype.rs b/src/meta/proto-conv/tests/it/v107_geography_datatype.rs index 43ddd7010f0a4..a6fabc939cfff 100644 --- a/src/meta/proto-conv/tests/it/v107_geography_datatype.rs +++ b/src/meta/proto-conv/tests/it/v107_geography_datatype.rs @@ -18,12 +18,12 @@ use std::sync::Arc; use chrono::TimeZone; use chrono::Utc; use databend_common_expression as ce; -use databend_common_expression::types::decimal::DecimalSize; -use databend_common_expression::types::DecimalDataType; -use databend_common_expression::types::NumberDataType; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; +use databend_common_expression::types::DecimalDataType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::decimal::DecimalSize; use databend_common_meta_app::schema as mt; use fastrace::func_name; use maplit::btreemap; diff --git a/src/meta/proto-conv/tests/it/v114_interval_datatype.rs b/src/meta/proto-conv/tests/it/v114_interval_datatype.rs index 029df20198205..b419f704f558c 100644 --- a/src/meta/proto-conv/tests/it/v114_interval_datatype.rs +++ b/src/meta/proto-conv/tests/it/v114_interval_datatype.rs @@ -17,12 +17,12 @@ use std::sync::Arc; use chrono::TimeZone; use chrono::Utc; -use databend_common_expression::types::decimal::DecimalSize; -use databend_common_expression::types::DecimalDataType; -use databend_common_expression::types::NumberDataType; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; +use databend_common_expression::types::DecimalDataType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::decimal::DecimalSize; use databend_common_meta_app::schema as mt; use fastrace::func_name; use maplit::btreemap; diff --git a/src/meta/proto-conv/tests/it/v115_add_udaf_script.rs b/src/meta/proto-conv/tests/it/v115_add_udaf_script.rs index d4e80f568707b..1f8c904815016 100644 --- a/src/meta/proto-conv/tests/it/v115_add_udaf_script.rs +++ b/src/meta/proto-conv/tests/it/v115_add_udaf_script.rs @@ -14,9 +14,9 @@ use chrono::DateTime; use chrono::Utc; +use databend_common_expression::DataField; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; -use databend_common_expression::DataField; use databend_common_meta_app::principal::UDAFScript; use databend_common_meta_app::principal::UDFDefinition; use databend_common_meta_app::principal::UserDefinedFunction; diff --git a/src/meta/proto-conv/tests/it/v129_vector_datatype.rs b/src/meta/proto-conv/tests/it/v129_vector_datatype.rs index b6fec43c8b2a9..b54b33f46cea8 100644 --- a/src/meta/proto-conv/tests/it/v129_vector_datatype.rs +++ b/src/meta/proto-conv/tests/it/v129_vector_datatype.rs @@ -17,17 +17,17 @@ use std::sync::Arc; use chrono::TimeZone; use chrono::Utc; -use databend_common_expression::types::decimal::DecimalSize; -use databend_common_expression::types::DecimalDataType; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::VectorDataType; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; +use databend_common_expression::VIRTUAL_COLUMN_ID_START; use databend_common_expression::VariantDataType; use databend_common_expression::VirtualDataField; use databend_common_expression::VirtualDataSchema; -use databend_common_expression::VIRTUAL_COLUMN_ID_START; +use databend_common_expression::types::DecimalDataType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::VectorDataType; +use databend_common_expression::types::decimal::DecimalSize; use databend_common_meta_app::schema as mt; use databend_common_meta_app::schema::TableIndex; use databend_common_meta_app::schema::TableIndexType; diff --git a/src/meta/proto-conv/tests/it/v143_udtf.rs b/src/meta/proto-conv/tests/it/v143_udtf.rs index d02259ef63ec7..fe77e2e1a1f82 100644 --- a/src/meta/proto-conv/tests/it/v143_udtf.rs +++ b/src/meta/proto-conv/tests/it/v143_udtf.rs @@ -16,8 +16,8 @@ use chrono::DateTime; use chrono::Utc; use databend_common_expression::types::DataType; use databend_common_meta_app::principal::UDFDefinition; -use databend_common_meta_app::principal::UserDefinedFunction; use databend_common_meta_app::principal::UDTF; +use databend_common_meta_app::principal::UserDefinedFunction; use fastrace::func_name; use crate::common; diff --git a/src/meta/proto-conv/tests/it/v145_opaque_data_type.rs b/src/meta/proto-conv/tests/it/v145_opaque_data_type.rs index a847b5e602c3f..f1ab95fd51f89 100644 --- a/src/meta/proto-conv/tests/it/v145_opaque_data_type.rs +++ b/src/meta/proto-conv/tests/it/v145_opaque_data_type.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_expression::types::NumberDataType; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; +use databend_common_expression::types::NumberDataType; use fastrace::func_name; use crate::common; diff --git a/src/meta/proto-conv/tests/it/v149_field_auto_increment.rs b/src/meta/proto-conv/tests/it/v149_field_auto_increment.rs index 170437eaac4ef..28ec620a88464 100644 --- a/src/meta/proto-conv/tests/it/v149_field_auto_increment.rs +++ b/src/meta/proto-conv/tests/it/v149_field_auto_increment.rs @@ -13,9 +13,9 @@ // limitations under the License. use databend_common_expression as ce; -use databend_common_expression::types::NumberDataType; use databend_common_expression::AutoIncrementExpr; use databend_common_expression::TableDataType; +use databend_common_expression::types::NumberDataType; use fastrace::func_name; use crate::common; diff --git a/src/meta/proto-conv/tests/it/v152_external_udf.rs b/src/meta/proto-conv/tests/it/v152_external_udf.rs index 12dcb3b81384e..07fb7ad92bf9d 100644 --- a/src/meta/proto-conv/tests/it/v152_external_udf.rs +++ b/src/meta/proto-conv/tests/it/v152_external_udf.rs @@ -16,8 +16,8 @@ use std::collections::BTreeMap; use chrono::DateTime; use chrono::Utc; -use databend_common_expression::types::DataType; use databend_common_expression::TableDataType; +use databend_common_expression::types::DataType; use databend_common_meta_app::principal::UDFDefinition; use databend_common_meta_app::principal::UDFServer; use databend_common_meta_app::principal::UserDefinedFunction; diff --git a/src/meta/protos/build.rs b/src/meta/protos/build.rs index 51213b9d52f97..681a13d5ac64b 100644 --- a/src/meta/protos/build.rs +++ b/src/meta/protos/build.rs @@ -72,11 +72,9 @@ fn build_proto() -> Result<()> { // Version before 3.12 doesn't support allow_proto3_optional if version < Version::new(3, 12, 0) { - return Err(Error::other( - format!( - "protoc version is outdated, expect: >= 3.12.0, actual: {version}, reason: need feature --experimental_allow_proto3_optional" - ), - )); + return Err(Error::other(format!( + "protoc version is outdated, expect: >= 3.12.0, actual: {version}, reason: need feature --experimental_allow_proto3_optional" + ))); } // allow_proto3_optional has been enabled by default since 3.15.0 if version < Version::new(3, 15, 0) { diff --git a/src/meta/raft-store/src/applier/applier_data/mod.rs b/src/meta/raft-store/src/applier/applier_data/mod.rs index 4567e54798ad8..c63924aff44de 100644 --- a/src/meta/raft-store/src/applier/applier_data/mod.rs +++ b/src/meta/raft-store/src/applier/applier_data/mod.rs @@ -17,8 +17,8 @@ use std::sync::Mutex; use std::time::Duration; use crate::leveled_store::view::StateMachineView; -use crate::sm_v003::writer_acquirer::WriterPermit; use crate::sm_v003::OnChange; +use crate::sm_v003::writer_acquirer::WriterPermit; pub(crate) struct ApplierData { /// Hold a unique permit to serialize all apply operations to the state machine. diff --git a/src/meta/raft-store/src/applier/mod.rs b/src/meta/raft-store/src/applier/mod.rs index 7efdbbb2a29b7..dcfa6adc42bcb 100644 --- a/src/meta/raft-store/src/applier/mod.rs +++ b/src/meta/raft-store/src/applier/mod.rs @@ -21,17 +21,6 @@ use std::future::ready; use std::io; use std::time::Duration; -use databend_common_meta_types::node::Node; -use databend_common_meta_types::protobuf as pb; -use databend_common_meta_types::protobuf::boolean_expression::CombiningOperator; -use databend_common_meta_types::protobuf::BooleanExpression; -use databend_common_meta_types::protobuf::FetchIncreaseU64; -use databend_common_meta_types::raft_types::Entry; -use databend_common_meta_types::raft_types::EntryPayload; -use databend_common_meta_types::raft_types::StoredMembership; -use databend_common_meta_types::sys_data::SysData; -use databend_common_meta_types::txn_condition::Target; -use databend_common_meta_types::txn_op::Request; use databend_common_meta_types::AppliedState; use databend_common_meta_types::Change; use databend_common_meta_types::Cmd; @@ -54,11 +43,22 @@ use databend_common_meta_types::TxnReply; use databend_common_meta_types::TxnRequest; use databend_common_meta_types::UpsertKV; use databend_common_meta_types::With; +use databend_common_meta_types::node::Node; +use databend_common_meta_types::protobuf as pb; +use databend_common_meta_types::protobuf::BooleanExpression; +use databend_common_meta_types::protobuf::FetchIncreaseU64; +use databend_common_meta_types::protobuf::boolean_expression::CombiningOperator; +use databend_common_meta_types::raft_types::Entry; +use databend_common_meta_types::raft_types::EntryPayload; +use databend_common_meta_types::raft_types::StoredMembership; +use databend_common_meta_types::sys_data::SysData; +use databend_common_meta_types::txn_condition::Target; +use databend_common_meta_types::txn_op::Request; use display_more::DisplayUnixTimeStampExt; use fastrace::func_name; use futures::stream::TryStreamExt; -use futures_util::future::BoxFuture; use futures_util::StreamExt; +use futures_util::future::BoxFuture; use log::debug; use log::error; use log::info; @@ -125,18 +125,18 @@ where SM: StateMachineApi + 'static *sys_data.last_applied_mut() = Some(*log_id); }); - let applied_state = match entry.payload { + let applied_state = match &entry.payload { EntryPayload::Blank => { info!("apply: blank: {}", log_id); AppliedState::None } - EntryPayload::Normal(ref data) => { + EntryPayload::Normal(data) => { info!("apply: normal: {} {}", log_id, data); self.apply_cmd(&data.cmd).await? } - EntryPayload::Membership(ref mem) => { + EntryPayload::Membership(mem) => { info!("apply: membership: {} {:?}", log_id, mem); let membership = StoredMembership::new(Some(*log_id), mem.clone()); @@ -217,11 +217,11 @@ where SM: StateMachineApi + 'static res } - Cmd::RemoveNode { ref node_id } => self.apply_remove_node(node_id), + Cmd::RemoveNode { node_id } => self.apply_remove_node(node_id), Cmd::SetFeature { feature, enable } => self.apply_set_feature(feature, *enable), - Cmd::UpsertKV(ref upsert_kv) => self.apply_upsert_kv(upsert_kv).await?, + Cmd::UpsertKV(upsert_kv) => self.apply_upsert_kv(upsert_kv).await?, Cmd::Transaction(txn) => self.apply_txn(txn).await?, }; @@ -735,7 +735,7 @@ where SM: StateMachineApi + 'static let to_clean = { let _timer = self.cmd_ctx.start_io_timer("expire_scan", "expired_keys"); let mut to_clean = vec![]; - let mut strm = self.sm.list_expire_index(log_time_ms).await?; + let strm = self.sm.list_expire_index(log_time_ms).await?; // Save the log time for next cleaning. // Avoid listing tombstone records. diff --git a/src/meta/raft-store/src/config.rs b/src/meta/raft-store/src/config.rs index e77925406b9cb..1a47acc1bcf4f 100644 --- a/src/meta/raft-store/src/config.rs +++ b/src/meta/raft-store/src/config.rs @@ -19,9 +19,9 @@ use std::path::Path; use databend_common_exception::Result; use databend_common_grpc::DNSResolver; -use databend_common_meta_types::raft_types::NodeId; use databend_common_meta_types::Endpoint; use databend_common_meta_types::MetaStartupError; +use databend_common_meta_types::raft_types::NodeId; use crate::ondisk::DATA_VERSION; use crate::raft_log_v004; diff --git a/src/meta/raft-store/src/immutable_compactor/mod.rs b/src/meta/raft-store/src/immutable_compactor/mod.rs index 90507bd40f3f4..1c08c567f29f5 100644 --- a/src/meta/raft-store/src/immutable_compactor/mod.rs +++ b/src/meta/raft-store/src/immutable_compactor/mod.rs @@ -20,9 +20,9 @@ use log::info; use seq_marked::InternalSeq; use crate::leveled_store::leveled_map::LeveledMap; +use crate::sm_v003::SMV003; use crate::sm_v003::compactor_acquirer::CompactorPermit; use crate::sm_v003::writer_acquirer::WriterPermit; -use crate::sm_v003::SMV003; /// Compact `ImmutableLevels` to reduce the number of levels. pub struct InMemoryCompactor { diff --git a/src/meta/raft-store/src/key_spaces.rs b/src/meta/raft-store/src/key_spaces.rs index 0f081358186de..b6a990a1f7584 100644 --- a/src/meta/raft-store/src/key_spaces.rs +++ b/src/meta/raft-store/src/key_spaces.rs @@ -26,19 +26,19 @@ //! //! V003 uses sled storage (legacy), V004 uses separate raft log with leveled state machine. -use databend_common_meta_sled_store::sled; use databend_common_meta_sled_store::SledKeySpace; use databend_common_meta_sled_store::SledOrderedSerde; use databend_common_meta_sled_store::SledSerde; +use databend_common_meta_sled_store::sled; use databend_common_meta_stoerr::MetaStorageError; +use databend_common_meta_types::SeqNum; +use databend_common_meta_types::SeqV; use databend_common_meta_types::node::Node; use databend_common_meta_types::raft_types::Entry; use databend_common_meta_types::raft_types::LogId; use databend_common_meta_types::raft_types::LogIndex; use databend_common_meta_types::raft_types::NodeId; use databend_common_meta_types::raft_types::Vote; -use databend_common_meta_types::SeqNum; -use databend_common_meta_types::SeqV; use serde::Deserialize; use serde::Serialize; use state_machine_api::ExpireKey; diff --git a/src/meta/raft-store/src/leveled_store/db_builder.rs b/src/meta/raft-store/src/leveled_store/db_builder.rs index cf1d9be707761..2f32146446d71 100644 --- a/src/meta/raft-store/src/leveled_store/db_builder.rs +++ b/src/meta/raft-store/src/leveled_store/db_builder.rs @@ -28,9 +28,9 @@ use rotbl::v001::RotblMeta; use rotbl::v001::SeqMarked; use crate::leveled_store::leveled_map::LeveledMap; -use crate::sm_v003::open_snapshot::OpenSnapshot; #[cfg(doc)] use crate::sm_v003::SnapshotStoreV004; +use crate::sm_v003::open_snapshot::OpenSnapshot; use crate::snapshot_config::SnapshotConfig; use crate::state_machine::MetaSnapshotId; @@ -67,7 +67,7 @@ impl DBBuilder { #[allow(dead_code)] pub async fn append_kv_stream( &mut self, - mut strm: impl Stream>, + strm: impl Stream>, ) -> Result<(), io::Error> { let mut strm = std::pin::pin!(strm); while let Some((k, v)) = strm.try_next().await? { diff --git a/src/meta/raft-store/src/leveled_store/db_exporter.rs b/src/meta/raft-store/src/leveled_store/db_exporter.rs index fff07b5dfed9c..cc83c1d4b33ee 100644 --- a/src/meta/raft-store/src/leveled_store/db_exporter.rs +++ b/src/meta/raft-store/src/leveled_store/db_exporter.rs @@ -17,14 +17,14 @@ use std::future; use std::io; -use databend_common_meta_types::snapshot_db::DB; use databend_common_meta_types::SeqNum; use databend_common_meta_types::SeqV; +use databend_common_meta_types::snapshot_db::DB; use futures_util::StreamExt; use futures_util::TryStreamExt; use log::info; -use map_api::mvcc::ScopedSeqBoundedRange; use map_api::IOResultStream; +use map_api::mvcc::ScopedSeqBoundedRange; use state_machine_api::ExpireKey; use state_machine_api::ExpireValue; use state_machine_api::UserKey; diff --git a/src/meta/raft-store/src/leveled_store/db_impl_scoped_seq_bounded_read.rs b/src/meta/raft-store/src/leveled_store/db_impl_scoped_seq_bounded_read.rs index 817736fc0bdd3..2d135e2fb2763 100644 --- a/src/meta/raft-store/src/leveled_store/db_impl_scoped_seq_bounded_read.rs +++ b/src/meta/raft-store/src/leveled_store/db_impl_scoped_seq_bounded_read.rs @@ -17,10 +17,10 @@ use std::ops::RangeBounds; use databend_common_meta_types::snapshot_db::DB; use futures_util::StreamExt; +use map_api::IOResultStream; use map_api::mvcc; use map_api::mvcc::ViewKey; use map_api::mvcc::ViewValue; -use map_api::IOResultStream; use rotbl::v001::SeqMarked; use crate::leveled_store::map_api::MapKey; diff --git a/src/meta/raft-store/src/leveled_store/immutable/compact.rs b/src/meta/raft-store/src/leveled_store/immutable/compact.rs index b9ab38b497005..6075a2250c2d9 100644 --- a/src/meta/raft-store/src/leveled_store/immutable/compact.rs +++ b/src/meta/raft-store/src/leveled_store/immutable/compact.rs @@ -124,7 +124,7 @@ mod tests { let mut want_level = Level::default(); want_level.with_sys_data(|s| s.update_seq(200)); // Higher seq wins - // Insert in sequence order: 10, 15, 20, 30 + // Insert in sequence order: 10, 15, 20, 30 want_level.kv.insert(k1(), 10, v1()).unwrap(); want_level.kv.insert(k2(), 15, v3()).unwrap(); want_level.kv.insert(k1(), 20, v2()).unwrap(); @@ -140,7 +140,7 @@ mod tests { let mut want_level = Level::default(); want_level.with_sys_data(|s| s.update_seq(200)); // Higher seq wins - // Based on actual compact behavior: seq 10 cleaned up, seq 20 preserved, seq 30 preserved + // Based on actual compact behavior: seq 10 cleaned up, seq 20 preserved, seq 30 preserved want_level.kv.insert(k2(), 15, v3()).unwrap(); // k2 seq 15 preserved want_level.kv.insert(k1(), 20, v2()).unwrap(); // k1 seq 20 preserved want_level.kv.insert_tombstone(k1(), 30).unwrap(); // k1 tombstone at seq 30 @@ -166,7 +166,7 @@ mod tests { let mut want_level = Level::default(); want_level.with_sys_data(|s| s.update_seq(200)); // Higher seq wins - // Insert in sequence order: 10, 15, 20, 30 (same result as Test 1) + // Insert in sequence order: 10, 15, 20, 30 (same result as Test 1) want_level.kv.insert(k1(), 10, v1()).unwrap(); want_level.kv.insert(k2(), 15, v3()).unwrap(); want_level.kv.insert(k1(), 20, v2()).unwrap(); diff --git a/src/meta/raft-store/src/leveled_store/immutable/mod.rs b/src/meta/raft-store/src/leveled_store/immutable/mod.rs index aabd61ad010b0..34730f9d6b31e 100644 --- a/src/meta/raft-store/src/leveled_store/immutable/mod.rs +++ b/src/meta/raft-store/src/leveled_store/immutable/mod.rs @@ -16,15 +16,15 @@ use std::io; use std::io::Error; use std::ops::Deref; use std::ops::RangeBounds; +use std::sync::Arc; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; -use std::sync::Arc; +use map_api::IOResultStream; use map_api::mvcc; use map_api::mvcc::ScopedSeqBoundedIntoRange; use map_api::mvcc::ViewKey; use map_api::mvcc::ViewValue; -use map_api::IOResultStream; use seq_marked::SeqMarked; use state_machine_api::ExpireKey; use state_machine_api::MetaValue; diff --git a/src/meta/raft-store/src/leveled_store/immutable_data/compact_into_stream.rs b/src/meta/raft-store/src/leveled_store/immutable_data/compact_into_stream.rs index 99596c8b72e91..3692135b1ebff 100644 --- a/src/meta/raft-store/src/leveled_store/immutable_data/compact_into_stream.rs +++ b/src/meta/raft-store/src/leveled_store/immutable_data/compact_into_stream.rs @@ -16,12 +16,12 @@ use std::fmt; use std::io; use databend_common_meta_types::sys_data::SysData; -use futures_util::future; use futures_util::StreamExt; use futures_util::TryStreamExt; -use map_api::mvcc::ScopedSeqBoundedRange; +use futures_util::future; use map_api::IOResultStream; use map_api::MapKV; +use map_api::mvcc::ScopedSeqBoundedRange; use seq_marked::SeqMarked; use state_machine_api::ExpireKey; use state_machine_api::UserKey; diff --git a/src/meta/raft-store/src/leveled_store/immutable_data/mod.rs b/src/meta/raft-store/src/leveled_store/immutable_data/mod.rs index 7f8b77df35d3b..d1e2cac429a18 100644 --- a/src/meta/raft-store/src/leveled_store/immutable_data/mod.rs +++ b/src/meta/raft-store/src/leveled_store/immutable_data/mod.rs @@ -22,17 +22,18 @@ use databend_common_meta_types::snapshot_db::DB; use display_more::DisplayOptionExt; use display_more::DisplaySliceExt; use futures_util::StreamExt; +use map_api::IOResultStream; +use map_api::MapKey; use map_api::mvcc; use map_api::mvcc::ViewKey; use map_api::mvcc::ViewValue; use map_api::util; -use map_api::IOResultStream; -use map_api::MapKey; use seq_marked::InternalSeq; use seq_marked::SeqMarked; use stream_more::KMerge; use stream_more::StreamMore; +use crate::leveled_store::ScopedSeqBoundedRead; use crate::leveled_store::immutable::Immutable; use crate::leveled_store::immutable_levels::ImmutableLevels; use crate::leveled_store::level::LevelStat; @@ -40,7 +41,6 @@ use crate::leveled_store::level_index::LevelIndex; use crate::leveled_store::map_api::MapKeyDecode; use crate::leveled_store::map_api::MapKeyEncode; use crate::leveled_store::persisted_codec::PersistedCodec; -use crate::leveled_store::ScopedSeqBoundedRead; mod compact_into_stream; diff --git a/src/meta/raft-store/src/leveled_store/immutable_levels/mod.rs b/src/meta/raft-store/src/leveled_store/immutable_levels/mod.rs index 6a4d3d70e0cad..30fb0f3bb317e 100644 --- a/src/meta/raft-store/src/leveled_store/immutable_levels/mod.rs +++ b/src/meta/raft-store/src/leveled_store/immutable_levels/mod.rs @@ -17,12 +17,12 @@ use std::io::Error; use std::ops::RangeBounds; use futures_util::StreamExt; +use map_api::IOResultStream; use map_api::mvcc; use map_api::mvcc::ScopedSeqBoundedIntoRange; use map_api::mvcc::ViewKey; use map_api::mvcc::ViewValue; use map_api::util; -use map_api::IOResultStream; use seq_marked::SeqMarked; use stream_more::KMerge; use stream_more::StreamMore; diff --git a/src/meta/raft-store/src/leveled_store/level/mod.rs b/src/meta/raft-store/src/leveled_store/level/mod.rs index b0b0402e530fd..f351ba7c3fd7a 100644 --- a/src/meta/raft-store/src/leveled_store/level/mod.rs +++ b/src/meta/raft-store/src/leveled_store/level/mod.rs @@ -16,17 +16,17 @@ use std::collections::BTreeMap; use std::io::Error; use std::ops::RangeBounds; +use databend_common_meta_types::Node; use databend_common_meta_types::raft_types::LogId; use databend_common_meta_types::raft_types::NodeId; use databend_common_meta_types::raft_types::StoredMembership; use databend_common_meta_types::sys_data::SysData; -use databend_common_meta_types::Node; use futures_util::StreamExt; +use map_api::IOResultStream; use map_api::mvcc; use map_api::mvcc::ScopedSeqBoundedRangeIter; use map_api::mvcc::ViewKey; use map_api::mvcc::ViewValue; -use map_api::IOResultStream; use seq_marked::InternalSeq; use seq_marked::SeqMarked; use state_machine_api::ExpireKey; @@ -259,7 +259,7 @@ mod tests { let mut want = Level::default(); want.with_sys_data(|s| s.update_seq(200)); // Higher seq wins - // Insert in sequence order: 10, 15, 20, 30 + // Insert in sequence order: 10, 15, 20, 30 want.kv.insert(k1(), 10, v1()).unwrap(); want.kv.insert(k2(), 15, v3()).unwrap(); want.kv.insert(k1(), 20, v2()).unwrap(); @@ -302,7 +302,7 @@ mod tests { let mut want = Level::default(); want.with_sys_data(|s| s.update_seq(200)); // Same higher seq - // Insert in sequence order: 10, 15, 20, 30 + // Insert in sequence order: 10, 15, 20, 30 want.kv.insert(k1(), 10, v1()).unwrap(); want.kv.insert(k2(), 15, v3()).unwrap(); want.kv.insert(k1(), 20, v2()).unwrap(); @@ -349,7 +349,7 @@ mod tests { let mut want = Level::default(); want.with_sys_data(|s| s.update_seq(200)); // Higher seq wins - // Insert in sequence order: 10, 15, 20, 30 + // Insert in sequence order: 10, 15, 20, 30 want.expire.insert(ek1(), 10, ev1()).unwrap(); want.expire.insert(ek2(), 15, ev3()).unwrap(); want.expire.insert(ek1(), 20, ev2()).unwrap(); @@ -391,7 +391,7 @@ mod tests { let mut want = Level::default(); want.with_sys_data(|s| s.update_seq(200)); // Same higher seq - // Insert in sequence order: 10, 15, 20, 30 + // Insert in sequence order: 10, 15, 20, 30 want.expire.insert(ek1(), 10, ev1()).unwrap(); want.expire.insert(ek2(), 15, ev3()).unwrap(); want.expire.insert(ek1(), 20, ev2()).unwrap(); diff --git a/src/meta/raft-store/src/leveled_store/leveled_map/impl_commit.rs b/src/meta/raft-store/src/leveled_store/leveled_map/impl_commit.rs index d22678c74ae33..64f331fed6d76 100644 --- a/src/meta/raft-store/src/leveled_store/leveled_map/impl_commit.rs +++ b/src/meta/raft-store/src/leveled_store/leveled_map/impl_commit.rs @@ -77,9 +77,9 @@ mod tests { use std::cmp::Reverse; use std::collections::BTreeMap; + use map_api::SeqMarked; use map_api::mvcc::Commit; use map_api::mvcc::Table; - use map_api::SeqMarked; use seq_marked::InternalSeq; use state_machine_api::ExpireKey; use state_machine_api::KVMeta; diff --git a/src/meta/raft-store/src/leveled_store/leveled_map/impl_scoped_seq_bounded_get.rs b/src/meta/raft-store/src/leveled_store/leveled_map/impl_scoped_seq_bounded_get.rs index 062d2dc7ebc11..4144cfcbb7f0f 100644 --- a/src/meta/raft-store/src/leveled_store/leveled_map/impl_scoped_seq_bounded_get.rs +++ b/src/meta/raft-store/src/leveled_store/leveled_map/impl_scoped_seq_bounded_get.rs @@ -14,10 +14,10 @@ use std::io; +use map_api::MapKey; use map_api::mvcc; use map_api::mvcc::ViewKey; use map_api::mvcc::ViewValue; -use map_api::MapKey; use seq_marked::SeqMarked; use crate::leveled_store::get_sub_table::GetSubTable; diff --git a/src/meta/raft-store/src/leveled_store/leveled_map/impl_scoped_seq_bounded_range.rs b/src/meta/raft-store/src/leveled_store/leveled_map/impl_scoped_seq_bounded_range.rs index 6a82f61fa4f01..e7fee5a40ae02 100644 --- a/src/meta/raft-store/src/leveled_store/leveled_map/impl_scoped_seq_bounded_range.rs +++ b/src/meta/raft-store/src/leveled_store/leveled_map/impl_scoped_seq_bounded_range.rs @@ -19,12 +19,12 @@ use std::time::Instant; use futures_util::StreamExt; use log::debug; use log::warn; +use map_api::IOResultStream; +use map_api::MapKey; use map_api::mvcc; use map_api::mvcc::ViewKey; use map_api::mvcc::ViewValue; use map_api::util; -use map_api::IOResultStream; -use map_api::MapKey; use seq_marked::SeqMarked; use stream_more::KMerge; use stream_more::StreamMore; diff --git a/src/meta/raft-store/src/leveled_store/leveled_map/impl_seq_bounded_range.rs b/src/meta/raft-store/src/leveled_store/leveled_map/impl_seq_bounded_range.rs index 728897ad22865..cd3466677296c 100644 --- a/src/meta/raft-store/src/leveled_store/leveled_map/impl_seq_bounded_range.rs +++ b/src/meta/raft-store/src/leveled_store/leveled_map/impl_seq_bounded_range.rs @@ -17,8 +17,8 @@ use std::ops::RangeBounds; use futures_util::StreamExt; use futures_util::TryStreamExt; -use map_api::mvcc; use map_api::IOResultStream; +use map_api::mvcc; use seq_marked::SeqMarked; use crate::leveled_store::leveled_map::LeveledMap; diff --git a/src/meta/raft-store/src/leveled_store/leveled_map/leveled_map_test.rs b/src/meta/raft-store/src/leveled_store/leveled_map/leveled_map_test.rs index 8f7dcfc55ea0a..be752b5c3ee2f 100644 --- a/src/meta/raft-store/src/leveled_store/leveled_map/leveled_map_test.rs +++ b/src/meta/raft-store/src/leveled_store/leveled_map/leveled_map_test.rs @@ -13,12 +13,12 @@ // limitations under the License. use futures_util::TryStreamExt; +use map_api::SeqMarked; use map_api::mvcc::ScopedGet; use map_api::mvcc::ScopedRange; use map_api::mvcc::ScopedSeqBoundedGet; use map_api::mvcc::ScopedSeqBoundedRange; use map_api::mvcc::ScopedSet; -use map_api::SeqMarked; use state_machine_api::KVMeta; use state_machine_api::UserKey; diff --git a/src/meta/raft-store/src/leveled_store/leveled_map/mod.rs b/src/meta/raft-store/src/leveled_store/leveled_map/mod.rs index cafcaed15a347..45ce564cc8c69 100644 --- a/src/meta/raft-store/src/leveled_store/leveled_map/mod.rs +++ b/src/meta/raft-store/src/leveled_store/leveled_map/mod.rs @@ -17,12 +17,12 @@ use std::sync::Arc; use std::sync::Mutex; use compactor::Compactor; +use databend_common_meta_types::Node; use databend_common_meta_types::raft_types::LogId; use databend_common_meta_types::raft_types::NodeId; use databend_common_meta_types::raft_types::StoredMembership; use databend_common_meta_types::snapshot_db::DB; use databend_common_meta_types::sys_data::SysData; -use databend_common_meta_types::Node; use display_more::DisplayOptionExt; use display_more::DisplaySliceExt; use log::info; diff --git a/src/meta/raft-store/src/leveled_store/map_api.rs b/src/meta/raft-store/src/leveled_store/map_api.rs index 52d990a0a1131..86162b6bf3619 100644 --- a/src/meta/raft-store/src/leveled_store/map_api.rs +++ b/src/meta/raft-store/src/leveled_store/map_api.rs @@ -16,11 +16,11 @@ use std::fmt; use std::fmt::Write; use std::io; +pub use map_api::BeforeAfter; +pub use map_api::IOResultStream; pub use map_api::map_key::MapKey; pub use map_api::map_value::MapValue; use map_api::mvcc; -pub use map_api::BeforeAfter; -pub use map_api::IOResultStream; use seq_marked::SeqMarked; use state_machine_api::KVMeta; use state_machine_api::MetaValue; diff --git a/src/meta/raft-store/src/leveled_store/rotbl_codec.rs b/src/meta/raft-store/src/leveled_store/rotbl_codec.rs index 7705142458401..542372937b45c 100644 --- a/src/meta/raft-store/src/leveled_store/rotbl_codec.rs +++ b/src/meta/raft-store/src/leveled_store/rotbl_codec.rs @@ -128,10 +128,10 @@ impl RotblCodec { #[cfg(test)] mod tests { - use state_machine_api::ExpireKey; - use state_machine_api::UserKey; use Bound::Excluded; use Bound::Included; + use state_machine_api::ExpireKey; + use state_machine_api::UserKey; use super::*; @@ -184,10 +184,10 @@ mod tests { assert!(RotblCodec::decode_key::("exp").is_err()); assert!(RotblCodec::decode_key::("exp-").is_err()); assert!(RotblCodec::decode_key::("exp-/").is_err()); - assert!(RotblCodec::decode_key::( - "exp-/00000000000000000012/00000000000000000034/" - ) - .is_err()); + assert!( + RotblCodec::decode_key::("exp-/00000000000000000012/00000000000000000034/") + .is_err() + ); } #[test] diff --git a/src/meta/raft-store/src/leveled_store/snapshot.rs b/src/meta/raft-store/src/leveled_store/snapshot.rs index 43ebafe26dc8b..0897c82188422 100644 --- a/src/meta/raft-store/src/leveled_store/snapshot.rs +++ b/src/meta/raft-store/src/leveled_store/snapshot.rs @@ -19,8 +19,8 @@ use std::ops::RangeBounds; use futures_util::StreamExt; use futures_util::TryStreamExt; -use map_api::mvcc; use map_api::IOResultStream; +use map_api::mvcc; use seq_marked::SeqMarked; use state_machine_api::ExpireKey; use state_machine_api::MetaValue; diff --git a/src/meta/raft-store/src/leveled_store/testing_data/mod.rs b/src/meta/raft-store/src/leveled_store/testing_data/mod.rs index a3ec2034d4cdf..bd16743ea6a2d 100644 --- a/src/meta/raft-store/src/leveled_store/testing_data/mod.rs +++ b/src/meta/raft-store/src/leveled_store/testing_data/mod.rs @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_meta_types::raft_types::Membership; -use databend_common_meta_types::raft_types::StoredMembership; use databend_common_meta_types::Endpoint; use databend_common_meta_types::Node; use databend_common_meta_types::UpsertKV; +use databend_common_meta_types::raft_types::Membership; +use databend_common_meta_types::raft_types::StoredMembership; use map_api::mvcc::ScopedSet; use maplit::btreemap; use openraft::testing::log_id; diff --git a/src/meta/raft-store/src/leveled_store/view.rs b/src/meta/raft-store/src/leveled_store/view.rs index 67aa77f573f0b..c11bed56a697e 100644 --- a/src/meta/raft-store/src/leveled_store/view.rs +++ b/src/meta/raft-store/src/leveled_store/view.rs @@ -19,8 +19,8 @@ use std::ops::RangeBounds; use futures_util::StreamExt; use futures_util::TryStreamExt; -use map_api::mvcc; use map_api::IOResultStream; +use map_api::mvcc; use seq_marked::SeqMarked; use state_machine_api::ExpireKey; use state_machine_api::MetaValue; diff --git a/src/meta/raft-store/src/lib.rs b/src/meta/raft-store/src/lib.rs index 83a661dc67c7e..bc97776f2339b 100644 --- a/src/meta/raft-store/src/lib.rs +++ b/src/meta/raft-store/src/lib.rs @@ -38,6 +38,9 @@ #![feature(impl_trait_in_assoc_type)] #![feature(try_blocks)] #![allow(clippy::diverging_sub_expression)] +#![allow(clippy::let_and_return)] +#![allow(clippy::manual_is_multiple_of)] +#![allow(clippy::collapsible_if)] pub mod applier; pub mod config; diff --git a/src/meta/raft-store/src/ondisk/data_version.rs b/src/meta/raft-store/src/ondisk/data_version.rs index f8e068e6efc9f..fcdd35dd39645 100644 --- a/src/meta/raft-store/src/ondisk/data_version.rs +++ b/src/meta/raft-store/src/ondisk/data_version.rs @@ -14,8 +14,8 @@ use std::fmt; -use crate::ondisk::version_info::VersionInfo; use crate::ondisk::version_info::VERSION_INFOS; +use crate::ondisk::version_info::VersionInfo; /// Available data versions this program can work upon. /// diff --git a/src/meta/raft-store/src/ondisk/header.rs b/src/meta/raft-store/src/ondisk/header.rs index 545c010adc4b1..3b533c04aaf57 100644 --- a/src/meta/raft-store/src/ondisk/header.rs +++ b/src/meta/raft-store/src/ondisk/header.rs @@ -14,12 +14,12 @@ use std::fmt; -use databend_common_meta_sled_store::sled; use databend_common_meta_sled_store::SledBytesError; use databend_common_meta_sled_store::SledSerde; +use databend_common_meta_sled_store::sled; -use crate::ondisk::DataVersion; use crate::ondisk::DATA_VERSION; +use crate::ondisk::DataVersion; #[derive( Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Serialize, serde::Deserialize, diff --git a/src/meta/raft-store/src/ondisk/mod.rs b/src/meta/raft-store/src/ondisk/mod.rs index 07fc68c97b7c3..d44e5511f670b 100644 --- a/src/meta/raft-store/src/ondisk/mod.rs +++ b/src/meta/raft-store/src/ondisk/mod.rs @@ -27,8 +27,8 @@ use std::path::Path; use std::path::PathBuf; pub use data_version::DataVersion; -use databend_common_meta_sled_store::init_get_sled_db; use databend_common_meta_sled_store::SledTree; +use databend_common_meta_sled_store::init_get_sled_db; use databend_common_meta_stoerr::MetaStorageError; pub use header::Header; use log::info; diff --git a/src/meta/raft-store/src/ondisk/upgrade_to_v004.rs b/src/meta/raft-store/src/ondisk/upgrade_to_v004.rs index 6ee56b5ce3710..b0f3bd1052610 100644 --- a/src/meta/raft-store/src/ondisk/upgrade_to_v004.rs +++ b/src/meta/raft-store/src/ondisk/upgrade_to_v004.rs @@ -18,9 +18,9 @@ use std::fs; use std::path::Path; use std::sync::Arc; +use databend_common_meta_sled_store::SledTree; use databend_common_meta_sled_store::drop_sled_db; use databend_common_meta_sled_store::init_get_sled_db; -use databend_common_meta_sled_store::SledTree; use databend_common_meta_stoerr::MetaStorageError; use fs_extra::dir::CopyOptions; use log::debug; @@ -32,8 +32,8 @@ use crate::key_spaces::LogMeta; use crate::key_spaces::RaftStoreEntry; use crate::ondisk::DataVersion; use crate::ondisk::OnDisk; -use crate::raft_log_v004::importer; use crate::raft_log_v004::RaftLogV004; +use crate::raft_log_v004::importer; use crate::sm_v003::SnapshotStoreV003; use crate::sm_v003::SnapshotStoreV004; use crate::state_machine::LogMetaKey; diff --git a/src/meta/raft-store/src/raft_log_v004/callback.rs b/src/meta/raft-store/src/raft_log_v004/callback.rs index 7e510d434234d..9c2b80e236d13 100644 --- a/src/meta/raft-store/src/raft_log_v004/callback.rs +++ b/src/meta/raft-store/src/raft_log_v004/callback.rs @@ -21,8 +21,8 @@ use log::info; use log::warn; use tokio::sync::oneshot; -use crate::raft_log_v004::callback_data::CallbackData; use crate::raft_log_v004::IODesc; +use crate::raft_log_v004::callback_data::CallbackData; /// The callback to be called when the IO is completed. /// diff --git a/src/meta/raft-store/src/raft_log_v004/codec_wrapper.rs b/src/meta/raft-store/src/raft_log_v004/codec_wrapper.rs index d8b41a1a99d26..a674bddead959 100644 --- a/src/meta/raft-store/src/raft_log_v004/codec_wrapper.rs +++ b/src/meta/raft-store/src/raft_log_v004/codec_wrapper.rs @@ -99,8 +99,8 @@ impl Cw { mod tests { use std::io; - use databend_common_meta_types::raft_types::new_log_id; use databend_common_meta_types::raft_types::LogId; + use databend_common_meta_types::raft_types::new_log_id; use raft_log::codeq::Decode; use raft_log::codeq::Encode; diff --git a/src/meta/raft-store/src/raft_log_v004/importer.rs b/src/meta/raft-store/src/raft_log_v004/importer.rs index e9962b0d01a4f..9b19ad4ddad95 100644 --- a/src/meta/raft-store/src/raft_log_v004/importer.rs +++ b/src/meta/raft-store/src/raft_log_v004/importer.rs @@ -18,10 +18,10 @@ use databend_common_meta_types::raft_types::LogId; use raft_log::api::raft_log_writer::RaftLogWriter; use crate::key_spaces::RaftStoreEntry; +use crate::raft_log_v004::RaftLogV004; use crate::raft_log_v004::codec_wrapper::Cw; use crate::raft_log_v004::log_store_meta::LogStoreMeta; use crate::raft_log_v004::util; -use crate::raft_log_v004::RaftLogV004; /// Import series of [`RaftStoreEntry`] record into [`RaftLogV004`]. /// diff --git a/src/meta/raft-store/src/raft_log_v004/util.rs b/src/meta/raft-store/src/raft_log_v004/util.rs index 3f487a62872fb..c9833c4465df9 100644 --- a/src/meta/raft-store/src/raft_log_v004/util.rs +++ b/src/meta/raft-store/src/raft_log_v004/util.rs @@ -17,9 +17,9 @@ use std::io; use raft_log::api::raft_log_writer::RaftLogWriter; use tokio::sync::oneshot; -use crate::raft_log_v004::callback::Callback; use crate::raft_log_v004::IODesc; use crate::raft_log_v004::RaftLogV004; +use crate::raft_log_v004::callback::Callback; pub async fn blocking_flush(rl: &mut RaftLogV004) -> Result<(), io::Error> { let (tx, rx) = oneshot::channel(); diff --git a/src/meta/raft-store/src/sm_v003/compact_with_db_test.rs b/src/meta/raft-store/src/sm_v003/compact_with_db_test.rs index 1e6b3b7a1adfd..cbfdc9248d5d3 100644 --- a/src/meta/raft-store/src/sm_v003/compact_with_db_test.rs +++ b/src/meta/raft-store/src/sm_v003/compact_with_db_test.rs @@ -15,11 +15,11 @@ use std::io; use std::sync::Arc; +use databend_common_meta_types::Endpoint; +use databend_common_meta_types::UpsertKV; use databend_common_meta_types::node::Node; use databend_common_meta_types::raft_types::Membership; use databend_common_meta_types::raft_types::StoredMembership; -use databend_common_meta_types::Endpoint; -use databend_common_meta_types::UpsertKV; use futures_util::TryStreamExt; use map_api::mvcc; use map_api::mvcc::ScopedSeqBoundedGet; @@ -33,12 +33,12 @@ use state_machine_api::ExpireKey; use state_machine_api::KVMeta; use state_machine_api::UserKey; +use crate::leveled_store::ScopedSeqBoundedRead; use crate::leveled_store::db_builder::DBBuilder; use crate::leveled_store::immutable_data::ImmutableData; use crate::leveled_store::immutable_levels::ImmutableLevels; use crate::leveled_store::leveled_map::LeveledMap; use crate::leveled_store::sys_data_api::SysDataApiRO; -use crate::leveled_store::ScopedSeqBoundedRead; use crate::sm_v003::sm_v003::SMV003; #[tokio::test(flavor = "multi_thread", worker_threads = 3)] diff --git a/src/meta/raft-store/src/sm_v003/receiver_v003.rs b/src/meta/raft-store/src/sm_v003/receiver_v003.rs index ac153c84a13e3..7203eaeffc1fa 100644 --- a/src/meta/raft-store/src/sm_v003/receiver_v003.rs +++ b/src/meta/raft-store/src/sm_v003/receiver_v003.rs @@ -171,7 +171,13 @@ impl ReceiverV003 { info!( "snapshot from {} is completely received, format: {}, vote: {:?}, meta: {:?}, size: {}; path: {}/{}", - self.remote_addr, format, vote, snapshot_meta, self.size_received, self.storage_path, self.temp_rel_path + self.remote_addr, + format, + vote, + snapshot_meta, + self.size_received, + self.storage_path, + self.temp_rel_path ); if format != "rotbl::v001" { diff --git a/src/meta/raft-store/src/sm_v003/sm_v003.rs b/src/meta/raft-store/src/sm_v003/sm_v003.rs index 9e889b1cb45d8..e01d8a81afef3 100644 --- a/src/meta/raft-store/src/sm_v003/sm_v003.rs +++ b/src/meta/raft-store/src/sm_v003/sm_v003.rs @@ -20,11 +20,11 @@ use std::sync::Arc; use std::sync::Mutex; use std::time::Duration; +use databend_common_meta_types::AppliedState; use databend_common_meta_types::raft_types::Entry; use databend_common_meta_types::raft_types::StorageError; use databend_common_meta_types::snapshot_db::DB; use databend_common_meta_types::sys_data::SysData; -use databend_common_meta_types::AppliedState; use log::debug; use log::info; use map_api::mvcc::ScopedGet; @@ -34,12 +34,12 @@ use state_machine_api::StateMachineApi; use state_machine_api::UserKey; use tokio::sync::Semaphore; -use crate::applier::applier_data::ApplierData; use crate::applier::Applier; +use crate::applier::applier_data::ApplierData; use crate::leveled_store::immutable_data::ImmutableData; +use crate::leveled_store::leveled_map::LeveledMap; use crate::leveled_store::leveled_map::compactor::Compactor; use crate::leveled_store::leveled_map::leveled_map_data::LeveledMapData; -use crate::leveled_store::leveled_map::LeveledMap; use crate::leveled_store::snapshot::StateMachineSnapshot; use crate::leveled_store::view::StateMachineView; use crate::sm_v003::compactor_acquirer::CompactorAcquirer; @@ -161,7 +161,7 @@ impl SMV003 { self.leveled_map.to_state_machine_snapshot() } - pub fn kv_api(&self) -> SMV003KVApi { + pub fn kv_api(&self) -> SMV003KVApi<'_> { SMV003KVApi { sm: self } } diff --git a/src/meta/raft-store/src/sm_v003/sm_v003_kv_api.rs b/src/meta/raft-store/src/sm_v003/sm_v003_kv_api.rs index 16f221b57b1ff..2494110e088fe 100644 --- a/src/meta/raft-store/src/sm_v003/sm_v003_kv_api.rs +++ b/src/meta/raft-store/src/sm_v003/sm_v003_kv_api.rs @@ -17,12 +17,12 @@ use std::io; use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::KVStream; -use databend_common_meta_types::protobuf::StreamItem; use databend_common_meta_types::Change; use databend_common_meta_types::SeqV; use databend_common_meta_types::TxnReply; use databend_common_meta_types::TxnRequest; use databend_common_meta_types::UpsertKV; +use databend_common_meta_types::protobuf::StreamItem; use futures_util::StreamExt; use futures_util::TryStreamExt; use map_api::mvcc::ScopedGet; diff --git a/src/meta/raft-store/src/sm_v003/sm_v003_test.rs b/src/meta/raft-store/src/sm_v003/sm_v003_test.rs index 00e965047fab9..fca467d85ea97 100644 --- a/src/meta/raft-store/src/sm_v003/sm_v003_test.rs +++ b/src/meta/raft-store/src/sm_v003/sm_v003_test.rs @@ -13,10 +13,10 @@ // limitations under the License. use databend_common_meta_kvapi::kvapi::KVApi; -use databend_common_meta_types::normalize_meta::NormalizeMeta; use databend_common_meta_types::CmdContext; use databend_common_meta_types::SeqV; use databend_common_meta_types::UpsertKV; +use databend_common_meta_types::normalize_meta::NormalizeMeta; use futures_util::TryStreamExt; use map_api::mvcc::ScopedRange; use pretty_assertions::assert_eq; diff --git a/src/meta/raft-store/src/sm_v003/snapshot_loader.rs b/src/meta/raft-store/src/sm_v003/snapshot_loader.rs index 966e31066f86b..129392b11b472 100644 --- a/src/meta/raft-store/src/sm_v003/snapshot_loader.rs +++ b/src/meta/raft-store/src/sm_v003/snapshot_loader.rs @@ -22,8 +22,8 @@ use log::info; use log::warn; use openraft::SnapshotId; -use crate::sm_v003::open_snapshot::OpenSnapshot; use crate::sm_v003::SnapshotStoreError; +use crate::sm_v003::open_snapshot::OpenSnapshot; use crate::snapshot_config::SnapshotConfig; use crate::state_machine::MetaSnapshotId; diff --git a/src/meta/raft-store/src/sm_v003/snapshot_store_v003.rs b/src/meta/raft-store/src/sm_v003/snapshot_store_v003.rs index 7e357f339f0b6..9168ebe05156f 100644 --- a/src/meta/raft-store/src/sm_v003/snapshot_store_v003.rs +++ b/src/meta/raft-store/src/sm_v003/snapshot_store_v003.rs @@ -21,9 +21,9 @@ use databend_common_meta_types::snapshot_db::DB; use crate::config::RaftConfig; use crate::ondisk::DataVersion; +use crate::sm_v003::WriterV003; use crate::sm_v003::receiver_v003::ReceiverV003; use crate::sm_v003::snapshot_loader::SnapshotLoader; -use crate::sm_v003::WriterV003; use crate::snapshot_config::SnapshotConfig; #[derive(Debug)] diff --git a/src/meta/raft-store/src/state/raft_state_kv.rs b/src/meta/raft-store/src/state/raft_state_kv.rs index 3d55b1f0afa49..6767ca6c7cdc3 100644 --- a/src/meta/raft-store/src/state/raft_state_kv.rs +++ b/src/meta/raft-store/src/state/raft_state_kv.rs @@ -14,10 +14,10 @@ use std::fmt; -use databend_common_meta_sled_store::sled; use databend_common_meta_sled_store::SledBytesError; use databend_common_meta_sled_store::SledOrderedSerde; use databend_common_meta_sled_store::SledSerde; +use databend_common_meta_sled_store::sled; use databend_common_meta_types::anyerror::AnyError; use databend_common_meta_types::raft_types::LogId; use databend_common_meta_types::raft_types::NodeId; diff --git a/src/meta/raft-store/src/state_machine/log_meta.rs b/src/meta/raft-store/src/state_machine/log_meta.rs index 72257981fbd42..c5c1107f25565 100644 --- a/src/meta/raft-store/src/state_machine/log_meta.rs +++ b/src/meta/raft-store/src/state_machine/log_meta.rs @@ -14,10 +14,10 @@ use std::fmt; -use databend_common_meta_sled_store::sled; use databend_common_meta_sled_store::SledBytesError; use databend_common_meta_sled_store::SledOrderedSerde; use databend_common_meta_sled_store::SledSerde; +use databend_common_meta_sled_store::sled; use databend_common_meta_types::anyerror::AnyError; use databend_common_meta_types::raft_types::LogId; use serde::Deserialize; diff --git a/src/meta/raft-store/src/state_machine/snapshot_id.rs b/src/meta/raft-store/src/state_machine/snapshot_id.rs index c9c8ae2e332d4..fe0a1656d0f94 100644 --- a/src/meta/raft-store/src/state_machine/snapshot_id.rs +++ b/src/meta/raft-store/src/state_machine/snapshot_id.rs @@ -17,8 +17,8 @@ use std::str::FromStr; use std::time::SystemTime; use std::time::UNIX_EPOCH; -use databend_common_meta_types::raft_types::new_log_id; use databend_common_meta_types::raft_types::LogId; +use databend_common_meta_types::raft_types::new_log_id; /// Structured snapshot id used by meta service #[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord)] diff --git a/src/meta/raft-store/src/state_machine/state_machine_meta.rs b/src/meta/raft-store/src/state_machine/state_machine_meta.rs index 448da6a3775af..65042ec78c552 100644 --- a/src/meta/raft-store/src/state_machine/state_machine_meta.rs +++ b/src/meta/raft-store/src/state_machine/state_machine_meta.rs @@ -14,10 +14,10 @@ use std::fmt; -use databend_common_meta_sled_store::sled; use databend_common_meta_sled_store::SledBytesError; use databend_common_meta_sled_store::SledOrderedSerde; use databend_common_meta_sled_store::SledSerde; +use databend_common_meta_sled_store::sled; use databend_common_meta_types::anyerror::AnyError; use databend_common_meta_types::raft_types::LogId; use databend_common_meta_types::raft_types::StoredMembership; diff --git a/src/meta/raft-store/src/state_machine/testing.rs b/src/meta/raft-store/src/state_machine/testing.rs index a8568f6185930..a3b344679d232 100644 --- a/src/meta/raft-store/src/state_machine/testing.rs +++ b/src/meta/raft-store/src/state_machine/testing.rs @@ -13,15 +13,15 @@ // limitations under the License. use databend_common_meta_sled_store::openraft; -use databend_common_meta_types::raft_types::new_log_id; -use databend_common_meta_types::raft_types::Entry; -use databend_common_meta_types::raft_types::EntryPayload; use databend_common_meta_types::Cmd; use databend_common_meta_types::LogEntry; use databend_common_meta_types::UpsertKV; +use databend_common_meta_types::raft_types::Entry; +use databend_common_meta_types::raft_types::EntryPayload; +use databend_common_meta_types::raft_types::new_log_id; use maplit::btreeset; -use openraft::entry::RaftEntry; use openraft::Membership; +use openraft::entry::RaftEntry; /// Logs and the expected snapshot for testing snapshot. pub fn snapshot_logs() -> (Vec, Vec) { diff --git a/src/meta/raft-store/src/state_machine_api_ext.rs b/src/meta/raft-store/src/state_machine_api_ext.rs index de73f75ecb502..377efb2329e7e 100644 --- a/src/meta/raft-store/src/state_machine_api_ext.rs +++ b/src/meta/raft-store/src/state_machine_api_ext.rs @@ -22,22 +22,22 @@ use std::io; use std::ops::RangeBounds; use std::time::Duration; -use databend_common_meta_types::sys_data::SysData; use databend_common_meta_types::CmdContext; use databend_common_meta_types::Expirable; use databend_common_meta_types::MatchSeqExt; use databend_common_meta_types::Operation; use databend_common_meta_types::UpsertKV; +use databend_common_meta_types::sys_data::SysData; use display_more::DisplayUnixTimeStampExt; use futures_util::StreamExt; use futures_util::TryStreamExt; use log::debug; use log::info; use log::warn; +use map_api::IOResultStream; use map_api::mvcc::ScopedGet; use map_api::mvcc::ScopedRange; use map_api::mvcc::ScopedSet; -use map_api::IOResultStream; use seq_marked::SeqMarked; use seq_marked::SeqValue; use state_machine_api::ExpireKey; @@ -106,8 +106,7 @@ pub trait StateMachineApiExt: StateMachineApi { warn!( "upsert_kv_primary_index: expired key inserted: {} < timestamp in log entry: {}; key: {}", Duration::from_millis(expire_ms).display_unix_timestamp_short(), - Duration::from_millis(curr_time_ms) - .display_unix_timestamp_short(), + Duration::from_millis(curr_time_ms).display_unix_timestamp_short(), upsert_kv.key ); // The record has expired, delete it at once. diff --git a/src/meta/raft-store/tests/it/types.rs b/src/meta/raft-store/tests/it/types.rs index 22a1ce82f2a60..4de4b35464290 100644 --- a/src/meta/raft-store/tests/it/types.rs +++ b/src/meta/raft-store/tests/it/types.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_meta_sled_store::sled; use databend_common_meta_sled_store::SledOrderedSerde; +use databend_common_meta_sled_store::sled; use databend_common_meta_types::raft_types::NodeId; #[test] diff --git a/src/meta/semaphore/src/acquirer/acquirer.rs b/src/meta/semaphore/src/acquirer/acquirer.rs index cd02a7023fdff..28046af529f72 100644 --- a/src/meta/semaphore/src/acquirer/acquirer.rs +++ b/src/meta/semaphore/src/acquirer/acquirer.rs @@ -22,10 +22,10 @@ use codeq::Encode; use databend_common_base::runtime::spawn_named; use databend_common_meta_client::ClientHandle; use databend_common_meta_kvapi::kvapi::KVApi; -use databend_common_meta_types::protobuf as pb; use databend_common_meta_types::MatchSeq; use databend_common_meta_types::UpsertKV; use databend_common_meta_types::With; +use databend_common_meta_types::protobuf as pb; use futures::FutureExt; use log::info; use log::warn; @@ -33,14 +33,14 @@ use seq_marked::SeqValue; use tokio::sync::mpsc; use tokio::sync::oneshot; +use crate::PermitEntry; +use crate::PermitKey; use crate::acquirer::Permit; use crate::acquirer::SharedAcquirerStat; use crate::errors::AcquireError; use crate::errors::ConnectionClosed; use crate::errors::EarlyRemoved; use crate::queue::PermitEvent; -use crate::PermitEntry; -use crate::PermitKey; #[derive(Clone, Debug)] pub(crate) enum SeqPolicy { diff --git a/src/meta/semaphore/src/acquirer/stat.rs b/src/meta/semaphore/src/acquirer/stat.rs index 1ea3218329a8a..b78d27a7c2222 100644 --- a/src/meta/semaphore/src/acquirer/stat.rs +++ b/src/meta/semaphore/src/acquirer/stat.rs @@ -21,9 +21,9 @@ use std::time::Instant; use itertools::EitherOrBoth; use itertools::Itertools; +use crate::PermitSeq; use crate::queue::EventDesc; use crate::queue::PermitEvent; -use crate::PermitSeq; /// The statistics about running the semaphore acquirer. #[derive(Debug, Clone)] diff --git a/src/meta/semaphore/src/errors/early_removed.rs b/src/meta/semaphore/src/errors/early_removed.rs index 275963c64935f..2abed8d5034c7 100644 --- a/src/meta/semaphore/src/errors/early_removed.rs +++ b/src/meta/semaphore/src/errors/early_removed.rs @@ -28,7 +28,9 @@ use crate::storage::PermitKey; /// This is typically a transient condition that can be resolved by retrying /// the acquisition with a fresh permit entry. #[derive(thiserror::Error, Debug, Clone, PartialEq, Eq)] -#[error("EarlyRemoved: Semaphore PermitEntry is removed before being acquired: key:{permit_key} entry:{permit_entry}")] +#[error( + "EarlyRemoved: Semaphore PermitEntry is removed before being acquired: key:{permit_key} entry:{permit_entry}" +)] pub struct EarlyRemoved { permit_key: PermitKey, permit_entry: PermitEntry, diff --git a/src/meta/semaphore/src/lib.rs b/src/meta/semaphore/src/lib.rs index 4f7384b224a76..66d1bd30b34bd 100644 --- a/src/meta/semaphore/src/lib.rs +++ b/src/meta/semaphore/src/lib.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(clippy::uninlined_format_args)] +#![allow(clippy::collapsible_if, clippy::uninlined_format_args)] //! Implement a semaphore upon meta-service watch API and upsert API //! diff --git a/src/meta/semaphore/src/meta_event_subscriber/processor.rs b/src/meta/semaphore/src/meta_event_subscriber/processor.rs index 146baf997d1fd..0e6078d4989f4 100644 --- a/src/meta/semaphore/src/meta_event_subscriber/processor.rs +++ b/src/meta/semaphore/src/meta_event_subscriber/processor.rs @@ -13,12 +13,14 @@ // limitations under the License. use codeq::Decode; -use databend_common_meta_types::protobuf::WatchResponse; use databend_common_meta_types::SeqV; +use databend_common_meta_types::protobuf::WatchResponse; use display_more::DisplaySliceExt; use log::warn; use tokio::sync::mpsc; +use crate::PermitEntry; +use crate::PermitKey; #[cfg(doc)] use crate::acquirer::Acquirer; use crate::errors::AcquirerClosed; @@ -26,8 +28,6 @@ use crate::errors::ConnectionClosed; use crate::errors::ProcessorError; use crate::queue::PermitEvent; use crate::queue::SemaphoreQueue; -use crate::PermitEntry; -use crate::PermitKey; /// Process the watch response from the meta-service and emit [`PermitEvent`]. pub(crate) struct Processor { diff --git a/src/meta/semaphore/src/meta_event_subscriber/subscriber.rs b/src/meta/semaphore/src/meta_event_subscriber/subscriber.rs index 3ae055dff7c21..04c9f876c1de6 100644 --- a/src/meta/semaphore/src/meta_event_subscriber/subscriber.rs +++ b/src/meta/semaphore/src/meta_event_subscriber/subscriber.rs @@ -127,7 +127,7 @@ impl MetaEventSubscriber { pub(crate) async fn process_meta_event_loop( &mut self, mut strm: impl Stream> + Send + Unpin + 'static, - mut cancel: impl Future + Send, + cancel: impl Future + Send, ) -> Result<(), ProcessorError> { // let mut c = std::pin::pin!(cancel); diff --git a/src/meta/semaphore/src/queue/semaphore_event.rs b/src/meta/semaphore/src/queue/semaphore_event.rs index 4b767b2c410bb..a732903561b6d 100644 --- a/src/meta/semaphore/src/queue/semaphore_event.rs +++ b/src/meta/semaphore/src/queue/semaphore_event.rs @@ -14,9 +14,9 @@ use std::fmt; -use crate::queue::event_desc::EventDesc; use crate::PermitEntry; use crate::PermitSeq; +use crate::queue::event_desc::EventDesc; // TODO: consider adding a state `Waiting` when a semaphore enters waiting queue? /// The event of a semaphore permit, diff --git a/src/meta/semaphore/src/queue/semaphore_queue.rs b/src/meta/semaphore/src/queue/semaphore_queue.rs index 3a8d288bb91ea..837f0b42835e2 100644 --- a/src/meta/semaphore/src/queue/semaphore_queue.rs +++ b/src/meta/semaphore/src/queue/semaphore_queue.rs @@ -15,9 +15,9 @@ use std::collections::BTreeMap; use std::fmt; -use crate::queue::semaphore_event::PermitEvent; use crate::PermitEntry; use crate::PermitSeq; +use crate::queue::semaphore_event::PermitEvent; /// Manage the acquired and waiting semaphore entries. /// @@ -208,10 +208,10 @@ fn removed(seq: PermitSeq, entry: PermitEntry) -> PermitEvent { mod tests { use std::collections::BTreeMap; + use crate::PermitEntry; use crate::queue::semaphore_queue::acquired; use crate::queue::semaphore_queue::removed; use crate::queue::*; - use crate::PermitEntry; #[test] fn test_display() { diff --git a/src/meta/service/src/analysis/count_prefix.rs b/src/meta/service/src/analysis/count_prefix.rs index d32d4741d9fc1..91f3ca40610fb 100644 --- a/src/meta/service/src/analysis/count_prefix.rs +++ b/src/meta/service/src/analysis/count_prefix.rs @@ -121,9 +121,9 @@ mod tests { use databend_common_meta_types::protobuf::KeysCount; use databend_common_meta_types::protobuf::KeysLayoutRequest; + use futures::StreamExt; use futures::stream::BoxStream; use futures::stream::{self}; - use futures::StreamExt; use super::count_prefix; diff --git a/src/meta/service/src/api/grpc/grpc_service.rs b/src/meta/service/src/api/grpc/grpc_service.rs index 43c3d9708bdca..733a936e9e9da 100644 --- a/src/meta/service/src/api/grpc/grpc_service.rs +++ b/src/meta/service/src/api/grpc/grpc_service.rs @@ -28,8 +28,11 @@ use databend_common_grpc::GrpcClaim; use databend_common_grpc::GrpcToken; use databend_common_meta_client::MetaGrpcReadReq; use databend_common_meta_client::MetaGrpcReq; +use databend_common_meta_types::Endpoint; +use databend_common_meta_types::GrpcHelper; +use databend_common_meta_types::TxnReply; +use databend_common_meta_types::TxnRequest; use databend_common_meta_types::protobuf as pb; -use databend_common_meta_types::protobuf::meta_service_server::MetaService; use databend_common_meta_types::protobuf::ClientInfo; use databend_common_meta_types::protobuf::ClusterStatus; use databend_common_meta_types::protobuf::Empty; @@ -45,42 +48,39 @@ use databend_common_meta_types::protobuf::RaftRequest; use databend_common_meta_types::protobuf::StreamItem; use databend_common_meta_types::protobuf::WatchRequest; use databend_common_meta_types::protobuf::WatchResponse; -use databend_common_meta_types::Endpoint; -use databend_common_meta_types::GrpcHelper; -use databend_common_meta_types::TxnReply; -use databend_common_meta_types::TxnRequest; +use databend_common_meta_types::protobuf::meta_service_server::MetaService; use databend_common_metrics::count::Count; use databend_common_tracing::start_trace_for_remote_request; use fastrace::func_name; use fastrace::func_path; use fastrace::prelude::*; -use futures::stream::TryChunksError; use futures::StreamExt; use futures::TryStreamExt; +use futures::stream::TryChunksError; use log::debug; use log::error; use log::info; use prost::Message; use tokio_stream; use tokio_stream::Stream; -use tonic::codegen::BoxStream; -use tonic::metadata::MetadataMap; -use tonic::server::NamedService; use tonic::Request; use tonic::Response; use tonic::Status; use tonic::Streaming; +use tonic::codegen::BoxStream; +use tonic::metadata::MetadataMap; +use tonic::server::NamedService; use watcher::watch_stream::WatchStreamSender; use crate::meta_node::meta_handle::MetaHandle; use crate::meta_service::watcher::DispatcherHandle; use crate::meta_service::watcher::WatchTypes; -use crate::metrics::network_metrics; use crate::metrics::InFlightRead; use crate::metrics::InFlightWrite; +use crate::metrics::network_metrics; +use crate::version::MIN_METACLI_SEMVER; use crate::version::from_digit_ver; use crate::version::to_digit_ver; -use crate::version::MIN_METACLI_SEMVER; /// Metrics collector that logs when dropped struct MetricsCollector { @@ -328,6 +328,7 @@ impl MetaService for MetaServiceImpl { type KvReadV1Stream = BoxStream; + #[allow(unused)] async fn kv_read_v1( &self, request: Request, @@ -349,12 +350,12 @@ impl MetaService for MetaServiceImpl { let (endpoint, strm) = self.handle_kv_read_v1(req).in_span(root).await?; // MetricsCollector logs metrics when dropped - let mut collector = MetricsCollector::new(req_str); + let mut _collector = MetricsCollector::new(req_str); let strm = strm.map(move |item| { let _g = &guard; // hold the guard until the stream is done. network_metrics::incr_stream_sent_item(req_typ); - collector.count += 1; + _collector.count += 1; item }); diff --git a/src/meta/service/src/api/grpc_server.rs b/src/meta/service/src/api/grpc_server.rs index 76fe0b267faeb..76188ba659240 100644 --- a/src/meta/service/src/api/grpc_server.rs +++ b/src/meta/service/src/api/grpc_server.rs @@ -16,18 +16,18 @@ use std::sync::Arc; use std::time::Duration; use anyerror::AnyError; +use databend_common_base::base::Stoppable; use databend_common_base::base::tokio; use databend_common_base::base::tokio::sync::oneshot; use databend_common_base::base::tokio::sync::oneshot::Sender; use databend_common_base::base::tokio::task::JoinHandle; -use databend_common_base::base::Stoppable; -use databend_common_meta_types::protobuf::meta_service_server::MetaServiceServer; -use databend_common_meta_types::protobuf::FILE_DESCRIPTOR_SET; use databend_common_meta_types::GrpcConfig; use databend_common_meta_types::MetaNetworkError; +use databend_common_meta_types::protobuf::FILE_DESCRIPTOR_SET; +use databend_common_meta_types::protobuf::meta_service_server::MetaServiceServer; use fastrace::prelude::*; -use futures::future::select; use futures::future::Either; +use futures::future::select; use log::info; use tonic::transport::Identity; use tonic::transport::Server; diff --git a/src/meta/service/src/api/http/v1/features.rs b/src/meta/service/src/api/http/v1/features.rs index d7ef7101d67e2..84bd16bbc2226 100644 --- a/src/meta/service/src/api/http/v1/features.rs +++ b/src/meta/service/src/api/http/v1/features.rs @@ -19,10 +19,10 @@ use databend_common_meta_sled_store::openraft::async_runtime::watch::WatchReceiv use http::StatusCode; use log::info; use log::warn; +use poem::IntoResponse; use poem::web::Data; use poem::web::Json; use poem::web::Query; -use poem::IntoResponse; use crate::meta_node::errors::MetaNodeStopped; use crate::meta_node::meta_handle::MetaHandle; diff --git a/src/meta/service/src/api/http_service.rs b/src/meta/service/src/api/http_service.rs index 79f7c17c219f3..6c7127c292a99 100644 --- a/src/meta/service/src/api/http_service.rs +++ b/src/meta/service/src/api/http_service.rs @@ -16,23 +16,23 @@ use std::net::SocketAddr; use std::sync::Arc; use anyerror::AnyError; -use databend_common_base::base::tokio::sync::broadcast; use databend_common_base::base::Stoppable; +use databend_common_base::base::tokio::sync::broadcast; +use databend_common_http::HttpError; +use databend_common_http::HttpShutdownHandler; use databend_common_http::health_handler; use databend_common_http::home::debug_home_handler; #[cfg(feature = "memory-profiling")] use databend_common_http::jeprof::debug_jeprof_dump_handler; use databend_common_http::pprof::debug_pprof_handler; -use databend_common_http::HttpError; -use databend_common_http::HttpShutdownHandler; use databend_common_meta_types::MetaNetworkError; use log::info; use log::warn; -use poem::get; -use poem::listener::OpensslTlsConfig; use poem::Endpoint; use poem::EndpointExt; use poem::Route; +use poem::get; +use poem::listener::OpensslTlsConfig; use crate::configs::Config; use crate::meta_node::meta_handle::MetaHandle; @@ -52,7 +52,7 @@ impl HttpService { }) } - fn build_router(&self) -> impl Endpoint { + fn build_router(&self) -> impl Endpoint + use<> { #[cfg_attr(not(feature = "memory-profiling"), allow(unused_mut))] let mut route = Route::new() .at("/v1/health", get(health_handler)) diff --git a/src/meta/service/src/configs/inner.rs b/src/meta/service/src/configs/inner.rs index ea92592e52462..d739e1240c72b 100644 --- a/src/meta/service/src/configs/inner.rs +++ b/src/meta/service/src/configs/inner.rs @@ -15,8 +15,8 @@ use std::net::SocketAddr; use databend_common_meta_raft_store::config::RaftConfig; -use databend_common_meta_types::node::Node; use databend_common_meta_types::MetaStartupError; +use databend_common_meta_types::node::Node; use databend_common_tracing::Config as LogConfig; use super::outer_v0::Config as OuterV0Config; diff --git a/src/meta/service/src/configs/outer_v0.rs b/src/meta/service/src/configs/outer_v0.rs index 85b211a178180..21deca4fbb781 100644 --- a/src/meta/service/src/configs/outer_v0.rs +++ b/src/meta/service/src/configs/outer_v0.rs @@ -19,11 +19,12 @@ use clap::ArgAction; use clap::Args; use clap::Parser; use databend_common_config::StorageConfig; -use databend_common_meta_raft_store::config::get_default_raft_advertise_host; use databend_common_meta_raft_store::config::RaftConfig as InnerRaftConfig; +use databend_common_meta_raft_store::config::get_default_raft_advertise_host; use databend_common_meta_raft_store::ondisk::DATA_VERSION; use databend_common_meta_types::MetaStartupError; use databend_common_storage::StorageConfig as InnerStorageConfig; +use databend_common_tracing::CONFIG_DEFAULT_LOG_LEVEL; use databend_common_tracing::Config as InnerLogConfig; use databend_common_tracing::FileConfig as InnerFileLogConfig; use databend_common_tracing::HistoryConfig as InnerLogHistoryConfig; @@ -34,7 +35,6 @@ use databend_common_tracing::QueryLogConfig; use databend_common_tracing::StderrConfig as InnerStderrLogConfig; use databend_common_tracing::StructLogConfig; use databend_common_tracing::TracingConfig; -use databend_common_tracing::CONFIG_DEFAULT_LOG_LEVEL; use databend_common_version::DATABEND_GIT_SEMVER; use databend_common_version::VERGEN_BUILD_TIMESTAMP; use databend_common_version::VERGEN_GIT_SHA; diff --git a/src/meta/service/src/lib.rs b/src/meta/service/src/lib.rs index 2bff5f710d9b3..c3cb257b6ab57 100644 --- a/src/meta/service/src/lib.rs +++ b/src/meta/service/src/lib.rs @@ -14,7 +14,15 @@ #![feature(try_blocks)] #![feature(coroutines)] -#![allow(clippy::uninlined_format_args)] +#![allow( + clippy::collapsible_if, + clippy::let_and_return, + clippy::manual_is_multiple_of, + clippy::redundant_closure, + clippy::unnecessary_unwrap, + clippy::uninlined_format_args, + clippy::useless_vec +)] pub mod analysis; pub mod api; diff --git a/src/meta/service/src/message.rs b/src/meta/service/src/message.rs index 42eb162718d1d..e08ee25b7533b 100644 --- a/src/meta/service/src/message.rs +++ b/src/meta/service/src/message.rs @@ -20,13 +20,13 @@ use databend_common_meta_kvapi::kvapi::ListKVReply; use databend_common_meta_kvapi::kvapi::ListKVReq; use databend_common_meta_kvapi::kvapi::MGetKVReply; use databend_common_meta_kvapi::kvapi::MGetKVReq; -use databend_common_meta_types::protobuf::RaftRequest; -use databend_common_meta_types::raft_types::NodeId; use databend_common_meta_types::AppliedState; use databend_common_meta_types::Endpoint; use databend_common_meta_types::GrpcHelper; use databend_common_meta_types::LogEntry; use databend_common_meta_types::MetaAPIError; +use databend_common_meta_types::protobuf::RaftRequest; +use databend_common_meta_types::raft_types::NodeId; #[derive(serde::Serialize, serde::Deserialize, Debug, Default, Clone, PartialEq, Eq)] pub struct JoinRequest { diff --git a/src/meta/service/src/meta_node/meta_handle.rs b/src/meta/service/src/meta_node/meta_handle.rs index 1af0eeb54673f..6f88b7e0e9615 100644 --- a/src/meta/service/src/meta_node/meta_handle.rs +++ b/src/meta/service/src/meta_node/meta_handle.rs @@ -26,6 +26,15 @@ use databend_common_meta_client::MetaGrpcReadReq; use databend_common_meta_kvapi::kvapi::KVApi; use databend_common_meta_kvapi::kvapi::UpsertKVReply; use databend_common_meta_raft_store::leveled_store::db_exporter::DBExporter; +use databend_common_meta_types::AppliedState; +use databend_common_meta_types::Cmd; +use databend_common_meta_types::Endpoint; +use databend_common_meta_types::LogEntry; +use databend_common_meta_types::MetaAPIError; +use databend_common_meta_types::Node; +use databend_common_meta_types::TxnReply; +use databend_common_meta_types::TxnRequest; +use databend_common_meta_types::UpsertKV; use databend_common_meta_types::protobuf::KeysCount; use databend_common_meta_types::protobuf::KeysLayoutRequest; use databend_common_meta_types::protobuf::MemberListRequest; @@ -37,17 +46,8 @@ use databend_common_meta_types::raft_types::NodeId; use databend_common_meta_types::raft_types::RaftMetrics; use databend_common_meta_types::raft_types::Wait; use databend_common_meta_types::sys_data::SysData; -use databend_common_meta_types::AppliedState; -use databend_common_meta_types::Cmd; -use databend_common_meta_types::Endpoint; -use databend_common_meta_types::LogEntry; -use databend_common_meta_types::MetaAPIError; -use databend_common_meta_types::Node; -use databend_common_meta_types::TxnReply; -use databend_common_meta_types::TxnRequest; -use databend_common_meta_types::UpsertKV; -use futures::stream::BoxStream; use futures::Stream; +use futures::stream::BoxStream; use tokio::sync::mpsc; use tokio::sync::oneshot; use tokio::sync::watch; diff --git a/src/meta/service/src/meta_node/meta_node.rs b/src/meta/service/src/meta_node/meta_node.rs index f4d4069b3f362..c680686a1b71e 100644 --- a/src/meta/service/src/meta_node/meta_node.rs +++ b/src/meta/service/src/meta_node/meta_node.rs @@ -16,47 +16,33 @@ use std::collections::BTreeMap; use std::collections::BTreeSet; use std::future; use std::net::Ipv4Addr; -use std::sync::atomic::AtomicI32; use std::sync::Arc; use std::sync::Weak; +use std::sync::atomic::AtomicI32; use std::time::Duration; use anyerror::AnyError; +use databend_common_base::base::BuildInfoRef; use databend_common_base::base::tokio; +use databend_common_base::base::tokio::sync::Mutex; use databend_common_base::base::tokio::sync::watch; use databend_common_base::base::tokio::sync::watch::error::RecvError; -use databend_common_base::base::tokio::sync::Mutex; use databend_common_base::base::tokio::task::JoinHandle; -use databend_common_base::base::tokio::time::sleep; use databend_common_base::base::tokio::time::Instant; -use databend_common_base::base::BuildInfoRef; +use databend_common_base::base::tokio::time::sleep; use databend_common_grpc::ConnectionFactory; use databend_common_grpc::DNSResolver; -use databend_common_meta_client::reply_to_api_result; use databend_common_meta_client::RequestFor; +use databend_common_meta_client::reply_to_api_result; +use databend_common_meta_raft_store::StateMachineFeature; use databend_common_meta_raft_store::config::RaftConfig; use databend_common_meta_raft_store::ondisk::DATA_VERSION; use databend_common_meta_raft_store::raft_log_v004::RaftLogStat; use databend_common_meta_raft_store::utils::seq_marked_to_seqv; -use databend_common_meta_raft_store::StateMachineFeature; use databend_common_meta_sled_store::openraft; -use databend_common_meta_sled_store::openraft::error::RaftError; use databend_common_meta_sled_store::openraft::ChangeMembers; +use databend_common_meta_sled_store::openraft::error::RaftError; use databend_common_meta_stoerr::MetaStorageError; -use databend_common_meta_types::node::Node; -use databend_common_meta_types::protobuf::raft_service_client::RaftServiceClient; -use databend_common_meta_types::protobuf::raft_service_server::RaftServiceServer; -use databend_common_meta_types::protobuf::watch_request::FilterType; -use databend_common_meta_types::protobuf::WatchRequest; -use databend_common_meta_types::protobuf::WatchResponse; -use databend_common_meta_types::raft_types::new_log_id; -use databend_common_meta_types::raft_types::ForwardToLeader; -use databend_common_meta_types::raft_types::InitializeError; -use databend_common_meta_types::raft_types::MembershipNode; -use databend_common_meta_types::raft_types::NodeId; -use databend_common_meta_types::raft_types::RaftMetrics; -use databend_common_meta_types::raft_types::TypeConfig; -use databend_common_meta_types::snapshot_db::DBStat; use databend_common_meta_types::AppliedState; use databend_common_meta_types::Cmd; use databend_common_meta_types::Endpoint; @@ -69,11 +55,25 @@ use databend_common_meta_types::MetaManagementError; use databend_common_meta_types::MetaNetworkError; use databend_common_meta_types::MetaOperationError; use databend_common_meta_types::MetaStartupError; +use databend_common_meta_types::node::Node; +use databend_common_meta_types::protobuf::WatchRequest; +use databend_common_meta_types::protobuf::WatchResponse; +use databend_common_meta_types::protobuf::raft_service_client::RaftServiceClient; +use databend_common_meta_types::protobuf::raft_service_server::RaftServiceServer; +use databend_common_meta_types::protobuf::watch_request::FilterType; +use databend_common_meta_types::raft_types::ForwardToLeader; +use databend_common_meta_types::raft_types::InitializeError; +use databend_common_meta_types::raft_types::MembershipNode; +use databend_common_meta_types::raft_types::NodeId; +use databend_common_meta_types::raft_types::RaftMetrics; +use databend_common_meta_types::raft_types::TypeConfig; +use databend_common_meta_types::raft_types::new_log_id; +use databend_common_meta_types::snapshot_db::DBStat; use fastrace::func_name; use fastrace::prelude::*; -use futures::stream::BoxStream; use futures::StreamExt; use futures::TryStreamExt; +use futures::stream::BoxStream; use itertools::Itertools; use log::debug; use log::error; @@ -88,6 +88,7 @@ use openraft::SnapshotPolicy; use state_machine_api::UserKey; use tokio::sync::mpsc; use tonic::Status; +use watcher::EventFilter; use watcher::dispatch::Command; use watcher::dispatch::Dispatcher; use watcher::key_range::build_key_range; @@ -95,7 +96,6 @@ use watcher::util::new_initialization_sink; use watcher::util::try_forward; use watcher::watch_stream::WatchStream; use watcher::watch_stream::WatchStreamSender; -use watcher::EventFilter; use crate::api::grpc::grpc_service::try_remove_sender; use crate::configs::Config as MetaConfig; @@ -105,23 +105,23 @@ use crate::message::ForwardResponse; use crate::message::JoinRequest; use crate::message::LeaveRequest; use crate::meta_node::meta_node_status::MetaNodeStatus; -use crate::meta_service::errors::grpc_error_to_network_err; -use crate::meta_service::meta_leader::MetaLeader; -use crate::meta_service::runtime_config::RuntimeConfig; -use crate::meta_service::watcher::DispatcherHandle; -use crate::meta_service::watcher::WatchTypes; use crate::meta_service::MetaForwarder; use crate::meta_service::MetaKVApi; use crate::meta_service::MetaKVApiOwned; use crate::meta_service::MetaNodeBuilder; use crate::meta_service::RaftServiceImpl; +use crate::meta_service::errors::grpc_error_to_network_err; +use crate::meta_service::meta_leader::MetaLeader; +use crate::meta_service::runtime_config::RuntimeConfig; +use crate::meta_service::watcher::DispatcherHandle; +use crate::meta_service::watcher::WatchTypes; use crate::metrics::network_metrics; use crate::metrics::server_metrics; use crate::request_handling::Forwarder; use crate::request_handling::Handler; +use crate::store::RaftStore; use crate::store::meta_raft_log::MetaRaftLog; use crate::store::meta_raft_state_machine::MetaRaftStateMachine; -use crate::store::RaftStore; pub type LogStore = MetaRaftLog; pub type SMStore = MetaRaftStateMachine; @@ -1623,7 +1623,7 @@ impl MetaNode { } /// Get a kvapi::KVApi implementation. - pub fn kv_api(&self) -> MetaKVApi { + pub fn kv_api(&self) -> MetaKVApi<'_> { MetaKVApi::new(self) } diff --git a/src/meta/service/src/meta_node/meta_node_builder.rs b/src/meta/service/src/meta_node/meta_node_builder.rs index 0cb49cbe8fcd8..b2c66e9712f2f 100644 --- a/src/meta/service/src/meta_node/meta_node_builder.rs +++ b/src/meta/service/src/meta_node/meta_node_builder.rs @@ -12,23 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::sync::atomic::AtomicI32; use std::sync::Arc; +use std::sync::atomic::AtomicI32; use databend_common_base::base::BuildInfoRef; use databend_common_meta_sled_store::openraft::Config; -use databend_common_meta_types::raft_types::NodeId; use databend_common_meta_types::Endpoint; use databend_common_meta_types::MetaStartupError; +use databend_common_meta_types::raft_types::NodeId; use log::info; -use tokio::sync::watch; use tokio::sync::Mutex; +use tokio::sync::watch; use watcher::dispatch::Dispatcher; use crate::meta_node::meta_node::MetaRaft; +use crate::meta_service::MetaNode; use crate::meta_service::runtime_config::RuntimeConfig; use crate::meta_service::watcher::DispatcherHandle; -use crate::meta_service::MetaNode; use crate::network::NetworkFactory; use crate::store::RaftStore; diff --git a/src/meta/service/src/meta_service/forwarder.rs b/src/meta/service/src/meta_service/forwarder.rs index 542c7a0682532..c540c6330bf3b 100644 --- a/src/meta/service/src/meta_service/forwarder.rs +++ b/src/meta/service/src/meta_service/forwarder.rs @@ -16,15 +16,15 @@ use databend_common_meta_api::reply::reply_to_api_result; use databend_common_meta_client::MetaGrpcReadReq; -use databend_common_meta_types::protobuf::raft_service_client::RaftServiceClient; -use databend_common_meta_types::protobuf::StreamItem; -use databend_common_meta_types::raft_types::NodeId; use databend_common_meta_types::ConnectionError; use databend_common_meta_types::Endpoint; use databend_common_meta_types::ForwardRPCError; use databend_common_meta_types::GrpcConfig; use databend_common_meta_types::MetaAPIError; use databend_common_meta_types::MetaNetworkError; +use databend_common_meta_types::protobuf::StreamItem; +use databend_common_meta_types::protobuf::raft_service_client::RaftServiceClient; +use databend_common_meta_types::raft_types::NodeId; use log::debug; use tonic::codegen::BoxStream; use tonic::transport::Channel; diff --git a/src/meta/service/src/meta_service/meta_leader.rs b/src/meta/service/src/meta_service/meta_leader.rs index c2330340585b2..9c608acf5904e 100644 --- a/src/meta/service/src/meta_service/meta_leader.rs +++ b/src/meta/service/src/meta_service/meta_leader.rs @@ -22,18 +22,18 @@ use databend_common_meta_kvapi::kvapi::KVApi; use databend_common_meta_kvapi::kvapi::KvApiExt; use databend_common_meta_sled_store::openraft::ChangeMembers; use databend_common_meta_stoerr::MetaStorageError; -use databend_common_meta_types::node::Node; -use databend_common_meta_types::protobuf::StreamItem; -use databend_common_meta_types::raft_types::ClientWriteError; -use databend_common_meta_types::raft_types::MembershipNode; -use databend_common_meta_types::raft_types::NodeId; -use databend_common_meta_types::raft_types::RaftError; use databend_common_meta_types::AppliedState; use databend_common_meta_types::Cmd; use databend_common_meta_types::LogEntry; use databend_common_meta_types::MetaDataError; use databend_common_meta_types::MetaDataReadError; use databend_common_meta_types::MetaOperationError; +use databend_common_meta_types::node::Node; +use databend_common_meta_types::protobuf::StreamItem; +use databend_common_meta_types::raft_types::ClientWriteError; +use databend_common_meta_types::raft_types::MembershipNode; +use databend_common_meta_types::raft_types::NodeId; +use databend_common_meta_types::raft_types::RaftError; use databend_common_metrics::count::Count; use futures::StreamExt; use futures::TryStreamExt; @@ -41,8 +41,8 @@ use log::debug; use log::info; use maplit::btreemap; use maplit::btreeset; -use tonic::codegen::BoxStream; use tonic::Status; +use tonic::codegen::BoxStream; use crate::message::ForwardRequest; use crate::message::ForwardRequestBody; @@ -51,8 +51,8 @@ use crate::message::JoinRequest; use crate::message::LeaveRequest; use crate::meta_node::meta_node::MetaRaft; use crate::meta_service::MetaNode; -use crate::metrics::server_metrics; use crate::metrics::ProposalPending; +use crate::metrics::server_metrics; use crate::request_handling::Handler; use crate::store::RaftStore; diff --git a/src/meta/service/src/meta_service/raft_service_impl.rs b/src/meta/service/src/meta_service/raft_service_impl.rs index be69f403655b5..78a2b632ba4f0 100644 --- a/src/meta/service/src/meta_service/raft_service_impl.rs +++ b/src/meta/service/src/meta_service/raft_service_impl.rs @@ -21,15 +21,15 @@ use std::sync::Arc; use databend_common_meta_client::MetaGrpcReadReq; use databend_common_meta_raft_store::leveled_store::persisted_codec::PersistedCodec; use databend_common_meta_raft_store::ondisk::DATA_VERSION; +use databend_common_meta_raft_store::sm_v003::WriterV003; use databend_common_meta_raft_store::sm_v003::open_snapshot::OpenSnapshot; use databend_common_meta_raft_store::sm_v003::received::Received; use databend_common_meta_raft_store::sm_v003::write_entry::WriteEntry; -use databend_common_meta_raft_store::sm_v003::WriterV003; use databend_common_meta_raft_store::snapshot_config::SnapshotConfig; use databend_common_meta_raft_store::state_machine::MetaSnapshotId; use databend_common_meta_sled_store::openraft::MessageSummary; +use databend_common_meta_types::GrpcHelper; use databend_common_meta_types::protobuf as pb; -use databend_common_meta_types::protobuf::raft_service_server::RaftService; use databend_common_meta_types::protobuf::Empty; use databend_common_meta_types::protobuf::InstallEntryV004; use databend_common_meta_types::protobuf::InstallSnapshotResponseV004; @@ -38,6 +38,7 @@ use databend_common_meta_types::protobuf::RaftRequest; use databend_common_meta_types::protobuf::SnapshotChunkRequestV003; use databend_common_meta_types::protobuf::SnapshotResponseV003; use databend_common_meta_types::protobuf::StreamItem; +use databend_common_meta_types::protobuf::raft_service_server::RaftService; use databend_common_meta_types::raft_types::AppendEntriesRequest; use databend_common_meta_types::raft_types::Snapshot; use databend_common_meta_types::raft_types::SnapshotMeta; @@ -46,7 +47,6 @@ use databend_common_meta_types::raft_types::Vote; use databend_common_meta_types::raft_types::VoteRequest; use databend_common_meta_types::snapshot_db::DB; use databend_common_meta_types::sys_data::SysData; -use databend_common_meta_types::GrpcHelper; use databend_common_metrics::count::Count; use fastrace::func_path; use fastrace::prelude::*; @@ -58,11 +58,11 @@ use raft_metrics::network::incr_snapshot_recvfrom_result; use seq_marked::SeqMarked; use state_machine_api::MetaValue; use state_machine_api::SeqV; -use tonic::codegen::BoxStream; use tonic::Request; use tonic::Response; use tonic::Status; use tonic::Streaming; +use tonic::codegen::BoxStream; use crate::message::ForwardRequest; use crate::message::ForwardRequestBody; diff --git a/src/meta/service/src/meta_service/runtime_config.rs b/src/meta/service/src/meta_service/runtime_config.rs index 38f3ffaf78c7a..b12ca24653f3b 100644 --- a/src/meta/service/src/meta_service/runtime_config.rs +++ b/src/meta/service/src/meta_service/runtime_config.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::sync::atomic::AtomicBool; use std::sync::Arc; +use std::sync::atomic::AtomicBool; pub struct RuntimeConfig { pub broadcast_state_machine_changes: Arc, diff --git a/src/meta/service/src/meta_service/watcher.rs b/src/meta/service/src/meta_service/watcher.rs index d09af760641f0..54cbff9306081 100644 --- a/src/meta/service/src/meta_service/watcher.rs +++ b/src/meta/service/src/meta_service/watcher.rs @@ -19,8 +19,8 @@ use std::future::Future; use std::io::Error; use std::ops::Deref; -use databend_common_meta_types::protobuf::WatchResponse; use databend_common_meta_types::SeqV; +use databend_common_meta_types::protobuf::WatchResponse; use log::debug; use tonic::Status; use watcher::dispatch::DispatcherHandle as GenericDispatcherHandle; diff --git a/src/meta/service/src/metrics/meta_metrics.rs b/src/meta/service/src/metrics/meta_metrics.rs index 1ff9bbfb6f7d5..56d9bfa2d8c49 100644 --- a/src/meta/service/src/metrics/meta_metrics.rs +++ b/src/meta/service/src/metrics/meta_metrics.rs @@ -415,8 +415,8 @@ pub mod raft_metrics { use prometheus_client::metrics::counter::Counter; use prometheus_client::metrics::family::Family; use prometheus_client::metrics::gauge::Gauge; - use prometheus_client::metrics::histogram::exponential_buckets; use prometheus_client::metrics::histogram::Histogram; + use prometheus_client::metrics::histogram::exponential_buckets; macro_rules! key { ($key: literal) => { diff --git a/src/meta/service/src/metrics/mod.rs b/src/meta/service/src/metrics/mod.rs index 6cf6cb8e8ab9f..3e74838a09d47 100644 --- a/src/meta/service/src/metrics/mod.rs +++ b/src/meta/service/src/metrics/mod.rs @@ -15,11 +15,11 @@ mod meta_metrics; mod registry; -pub use meta_metrics::meta_metrics_to_prometheus_string; -pub use meta_metrics::network_metrics; -pub use meta_metrics::raft_metrics; -pub use meta_metrics::server_metrics; pub(crate) use meta_metrics::InFlightRead; pub(crate) use meta_metrics::InFlightWrite; pub(crate) use meta_metrics::ProposalPending; pub(crate) use meta_metrics::SnapshotBuilding; +pub use meta_metrics::meta_metrics_to_prometheus_string; +pub use meta_metrics::network_metrics; +pub use meta_metrics::raft_metrics; +pub use meta_metrics::server_metrics; diff --git a/src/meta/service/src/network.rs b/src/meta/service/src/network.rs index 9c349bcae039e..c35dfc9b22f01 100644 --- a/src/meta/service/src/network.rs +++ b/src/meta/service/src/network.rs @@ -29,13 +29,17 @@ use databend_common_base::runtime; use databend_common_base::runtime::spawn_named; use databend_common_meta_raft_store::leveled_store::persisted_codec::PersistedCodec; use databend_common_meta_sled_store::openraft; +use databend_common_meta_sled_store::openraft::MessageSummary; +use databend_common_meta_sled_store::openraft::RaftNetworkFactory; use databend_common_meta_sled_store::openraft::error::PayloadTooLarge; use databend_common_meta_sled_store::openraft::error::ReplicationClosed; use databend_common_meta_sled_store::openraft::error::Unreachable; -use databend_common_meta_sled_store::openraft::network::v2::RaftNetworkV2; use databend_common_meta_sled_store::openraft::network::RPCOption; -use databend_common_meta_sled_store::openraft::MessageSummary; -use databend_common_meta_sled_store::openraft::RaftNetworkFactory; +use databend_common_meta_sled_store::openraft::network::v2::RaftNetworkV2; +use databend_common_meta_types::Endpoint; +use databend_common_meta_types::GrpcConfig; +use databend_common_meta_types::GrpcHelper; +use databend_common_meta_types::MetaNetworkError; use databend_common_meta_types::protobuf as pb; use databend_common_meta_types::protobuf::InstallEntryV004; use databend_common_meta_types::protobuf::RaftReply; @@ -57,10 +61,6 @@ use databend_common_meta_types::raft_types::TypeConfig; use databend_common_meta_types::raft_types::Vote; use databend_common_meta_types::raft_types::VoteRequest; use databend_common_meta_types::raft_types::VoteResponse; -use databend_common_meta_types::Endpoint; -use databend_common_meta_types::GrpcConfig; -use databend_common_meta_types::GrpcHelper; -use databend_common_meta_types::MetaNetworkError; use databend_common_metrics::count::Count; use fastrace::func_name; use futures::FutureExt; @@ -73,8 +73,8 @@ use log::warn; use seq_marked::SeqData; use seq_marked::SeqV; use state_machine_api::MetaValue; -use tokio::sync::oneshot; use tokio::sync::Mutex; +use tokio::sync::oneshot; use tokio_stream::wrappers::ReceiverStream; use crate::metrics::raft_metrics; @@ -328,7 +328,7 @@ impl Network { } } - pub(crate) fn back_off(&self) -> impl Iterator { + pub(crate) fn back_off(&self) -> impl Iterator + use<> { let policy = ExponentialBuilder::default() .with_factor(self.backoff.back_off_ratio) .with_min_delay(self.backoff.back_off_min_delay) @@ -351,7 +351,7 @@ impl Network { E: std::error::Error, { // Return status error - let resp = grpc_res.map_err(|e| self.status_to_unreachable(e))?; + let resp = grpc_res.map_err(|e| RPCError::Unreachable(self.status_to_unreachable(e)))?; // Parse serialized response into `Result` let raft_res = GrpcHelper::parse_raft_reply::(resp).map_err(|serde_err| { @@ -645,7 +645,8 @@ impl Network { } } } - let grpc_response = grpc_res.map_err(|e| self.status_to_unreachable(e))?; + let grpc_response = + grpc_res.map_err(|e| StreamingError::Unreachable(self.status_to_unreachable(e)))?; let snapshot_response = grpc_response.into_inner(); // Convert protobuf Vote back to internal Vote @@ -727,9 +728,12 @@ impl Network { } } - let grpc_response = grpc_res.map_err(|e| self.status_to_unreachable(e))?; + let grpc_response = + grpc_res.map_err(|e| StreamingError::Unreachable(self.status_to_unreachable(e)))?; let snapshot_response = grpc_response.into_inner(); - let vote = snapshot_response.to_vote()?; + let vote = snapshot_response + .to_vote() + .map_err(|e| StreamingError::Network(e))?; SnapshotResponse { vote } }; @@ -956,7 +960,7 @@ impl RaftNetworkV2 for Network { } } - grpc_res.map_err(|e| self.status_to_unreachable(e))?; + grpc_res.map_err(|e| RPCError::Unreachable(self.status_to_unreachable(e)))?; Ok(()) } diff --git a/src/meta/service/src/raft_client.rs b/src/meta/service/src/raft_client.rs index 90cc3865a79d2..01befeccfbeb7 100644 --- a/src/meta/service/src/raft_client.rs +++ b/src/meta/service/src/raft_client.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_meta_types::protobuf::raft_service_client::RaftServiceClient; -use databend_common_meta_types::raft_types::NodeId; use databend_common_meta_types::Endpoint; use databend_common_meta_types::GrpcConfig; +use databend_common_meta_types::protobuf::raft_service_client::RaftServiceClient; +use databend_common_meta_types::raft_types::NodeId; use databend_common_metrics::count; use log::debug; use tonic::transport::channel::Channel; diff --git a/src/meta/service/src/request_handling.rs b/src/meta/service/src/request_handling.rs index 88b625755ac82..7e9c461e394c2 100644 --- a/src/meta/service/src/request_handling.rs +++ b/src/meta/service/src/request_handling.rs @@ -13,10 +13,10 @@ // limitations under the License. use databend_common_meta_client::RequestFor; -use databend_common_meta_types::raft_types::NodeId; use databend_common_meta_types::Endpoint; use databend_common_meta_types::ForwardRPCError; use databend_common_meta_types::MetaOperationError; +use databend_common_meta_types::raft_types::NodeId; use crate::message::ForwardRequest; use crate::message::ForwardRequestBody; diff --git a/src/meta/service/src/store/meta_raft_log/impl_raft_log_storage.rs b/src/meta/service/src/store/meta_raft_log/impl_raft_log_storage.rs index b25d3ddd62bb9..971c1fcd183db 100644 --- a/src/meta/service/src/store/meta_raft_log/impl_raft_log_storage.rs +++ b/src/meta/service/src/store/meta_raft_log/impl_raft_log_storage.rs @@ -20,13 +20,13 @@ use databend_common_base::base::tokio::sync::oneshot; use databend_common_meta_raft_store::raft_log_v004; use databend_common_meta_raft_store::raft_log_v004::codec_wrapper::Cw; use databend_common_meta_raft_store::raft_log_v004::io_desc::IODesc; -use databend_common_meta_sled_store::openraft::entry::RaftEntry; -use databend_common_meta_sled_store::openraft::storage::RaftLogStorage; use databend_common_meta_sled_store::openraft::EntryPayload; use databend_common_meta_sled_store::openraft::LogIdOptionExt; use databend_common_meta_sled_store::openraft::LogState; use databend_common_meta_sled_store::openraft::OptionalSend; use databend_common_meta_sled_store::openraft::RaftLogReader; +use databend_common_meta_sled_store::openraft::entry::RaftEntry; +use databend_common_meta_sled_store::openraft::storage::RaftLogStorage; use databend_common_meta_types::raft_types::Entry; use databend_common_meta_types::raft_types::IOFlushed; use databend_common_meta_types::raft_types::LogId; diff --git a/src/meta/service/src/store/meta_raft_state_machine/mod.rs b/src/meta/service/src/store/meta_raft_state_machine/mod.rs index b0577955aef89..cf42f4964a9f5 100644 --- a/src/meta/service/src/store/meta_raft_state_machine/mod.rs +++ b/src/meta/service/src/store/meta_raft_state_machine/mod.rs @@ -22,9 +22,9 @@ use std::time::Duration; use databend_common_base::runtime::spawn_named; use databend_common_meta_raft_store::config::RaftConfig; use databend_common_meta_raft_store::immutable_compactor::InMemoryCompactor; +use databend_common_meta_raft_store::sm_v003::SMV003; use databend_common_meta_raft_store::sm_v003::SnapshotStoreV004; use databend_common_meta_raft_store::sm_v003::WriteEntry; -use databend_common_meta_raft_store::sm_v003::SMV003; use databend_common_meta_raft_store::state_machine::MetaSnapshotId; use databend_common_meta_stoerr::MetaStorageError; use databend_common_meta_types::raft_types::NodeId; @@ -40,8 +40,8 @@ use log::info; use seq_marked::InternalSeq; use tokio::sync::oneshot; -use crate::metrics::raft_metrics; use crate::metrics::SnapshotBuilding; +use crate::metrics::raft_metrics; mod raft_state_machine_impl; diff --git a/src/meta/service/src/store/meta_raft_state_machine/raft_state_machine_impl.rs b/src/meta/service/src/store/meta_raft_state_machine/raft_state_machine_impl.rs index b6247a2f50e93..7e079cee5692b 100644 --- a/src/meta/service/src/store/meta_raft_state_machine/raft_state_machine_impl.rs +++ b/src/meta/service/src/store/meta_raft_state_machine/raft_state_machine_impl.rs @@ -12,11 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_meta_raft_store::sm_v003::open_snapshot::OpenSnapshot; use databend_common_meta_raft_store::sm_v003::SnapshotStoreV004; -use databend_common_meta_sled_store::openraft::storage::RaftStateMachine; +use databend_common_meta_raft_store::sm_v003::open_snapshot::OpenSnapshot; use databend_common_meta_sled_store::openraft::OptionalSend; use databend_common_meta_sled_store::openraft::RaftSnapshotBuilder; +use databend_common_meta_sled_store::openraft::storage::RaftStateMachine; +use databend_common_meta_types::AppliedState; use databend_common_meta_types::raft_types::Entry; use databend_common_meta_types::raft_types::LogId; use databend_common_meta_types::raft_types::Snapshot; @@ -25,7 +26,6 @@ use databend_common_meta_types::raft_types::StorageError; use databend_common_meta_types::raft_types::StoredMembership; use databend_common_meta_types::raft_types::TypeConfig; use databend_common_meta_types::snapshot_db::DB; -use databend_common_meta_types::AppliedState; use log::debug; use log::error; use log::info; @@ -75,7 +75,9 @@ impl RaftStateMachine for MetaRaftStateMachine { // This method is not used #[fastrace::trace] async fn begin_receiving_snapshot(&mut self) -> Result { - unreachable!("begin_receiving_snapshot is only required when using OpenRaft Chunked snapshot transmit"); + unreachable!( + "begin_receiving_snapshot is only required when using OpenRaft Chunked snapshot transmit" + ); } #[fastrace::trace] diff --git a/src/meta/service/src/store/store.rs b/src/meta/service/src/store/store.rs index d12f146ef2aa6..77c7fd6a31132 100644 --- a/src/meta/service/src/store/store.rs +++ b/src/meta/service/src/store/store.rs @@ -26,22 +26,22 @@ use databend_common_meta_raft_store::leveled_store::db_exporter::DBExporter; use databend_common_meta_raft_store::ondisk::Header; use databend_common_meta_raft_store::ondisk::TREE_HEADER; use databend_common_meta_raft_store::raft_log_v004; -use databend_common_meta_raft_store::raft_log_v004::util; use databend_common_meta_raft_store::raft_log_v004::Cw; use databend_common_meta_raft_store::raft_log_v004::RaftLogV004; -use databend_common_meta_raft_store::sm_v003::compactor_acquirer::CompactorAcquirer; -use databend_common_meta_raft_store::sm_v003::SnapshotStoreV004; +use databend_common_meta_raft_store::raft_log_v004::util; use databend_common_meta_raft_store::sm_v003::SMV003; +use databend_common_meta_raft_store::sm_v003::SnapshotStoreV004; +use databend_common_meta_raft_store::sm_v003::compactor_acquirer::CompactorAcquirer; use databend_common_meta_raft_store::state_machine::MetaSnapshotId; use databend_common_meta_stoerr::MetaStorageError; +use databend_common_meta_types::Endpoint; +use databend_common_meta_types::MetaStartupError; +use databend_common_meta_types::Node; use databend_common_meta_types::raft_types::Entry; use databend_common_meta_types::raft_types::Membership; use databend_common_meta_types::raft_types::NodeId; -use databend_common_meta_types::snapshot_db::DBStat; use databend_common_meta_types::snapshot_db::DB; -use databend_common_meta_types::Endpoint; -use databend_common_meta_types::MetaStartupError; -use databend_common_meta_types::Node; +use databend_common_meta_types::snapshot_db::DBStat; use futures::TryStreamExt; use log::debug; use log::info; diff --git a/src/meta/service/src/version.rs b/src/meta/service/src/version.rs index a110f71ac4fc5..26a5d371e5361 100644 --- a/src/meta/service/src/version.rs +++ b/src/meta/service/src/version.rs @@ -60,10 +60,10 @@ pub static MIN_META_SEMVER: Version = Version::new(0, 9, 41); /// ``` pub(crate) mod raft { pub(crate) mod server { - use feature_set::add_provide; - use feature_set::del_provide; use feature_set::Action; use feature_set::Provide; + use feature_set::add_provide; + use feature_set::del_provide; /// Removed and unused anymore. #[rustfmt::skip] @@ -97,11 +97,11 @@ pub(crate) mod raft { } pub(crate) mod client { + use feature_set::Action; + use feature_set::Require; use feature_set::add_optional; use feature_set::add_require; use feature_set::del_require; - use feature_set::Action; - use feature_set::Require; /// Removed and unused anymore. #[rustfmt::skip] diff --git a/src/meta/service/tests/it/api/http/cluster_state_test.rs b/src/meta/service/tests/it/api/http/cluster_state_test.rs index bf27a0a54ca1b..a053ae55e8c54 100644 --- a/src/meta/service/tests/it/api/http/cluster_state_test.rs +++ b/src/meta/service/tests/it/api/http/cluster_state_test.rs @@ -18,27 +18,27 @@ use std::io::Read; use std::sync::Arc; use std::time::Duration; +use databend_common_base::base::Stoppable; use databend_common_base::base::tokio; use databend_common_base::base::tokio::time::Instant; -use databend_common_base::base::Stoppable; use databend_common_meta_kvapi::kvapi::KVApi; +use databend_common_meta_types::UpsertKV; use databend_common_meta_types::node::Node; use databend_common_meta_types::raft_types::new_log_id; -use databend_common_meta_types::UpsertKV; use databend_common_version::BUILD_INFO; -use databend_meta::api::http::v1::cluster_state::nodes_handler; use databend_meta::api::HttpService; +use databend_meta::api::http::v1::cluster_state::nodes_handler; use databend_meta::meta_node::meta_worker::MetaWorker; use databend_meta::meta_service::MetaNode; use http::Method; use http::StatusCode; use http::Uri; use log::info; -use poem::get; use poem::Endpoint; use poem::EndpointExt; use poem::Request; use poem::Route; +use poem::get; use pretty_assertions::assert_eq; use test_harness::test; diff --git a/src/meta/service/tests/it/api/http/config.rs b/src/meta/service/tests/it/api/http/config.rs index c9622bc408908..11deca094cb3f 100644 --- a/src/meta/service/tests/it/api/http/config.rs +++ b/src/meta/service/tests/it/api/http/config.rs @@ -18,11 +18,11 @@ use databend_meta::configs::Config; use http::Method; use http::StatusCode; use http::Uri; -use poem::get; use poem::Endpoint; use poem::EndpointExt; use poem::Request; use poem::Route; +use poem::get; use pretty_assertions::assert_eq; #[tokio::test(flavor = "multi_thread")] diff --git a/src/meta/service/tests/it/api/http/features.rs b/src/meta/service/tests/it/api/http/features.rs index 16a01437343bc..a1c2e27d6126e 100644 --- a/src/meta/service/tests/it/api/http/features.rs +++ b/src/meta/service/tests/it/api/http/features.rs @@ -12,15 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![allow(clippy::unnecessary_unwrap)] + use std::time::Duration; +use databend_common_base::base::Stoppable; use databend_common_base::base::tokio; use databend_common_base::base::tokio::time::Instant; -use databend_common_base::base::Stoppable; use databend_common_meta_raft_store::StateMachineFeature; use databend_common_meta_sled_store::openraft::async_runtime::watch::WatchReceiver; -use databend_meta::api::http::v1::features::FeatureResponse; use databend_meta::api::HttpService; +use databend_meta::api::http::v1::features::FeatureResponse; use log::info; use pretty_assertions::assert_eq; use test_harness::test; diff --git a/src/meta/service/tests/it/api/http/metrics.rs b/src/meta/service/tests/it/api/http/metrics.rs index 96af4f74ab67d..15f3a72203c39 100644 --- a/src/meta/service/tests/it/api/http/metrics.rs +++ b/src/meta/service/tests/it/api/http/metrics.rs @@ -21,11 +21,11 @@ use http::StatusCode; use http::Uri; use log::info; use maplit::btreeset; -use poem::get; use poem::Endpoint; use poem::EndpointExt; use poem::Request; use poem::Route; +use poem::get; use pretty_assertions::assert_eq; use test_harness::test; diff --git a/src/meta/service/tests/it/api/http/transfer_leader.rs b/src/meta/service/tests/it/api/http/transfer_leader.rs index 43c643ccf9516..f943d1c0fb0bc 100644 --- a/src/meta/service/tests/it/api/http/transfer_leader.rs +++ b/src/meta/service/tests/it/api/http/transfer_leader.rs @@ -14,9 +14,9 @@ use std::time::Duration; +use databend_common_base::base::Stoppable; use databend_common_base::base::tokio; use databend_common_base::base::tokio::time::Instant; -use databend_common_base::base::Stoppable; use databend_common_meta_sled_store::openraft::async_runtime::watch::WatchReceiver; use databend_meta::api::HttpService; use log::info; diff --git a/src/meta/service/tests/it/grpc/metasrv_connection_error.rs b/src/meta/service/tests/it/grpc/metasrv_connection_error.rs index 9db576fdff9e5..37a141dfadb32 100644 --- a/src/meta/service/tests/it/grpc/metasrv_connection_error.rs +++ b/src/meta/service/tests/it/grpc/metasrv_connection_error.rs @@ -20,9 +20,9 @@ use std::sync::Arc; use std::time::Duration; use databend_common_base::base::Stoppable; -use databend_common_meta_client::errors::CreationError; use databend_common_meta_client::ClientHandle; use databend_common_meta_client::MetaGrpcClient; +use databend_common_meta_client::errors::CreationError; use databend_common_meta_kvapi::kvapi::KVApi; use databend_common_meta_kvapi::kvapi::KvApiExt; use databend_common_meta_types::UpsertKV; diff --git a/src/meta/service/tests/it/grpc/metasrv_grpc_api.rs b/src/meta/service/tests/it/grpc/metasrv_grpc_api.rs index 492af792c565b..90bab762b4b8b 100644 --- a/src/meta/service/tests/it/grpc/metasrv_grpc_api.rs +++ b/src/meta/service/tests/it/grpc/metasrv_grpc_api.rs @@ -15,14 +15,14 @@ //! Test arrow-grpc API of metasrv use std::collections::HashSet; -use databend_common_base::base::tokio; use databend_common_base::base::Stoppable; +use databend_common_base::base::tokio; use databend_common_meta_kvapi::kvapi::KVApi; use databend_common_meta_kvapi::kvapi::KvApiExt; use databend_common_meta_kvapi::kvapi::UpsertKVReply; -use databend_common_meta_types::normalize_meta::NormalizeMeta; use databend_common_meta_types::SeqV; use databend_common_meta_types::UpsertKV; +use databend_common_meta_types::normalize_meta::NormalizeMeta; use log::debug; use log::info; use pretty_assertions::assert_eq; diff --git a/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs b/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs index 02368408c22b1..659148f44d845 100644 --- a/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs +++ b/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs @@ -16,8 +16,8 @@ use std::time::Duration; use databend_common_base::base::tokio::time::sleep; use databend_common_meta_kvapi::kvapi::KVApi; -use databend_common_meta_types::protobuf as pb; use databend_common_meta_types::UpsertKV; +use databend_common_meta_types::protobuf as pb; use log::info; use pretty_assertions::assert_eq; use regex::Regex; diff --git a/src/meta/service/tests/it/grpc/metasrv_grpc_handshake.rs b/src/meta/service/tests/it/grpc/metasrv_grpc_handshake.rs index d83ba1fcb5185..8a56d8bb131fd 100644 --- a/src/meta/service/tests/it/grpc/metasrv_grpc_handshake.rs +++ b/src/meta/service/tests/it/grpc/metasrv_grpc_handshake.rs @@ -18,11 +18,11 @@ use std::time::Duration; use databend_common_grpc::ConnectionFactory; -use databend_common_meta_client::from_digit_ver; -use databend_common_meta_client::to_digit_ver; +use databend_common_meta_client::MIN_METASRV_SEMVER; use databend_common_meta_client::MetaChannelManager; use databend_common_meta_client::MetaGrpcClient; -use databend_common_meta_client::MIN_METASRV_SEMVER; +use databend_common_meta_client::from_digit_ver; +use databend_common_meta_client::to_digit_ver; use databend_common_version::DATABEND_SEMVER; use databend_meta::version::MIN_METACLI_SEMVER; use log::debug; diff --git a/src/meta/service/tests/it/grpc/metasrv_grpc_kv_api_restart_cluster.rs b/src/meta/service/tests/it/grpc/metasrv_grpc_kv_api_restart_cluster.rs index 0d13c74d7e39c..73cc92f761400 100644 --- a/src/meta/service/tests/it/grpc/metasrv_grpc_kv_api_restart_cluster.rs +++ b/src/meta/service/tests/it/grpc/metasrv_grpc_kv_api_restart_cluster.rs @@ -29,8 +29,8 @@ use test_harness::test; use tokio::time::sleep; use crate::testing::meta_service_test_harness; -use crate::tests::service::start_metasrv_cluster; use crate::tests::service::MetaSrvTestContext; +use crate::tests::service::start_metasrv_cluster; use crate::tests::start_metasrv_with_context; /// - Start a cluster of 3. diff --git a/src/meta/service/tests/it/grpc/metasrv_grpc_kv_read_v1.rs b/src/meta/service/tests/it/grpc/metasrv_grpc_kv_read_v1.rs index c73190189533d..c4ec91184afb6 100644 --- a/src/meta/service/tests/it/grpc/metasrv_grpc_kv_read_v1.rs +++ b/src/meta/service/tests/it/grpc/metasrv_grpc_kv_read_v1.rs @@ -22,13 +22,13 @@ use databend_common_meta_client::Streamed; use databend_common_meta_kvapi::kvapi::KVApi; use databend_common_meta_kvapi::kvapi::ListKVReq; use databend_common_meta_kvapi::kvapi::MGetKVReq; -use databend_common_meta_types::normalize_meta::NormalizeMeta; -use databend_common_meta_types::protobuf as pb; use databend_common_meta_types::MetaSpec; use databend_common_meta_types::UpsertKV; use databend_common_meta_types::With; -use futures::stream::StreamExt; +use databend_common_meta_types::normalize_meta::NormalizeMeta; +use databend_common_meta_types::protobuf as pb; use futures::TryStreamExt; +use futures::stream::StreamExt; use log::info; use pretty_assertions::assert_eq; use state_machine_api::KVMeta; diff --git a/src/meta/service/tests/it/grpc/metasrv_grpc_member_list.rs b/src/meta/service/tests/it/grpc/metasrv_grpc_member_list.rs index a46516d7d464a..ac32557f37422 100644 --- a/src/meta/service/tests/it/grpc/metasrv_grpc_member_list.rs +++ b/src/meta/service/tests/it/grpc/metasrv_grpc_member_list.rs @@ -14,8 +14,8 @@ use std::collections::HashSet; -use databend_common_base::base::tokio::time::sleep; use databend_common_base::base::tokio::time::Duration; +use databend_common_base::base::tokio::time::sleep; use databend_common_meta_sled_store::openraft::ServerState; use databend_common_version::BUILD_INFO; use databend_meta::message::ForwardRequest; diff --git a/src/meta/service/tests/it/grpc/metasrv_grpc_watch.rs b/src/meta/service/tests/it/grpc/metasrv_grpc_watch.rs index c1b6eb967fffe..2b70057d1c55c 100644 --- a/src/meta/service/tests/it/grpc/metasrv_grpc_watch.rs +++ b/src/meta/service/tests/it/grpc/metasrv_grpc_watch.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![allow(clippy::collapsible_if, clippy::useless_vec)] + use std::collections::BTreeMap; use std::sync::Arc; use std::sync::Mutex; @@ -25,22 +27,22 @@ use databend_common_meta_client::ClientHandle; use databend_common_meta_client::MetaGrpcClient; use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::KVApi; +use databend_common_meta_types::ConditionResult; +use databend_common_meta_types::MatchSeq; +use databend_common_meta_types::Operation; +use databend_common_meta_types::TxnCondition; +use databend_common_meta_types::TxnDeleteByPrefixRequest; +use databend_common_meta_types::TxnOp; +use databend_common_meta_types::UpsertKV; use databend_common_meta_types::normalize_meta::NormalizeMeta; -use databend_common_meta_types::protobuf::watch_request::FilterType; use databend_common_meta_types::protobuf::Event; use databend_common_meta_types::protobuf::KvMeta; use databend_common_meta_types::protobuf::SeqV; use databend_common_meta_types::protobuf::TxnRequest; use databend_common_meta_types::protobuf::WatchRequest; +use databend_common_meta_types::protobuf::watch_request::FilterType; use databend_common_meta_types::txn_condition; use databend_common_meta_types::txn_op; -use databend_common_meta_types::ConditionResult; -use databend_common_meta_types::MatchSeq; -use databend_common_meta_types::Operation; -use databend_common_meta_types::TxnCondition; -use databend_common_meta_types::TxnDeleteByPrefixRequest; -use databend_common_meta_types::TxnOp; -use databend_common_meta_types::UpsertKV; use databend_common_version::BUILD_INFO; use databend_meta::meta_service::MetaNode; use log::info; diff --git a/src/meta/service/tests/it/grpc/t51_metasrv_grpc_semaphore.rs b/src/meta/service/tests/it/grpc/t51_metasrv_grpc_semaphore.rs index 19fa629be0001..c8ba8caaaf49a 100644 --- a/src/meta/service/tests/it/grpc/t51_metasrv_grpc_semaphore.rs +++ b/src/meta/service/tests/it/grpc/t51_metasrv_grpc_semaphore.rs @@ -611,9 +611,10 @@ async fn test_time_based_connection_closed_error() -> anyhow::Result<()> { info!("permit2: {:?}", res); println!("permit2: {:?}", res); let err = res.unwrap().unwrap().unwrap_err(); - assert!(err - .to_string() - .contains("distributed-Semaphore connection closed")); + assert!( + err.to_string() + .contains("distributed-Semaphore connection closed") + ); Ok(()) } diff --git a/src/meta/service/tests/it/grpc/t52_metasrv_grpc_cache.rs b/src/meta/service/tests/it/grpc/t52_metasrv_grpc_cache.rs index 22261b8373573..f2ba323c33648 100644 --- a/src/meta/service/tests/it/grpc/t52_metasrv_grpc_cache.rs +++ b/src/meta/service/tests/it/grpc/t52_metasrv_grpc_cache.rs @@ -19,9 +19,9 @@ use std::time::Duration; use databend_common_base::base::Stoppable; use databend_common_meta_cache::Cache; use databend_common_meta_kvapi::kvapi::KVApi; -use databend_common_meta_types::normalize_meta::NormalizeMeta; use databend_common_meta_types::SeqV; use databend_common_meta_types::UpsertKV; +use databend_common_meta_types::normalize_meta::NormalizeMeta; use log::debug; use test_harness::test; use tokio::time::sleep; diff --git a/src/meta/service/tests/it/grpc/t53_metasrv_grpc_snapshot_keys_layout.rs b/src/meta/service/tests/it/grpc/t53_metasrv_grpc_snapshot_keys_layout.rs index 9782c121b2570..77162b16bbc98 100644 --- a/src/meta/service/tests/it/grpc/t53_metasrv_grpc_snapshot_keys_layout.rs +++ b/src/meta/service/tests/it/grpc/t53_metasrv_grpc_snapshot_keys_layout.rs @@ -16,8 +16,8 @@ use std::time::Duration; use databend_common_base::base::tokio::time::sleep; use databend_common_meta_kvapi::kvapi::KVApi; -use databend_common_meta_types::protobuf as pb; use databend_common_meta_types::UpsertKV; +use databend_common_meta_types::protobuf as pb; use log::info; use pretty_assertions::assert_eq; use test_harness::test; diff --git a/src/meta/service/tests/it/meta_node/meta_node_lifecycle.rs b/src/meta/service/tests/it/meta_node/meta_node_lifecycle.rs index 4974b310698ef..ac12fe983deac 100644 --- a/src/meta/service/tests/it/meta_node/meta_node_lifecycle.rs +++ b/src/meta/service/tests/it/meta_node/meta_node_lifecycle.rs @@ -20,13 +20,13 @@ use databend_common_meta_kvapi::kvapi::KvApiExt; use databend_common_meta_sled_store::openraft::LogIdOptionExt; use databend_common_meta_sled_store::openraft::RaftLogReader; use databend_common_meta_sled_store::openraft::ServerState; -use databend_common_meta_types::protobuf::raft_service_client::RaftServiceClient; -use databend_common_meta_types::raft_types::new_log_id; -use databend_common_meta_types::raft_types::NodeId; use databend_common_meta_types::Cmd; use databend_common_meta_types::Endpoint; use databend_common_meta_types::LogEntry; use databend_common_meta_types::UpsertKV; +use databend_common_meta_types::protobuf::raft_service_client::RaftServiceClient; +use databend_common_meta_types::raft_types::NodeId; +use databend_common_meta_types::raft_types::new_log_id; use databend_common_version::BUILD_INFO; use databend_meta::configs; use databend_meta::message::ForwardRequest; diff --git a/src/meta/service/tests/it/meta_node/meta_node_replication.rs b/src/meta/service/tests/it/meta_node/meta_node_replication.rs index 4b048229a0299..c3c34eec72ab8 100644 --- a/src/meta/service/tests/it/meta_node/meta_node_replication.rs +++ b/src/meta/service/tests/it/meta_node/meta_node_replication.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![allow(clippy::let_and_return)] + use std::fs; use std::io::Read; @@ -19,9 +21,13 @@ use databend_common_meta_client::MetaGrpcReadReq; use databend_common_meta_kvapi::kvapi::GetKVReq; use databend_common_meta_raft_store::sm_v003::SnapshotStoreV004; use databend_common_meta_raft_store::state_machine::MetaSnapshotId; -use databend_common_meta_sled_store::openraft::testing::log_id; use databend_common_meta_sled_store::openraft::LogIdOptionExt; use databend_common_meta_sled_store::openraft::ServerState; +use databend_common_meta_sled_store::openraft::testing::log_id; +use databend_common_meta_types::Cmd; +use databend_common_meta_types::LogEntry; +use databend_common_meta_types::SeqV; +use databend_common_meta_types::UpsertKV; use databend_common_meta_types::protobuf as pb; use databend_common_meta_types::protobuf::SnapshotChunkRequestV003; use databend_common_meta_types::raft_types::SnapshotMeta; @@ -29,15 +35,11 @@ use databend_common_meta_types::raft_types::SnapshotResponse; use databend_common_meta_types::raft_types::StoredMembership; use databend_common_meta_types::raft_types::Vote; use databend_common_meta_types::sys_data::SysData; -use databend_common_meta_types::Cmd; -use databend_common_meta_types::LogEntry; -use databend_common_meta_types::SeqV; -use databend_common_meta_types::UpsertKV; use databend_common_version::BUILD_INFO; use databend_meta::message::ForwardRequest; use databend_meta::meta_service::MetaNode; -use futures::stream; use futures::TryStreamExt; +use futures::stream; use itertools::Itertools; use log::info; use maplit::btreeset; diff --git a/src/meta/service/tests/it/meta_node/meta_node_request_forwarding.rs b/src/meta/service/tests/it/meta_node/meta_node_request_forwarding.rs index ec6364b5e2244..552258198f10a 100644 --- a/src/meta/service/tests/it/meta_node/meta_node_request_forwarding.rs +++ b/src/meta/service/tests/it/meta_node/meta_node_request_forwarding.rs @@ -15,13 +15,13 @@ use std::sync::Arc; use databend_common_meta_sled_store::openraft::error::RaftError; -use databend_common_meta_types::raft_types::ClientWriteError; -use databend_common_meta_types::raft_types::ForwardToLeader; use databend_common_meta_types::Cmd; use databend_common_meta_types::LogEntry; use databend_common_meta_types::UpsertKV; -use databend_meta::meta_service::meta_leader::MetaLeader; +use databend_common_meta_types::raft_types::ClientWriteError; +use databend_common_meta_types::raft_types::ForwardToLeader; use databend_meta::meta_service::MetaNode; +use databend_meta::meta_service::meta_leader::MetaLeader; use maplit::btreeset; use test_harness::test; diff --git a/src/meta/service/tests/it/meta_node/t90_time_revert_cross_snapshot_boundary.rs b/src/meta/service/tests/it/meta_node/t90_time_revert_cross_snapshot_boundary.rs index 4eb405a63286a..3e11b5de70a75 100644 --- a/src/meta/service/tests/it/meta_node/t90_time_revert_cross_snapshot_boundary.rs +++ b/src/meta/service/tests/it/meta_node/t90_time_revert_cross_snapshot_boundary.rs @@ -15,12 +15,12 @@ use std::time::Duration; use databend_common_meta_kvapi::kvapi::KvApiExt; -use databend_common_meta_types::raft_types::new_log_id; use databend_common_meta_types::Cmd; use databend_common_meta_types::LogEntry; use databend_common_meta_types::SeqV; use databend_common_meta_types::TxnOp; use databend_common_meta_types::TxnRequest; +use databend_common_meta_types::raft_types::new_log_id; use databend_common_version::BUILD_INFO; use databend_meta::meta_service::MetaNode; use log::info; @@ -63,11 +63,9 @@ async fn test_meta_node_log_time_revert_cross_snapshot_boundary() -> anyhow::Res // Log with earlier timestamp (T+60s) but expires at T+120s - not in snapshot let log_earlier = LogEntry { time_ms: Some(now_ms + 60_000), - cmd: Cmd::Transaction(TxnRequest::new(vec![], vec![TxnOp::put( - "k1", - b"v2".to_vec(), - ) - .with_expires_at_ms(Some(now_ms + 120_000))])), + cmd: Cmd::Transaction(TxnRequest::new(vec![], vec![ + TxnOp::put("k1", b"v2".to_vec()).with_expires_at_ms(Some(now_ms + 120_000)), + ])), }; let result_with_restart = write_two_logs(log_later.clone(), log_earlier.clone(), true).await?; diff --git a/src/meta/service/tests/it/store.rs b/src/meta/service/tests/it/store.rs index 4e0f3c5712a29..9ed82ffec4f42 100644 --- a/src/meta/service/tests/it/store.rs +++ b/src/meta/service/tests/it/store.rs @@ -16,6 +16,8 @@ use std::io; use databend_common_meta_raft_store::leveled_store::db_exporter::DBExporter; use databend_common_meta_raft_store::state_machine::testing::snapshot_logs; +use databend_common_meta_sled_store::openraft::RaftLogReader; +use databend_common_meta_sled_store::openraft::RaftSnapshotBuilder; use databend_common_meta_sled_store::openraft::entry::RaftEntry; use databend_common_meta_sled_store::openraft::storage::RaftLogReaderExt; use databend_common_meta_sled_store::openraft::storage::RaftLogStorage; @@ -23,15 +25,13 @@ use databend_common_meta_sled_store::openraft::storage::RaftLogStorageExt; use databend_common_meta_sled_store::openraft::storage::RaftStateMachine; use databend_common_meta_sled_store::openraft::testing::log::StoreBuilder; use databend_common_meta_sled_store::openraft::testing::log_id; -use databend_common_meta_sled_store::openraft::RaftLogReader; -use databend_common_meta_sled_store::openraft::RaftSnapshotBuilder; -use databend_common_meta_types::raft_types::new_log_id; use databend_common_meta_types::raft_types::Entry; use databend_common_meta_types::raft_types::Membership; use databend_common_meta_types::raft_types::StorageError; use databend_common_meta_types::raft_types::StoredMembership; use databend_common_meta_types::raft_types::TypeConfig; use databend_common_meta_types::raft_types::Vote; +use databend_common_meta_types::raft_types::new_log_id; use databend_common_meta_types::snapshot_db::DB; use databend_meta::meta_node::meta_node::LogStore; use databend_meta::meta_node::meta_node::SMStore; diff --git a/src/meta/service/tests/it/testing.rs b/src/meta/service/tests/it/testing.rs index 26ec7fa05e31b..6db0c071deacf 100644 --- a/src/meta/service/tests/it/testing.rs +++ b/src/meta/service/tests/it/testing.rs @@ -18,9 +18,9 @@ use std::time::Duration; use std::time::SystemTime; use databend_common_base::base::tokio; +use databend_common_tracing::Config; use databend_common_tracing::closure_name; use databend_common_tracing::init_logging; -use databend_common_tracing::Config; use fastrace::prelude::*; pub fn meta_service_test_harness(test: F) diff --git a/src/meta/service/tests/it/tests/meta_node.rs b/src/meta/service/tests/it/tests/meta_node.rs index 46f4db985a7bd..f9f4b69fca8b1 100644 --- a/src/meta/service/tests/it/tests/meta_node.rs +++ b/src/meta/service/tests/it/tests/meta_node.rs @@ -19,9 +19,9 @@ use std::sync::Arc; use std::time::Duration; use databend_common_meta_sled_store::openraft::ServerState; +use databend_common_meta_types::AppliedState; use databend_common_meta_types::node::Node; use databend_common_meta_types::raft_types::NodeId; -use databend_common_meta_types::AppliedState; use databend_common_version::BUILD_INFO; use databend_meta::meta_service::MetaNode; use log::info; diff --git a/src/meta/service/tests/it/tests/service.rs b/src/meta/service/tests/it/tests/service.rs index 6e727c44cfb6d..4046971446a11 100644 --- a/src/meta/service/tests/it/tests/service.rs +++ b/src/meta/service/tests/it/tests/service.rs @@ -20,12 +20,12 @@ use std::time::Duration; use anyhow::Result; use async_trait::async_trait; -use databend_common_base::base::tokio; use databend_common_base::base::GlobalSequence; use databend_common_base::base::Stoppable; -use databend_common_meta_client::errors::CreationError; +use databend_common_base::base::tokio; use databend_common_meta_client::ClientHandle; use databend_common_meta_client::MetaGrpcClient; +use databend_common_meta_client::errors::CreationError; use databend_common_meta_kvapi::kvapi; use databend_common_meta_types::protobuf::raft_service_client::RaftServiceClient; use databend_common_meta_types::raft_types::NodeId; diff --git a/src/meta/sled-store/src/sled_serde.rs b/src/meta/sled-store/src/sled_serde.rs index fde0466c6231c..c5105373ed320 100644 --- a/src/meta/sled-store/src/sled_serde.rs +++ b/src/meta/sled-store/src/sled_serde.rs @@ -16,8 +16,8 @@ use std::mem::size_of_val; use byteorder::BigEndian; use byteorder::ByteOrder; -use serde::de::DeserializeOwned; use serde::Serialize; +use serde::de::DeserializeOwned; use sled::IVec; use state_machine_api::ExpireKey; diff --git a/src/meta/sled-store/src/sled_serde_impl.rs b/src/meta/sled-store/src/sled_serde_impl.rs index 62b11017fb17e..532c71826813c 100644 --- a/src/meta/sled-store/src/sled_serde_impl.rs +++ b/src/meta/sled-store/src/sled_serde_impl.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_meta_types::SeqNum; +use databend_common_meta_types::SeqV; use databend_common_meta_types::node::Node; use databend_common_meta_types::raft_types::Entry; use databend_common_meta_types::raft_types::EntryPayload; @@ -20,8 +22,6 @@ use databend_common_meta_types::raft_types::Membership; use databend_common_meta_types::raft_types::SnapshotMeta; use databend_common_meta_types::raft_types::StoredMembership; use databend_common_meta_types::raft_types::Vote; -use databend_common_meta_types::SeqNum; -use databend_common_meta_types::SeqV; use state_machine_api::ExpireValue; use crate::SledBytesError; diff --git a/src/meta/sled-store/src/sled_tree.rs b/src/meta/sled-store/src/sled_tree.rs index af8de77fbf00f..271d59ab788ba 100644 --- a/src/meta/sled-store/src/sled_tree.rs +++ b/src/meta/sled-store/src/sled_tree.rs @@ -92,7 +92,7 @@ impl SledTree { } /// Borrows the SledTree and creates a wrapper with access limited to a specified key space `KV`. - pub fn key_space(&self) -> AsKeySpace { + pub fn key_space(&self) -> AsKeySpace<'_, KV> { AsKeySpace:: { inner: self, phantom: PhantomData, diff --git a/src/meta/store/src/lib.rs b/src/meta/store/src/lib.rs index 3582d4294ba3f..03c23f1eaf457 100644 --- a/src/meta/store/src/lib.rs +++ b/src/meta/store/src/lib.rs @@ -21,22 +21,22 @@ use std::time::Duration; use std::time::SystemTime; use std::time::UNIX_EPOCH; -use databend_common_base::base::tokio::time::timeout; use databend_common_base::base::BuildInfoRef; +use databend_common_base::base::tokio::time::timeout; use databend_common_grpc::RpcClientConf; -use databend_common_meta_client::errors::CreationError; use databend_common_meta_client::ClientHandle; use databend_common_meta_client::MetaGrpcClient; +use databend_common_meta_client::errors::CreationError; +use databend_common_meta_semaphore::Semaphore; use databend_common_meta_semaphore::acquirer::Permit; use databend_common_meta_semaphore::errors::AcquireError; -use databend_common_meta_semaphore::Semaphore; -use databend_common_meta_types::protobuf::WatchResponse; use databend_common_meta_types::MetaError; +use databend_common_meta_types::protobuf::WatchResponse; pub use local::LocalMetaService; use log::info; use log::warn; -use tokio::time::error::Elapsed; use tokio::time::Instant; +use tokio::time::error::Elapsed; use tokio_stream::Stream; pub type WatchStream = diff --git a/src/meta/store/src/local.rs b/src/meta/store/src/local.rs index 8f6523d032385..8a6cafd86205f 100644 --- a/src/meta/store/src/local.rs +++ b/src/meta/store/src/local.rs @@ -22,9 +22,9 @@ use std::time::Duration; use databend_common_base::base::BuildInfoRef; use databend_common_base::base::GlobalSequence; use databend_common_base::base::Stoppable; -use databend_common_meta_client::errors::CreationError; use databend_common_meta_client::ClientHandle; use databend_common_meta_client::MetaGrpcClient; +use databend_common_meta_client::errors::CreationError; use databend_common_meta_types::protobuf::raft_service_client::RaftServiceClient; use databend_meta::api::GrpcServer; use databend_meta::configs; diff --git a/src/meta/types/src/applied_state.rs b/src/meta/types/src/applied_state.rs index bd79d8223dfa6..e557f57fae259 100644 --- a/src/meta/types/src/applied_state.rs +++ b/src/meta/types/src/applied_state.rs @@ -15,10 +15,10 @@ use std::fmt; use std::fmt::Formatter; -use crate::node::Node; -use crate::protobuf::RaftReply; use crate::Change; use crate::TxnReply; +use crate::node::Node; +use crate::protobuf::RaftReply; /// The state of an applied raft log. /// Normally it includes two fields: the state before applying and the state after applying the log. @@ -70,11 +70,8 @@ impl AppliedState { /// Whether the state changed pub fn changed(&self) -> bool { match self { - AppliedState::Node { - ref prev, - ref result, - } => prev != result, - AppliedState::KV(ref ch) => ch.is_changed(), + AppliedState::Node { prev, result } => prev != result, + AppliedState::KV(ch) => ch.is_changed(), AppliedState::None => false, AppliedState::TxnReply(txn) => txn.success, } @@ -98,8 +95,8 @@ impl AppliedState { pub fn prev_is_none(&self) -> bool { match self { - AppliedState::Node { ref prev, .. } => prev.is_none(), - AppliedState::KV(Change { ref prev, .. }) => prev.is_none(), + AppliedState::Node { prev, .. } => prev.is_none(), + AppliedState::KV(Change { prev, .. }) => prev.is_none(), AppliedState::None => true, AppliedState::TxnReply(_txn) => true, } @@ -107,8 +104,8 @@ impl AppliedState { pub fn result_is_none(&self) -> bool { match self { - AppliedState::Node { ref result, .. } => result.is_none(), - AppliedState::KV(Change { ref result, .. }) => result.is_none(), + AppliedState::Node { result, .. } => result.is_none(), + AppliedState::KV(Change { result, .. }) => result.is_none(), AppliedState::None => true, AppliedState::TxnReply(txn) => !txn.success, } diff --git a/src/meta/types/src/cmd/cmd_context.rs b/src/meta/types/src/cmd/cmd_context.rs index e61f251180dc2..f7a8b0e0ec73a 100644 --- a/src/meta/types/src/cmd/cmd_context.rs +++ b/src/meta/types/src/cmd/cmd_context.rs @@ -19,10 +19,10 @@ use std::time::Duration; use display_more::DisplayUnixTimeStampExt; +use crate::Time; use crate::cmd::io_timing::IoTimer; use crate::cmd::io_timing::IoTiming; use crate::raft_types::LogId; -use crate::Time; /// A context used when executing a [`Cmd`], to provide additional environment information. /// @@ -110,7 +110,7 @@ impl CmdContext { &self, op_type: impl Into, details: impl Into, - ) -> IoTimer { + ) -> IoTimer<'_> { IoTimer::new(self, op_type, details) } } diff --git a/src/meta/types/src/cmd/meta_spec.rs b/src/meta/types/src/cmd/meta_spec.rs index a4ab97ab67d29..b1efab53cd18a 100644 --- a/src/meta/types/src/cmd/meta_spec.rs +++ b/src/meta/types/src/cmd/meta_spec.rs @@ -21,8 +21,8 @@ use display_more::DisplayUnixTimeStampExt; use state_machine_api::KVMeta; use crate::cmd::CmdContext; -use crate::time::flexible_timestamp_to_duration; use crate::time::Interval; +use crate::time::flexible_timestamp_to_duration; /// Specifies the metadata associated with a kv record, used in an `upsert` cmd. /// @@ -128,8 +128,8 @@ mod tests { use state_machine_api::KVMeta; use super::MetaSpec; - use crate::cmd::CmdContext; use crate::Time; + use crate::cmd::CmdContext; #[test] fn test_serde() { diff --git a/src/meta/types/src/cmd/mod.rs b/src/meta/types/src/cmd/mod.rs index 9517ec7a99ac6..e13268641b869 100644 --- a/src/meta/types/src/cmd/mod.rs +++ b/src/meta/types/src/cmd/mod.rs @@ -17,9 +17,9 @@ use std::fmt; use serde::Deserialize; use serde::Serialize; +use crate::TxnRequest; use crate::node::Node; use crate::raft_types::NodeId; -use crate::TxnRequest; mod cmd_context; mod io_timing; diff --git a/src/meta/types/src/cmd/upsert_kv.rs b/src/meta/types/src/cmd/upsert_kv.rs index fa270edbaa29b..a371a1a9cc333 100644 --- a/src/meta/types/src/cmd/upsert_kv.rs +++ b/src/meta/types/src/cmd/upsert_kv.rs @@ -19,10 +19,10 @@ use display_more::DisplayOptionExt; use serde::Deserialize; use serde::Serialize; -use crate::match_seq::MatchSeq; use crate::MetaSpec; use crate::Operation; use crate::With; +use crate::match_seq::MatchSeq; /// Update or insert a general purpose kv store #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, deepsize::DeepSizeOf)] diff --git a/src/meta/types/src/errors/meta_api_errors.rs b/src/meta/types/src/errors/meta_api_errors.rs index 3004a814cb650..e38012c16dceb 100644 --- a/src/meta/types/src/errors/meta_api_errors.rs +++ b/src/meta/types/src/errors/meta_api_errors.rs @@ -19,15 +19,15 @@ use anyerror::AnyError; use databend_common_exception::ErrorCode; use tonic::Status; +use crate::InvalidArgument; +use crate::InvalidReply; +use crate::MetaNetworkError; use crate::errors; use crate::raft_types::ChangeMembershipError; use crate::raft_types::ClientWriteError; use crate::raft_types::Fatal; use crate::raft_types::ForwardToLeader; use crate::raft_types::RaftError; -use crate::InvalidArgument; -use crate::InvalidReply; -use crate::MetaNetworkError; /// Errors raised when meta-service handling a request. #[derive(thiserror::Error, serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Eq)] diff --git a/src/meta/types/src/errors/meta_errors.rs b/src/meta/types/src/errors/meta_errors.rs index d0a30b61934b6..b863d2e2d8429 100644 --- a/src/meta/types/src/errors/meta_errors.rs +++ b/src/meta/types/src/errors/meta_errors.rs @@ -18,13 +18,13 @@ use databend_common_exception::ErrorCode; use databend_common_meta_stoerr::MetaStorageError; use thiserror::Error; -use crate::errors; use crate::InvalidArgument; use crate::InvalidReply; use crate::MetaAPIError; use crate::MetaClientError; use crate::MetaHandshakeError; use crate::MetaNetworkError; +use crate::errors; /// Top level error MetaNode would return. #[derive(Error, Debug, Clone, PartialEq, Eq)] diff --git a/src/meta/types/src/errors/meta_raft_errors.rs b/src/meta/types/src/errors/meta_raft_errors.rs index 2af33338aeb9a..4a9ec0979e743 100644 --- a/src/meta/types/src/errors/meta_raft_errors.rs +++ b/src/meta/types/src/errors/meta_raft_errors.rs @@ -17,10 +17,10 @@ pub use openraft::error::EmptyMembership; pub use openraft::error::InProgress; pub use openraft::error::InitializeError; -use crate::raft_types::ClientWriteError; -use crate::raft_types::RaftError; use crate::MetaDataError; use crate::MetaOperationError; +use crate::raft_types::ClientWriteError; +use crate::raft_types::RaftError; // Collection of errors that occur when change membership on local raft node. pub type RaftChangeMembershipError = ClientWriteError; diff --git a/src/meta/types/src/errors/meta_startup_errors.rs b/src/meta/types/src/errors/meta_startup_errors.rs index f4836f2a3065c..40c16e3d9f816 100644 --- a/src/meta/types/src/errors/meta_startup_errors.rs +++ b/src/meta/types/src/errors/meta_startup_errors.rs @@ -15,9 +15,9 @@ use anyerror::AnyError; use databend_common_meta_stoerr::MetaStorageError; +use crate::MetaNetworkError; use crate::raft_types::InitializeError; use crate::raft_types::RaftError; -use crate::MetaNetworkError; /// Error raised when meta-server startup. #[derive(thiserror::Error, Debug, Clone, PartialEq, Eq)] diff --git a/src/meta/types/src/grpc_helper.rs b/src/meta/types/src/grpc_helper.rs index 3eba3851d84c6..d7c1ef0c63bd8 100644 --- a/src/meta/types/src/grpc_helper.rs +++ b/src/meta/types/src/grpc_helper.rs @@ -20,10 +20,10 @@ use std::str::FromStr; use log::error; use tonic::metadata::MetadataValue; +use crate::Endpoint; use crate::protobuf::RaftReply; use crate::protobuf::RaftRequest; use crate::raft_types::RaftError; -use crate::Endpoint; const HEADER_LEADER: &str = "x-databend-meta-leader-grpc-endpoint"; // const HEADER_LEADER_BIN: &str = "x-databend-meta-leader-grpc-endpoint-bin"; diff --git a/src/meta/types/src/lib.rs b/src/meta/types/src/lib.rs index d7561b1f77892..13840aa5d96e5 100644 --- a/src/meta/types/src/lib.rs +++ b/src/meta/types/src/lib.rs @@ -13,8 +13,8 @@ // limitations under the License. #![allow(clippy::uninlined_format_args)] +#![allow(clippy::collapsible_if)] #![allow(non_local_definitions)] -#![feature(no_sanitize)] //! This crate defines data types used in meta data storage service. @@ -77,9 +77,9 @@ pub use grpc_config::GrpcConfig; pub use log_entry::LogEntry; pub use map_api::Expirable; pub mod match_seq { - pub use map_api::match_seq::errors::ConflictSeq; pub use map_api::match_seq::MatchSeq; pub use map_api::match_seq::MatchSeqExt; + pub use map_api::match_seq::errors::ConflictSeq; } pub use match_seq::ConflictSeq; pub use match_seq::MatchSeq; @@ -87,10 +87,6 @@ pub use match_seq::MatchSeqExt; pub use node::Node; pub use operation::MetaId; pub use operation::Operation; -pub use protobuf::txn_condition; -pub use protobuf::txn_condition::ConditionResult; -pub use protobuf::txn_op; -pub use protobuf::txn_op_response; pub use protobuf::TxnCondition; pub use protobuf::TxnDeleteByPrefixRequest; pub use protobuf::TxnDeleteByPrefixResponse; @@ -104,6 +100,10 @@ pub use protobuf::TxnPutRequest; pub use protobuf::TxnPutResponse; pub use protobuf::TxnReply; pub use protobuf::TxnRequest; +pub use protobuf::txn_condition; +pub use protobuf::txn_condition::ConditionResult; +pub use protobuf::txn_op; +pub use protobuf::txn_op_response; pub use seq_num::SeqNum; pub use state_machine_api::SeqV; pub use time::Interval; diff --git a/src/meta/types/src/message.rs b/src/meta/types/src/message.rs index 7b6fd816db855..fabc71ea23de6 100644 --- a/src/meta/types/src/message.rs +++ b/src/meta/types/src/message.rs @@ -14,14 +14,14 @@ use serde::Serialize; +use crate::GrpcHelper; +use crate::LogEntry; +use crate::MetaAPIError; use crate::protobuf::RaftReply; use crate::protobuf::RaftRequest; use crate::raft_types::AppendEntriesRequest; use crate::raft_types::InstallSnapshotRequest; use crate::raft_types::VoteRequest; -use crate::GrpcHelper; -use crate::LogEntry; -use crate::MetaAPIError; impl tonic::IntoRequest for LogEntry { fn into_request(self) -> tonic::Request { diff --git a/src/meta/types/src/proto_display/watch_display.rs b/src/meta/types/src/proto_display/watch_display.rs index f383b6714b320..8465e1e1614fa 100644 --- a/src/meta/types/src/proto_display/watch_display.rs +++ b/src/meta/types/src/proto_display/watch_display.rs @@ -47,9 +47,9 @@ impl fmt::Display for WatchRequest { mod tests { use super::*; use crate::protobuf as pb; - use crate::protobuf::watch_request::FilterType; use crate::protobuf::KvMeta; use crate::protobuf::SeqV; + use crate::protobuf::watch_request::FilterType; #[test] fn test_watch_response_display() { @@ -72,10 +72,16 @@ mod tests { }), is_initialization: false, }; - assert_eq!(watch_response.to_string(), "CHANGE:(test_key: (seq=1 [expire=2024-08-08T07:40:19.000, proposed=2024-08-08T07:40:00.000] 'test_prev') -> (seq=2 [] 'test_current'))"); + assert_eq!( + watch_response.to_string(), + "CHANGE:(test_key: (seq=1 [expire=2024-08-08T07:40:19.000, proposed=2024-08-08T07:40:00.000] 'test_prev') -> (seq=2 [] 'test_current'))" + ); watch_response.is_initialization = true; - assert_eq!(watch_response.to_string(), "INIT:(test_key: (seq=1 [expire=2024-08-08T07:40:19.000, proposed=2024-08-08T07:40:00.000] 'test_prev') -> (seq=2 [] 'test_current'))"); + assert_eq!( + watch_response.to_string(), + "INIT:(test_key: (seq=1 [expire=2024-08-08T07:40:19.000, proposed=2024-08-08T07:40:00.000] 'test_prev') -> (seq=2 [] 'test_current'))" + ); let watch_response = WatchResponse { event: None, diff --git a/src/meta/types/src/proto_ext/conditional_operation_ext.rs b/src/meta/types/src/proto_ext/conditional_operation_ext.rs index c36281fada7e6..cb9ffa2d3d045 100644 --- a/src/meta/types/src/proto_ext/conditional_operation_ext.rs +++ b/src/meta/types/src/proto_ext/conditional_operation_ext.rs @@ -170,7 +170,10 @@ mod tests { let cond_op = pb::ConditionalOperation::new(predicate, operations); let display_str = cond_op.to_string(); - assert_eq!(display_str, "if:(key == seq(1)) then:[Put(Put key=key0),Put(Put key=key1),Put(Put key=key2),Put(Put key=key3),Put(Put key=key4)]"); + assert_eq!( + display_str, + "if:(key == seq(1)) then:[Put(Put key=key0),Put(Put key=key1),Put(Put key=key2),Put(Put key=key3),Put(Put key=key4)]" + ); } #[test] diff --git a/src/meta/types/src/proto_ext/fetch_increase_u64_response_ext.rs b/src/meta/types/src/proto_ext/fetch_increase_u64_response_ext.rs index 5e9729827eef6..39329c4c2c59a 100644 --- a/src/meta/types/src/proto_ext/fetch_increase_u64_response_ext.rs +++ b/src/meta/types/src/proto_ext/fetch_increase_u64_response_ext.rs @@ -15,8 +15,8 @@ use std::fmt::Display; use std::fmt::Formatter; -use crate::protobuf as pb; use crate::SeqV; +use crate::protobuf as pb; impl pb::FetchIncreaseU64Response { pub fn new(key: impl ToString, before: SeqV, after: SeqV) -> Self { diff --git a/src/meta/types/src/proto_ext/seq_v_ext.rs b/src/meta/types/src/proto_ext/seq_v_ext.rs index 3b6ae4ccec409..b25641c79d00a 100644 --- a/src/meta/types/src/proto_ext/seq_v_ext.rs +++ b/src/meta/types/src/proto_ext/seq_v_ext.rs @@ -14,9 +14,9 @@ use std::time::Duration; -use map_api::match_seq::errors::ConflictSeq; use map_api::match_seq::MatchSeq; use map_api::match_seq::MatchSeqExt; +use map_api::match_seq::errors::ConflictSeq; use state_machine_api::KVMeta; use state_machine_api::SeqV; @@ -74,7 +74,7 @@ impl pb::KvMeta { let a = a.as_micros() as u64; let b = b.as_micros() as u64; - let diff = if a > b { a - b } else { b - a }; + let diff = a.abs_diff(b); diff <= tolerance.as_micros() as u64 } } diff --git a/src/meta/types/src/proto_ext/txn_op_response_ext.rs b/src/meta/types/src/proto_ext/txn_op_response_ext.rs index d1edd297688de..7259c497cac2e 100644 --- a/src/meta/types/src/proto_ext/txn_op_response_ext.rs +++ b/src/meta/types/src/proto_ext/txn_op_response_ext.rs @@ -19,8 +19,8 @@ use std::fmt::Formatter; use display_more::DisplayOptionExt; -use crate::protobuf as pb; use crate::SeqV; +use crate::protobuf as pb; impl pb::TxnOpResponse { /// Create a new `TxnOpResponse` from a `Response` variant. diff --git a/src/meta/types/src/proto_ext/txn_put_response_ext.rs b/src/meta/types/src/proto_ext/txn_put_response_ext.rs index eb758e1573726..371935a7872bf 100644 --- a/src/meta/types/src/proto_ext/txn_put_response_ext.rs +++ b/src/meta/types/src/proto_ext/txn_put_response_ext.rs @@ -17,9 +17,9 @@ use std::fmt::Formatter; use display_more::DisplayOptionExt; -use crate::protobuf as pb; use crate::SeqV; use crate::TxnPutResponse; +use crate::protobuf as pb; impl Display for TxnPutResponse { fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { diff --git a/src/meta/types/src/proto_ext/txn_reply_ext.rs b/src/meta/types/src/proto_ext/txn_reply_ext.rs index c78e9437834ef..9335e12332816 100644 --- a/src/meta/types/src/proto_ext/txn_reply_ext.rs +++ b/src/meta/types/src/proto_ext/txn_reply_ext.rs @@ -19,10 +19,10 @@ use anyerror::AnyError; use display_more::DisplaySliceExt; use map_api::SeqV; -use crate::protobuf as pb; -use crate::txn_op_response; use crate::Change; use crate::InvalidReply; +use crate::protobuf as pb; +use crate::txn_op_response; impl pb::TxnReply { pub fn new(execution_path: impl ToString) -> Self { @@ -184,7 +184,10 @@ mod tests { ]; let display_str = reply.to_string(); - assert_eq!(display_str, "TxnReply{ success: true, responses: [TxnOpResponse: Get: Get-resp: key=key1, prev_seq=Some(1),TxnOpResponse: Put: Put-resp: key=key2, prev_seq=None, current_seq=2,TxnOpResponse: Delete: Delete-resp: success: true, key=key3, prev_seq=None] }"); + assert_eq!( + display_str, + "TxnReply{ success: true, responses: [TxnOpResponse: Get: Get-resp: key=key1, prev_seq=Some(1),TxnOpResponse: Put: Put-resp: key=key2, prev_seq=None, current_seq=2,TxnOpResponse: Delete: Delete-resp: success: true, key=key3, prev_seq=None] }" + ); } #[test] @@ -341,10 +344,11 @@ mod tests { reply.responses = vec![pb::TxnOpResponse { response: None }]; let res = reply.into_upsert_reply(); - assert!(res - .unwrap_err() - .to_string() - .contains("Empty response in TxnReply")); + assert!( + res.unwrap_err() + .to_string() + .contains("Empty response in TxnReply") + ); } #[test] @@ -381,9 +385,10 @@ mod tests { let reply = pb::TxnReply::new("operation:0"); // responses vector is empty by default let res = reply.into_upsert_reply(); - assert!(res - .unwrap_err() - .to_string() - .contains("No responses in TxnReply")); + assert!( + res.unwrap_err() + .to_string() + .contains("No responses in TxnReply") + ); } } diff --git a/src/meta/types/src/proto_ext/txn_request_ext.rs b/src/meta/types/src/proto_ext/txn_request_ext.rs index 55d88ec2872a9..b3e0768409387 100644 --- a/src/meta/types/src/proto_ext/txn_request_ext.rs +++ b/src/meta/types/src/proto_ext/txn_request_ext.rs @@ -18,11 +18,11 @@ use std::fmt::Formatter; use display_more::DisplaySliceExt; use map_api::match_seq::MatchSeq; -use crate::protobuf as pb; -use crate::time::flexible_timestamp_to_duration; use crate::ConditionResult; use crate::Operation; use crate::UpsertKV; +use crate::protobuf as pb; +use crate::time::flexible_timestamp_to_duration; impl pb::TxnRequest { /// Build a transaction request from an upsert operation. @@ -182,7 +182,7 @@ mod tests { assert_eq!( format!("{}", req), - "TxnRequest{{ if:(k1 == seq(1) AND k2 == seq(2)) then:[Put(Put key=k1),Put(Put key=k2)] }, if:[k1 == seq(1),k2 == seq(2)] then:[Put(Put key=k1),Put(Put key=k2)] else:[Put(Put key=k3)]}", + "TxnRequest{{ if:(k1 == seq(1) AND k2 == seq(2)) then:[Put(Put key=k1),Put(Put key=k2)] }, if:[k1 == seq(1),k2 == seq(2)] then:[Put(Put key=k1),Put(Put key=k2)] else:[Put(Put key=k3)]}", ); } diff --git a/src/meta/types/src/proto_ext/watch_ext.rs b/src/meta/types/src/proto_ext/watch_ext.rs index f56e5e900fcff..22172e4fcf155 100644 --- a/src/meta/types/src/proto_ext/watch_ext.rs +++ b/src/meta/types/src/proto_ext/watch_ext.rs @@ -14,12 +14,12 @@ mod event_ext; +use crate::Change; +use crate::SeqV; use crate::protobuf as pb; -use crate::protobuf::watch_request::FilterType; use crate::protobuf::WatchRequest; use crate::protobuf::WatchResponse; -use crate::Change; -use crate::SeqV; +use crate::protobuf::watch_request::FilterType; impl WatchRequest { pub fn new(key: String, key_end: Option) -> Self { diff --git a/src/meta/types/src/proto_ext/watch_ext/event_ext.rs b/src/meta/types/src/proto_ext/watch_ext/event_ext.rs index 2588f8afababd..c7451af487841 100644 --- a/src/meta/types/src/proto_ext/watch_ext/event_ext.rs +++ b/src/meta/types/src/proto_ext/watch_ext/event_ext.rs @@ -132,7 +132,10 @@ mod tests { meta: None, }), }; - assert_eq!(event.to_string(), "(test_key: (seq=1 [expire=2024-08-08T07:40:19.000, proposed=2024-08-08T07:40:00.000] 'test_prev') -> (seq=2 [] 'test_current'))"); + assert_eq!( + event.to_string(), + "(test_key: (seq=1 [expire=2024-08-08T07:40:19.000, proposed=2024-08-08T07:40:00.000] 'test_prev') -> (seq=2 [] 'test_current'))" + ); } #[test] diff --git a/src/meta/types/src/raft_types.rs b/src/meta/types/src/raft_types.rs index 12bcc8a82f2e5..a4ef9c403eec7 100644 --- a/src/meta/types/src/raft_types.rs +++ b/src/meta/types/src/raft_types.rs @@ -14,16 +14,16 @@ //! This mod wraps openraft types that have generics parameter with concrete types. +use openraft::RaftTypeConfig; +use openraft::TokioRuntime; use openraft::error::Infallible; use openraft::impls::OneshotResponder; -pub use openraft::vote::leader_id_adv::CommittedLeaderId; use openraft::vote::RaftLeaderId; -use openraft::RaftTypeConfig; -use openraft::TokioRuntime; +pub use openraft::vote::leader_id_adv::CommittedLeaderId; -use crate::snapshot_db::DB; use crate::AppliedState; use crate::LogEntry; +use crate::snapshot_db::DB; pub type NodeId = u64; pub type MembershipNode = openraft::EmptyNode; pub type LogIndex = u64; diff --git a/src/meta/types/src/snapshot_db.rs b/src/meta/types/src/snapshot_db.rs index 6b4c384b17cf7..9a990c580d4cd 100644 --- a/src/meta/types/src/snapshot_db.rs +++ b/src/meta/types/src/snapshot_db.rs @@ -21,9 +21,9 @@ use std::sync::Arc; use futures_util::stream::BoxStream; use log::info; use openraft::SnapshotId; -use rotbl::v001::stat::RotblStat; use rotbl::v001::Rotbl; use rotbl::v001::SeqMarked; +use rotbl::v001::stat::RotblStat; use crate::raft_types::SnapshotMeta; use crate::sys_data::SysData; diff --git a/src/meta/types/src/sys_data.rs b/src/meta/types/src/sys_data.rs index 1d942602d9062..52db309451a35 100644 --- a/src/meta/types/src/sys_data.rs +++ b/src/meta/types/src/sys_data.rs @@ -148,9 +148,9 @@ mod tests { use pretty_assertions::assert_eq; use super::*; - use crate::raft_types::new_log_id; - use crate::raft_types::Membership; use crate::Endpoint; + use crate::raft_types::Membership; + use crate::raft_types::new_log_id; /// Fields with default value should not be serialized. #[test] diff --git a/src/meta/types/tests/it/txn_serde.rs b/src/meta/types/tests/it/txn_serde.rs index ad9286522e7c5..2b87a95e55095 100644 --- a/src/meta/types/tests/it/txn_serde.rs +++ b/src/meta/types/tests/it/txn_serde.rs @@ -14,10 +14,10 @@ use std::time::Duration; -use databend_common_meta_types::protobuf::BooleanExpression; use databend_common_meta_types::TxnCondition; use databend_common_meta_types::TxnOp; use databend_common_meta_types::TxnRequest; +use databend_common_meta_types::protobuf::BooleanExpression; #[test] fn test_txn_request_serde() -> anyhow::Result<()> { diff --git a/src/query/ast/benches/bench.rs b/src/query/ast/benches/bench.rs index 701219e97939b..66ff1ef6aeda6 100644 --- a/src/query/ast/benches/bench.rs +++ b/src/query/ast/benches/bench.rs @@ -27,10 +27,10 @@ fn main() { #[divan::bench_group(max_time = 0.5)] mod dummy { + use databend_common_ast::parser::Dialect; use databend_common_ast::parser::parse_expr; use databend_common_ast::parser::parse_sql; use databend_common_ast::parser::tokenize_sql; - use databend_common_ast::parser::Dialect; #[divan::bench] fn large_statement() { diff --git a/src/query/ast/fuzz/Cargo.toml b/src/query/ast/fuzz/Cargo.toml index a977b62b5bc74..d12c326ad474d 100644 --- a/src/query/ast/fuzz/Cargo.toml +++ b/src/query/ast/fuzz/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "databend-fuzz" version = "0.1.0" -edition = "2021" +edition = "2024" publish = false # cargo can't build fuzz targets with afl diff --git a/src/query/ast/src/ast/common.rs b/src/query/ast/src/ast/common.rs index e8d99464acc91..d98860ab2da57 100644 --- a/src/query/ast/src/ast/common.rs +++ b/src/query/ast/src/ast/common.rs @@ -20,9 +20,9 @@ use derive_visitor::DriveMut; use ethnum::i256; use super::quote::QuotedString; -use crate::ast::quote::QuotedIdent; -use crate::ast::WithOptions; use crate::Span; +use crate::ast::WithOptions; +use crate::ast::quote::QuotedIdent; // Identifier of table name or column name. #[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] diff --git a/src/query/ast/src/ast/expr.rs b/src/query/ast/src/ast/expr.rs index 6fd85d563bc93..fb5f907f523bb 100644 --- a/src/query/ast/src/ast/expr.rs +++ b/src/query/ast/src/ast/expr.rs @@ -25,17 +25,17 @@ use pratt::Associativity; use super::ColumnRef; use super::OrderByExpr; +use crate::ParseError; +use crate::Result; +use crate::Span; +use crate::ast::Identifier; +use crate::ast::Indirection; +use crate::ast::Query; use crate::ast::display_decimal_256; use crate::ast::quote::QuotedString; use crate::ast::write_comma_separated_list; use crate::ast::write_dot_separated_list; -use crate::ast::Identifier; -use crate::ast::Indirection; -use crate::ast::Query; use crate::span::merge_span; -use crate::ParseError; -use crate::Result; -use crate::Span; #[derive(Educe, Drive, DriveMut)] #[educe( diff --git a/src/query/ast/src/ast/query.rs b/src/query/ast/src/ast/query.rs index ccab4ded2c831..8411045481750 100644 --- a/src/query/ast/src/ast/query.rs +++ b/src/query/ast/src/ast/query.rs @@ -20,10 +20,9 @@ use derive_visitor::Drive; use derive_visitor::DriveMut; use educe::Educe; -use crate::ast::quote::QuotedString; -use crate::ast::write_comma_separated_list; -use crate::ast::write_comma_separated_string_map; -use crate::ast::write_dot_separated_list; +use crate::ParseError; +use crate::Result; +use crate::Span; use crate::ast::Expr; use crate::ast::FileLocation; use crate::ast::Hint; @@ -31,9 +30,10 @@ use crate::ast::Identifier; use crate::ast::Lambda; use crate::ast::SelectStageOptions; use crate::ast::WindowDefinition; -use crate::ParseError; -use crate::Result; -use crate::Span; +use crate::ast::quote::QuotedString; +use crate::ast::write_comma_separated_list; +use crate::ast::write_comma_separated_string_map; +use crate::ast::write_dot_separated_list; /// Root node of a query tree #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] @@ -197,9 +197,9 @@ impl Display for SelectStmt { } // GROUP BY clause - if self.group_by.is_some() { + if let Some(g) = &self.group_by { write!(f, " GROUP BY ")?; - write!(f, "{}", self.group_by.as_ref().unwrap())?; + write!(f, "{}", g)?; } // HAVING clause if let Some(having) = &self.having { diff --git a/src/query/ast/src/ast/statements/call.rs b/src/query/ast/src/ast/statements/call.rs index 1d78018d13c8a..f6276eecf8b5e 100644 --- a/src/query/ast/src/ast/statements/call.rs +++ b/src/query/ast/src/ast/statements/call.rs @@ -18,8 +18,8 @@ use std::fmt::Formatter; use derive_visitor::Drive; use derive_visitor::DriveMut; -use crate::ast::write_comma_separated_string_list; use crate::ast::Identifier; +use crate::ast::write_comma_separated_string_list; #[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] pub struct CallStmt { diff --git a/src/query/ast/src/ast/statements/catalog.rs b/src/query/ast/src/ast/statements/catalog.rs index 36036f0ab4bed..50419bd5e8231 100644 --- a/src/query/ast/src/ast/statements/catalog.rs +++ b/src/query/ast/src/ast/statements/catalog.rs @@ -19,10 +19,10 @@ use std::fmt::Formatter; use derive_visitor::Drive; use derive_visitor::DriveMut; -use crate::ast::write_comma_separated_string_map; use crate::ast::CatalogType; use crate::ast::Identifier; use crate::ast::ShowLimit; +use crate::ast::write_comma_separated_string_map; #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub struct ShowCatalogsStmt { diff --git a/src/query/ast/src/ast/statements/copy.rs b/src/query/ast/src/ast/statements/copy.rs index 7109b08bf5a32..68e14df1ad4ca 100644 --- a/src/query/ast/src/ast/statements/copy.rs +++ b/src/query/ast/src/ast/statements/copy.rs @@ -25,19 +25,19 @@ use itertools::Itertools; use percent_encoding::percent_decode_str; use url::Url; -use crate::ast::quote::QuotedString; -use crate::ast::write_comma_separated_list; -use crate::ast::write_comma_separated_map; -use crate::ast::write_comma_separated_string_list; -use crate::ast::write_comma_separated_string_map; +use crate::ParseError; +use crate::Result; use crate::ast::Hint; use crate::ast::Identifier; use crate::ast::Query; use crate::ast::SelectTarget; use crate::ast::TableRef; use crate::ast::With; -use crate::ParseError; -use crate::Result; +use crate::ast::quote::QuotedString; +use crate::ast::write_comma_separated_list; +use crate::ast::write_comma_separated_map; +use crate::ast::write_comma_separated_string_list; +use crate::ast::write_comma_separated_string_map; /// CopyIntoTableStmt is the parsed statement of `COPY into from `. /// diff --git a/src/query/ast/src/ast/statements/data_mask.rs b/src/query/ast/src/ast/statements/data_mask.rs index 9646f5df08d37..147ec74f9feb7 100644 --- a/src/query/ast/src/ast/statements/data_mask.rs +++ b/src/query/ast/src/ast/statements/data_mask.rs @@ -18,9 +18,9 @@ use std::fmt::Formatter; use derive_visitor::Drive; use derive_visitor::DriveMut; -use crate::ast::quote::QuotedString; use crate::ast::Expr; use crate::ast::TypeName; +use crate::ast::quote::QuotedString; #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub struct DataMaskArg { diff --git a/src/query/ast/src/ast/statements/database.rs b/src/query/ast/src/ast/statements/database.rs index 5636a9cbf0cba..0006e069742ad 100644 --- a/src/query/ast/src/ast/statements/database.rs +++ b/src/query/ast/src/ast/statements/database.rs @@ -18,11 +18,11 @@ use std::fmt::Formatter; use derive_visitor::Drive; use derive_visitor::DriveMut; -use crate::ast::statements::show::ShowLimit; -use crate::ast::write_dot_separated_list; use crate::ast::CreateOption; use crate::ast::DatabaseRef; use crate::ast::Identifier; +use crate::ast::statements::show::ShowLimit; +use crate::ast::write_dot_separated_list; #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub struct ShowDatabasesStmt { diff --git a/src/query/ast/src/ast/statements/dictionary.rs b/src/query/ast/src/ast/statements/dictionary.rs index c58c2d4acd9ec..8cd8531105a61 100644 --- a/src/query/ast/src/ast/statements/dictionary.rs +++ b/src/query/ast/src/ast/statements/dictionary.rs @@ -20,13 +20,13 @@ use derive_visitor::Drive; use derive_visitor::DriveMut; use super::ShowLimit; +use crate::ast::ColumnDefinition; +use crate::ast::CreateOption; +use crate::ast::Identifier; use crate::ast::quote::QuotedString; use crate::ast::write_comma_separated_list; use crate::ast::write_dot_separated_list; use crate::ast::write_space_separated_string_map; -use crate::ast::ColumnDefinition; -use crate::ast::CreateOption; -use crate::ast::Identifier; #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub struct CreateDictionaryStmt { diff --git a/src/query/ast/src/ast/statements/dynamic_table.rs b/src/query/ast/src/ast/statements/dynamic_table.rs index 1060bc71b12c7..9737a16050139 100644 --- a/src/query/ast/src/ast/statements/dynamic_table.rs +++ b/src/query/ast/src/ast/statements/dynamic_table.rs @@ -19,14 +19,14 @@ use std::fmt::Formatter; use derive_visitor::Drive; use derive_visitor::DriveMut; -use crate::ast::write_dot_separated_list; -use crate::ast::write_space_separated_string_map; use crate::ast::ClusterOption; use crate::ast::CreateOption; use crate::ast::CreateTableSource; use crate::ast::Identifier; use crate::ast::Query; use crate::ast::WarehouseOptions; +use crate::ast::write_dot_separated_list; +use crate::ast::write_space_separated_string_map; #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub enum TargetLag { diff --git a/src/query/ast/src/ast/statements/index.rs b/src/query/ast/src/ast/statements/index.rs index 251b4f9f929e4..3316be13ac5bd 100644 --- a/src/query/ast/src/ast/statements/index.rs +++ b/src/query/ast/src/ast/statements/index.rs @@ -19,12 +19,12 @@ use std::fmt::Formatter; use derive_visitor::Drive; use derive_visitor::DriveMut; -use crate::ast::write_comma_separated_list; -use crate::ast::write_dot_separated_list; -use crate::ast::write_space_separated_string_map; use crate::ast::CreateOption; use crate::ast::Identifier; use crate::ast::Query; +use crate::ast::write_comma_separated_list; +use crate::ast::write_dot_separated_list; +use crate::ast::write_space_separated_string_map; #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub struct CreateIndexStmt { diff --git a/src/query/ast/src/ast/statements/insert.rs b/src/query/ast/src/ast/statements/insert.rs index 39242d6c7365c..51de669d03266 100644 --- a/src/query/ast/src/ast/statements/insert.rs +++ b/src/query/ast/src/ast/statements/insert.rs @@ -18,14 +18,14 @@ use std::fmt::Formatter; use derive_visitor::Drive; use derive_visitor::DriveMut; -use crate::ast::write_comma_separated_list; -use crate::ast::write_dot_separated_list; use crate::ast::Expr; use crate::ast::FileFormatOptions; use crate::ast::Hint; use crate::ast::Identifier; use crate::ast::Query; use crate::ast::With; +use crate::ast::write_comma_separated_list; +use crate::ast::write_dot_separated_list; #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub struct InsertStmt { diff --git a/src/query/ast/src/ast/statements/insert_multi_table.rs b/src/query/ast/src/ast/statements/insert_multi_table.rs index ef63b4e1fd047..5599e536bfae5 100644 --- a/src/query/ast/src/ast/statements/insert_multi_table.rs +++ b/src/query/ast/src/ast/statements/insert_multi_table.rs @@ -17,10 +17,10 @@ use std::fmt::Display; use derive_visitor::Drive; use derive_visitor::DriveMut; -use crate::ast::write_comma_separated_list; use crate::ast::Expr; use crate::ast::Identifier; use crate::ast::Query; +use crate::ast::write_comma_separated_list; #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub struct IntoClause { pub catalog: Option, diff --git a/src/query/ast/src/ast/statements/merge_into.rs b/src/query/ast/src/ast/statements/merge_into.rs index 19551c9baa337..3452b7c74c867 100644 --- a/src/query/ast/src/ast/statements/merge_into.rs +++ b/src/query/ast/src/ast/statements/merge_into.rs @@ -18,8 +18,6 @@ use std::fmt::Formatter; use derive_visitor::Drive; use derive_visitor::DriveMut; -use crate::ast::write_comma_separated_list; -use crate::ast::write_dot_separated_list; use crate::ast::Expr; use crate::ast::Hint; use crate::ast::Identifier; @@ -27,6 +25,8 @@ use crate::ast::Query; use crate::ast::TableAlias; use crate::ast::TableReference; use crate::ast::WithOptions; +use crate::ast::write_comma_separated_list; +use crate::ast::write_dot_separated_list; #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub struct MutationUpdateExpr { @@ -140,12 +140,12 @@ impl Display for MergeIntoStmt { } write!(f, "THEN INSERT")?; - if let Some(columns) = &unmatch_clause.insert_operation.columns { - if !columns.is_empty() { - write!(f, " (")?; - write_comma_separated_list(f, columns)?; - write!(f, ")")?; - } + if let Some(columns) = &unmatch_clause.insert_operation.columns + && !columns.is_empty() + { + write!(f, " (")?; + write_comma_separated_list(f, columns)?; + write!(f, ")")?; } if unmatch_clause.insert_operation.is_star { diff --git a/src/query/ast/src/ast/statements/network_policy.rs b/src/query/ast/src/ast/statements/network_policy.rs index 5d2f49de372ff..efac3c9f4c686 100644 --- a/src/query/ast/src/ast/statements/network_policy.rs +++ b/src/query/ast/src/ast/statements/network_policy.rs @@ -18,8 +18,8 @@ use std::fmt::Formatter; use derive_visitor::Drive; use derive_visitor::DriveMut; -use crate::ast::quote::QuotedString; use crate::ast::CreateOption; +use crate::ast::quote::QuotedString; #[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] pub struct CreateNetworkPolicyStmt { diff --git a/src/query/ast/src/ast/statements/password_policy.rs b/src/query/ast/src/ast/statements/password_policy.rs index aeb968d088db1..848db98284bbf 100644 --- a/src/query/ast/src/ast/statements/password_policy.rs +++ b/src/query/ast/src/ast/statements/password_policy.rs @@ -18,8 +18,8 @@ use std::fmt::Formatter; use derive_visitor::Drive; use derive_visitor::DriveMut; -use crate::ast::quote::QuotedString; use crate::ast::CreateOption; +use crate::ast::quote::QuotedString; #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub struct CreatePasswordPolicyStmt { diff --git a/src/query/ast/src/ast/statements/presign.rs b/src/query/ast/src/ast/statements/presign.rs index cace1db5b873e..9f82747ebbf6f 100644 --- a/src/query/ast/src/ast/statements/presign.rs +++ b/src/query/ast/src/ast/statements/presign.rs @@ -21,18 +21,13 @@ use derive_visitor::DriveMut; use crate::ast::quote::AtString; -#[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] +#[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut, Default)] pub enum PresignAction { + #[default] Download, Upload, } -impl Default for PresignAction { - fn default() -> Self { - Self::Download - } -} - impl Display for PresignAction { fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { match self { diff --git a/src/query/ast/src/ast/statements/principal.rs b/src/query/ast/src/ast/statements/principal.rs index d4c0accef9901..62251bf31bf9d 100644 --- a/src/query/ast/src/ast/statements/principal.rs +++ b/src/query/ast/src/ast/statements/principal.rs @@ -18,8 +18,8 @@ use std::fmt::Formatter; use derive_visitor::Drive; use derive_visitor::DriveMut; -use crate::ast::quote::QuotedString; use crate::ast::Identifier; +use crate::ast::quote::QuotedString; #[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] pub struct ShareNameIdent { diff --git a/src/query/ast/src/ast/statements/procedure.rs b/src/query/ast/src/ast/statements/procedure.rs index 0467718ae1309..98aa85abe996d 100644 --- a/src/query/ast/src/ast/statements/procedure.rs +++ b/src/query/ast/src/ast/statements/procedure.rs @@ -18,11 +18,11 @@ use std::fmt::Formatter; use derive_visitor::Drive; use derive_visitor::DriveMut; -use crate::ast::write_comma_separated_list; use crate::ast::CreateOption; use crate::ast::Expr; use crate::ast::Identifier; use crate::ast::TypeName; +use crate::ast::write_comma_separated_list; #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub struct ExecuteImmediateStmt { diff --git a/src/query/ast/src/ast/statements/replace.rs b/src/query/ast/src/ast/statements/replace.rs index ecbc6b4999c7d..1acc51f8dd04c 100644 --- a/src/query/ast/src/ast/statements/replace.rs +++ b/src/query/ast/src/ast/statements/replace.rs @@ -18,12 +18,12 @@ use std::fmt::Formatter; use derive_visitor::Drive; use derive_visitor::DriveMut; -use crate::ast::write_comma_separated_list; -use crate::ast::write_dot_separated_list; use crate::ast::Expr; use crate::ast::Hint; use crate::ast::Identifier; use crate::ast::InsertSource; +use crate::ast::write_comma_separated_list; +use crate::ast::write_dot_separated_list; #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub struct ReplaceStmt { diff --git a/src/query/ast/src/ast/statements/row_access_policy.rs b/src/query/ast/src/ast/statements/row_access_policy.rs index e7a7115cfe833..f2ed60ce280fd 100644 --- a/src/query/ast/src/ast/statements/row_access_policy.rs +++ b/src/query/ast/src/ast/statements/row_access_policy.rs @@ -18,10 +18,10 @@ use std::fmt::Formatter; use derive_visitor::Drive; use derive_visitor::DriveMut; -use crate::ast::write_comma_separated_list; use crate::ast::Expr; use crate::ast::Identifier; use crate::ast::TypeName; +use crate::ast::write_comma_separated_list; #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub struct RowAccessPolicyType { diff --git a/src/query/ast/src/ast/statements/script.rs b/src/query/ast/src/ast/statements/script.rs index 0574ee416b8e5..87b9164198210 100644 --- a/src/query/ast/src/ast/statements/script.rs +++ b/src/query/ast/src/ast/statements/script.rs @@ -15,11 +15,11 @@ use std::fmt::Display; use std::fmt::Formatter; +use crate::Span; use crate::ast::Expr; use crate::ast::Identifier; use crate::ast::Statement; use crate::ast::TypeName; -use crate::Span; const INDENT_DEPTH: usize = 4; diff --git a/src/query/ast/src/ast/statements/sequence.rs b/src/query/ast/src/ast/statements/sequence.rs index a57955e31ce37..f176f1bbd7e71 100644 --- a/src/query/ast/src/ast/statements/sequence.rs +++ b/src/query/ast/src/ast/statements/sequence.rs @@ -18,9 +18,9 @@ use std::fmt::Formatter; use derive_visitor::Drive; use derive_visitor::DriveMut; -use crate::ast::quote::QuotedString; use crate::ast::CreateOption; use crate::ast::Identifier; +use crate::ast::quote::QuotedString; #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub struct CreateSequenceStmt { diff --git a/src/query/ast/src/ast/statements/stage.rs b/src/query/ast/src/ast/statements/stage.rs index 196c757d9c438..c7b80f555abab 100644 --- a/src/query/ast/src/ast/statements/stage.rs +++ b/src/query/ast/src/ast/statements/stage.rs @@ -20,12 +20,12 @@ use std::fmt::Formatter; use derive_visitor::Drive; use derive_visitor::DriveMut; -use crate::ast::write_comma_separated_string_list; -use crate::ast::write_comma_separated_string_map; use crate::ast::CreateOption; use crate::ast::FileFormatOptions; use crate::ast::LiteralStringOrVariable; use crate::ast::UriLocation; +use crate::ast::write_comma_separated_string_list; +use crate::ast::write_comma_separated_string_map; #[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] pub struct CreateStageStmt { diff --git a/src/query/ast/src/ast/statements/statement.rs b/src/query/ast/src/ast/statements/statement.rs index 29b4e65446c33..58788e2ef83c0 100644 --- a/src/query/ast/src/ast/statements/statement.rs +++ b/src/query/ast/src/ast/statements/statement.rs @@ -24,6 +24,10 @@ use itertools::Itertools; use super::merge_into::MergeIntoStmt; use super::*; +use crate::Span; +use crate::ast::CreateOption; +use crate::ast::Identifier; +use crate::ast::Query; use crate::ast::quote::QuotedString; use crate::ast::statements::connection::CreateConnectionStmt; use crate::ast::statements::pipe::CreatePipeStmt; @@ -37,10 +41,6 @@ use crate::ast::statements::workload::RenameWorkloadGroupStmt; use crate::ast::statements::workload::SetWorkloadGroupQuotasStmt; use crate::ast::statements::workload::ShowWorkloadGroupsStmt; use crate::ast::write_comma_separated_list; -use crate::ast::CreateOption; -use crate::ast::Identifier; -use crate::ast::Query; -use crate::Span; // SQL statement #[allow(clippy::large_enum_variant)] diff --git a/src/query/ast/src/ast/statements/stream.rs b/src/query/ast/src/ast/statements/stream.rs index 91413e9b7b7f4..e2d039cd46f79 100644 --- a/src/query/ast/src/ast/statements/stream.rs +++ b/src/query/ast/src/ast/statements/stream.rs @@ -18,12 +18,12 @@ use std::fmt::Formatter; use derive_visitor::Drive; use derive_visitor::DriveMut; -use crate::ast::quote::QuotedString; -use crate::ast::write_dot_separated_list; use crate::ast::CreateOption; use crate::ast::Identifier; use crate::ast::ShowLimit; use crate::ast::TimeTravelPoint; +use crate::ast::quote::QuotedString; +use crate::ast::write_dot_separated_list; #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub struct CreateStreamStmt { diff --git a/src/query/ast/src/ast/statements/table.rs b/src/query/ast/src/ast/statements/table.rs index abe58053be3be..69a3e1dfc229e 100644 --- a/src/query/ast/src/ast/statements/table.rs +++ b/src/query/ast/src/ast/statements/table.rs @@ -20,13 +20,6 @@ use std::time::Duration; use derive_visitor::Drive; use derive_visitor::DriveMut; -use crate::ast::quote::QuotedString; -use crate::ast::statements::constraint::ConstraintType; -use crate::ast::statements::show::ShowLimit; -use crate::ast::write_comma_separated_list; -use crate::ast::write_comma_separated_string_map; -use crate::ast::write_dot_separated_list; -use crate::ast::write_space_separated_string_map; use crate::ast::CreateOption; use crate::ast::Expr; use crate::ast::Identifier; @@ -36,6 +29,13 @@ use crate::ast::TableReference; use crate::ast::TimeTravelPoint; use crate::ast::TypeName; use crate::ast::UriLocation; +use crate::ast::quote::QuotedString; +use crate::ast::statements::constraint::ConstraintType; +use crate::ast::statements::show::ShowLimit; +use crate::ast::write_comma_separated_list; +use crate::ast::write_comma_separated_string_map; +use crate::ast::write_dot_separated_list; +use crate::ast::write_space_separated_string_map; #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub struct ShowTablesStmt { diff --git a/src/query/ast/src/ast/statements/task.rs b/src/query/ast/src/ast/statements/task.rs index 2dff48c74955b..cbe4b0ab32c4d 100644 --- a/src/query/ast/src/ast/statements/task.rs +++ b/src/query/ast/src/ast/statements/task.rs @@ -20,12 +20,12 @@ use derive_visitor::Drive; use derive_visitor::DriveMut; use super::CreateOption; -use crate::ast::quote::QuotedString; -use crate::ast::write_comma_separated_string_list; -use crate::ast::write_comma_separated_string_map; use crate::ast::Expr; use crate::ast::Identifier; use crate::ast::ShowLimit; +use crate::ast::quote::QuotedString; +use crate::ast::write_comma_separated_string_list; +use crate::ast::write_comma_separated_string_map; #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub enum TaskSql { diff --git a/src/query/ast/src/ast/statements/udf.rs b/src/query/ast/src/ast/statements/udf.rs index e7208c15d9a4e..d4fd35e0e357a 100644 --- a/src/query/ast/src/ast/statements/udf.rs +++ b/src/query/ast/src/ast/statements/udf.rs @@ -20,12 +20,12 @@ use derive_visitor::Drive; use derive_visitor::DriveMut; use itertools::Itertools; -use crate::ast::quote::QuotedString; -use crate::ast::write_comma_separated_list; use crate::ast::CreateOption; use crate::ast::Expr; use crate::ast::Identifier; use crate::ast::TypeName; +use crate::ast::quote::QuotedString; +use crate::ast::write_comma_separated_list; #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub enum UDFArgs { diff --git a/src/query/ast/src/ast/statements/update.rs b/src/query/ast/src/ast/statements/update.rs index 2dbf27d33b18f..8429747d93de5 100644 --- a/src/query/ast/src/ast/statements/update.rs +++ b/src/query/ast/src/ast/statements/update.rs @@ -18,8 +18,6 @@ use std::fmt::Formatter; use derive_visitor::Drive; use derive_visitor::DriveMut; -use crate::ast::write_comma_separated_list; -use crate::ast::write_dot_separated_list; use crate::ast::Expr; use crate::ast::Hint; use crate::ast::Identifier; @@ -27,6 +25,8 @@ use crate::ast::MutationSource; use crate::ast::MutationUpdateExpr; use crate::ast::TableAlias; use crate::ast::With; +use crate::ast::write_comma_separated_list; +use crate::ast::write_dot_separated_list; #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub struct UpdateStmt { diff --git a/src/query/ast/src/ast/statements/user.rs b/src/query/ast/src/ast/statements/user.rs index 0bd617f3800c2..4148a433cf5e2 100644 --- a/src/query/ast/src/ast/statements/user.rs +++ b/src/query/ast/src/ast/statements/user.rs @@ -18,7 +18,6 @@ use std::fmt::Formatter; use derive_visitor::Drive; use derive_visitor::DriveMut; -use crate::ast::write_comma_separated_list; use crate::ast::AuthType; use crate::ast::CreateOption; use crate::ast::PrincipalIdentity; @@ -26,6 +25,7 @@ use crate::ast::ProcedureIdentity; use crate::ast::ShowOptions; use crate::ast::UserIdentity; use crate::ast::UserPrivilegeType; +use crate::ast::write_comma_separated_list; #[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] pub struct CreateUserStmt { @@ -231,20 +231,18 @@ impl Display for AccountMgrSource { if privileges.len() == 1 && privileges[0] == UserPrivilegeType::ApplyMaskingPolicy && matches!(level, AccountMgrLevel::MaskingPolicy(_)) + && let AccountMgrLevel::MaskingPolicy(policy) = level { - if let AccountMgrLevel::MaskingPolicy(policy) = level { - write!(f, " APPLY ON MASKING POLICY {policy}")?; - return Ok(()); - } + write!(f, " APPLY ON MASKING POLICY {policy}")?; + return Ok(()); } if privileges.len() == 1 && privileges[0] == UserPrivilegeType::ApplyRowAccessPolicy && matches!(level, AccountMgrLevel::RowAccessPolicy(_)) + && let AccountMgrLevel::RowAccessPolicy(policy) = level { - if let AccountMgrLevel::RowAccessPolicy(policy) = level { - write!(f, " APPLY ON ROW ACCESS POLICY {policy}")?; - return Ok(()); - } + write!(f, " APPLY ON ROW ACCESS POLICY {policy}")?; + return Ok(()); } write!(f, " ")?; write_comma_separated_list(f, privileges.iter().map(|p| p.to_string()))?; diff --git a/src/query/ast/src/ast/statements/view.rs b/src/query/ast/src/ast/statements/view.rs index f341faf5203d1..53d128f535505 100644 --- a/src/query/ast/src/ast/statements/view.rs +++ b/src/query/ast/src/ast/statements/view.rs @@ -18,12 +18,12 @@ use std::fmt::Formatter; use derive_visitor::Drive; use derive_visitor::DriveMut; -use crate::ast::write_comma_separated_list; -use crate::ast::write_dot_separated_list; use crate::ast::CreateOption; use crate::ast::Identifier; use crate::ast::Query; use crate::ast::ShowLimit; +use crate::ast::write_comma_separated_list; +use crate::ast::write_dot_separated_list; #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub struct CreateViewStmt { diff --git a/src/query/ast/src/ast/statements/virtual_column.rs b/src/query/ast/src/ast/statements/virtual_column.rs index 781b41a0286d3..e18ae0a587133 100644 --- a/src/query/ast/src/ast/statements/virtual_column.rs +++ b/src/query/ast/src/ast/statements/virtual_column.rs @@ -18,10 +18,10 @@ use std::fmt::Formatter; use derive_visitor::Drive; use derive_visitor::DriveMut; -use crate::ast::write_dot_separated_list; use crate::ast::Expr; use crate::ast::Identifier; use crate::ast::ShowLimit; +use crate::ast::write_dot_separated_list; #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub struct RefreshVirtualColumnStmt { diff --git a/src/query/ast/src/ast/statements/workload.rs b/src/query/ast/src/ast/statements/workload.rs index e7eecef32c3b2..c8fa8d2ebed45 100644 --- a/src/query/ast/src/ast/statements/workload.rs +++ b/src/query/ast/src/ast/statements/workload.rs @@ -48,10 +48,10 @@ impl QuotaValueStmt { if v.ends_with('%') { let num = v.trim_end_matches('%').trim(); - if let Ok(value) = num.parse::() { - if value <= 100 { - return Some(QuotaValueStmt::Percentage(value)); - } + if let Ok(value) = num.parse::() + && value <= 100 + { + return Some(QuotaValueStmt::Percentage(value)); } } None diff --git a/src/query/ast/src/parser/comment.rs b/src/query/ast/src/parser/comment.rs index 85e6391444286..ef98b9087a5ee 100644 --- a/src/query/ast/src/parser/comment.rs +++ b/src/query/ast/src/parser/comment.rs @@ -16,14 +16,14 @@ use nom::Parser; use nom_rule::rule; use super::expr::literal_string; -use crate::ast::statements::password_policy::AlterPasswordAction; -use crate::ast::statements::password_policy::PasswordSetOptions; use crate::ast::AlterTableAction; use crate::ast::AlterTableStmt; use crate::ast::Statement; +use crate::ast::statements::password_policy::AlterPasswordAction; +use crate::ast::statements::password_policy::PasswordSetOptions; use crate::ast::*; -use crate::parser::common::map_res; use crate::parser::common::IResult; +use crate::parser::common::map_res; use crate::parser::common::*; use crate::parser::input::Input; use crate::parser::token::*; diff --git a/src/query/ast/src/parser/common.rs b/src/query/ast/src/parser/common.rs index 5db06eeef6eac..a996e31871880 100644 --- a/src/query/ast/src/parser/common.rs +++ b/src/query/ast/src/parser/common.rs @@ -15,6 +15,8 @@ use std::cell::RefCell; use std::rc::Rc; +use nom::Offset; +use nom::Parser; pub use nom::branch::alt; pub use nom::branch::permutation; pub use nom::combinator::consumed; @@ -23,8 +25,6 @@ pub use nom::combinator::not; pub use nom::combinator::value; pub use nom::multi::many1; use nom::sequence::terminated; -use nom::Offset; -use nom::Parser; use nom_rule::rule; use pratt::PrattError; use pratt::PrattParser; @@ -35,7 +35,8 @@ where P: nom::Parser, Output = O, Error = Error<'a>> { move |input| parser.parse(input) } -use crate::ast::quote::QuotedIdent; +use crate::Range; +use crate::Span; use crate::ast::ColumnID; use crate::ast::DatabaseRef; use crate::ast::Identifier; @@ -43,14 +44,13 @@ use crate::ast::IdentifierType; use crate::ast::SetType; use crate::ast::TableRef; use crate::ast::TableReference; +use crate::ast::quote::QuotedIdent; +use crate::parser::Error; +use crate::parser::ErrorKind; use crate::parser::input::Input; use crate::parser::input::WithSpan; use crate::parser::query::with_options; use crate::parser::token::*; -use crate::parser::Error; -use crate::parser::ErrorKind; -use crate::Range; -use crate::Span; pub type IResult<'a, Output> = nom::IResult, Output, Error<'a>>; @@ -74,7 +74,7 @@ pub fn match_token(kind: TokenKind) -> impl FnMut(Input) -> IResult<&Token> { } } -pub fn any_token(i: Input) -> IResult<&Token> { +pub fn any_token(i: Input<'_>) -> IResult<'_, &Token<'_>> { match i.tokens.first().filter(|token| token.kind != EOI) { Some(token) => Ok((i.slice(1..), token)), _ => Err(nom::Err::Error(Error::from_error_kind( @@ -139,7 +139,7 @@ fn plain_identifier( | #non_reserved_keyword(is_reserved_keyword) }, |token| Identifier { - span: transform_span(&[token.clone()]), + span: transform_span(std::slice::from_ref(token)), name: token.text().to_string(), quote: None, ident_type: IdentifierType::None, @@ -165,7 +165,7 @@ fn quoted_identifier(i: Input) -> IResult { )) })?; Ok((i2, Identifier { - span: transform_span(&[token.clone()]), + span: transform_span(std::slice::from_ref(token)), name: ident, quote: Some(quote), ident_type: IdentifierType::None, @@ -353,7 +353,7 @@ pub fn column_row(i: Input) -> IResult { // ROW could be a column name for compatibility map_res(rule! {ROW}, |token| { Ok(ColumnID::Name(Identifier::from_name( - transform_span(&[token.clone()]), + transform_span(std::slice::from_ref(token)), "row", ))) }) diff --git a/src/query/ast/src/parser/copy.rs b/src/query/ast/src/parser/copy.rs index 021f12dc903fb..5dfa2b85909cd 100644 --- a/src/query/ast/src/parser/copy.rs +++ b/src/query/ast/src/parser/copy.rs @@ -27,11 +27,13 @@ use crate::ast::CopyIntoTableStmt; use crate::ast::LiteralStringOrVariable; use crate::ast::Statement; use crate::ast::Statement::CopyIntoLocation; +use crate::parser::ErrorKind; +use crate::parser::Input; +use crate::parser::common::IResult; use crate::parser::common::comma_separated_list0; use crate::parser::common::comma_separated_list1; use crate::parser::common::ident; use crate::parser::common::table_ref; -use crate::parser::common::IResult; use crate::parser::common::*; use crate::parser::expr::literal_bool; use crate::parser::expr::literal_string; @@ -42,8 +44,6 @@ use crate::parser::stage::file_location; use crate::parser::statement::hint; use crate::parser::token::TokenKind::COPY; use crate::parser::token::TokenKind::*; -use crate::parser::ErrorKind; -use crate::parser::Input; pub fn copy_into_table(i: Input) -> IResult { let copy_into_table_source = alt(( diff --git a/src/query/ast/src/parser/dynamic_table.rs b/src/query/ast/src/parser/dynamic_table.rs index 2e6af1dc350e6..93ec7c15f1df9 100644 --- a/src/query/ast/src/parser/dynamic_table.rs +++ b/src/query/ast/src/parser/dynamic_table.rs @@ -23,10 +23,11 @@ use crate::ast::RefreshMode; use crate::ast::Statement; use crate::ast::TargetLag; use crate::ast::WarehouseOptions; +use crate::parser::Input; +use crate::parser::common::IResult; use crate::parser::common::comma_separated_list1; use crate::parser::common::dot_separated_idents_1_to_3; use crate::parser::common::map_res; -use crate::parser::common::IResult; use crate::parser::common::*; use crate::parser::expr::expr; use crate::parser::expr::literal_u64; @@ -37,7 +38,6 @@ use crate::parser::statement::parse_create_option; use crate::parser::statement::table_option; use crate::parser::statement::task_warehouse_option; use crate::parser::token::TokenKind::*; -use crate::parser::Input; pub fn dynamic_table(i: Input) -> IResult { rule!( diff --git a/src/query/ast/src/parser/error.rs b/src/query/ast/src/parser/error.rs index a66576ee6fa5d..4837424dbc172 100644 --- a/src/query/ast/src/parser/error.rs +++ b/src/query/ast/src/parser/error.rs @@ -21,11 +21,11 @@ use std::num::ParseIntError; use itertools::Itertools; use ordered_float::OrderedFloat; +use crate::Range; use crate::parser::common::transform_span; use crate::parser::input::Input; use crate::parser::token::*; use crate::span::pretty_print_error; -use crate::Range; const MAX_DISPLAY_ERROR_COUNT: usize = 60; diff --git a/src/query/ast/src/parser/expr.rs b/src/query/ast/src/parser/expr.rs index 24fbadcf94c2b..8fdf2a1718d0d 100644 --- a/src/query/ast/src/parser/expr.rs +++ b/src/query/ast/src/parser/expr.rs @@ -14,30 +14,30 @@ use ethnum::i256; use itertools::Itertools; +use nom::Parser; use nom::combinator::consumed; use nom::combinator::verify; use nom::error::context; -use nom::Parser; use nom_rule::rule; use pratt::Affix; use pratt::Associativity; use pratt::PrattParser; use pratt::Precedence; +use crate::Span; use crate::ast::quote::AtString; use crate::ast::*; +use crate::parser::Error; +use crate::parser::ErrorKind; use crate::parser::common::*; use crate::parser::input::Input; use crate::parser::input::WithSpan; use crate::parser::query::*; use crate::parser::token::*; -use crate::parser::Error; -use crate::parser::ErrorKind; use crate::span::merge_span; -use crate::Span; macro_rules! with_span { - ($parser:expr) => { + ($parser:expr_2021) => { map(consumed($parser), |(span, elem)| WithSpan { span, elem }) }; } @@ -873,13 +873,13 @@ impl<'a, I: Iterator>> PrattParser for ExprP ExprElement::DotAccess { key } => { // `database.table.column` is parsed into [database] [.table] [.column], // so we need to transform it into the right `ColumnRef` form. - if let Expr::ColumnRef { column, .. } = &mut lhs { - if let ColumnID::Name(name) = &column.column { - column.database = column.table.take(); - column.table = Some(name.clone()); - column.column = key.clone(); - return Ok(lhs); - } + if let Expr::ColumnRef { column, .. } = &mut lhs + && let ColumnID::Name(name) = &column.column + { + column.database = column.table.take(); + column.table = Some(name.clone()); + column.column = key.clone(); + return Ok(lhs); } match key { @@ -1786,7 +1786,7 @@ fn return_op(i: Input, start: usize, op: T) -> IResult { } macro_rules! op_branch { - ($i:ident, $token_0:ident, $($kind:ident => $op:expr),+ $(,)?) => { + ($i:ident, $token_0:ident, $($kind:ident => $op:expr_2021),+ $(,)?) => { match $token_0.kind { $( TokenKind::$kind => return return_op($i, 1, $op), @@ -1854,7 +1854,7 @@ pub fn binary_op(i: Input) -> IResult { return_op(i, 2, BinaryOperator::LikeAny(None)) } else { return_op(i, 1, BinaryOperator::Like(None)) - } + }; } NOT => match i.tokens.get(1).map(|first| first.kind) { Some(LIKE) => { @@ -1876,7 +1876,12 @@ pub fn binary_op(i: Input) -> IResult { _ => (), } } - Err(nom::Err::Error(Error::from_error_kind(i, ErrorKind::Other("expecting `IS`, `IN`, `LIKE`, `EXISTS`, `BETWEEN`, `+`, `-`, `*`, `/`, `//`, `DIV`, `%`, `||`, `<=>`, `<+>`, `<->`, `>`, `<`, `>=`, `<=`, `=`, `<>`, `!=`, `^`, `AND`, `OR`, `XOR`, `NOT`, `REGEXP`, `RLIKE`, `SOUNDS`, or more ...")))) + Err(nom::Err::Error(Error::from_error_kind( + i, + ErrorKind::Other( + "expecting `IS`, `IN`, `LIKE`, `EXISTS`, `BETWEEN`, `+`, `-`, `*`, `/`, `//`, `DIV`, `%`, `||`, `<=>`, `<+>`, `<->`, `>`, `<`, `>=`, `<=`, `=`, `<>`, `!=`, `^`, `AND`, `OR`, `XOR`, `NOT`, `REGEXP`, `RLIKE`, `SOUNDS`, or more ...", + ), + ))) } pub fn json_op(i: Input) -> IResult { @@ -1897,7 +1902,12 @@ pub fn json_op(i: Input) -> IResult { HashMinus => JsonOperator::HashMinus, ); } - Err(nom::Err::Error(Error::from_error_kind(i, ErrorKind::Other("expecting `->`, '->>', '#>', '#>>', '?', '?|', '?&', '@>', '<@', '@?', '@@', '#-', or more ...")))) + Err(nom::Err::Error(Error::from_error_kind( + i, + ErrorKind::Other( + "expecting `->`, '->>', '#>', '#>>', '?', '?|', '?&', '@>', '<@', '@?', '@@', '#-', or more ...", + ), + ))) } pub fn literal(i: Input) -> IResult { @@ -1931,10 +1941,15 @@ pub fn literal(i: Input) -> IResult { NULL => null.parse(i), ); - Err(nom::Err::Error(Error::from_error_kind(i, ErrorKind::Other("expecting ``, '', '', '', 'TRUE', 'FALSE', or more ...")))) + Err(nom::Err::Error(Error::from_error_kind( + i, + ErrorKind::Other( + "expecting ``, '', '', '', 'TRUE', 'FALSE', or more ...", + ), + ))) } -pub fn literal_hex_str(i: Input) -> IResult<&str> { +pub fn literal_hex_str(i: Input<'_>) -> IResult<'_, &str> { // 0XFFFF let mysql_hex = map( rule! { @@ -2504,10 +2519,10 @@ pub fn interval_kind(i: Input) -> IResult { fn map_access_dot_number(i: Input) -> IResult { map_res(rule! { LiteralFloat }, |key| { - if key.text().starts_with('.') { - if let Ok(key) = (key.text()[1..]).parse::() { - return Ok(MapAccessor::DotNumber { key }); - } + if key.text().starts_with('.') + && let Ok(key) = (key.text()[1..]).parse::() + { + return Ok(MapAccessor::DotNumber { key }); } Err(nom::Err::Error(ErrorKind::ExpectText("."))) }) diff --git a/src/query/ast/src/parser/input.rs b/src/query/ast/src/parser/input.rs index c38b8f1d34436..c0b57873db020 100644 --- a/src/query/ast/src/parser/input.rs +++ b/src/query/ast/src/parser/input.rs @@ -19,8 +19,8 @@ use std::ops::RangeBounds; use enum_as_inner::EnumAsInner; use nom::Needed; -use crate::parser::token::Token; use crate::parser::Backtrace; +use crate::parser::token::Token; /// Input tokens slice with a backtrace that records all errors including /// the optional branch. diff --git a/src/query/ast/src/parser/mod.rs b/src/query/ast/src/parser/mod.rs index 6d7ac8c96384b..2af86b08e66d7 100644 --- a/src/query/ast/src/parser/mod.rs +++ b/src/query/ast/src/parser/mod.rs @@ -13,7 +13,7 @@ // limitations under the License. macro_rules! try_dispatch { - ($input:expr, $return_if_ok:literal, $($pat:pat => $body:expr),+ $(,)?) => {{ + ($input:expr_2021, $return_if_ok:literal, $($pat:pat => $body:expr_2021),+ $(,)?) => {{ if let Some(token_0) = $input.tokens.first() { use TokenKind::*; @@ -48,13 +48,13 @@ pub mod statement; pub mod stream; pub mod token; +pub use common::IResult; pub use common::match_text; pub use common::match_token; -pub use common::IResult; -pub use error::display_parser_error; pub use error::Backtrace; pub use error::Error; pub use error::ErrorKind; +pub use error::display_parser_error; pub use error_suggestion::suggest_correction; pub use input::Dialect; pub use input::Input; diff --git a/src/query/ast/src/parser/parser.rs b/src/query/ast/src/parser/parser.rs index 9253b18d27df6..94cbecd4c8bab 100644 --- a/src/query/ast/src/parser/parser.rs +++ b/src/query/ast/src/parser/parser.rs @@ -17,6 +17,9 @@ use derive_visitor::VisitorMut; use nom::Parser; use pretty_assertions::assert_eq; +use crate::ParseError; +use crate::Range; +use crate::Result; use crate::ast::ExplainKind; use crate::ast::Expr; use crate::ast::Identifier; @@ -24,11 +27,12 @@ use crate::ast::Literal; use crate::ast::SelectTarget; use crate::ast::Statement; use crate::ast::StatementWithFormat; +use crate::parser::Backtrace; +use crate::parser::common::IResult; use crate::parser::common::comma_separated_list0; use crate::parser::common::comma_separated_list1; use crate::parser::common::ident; use crate::parser::common::transform_span; -use crate::parser::common::IResult; use crate::parser::error::display_parser_error; use crate::parser::expr::expr; use crate::parser::expr::values; @@ -41,12 +45,8 @@ use crate::parser::statement::statement; use crate::parser::token::Token; use crate::parser::token::TokenKind; use crate::parser::token::Tokenizer; -use crate::parser::Backtrace; -use crate::ParseError; -use crate::Range; -use crate::Result; -pub fn tokenize_sql(sql: &str) -> Result> { +pub fn tokenize_sql(sql: &str) -> Result>> { Tokenizer::new(sql).collect::>>() } diff --git a/src/query/ast/src/parser/query.rs b/src/query/ast/src/parser/query.rs index 3931829f0a238..3d75eb669bf5c 100644 --- a/src/query/ast/src/parser/query.rs +++ b/src/query/ast/src/parser/query.rs @@ -14,15 +14,17 @@ use std::collections::BTreeMap; -use nom::error::context; use nom::Parser; +use nom::error::context; use nom_rule::rule; use pratt::Affix; use pratt::Associativity; use pratt::PrattParser; use pratt::Precedence; +use crate::Range; use crate::ast::*; +use crate::parser::ErrorKind; use crate::parser::common::*; use crate::parser::expr::*; use crate::parser::input::Input; @@ -33,8 +35,6 @@ use crate::parser::statement::hint; use crate::parser::statement::set_table_option; use crate::parser::statement::top_n; use crate::parser::token::*; -use crate::parser::ErrorKind; -use crate::Range; pub fn query(i: Input) -> IResult { context( diff --git a/src/query/ast/src/parser/sequence.rs b/src/query/ast/src/parser/sequence.rs index 865892d6a58af..0ab715a742ee9 100644 --- a/src/query/ast/src/parser/sequence.rs +++ b/src/query/ast/src/parser/sequence.rs @@ -22,8 +22,8 @@ use super::statement::parse_create_option; use crate::ast::CreateSequenceStmt; use crate::ast::DropSequenceStmt; use crate::ast::Statement; -use crate::parser::common::map_res; use crate::parser::common::IResult; +use crate::parser::common::map_res; use crate::parser::common::*; use crate::parser::input::Input; use crate::parser::statement::show_options; diff --git a/src/query/ast/src/parser/stage.rs b/src/query/ast/src/parser/stage.rs index dfd502ef00adb..780a08202c3e5 100644 --- a/src/query/ast/src/parser/stage.rs +++ b/src/query/ast/src/parser/stage.rs @@ -22,12 +22,12 @@ use crate::ast::FileFormatValue; use crate::ast::FileLocation; use crate::ast::SelectStageOption; use crate::ast::UriLocation; +use crate::parser::ErrorKind; use crate::parser::common::*; use crate::parser::copy::literal_string_or_variable; use crate::parser::expr::*; use crate::parser::input::Input; use crate::parser::token::*; -use crate::parser::ErrorKind; pub fn parameter_to_grant_string(i: Input) -> IResult { let ident_to_string = |i| map_res(grant_ident, |ident| Ok(ident.name))(i); diff --git a/src/query/ast/src/parser/statement.rs b/src/query/ast/src/parser/statement.rs index 869d7b008ac3c..49f2a3cd6684b 100644 --- a/src/query/ast/src/parser/statement.rs +++ b/src/query/ast/src/parser/statement.rs @@ -24,6 +24,8 @@ use super::stream::create_stream; use super::stream::describe_stream; use super::stream::show_streams; use crate::ast::*; +use crate::parser::Error; +use crate::parser::ErrorKind; use crate::parser::comment::comment; use crate::parser::common::*; use crate::parser::copy::copy_into; @@ -37,8 +39,6 @@ use crate::parser::query::*; use crate::parser::stage::*; use crate::parser::stream::drop_stream; use crate::parser::token::*; -use crate::parser::Error; -use crate::parser::ErrorKind; use crate::span::merge_span; pub enum ShowGrantOption { @@ -5611,7 +5611,7 @@ pub fn udf_definition(i: Input) -> IResult { (ReturnBody::Scalar(_), FuncBody::Server { .. }) => { return Err(nom::Err::Failure(ErrorKind::Other( "ScalarUDF unsupported external Server", - ))) + ))); } (ReturnBody::Table(return_types), FuncBody::Sql(sql)) => UDFDefinition::UDTFSql { arg_types, diff --git a/src/query/ast/src/parser/stream.rs b/src/query/ast/src/parser/stream.rs index 9a9fa8c2ce19e..d0688126fb718 100644 --- a/src/query/ast/src/parser/stream.rs +++ b/src/query/ast/src/parser/stream.rs @@ -20,10 +20,11 @@ use crate::ast::DescribeStreamStmt; use crate::ast::DropStreamStmt; use crate::ast::ShowStreamsStmt; use crate::ast::Statement; +use crate::parser::Input; +use crate::parser::common::IResult; use crate::parser::common::dot_separated_idents_1_to_2; use crate::parser::common::dot_separated_idents_1_to_3; use crate::parser::common::map_res; -use crate::parser::common::IResult; use crate::parser::common::*; use crate::parser::expr::literal_bool; use crate::parser::expr::literal_string; @@ -31,7 +32,6 @@ use crate::parser::query::travel_point; use crate::parser::statement::parse_create_option; use crate::parser::statement::show_limit; use crate::parser::token::TokenKind::*; -use crate::parser::Input; pub fn create_stream(i: Input) -> IResult { map_res( diff --git a/src/query/ast/src/parser_error.rs b/src/query/ast/src/parser_error.rs index 2905ac1038893..dd3d5fea8dc3e 100644 --- a/src/query/ast/src/parser_error.rs +++ b/src/query/ast/src/parser_error.rs @@ -15,8 +15,8 @@ use std::fmt::Display; use std::fmt::Formatter; -use crate::span::pretty_print_error; use crate::span::Span; +use crate::span::pretty_print_error; #[derive(Debug)] pub struct ParseError(pub Span, pub String); diff --git a/src/query/ast/src/span.rs b/src/query/ast/src/span.rs index 2cd1849ec6c89..53793a228e937 100644 --- a/src/query/ast/src/span.rs +++ b/src/query/ast/src/span.rs @@ -90,9 +90,9 @@ pub fn pretty_print_error(source: &str, labels: Vec<(Range, String)>) -> String use rspack_codespan_reporting::diagnostic::Label; use rspack_codespan_reporting::files::SimpleFile; use rspack_codespan_reporting::term; - use rspack_codespan_reporting::term::termcolor::Buffer; use rspack_codespan_reporting::term::Chars; use rspack_codespan_reporting::term::Config; + use rspack_codespan_reporting::term::termcolor::Buffer; let mut writer = Buffer::no_color(); let file = SimpleFile::new("SQL", source); diff --git a/src/query/ast/tests/it/decimal.rs b/src/query/ast/tests/it/decimal.rs index c0d22a3d06e46..356a8f08d8f64 100644 --- a/src/query/ast/tests/it/decimal.rs +++ b/src/query/ast/tests/it/decimal.rs @@ -19,7 +19,7 @@ use ethnum::i256; #[test] fn test_decimal() { - let cases = vec![ + let cases = [ ("1.1".to_string(), Literal::Decimal256 { value: 11.into(), precision: 76, @@ -52,7 +52,7 @@ fn test_decimal() { fn test_decimal_uint() { let min_decimal256 = i256::from(u64::MAX) + 1; let float_str = "1".to_string() + &vec!["0"; 76].join(""); - let cases = vec![ + let cases = [ ("1".to_string(), Literal::UInt64(1)), (u64::MAX.to_string(), Literal::UInt64(u64::MAX)), (min_decimal256.to_string(), Literal::Decimal256 { diff --git a/src/query/ast/tests/it/display.rs b/src/query/ast/tests/it/display.rs index 159a344a53c50..004c8f15e5def 100644 --- a/src/query/ast/tests/it/display.rs +++ b/src/query/ast/tests/it/display.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_ast::parser::Dialect; use databend_common_ast::parser::parse_sql; use databend_common_ast::parser::tokenize_sql; -use databend_common_ast::parser::Dialect; fn test_stmt_display(sql: &str) { let tokens = tokenize_sql(sql).unwrap(); diff --git a/src/query/ast/tests/it/parser.rs b/src/query/ast/tests/it/parser.rs index 7aeeeef948bf9..6ce84bef94703 100644 --- a/src/query/ast/tests/it/parser.rs +++ b/src/query/ast/tests/it/parser.rs @@ -16,8 +16,8 @@ use std::fmt::Debug; use std::fmt::Display; use std::io::Write; -use databend_common_ast::ast::quote::ident_needs_quote; use databend_common_ast::ast::quote::QuotedIdent; +use databend_common_ast::ast::quote::ident_needs_quote; use databend_common_ast::parser::expr::*; use databend_common_ast::parser::query::*; use databend_common_ast::parser::script::script_block; diff --git a/src/query/ast/tests/it/token.rs b/src/query/ast/tests/it/token.rs index af6ba131322e4..3eac62718a80c 100644 --- a/src/query/ast/tests/it/token.rs +++ b/src/query/ast/tests/it/token.rs @@ -15,8 +15,8 @@ use std::fs::File; use std::io::Write; -use databend_common_ast::parser::token::*; use databend_common_ast::Result; +use databend_common_ast::parser::token::*; use goldenfile::Mint; fn run_lexer(file: &mut File, source: &str) { diff --git a/src/query/catalog/src/catalog/interface.rs b/src/query/catalog/src/catalog/interface.rs index 64a5c9c11a31a..27724a7cea48e 100644 --- a/src/query/catalog/src/catalog/interface.rs +++ b/src/query/catalog/src/catalog/interface.rs @@ -22,9 +22,6 @@ use databend_common_exception::ErrorCode; use databend_common_exception::ErrorCodeResultExt; use databend_common_exception::Result; use databend_common_meta_app::principal::UDTFServer; -use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; -use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; -use databend_common_meta_app::schema::least_visible_time_ident::LeastVisibleTimeIdent; use databend_common_meta_app::schema::CatalogInfo; use databend_common_meta_app::schema::CommitTableMetaReply; use databend_common_meta_app::schema::CommitTableMetaReq; @@ -112,6 +109,9 @@ use databend_common_meta_app::schema::UpdateTableMetaReq; use databend_common_meta_app::schema::UpdateTempTableReq; use databend_common_meta_app::schema::UpsertTableOptionReply; use databend_common_meta_app::schema::UpsertTableOptionReq; +use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; +use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; +use databend_common_meta_app::schema::least_visible_time_ident::LeastVisibleTimeIdent; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_types::MetaId; use databend_common_meta_types::SeqV; @@ -418,7 +418,10 @@ pub trait Catalog: DynClone + Send + Sync + Debug { Err(failed_tables) => { let err_msg = format!( "Due to concurrent transactions, transaction commit failed. Conflicting table IDs: {:?}", - failed_tables.iter().map(|(tid, _, _)| tid).collect::>() + failed_tables + .iter() + .map(|(tid, _, _)| tid) + .collect::>() ); info!( "Due to concurrent transactions, transaction commit failed. Conflicting tables: {:?}", diff --git a/src/query/catalog/src/catalog/mod.rs b/src/query/catalog/src/catalog/mod.rs index 8befa374491f3..07d8431f96eec 100644 --- a/src/query/catalog/src/catalog/mod.rs +++ b/src/query/catalog/src/catalog/mod.rs @@ -20,5 +20,5 @@ mod manager; pub use interface::Catalog; pub use interface::CatalogCreator; pub use interface::StorageDescription; -pub use manager::CatalogManager; pub use manager::CATALOG_DEFAULT; +pub use manager::CatalogManager; diff --git a/src/query/catalog/src/database.rs b/src/query/catalog/src/database.rs index ce012ccd4be7e..96ba04a6930da 100644 --- a/src/query/catalog/src/database.rs +++ b/src/query/catalog/src/database.rs @@ -17,6 +17,7 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_meta_app::KeyWithTenant; use databend_common_meta_app::schema::CommitTableMetaReply; use databend_common_meta_app::schema::CommitTableMetaReq; use databend_common_meta_app::schema::CreateTableReply; @@ -40,7 +41,6 @@ use databend_common_meta_app::schema::UndropTableReq; use databend_common_meta_app::schema::UpsertTableOptionReply; use databend_common_meta_app::schema::UpsertTableOptionReq; use databend_common_meta_app::tenant::Tenant; -use databend_common_meta_app::KeyWithTenant; use dyn_clone::DynClone; use crate::table::Table; diff --git a/src/query/catalog/src/lib.rs b/src/query/catalog/src/lib.rs index ad20e974ee72d..b0ef2b918f365 100644 --- a/src/query/catalog/src/lib.rs +++ b/src/query/catalog/src/lib.rs @@ -13,8 +13,12 @@ // limitations under the License. #![feature(portable_simd)] -#![allow(clippy::uninlined_format_args)] -#![allow(clippy::large_enum_variant)] +#![allow( + clippy::collapsible_if, + clippy::large_enum_variant, + clippy::unnecessary_unwrap, + clippy::uninlined_format_args +)] pub mod catalog; pub mod catalog_kind; diff --git a/src/query/catalog/src/partition_columns/pushdown_transform.rs b/src/query/catalog/src/partition_columns/pushdown_transform.rs index f95db01e446a8..41d9a2c6eb64b 100644 --- a/src/query/catalog/src/partition_columns/pushdown_transform.rs +++ b/src/query/catalog/src/partition_columns/pushdown_transform.rs @@ -28,13 +28,13 @@ pub fn get_pushdown_without_partition_columns( if partition_columns.is_empty() { return Ok(pushdown); } - if let Some(ref mut p) = &mut pushdown.projection { + if let Some(p) = &mut pushdown.projection { *p = shift_projection(p.clone(), partition_columns)?; } - if let Some(ref mut p) = &mut pushdown.output_columns { + if let Some(p) = &mut pushdown.output_columns { *p = shift_projection(p.clone(), partition_columns)?; } - if let Some(ref mut p) = &mut pushdown.prewhere { + if let Some(p) = &mut pushdown.prewhere { p.output_columns = shift_projection(p.output_columns.clone(), partition_columns)?; p.prewhere_columns = shift_projection(p.prewhere_columns.clone(), partition_columns)?; p.remain_columns = shift_projection(p.remain_columns.clone(), partition_columns)?; diff --git a/src/query/catalog/src/partition_columns/values_serde.rs b/src/query/catalog/src/partition_columns/values_serde.rs index 005c48eed91d0..f989960baac1d 100644 --- a/src/query/catalog/src/partition_columns/values_serde.rs +++ b/src/query/catalog/src/partition_columns/values_serde.rs @@ -17,10 +17,10 @@ use databend_common_base::base::OrderedFloat; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::Scalar; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::NumberScalar; -use databend_common_expression::Scalar; pub fn str_to_scalar(value: &str, data_type: &DataType) -> Result { if value.is_empty() { diff --git a/src/query/catalog/src/plan/agg_index.rs b/src/query/catalog/src/plan/agg_index.rs index 65038a419ca8f..b4c9e431cc1c5 100644 --- a/src/query/catalog/src/plan/agg_index.rs +++ b/src/query/catalog/src/plan/agg_index.rs @@ -14,11 +14,11 @@ use std::fmt::Debug; -use databend_common_expression::local_block_meta_serde; use databend_common_expression::BlockMetaInfo; use databend_common_expression::BlockMetaInfoPtr; use databend_common_expression::RemoteExpr; use databend_common_expression::TableSchemaRef; +use databend_common_expression::local_block_meta_serde; use super::Projection; diff --git a/src/query/catalog/src/plan/datasource/datasource_info/data_source_info.rs b/src/query/catalog/src/plan/datasource/datasource_info/data_source_info.rs index 237f57eb7a5b7..4b935b131fe3b 100644 --- a/src/query/catalog/src/plan/datasource/datasource_info/data_source_info.rs +++ b/src/query/catalog/src/plan/datasource/datasource_info/data_source_info.rs @@ -18,10 +18,10 @@ use databend_common_expression::TableSchema; use databend_common_meta_app::schema::TableInfo; use crate::catalog::CATALOG_DEFAULT; -use crate::plan::datasource::datasource_info::orc::OrcTableInfo; use crate::plan::ParquetTableInfo; use crate::plan::ResultScanTableInfo; use crate::plan::StageTableInfo; +use crate::plan::datasource::datasource_info::orc::OrcTableInfo; #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] pub enum DataSourceInfo { diff --git a/src/query/catalog/src/plan/datasource/datasource_info/mod.rs b/src/query/catalog/src/plan/datasource/datasource_info/mod.rs index 0e84f4837252e..4a9c9b259c153 100644 --- a/src/query/catalog/src/plan/datasource/datasource_info/mod.rs +++ b/src/query/catalog/src/plan/datasource/datasource_info/mod.rs @@ -25,5 +25,5 @@ pub use parquet::FullParquetMeta; pub use parquet::ParquetTableInfo; pub use parquet_read_options::ParquetReadOptions; pub use result_scan::ResultScanTableInfo; -pub use stage::list_stage_files; pub use stage::StageTableInfo; +pub use stage::list_stage_files; diff --git a/src/query/catalog/src/plan/datasource/datasource_info/stage.rs b/src/query/catalog/src/plan/datasource/datasource_info/stage.rs index e979bd38468a8..5f1a0711fac88 100644 --- a/src/query/catalog/src/plan/datasource/datasource_info/stage.rs +++ b/src/query/catalog/src/plan/datasource/datasource_info/stage.rs @@ -23,9 +23,9 @@ use databend_common_expression::RemoteDefaultExpr; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRef; use databend_common_meta_app::principal::StageInfo; -use databend_common_storage::init_stage_operator; use databend_common_storage::StageFileInfo; use databend_common_storage::StageFilesInfo; +use databend_common_storage::init_stage_operator; use crate::plan::FullParquetMeta; diff --git a/src/query/catalog/src/plan/datasource/datasource_plan.rs b/src/query/catalog/src/plan/datasource/datasource_plan.rs index 209856113b05b..3da5a4104947b 100644 --- a/src/query/catalog/src/plan/datasource/datasource_plan.rs +++ b/src/query/catalog/src/plan/datasource/datasource_plan.rs @@ -18,11 +18,11 @@ use databend_common_expression::FieldIndex; use databend_common_expression::Scalar; use databend_common_expression::TableSchemaRef; -use crate::plan::datasource::datasource_info::DataSourceInfo; use crate::plan::InternalColumn; use crate::plan::PartStatistics; use crate::plan::Partitions; use crate::plan::PushDownInfo; +use crate::plan::datasource::datasource_info::DataSourceInfo; use crate::table_args::TableArgs; #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] diff --git a/src/query/catalog/src/plan/internal_column.rs b/src/query/catalog/src/plan/internal_column.rs index 2fe15da87d034..a6177fc3701df 100644 --- a/src/query/catalog/src/plan/internal_column.rs +++ b/src/query/catalog/src/plan/internal_column.rs @@ -14,37 +14,37 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::number::F32; -use databend_common_expression::types::DataType; -use databend_common_expression::types::DecimalDataType; -use databend_common_expression::types::DecimalSize; -use databend_common_expression::types::Float32Type; -use databend_common_expression::types::MutableBitmap; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::UInt64Type; +use databend_common_expression::BASE_BLOCK_IDS_COLUMN_ID; +use databend_common_expression::BASE_ROW_ID_COLUMN_ID; +use databend_common_expression::BLOCK_NAME_COLUMN_ID; use databend_common_expression::BlockEntry; use databend_common_expression::BlockMetaInfo; use databend_common_expression::BlockMetaInfoDowncast; use databend_common_expression::BlockMetaInfoPtr; use databend_common_expression::Column; use databend_common_expression::ColumnId; -use databend_common_expression::FromData; -use databend_common_expression::Scalar; -use databend_common_expression::TableDataType; -use databend_common_expression::BASE_BLOCK_IDS_COLUMN_ID; -use databend_common_expression::BASE_ROW_ID_COLUMN_ID; -use databend_common_expression::BLOCK_NAME_COLUMN_ID; -use databend_common_expression::FILENAME_COLUMN_ID; use databend_common_expression::FILE_ROW_NUMBER_COLUMN_ID; +use databend_common_expression::FILENAME_COLUMN_ID; +use databend_common_expression::FromData; use databend_common_expression::ROW_ID_COLUMN_ID; use databend_common_expression::SEARCH_MATCHED_COLUMN_ID; use databend_common_expression::SEARCH_SCORE_COLUMN_ID; use databend_common_expression::SEGMENT_NAME_COLUMN_ID; use databend_common_expression::SNAPSHOT_NAME_COLUMN_ID; +use databend_common_expression::Scalar; +use databend_common_expression::TableDataType; use databend_common_expression::VECTOR_SCORE_COLUMN_ID; -use databend_storages_common_table_meta::meta::try_extract_uuid_str_from_path; +use databend_common_expression::types::DataType; +use databend_common_expression::types::DecimalDataType; +use databend_common_expression::types::DecimalSize; +use databend_common_expression::types::Float32Type; +use databend_common_expression::types::MutableBitmap; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::UInt64Type; +use databend_common_expression::types::number::F32; use databend_storages_common_table_meta::meta::NUM_BLOCK_ID_BITS; +use databend_storages_common_table_meta::meta::try_extract_uuid_str_from_path; use roaring::RoaringTreemap; // Segment and Block id Bits when generate internal column `_row_id` diff --git a/src/query/catalog/src/plan/partition.rs b/src/query/catalog/src/plan/partition.rs index dbecc0980d831..12ee7273af816 100644 --- a/src/query/catalog/src/plan/partition.rs +++ b/src/query/catalog/src/plan/partition.rs @@ -13,9 +13,9 @@ // limitations under the License. use std::any::Any; -use std::collections::hash_map::DefaultHasher; use std::collections::HashMap; use std::collections::VecDeque; +use std::collections::hash_map::DefaultHasher; use std::fmt::Debug; use std::fmt::Formatter; use std::hash::Hash; diff --git a/src/query/catalog/src/plan/projection.rs b/src/query/catalog/src/plan/projection.rs index 4af4dbff19d14..a0f1bb6494f50 100644 --- a/src/query/catalog/src/plan/projection.rs +++ b/src/query/catalog/src/plan/projection.rs @@ -18,10 +18,10 @@ use std::fmt::Formatter; use databend_common_exception::Result; use databend_common_expression::FieldIndex; use databend_common_expression::TableSchema; -use databend_common_storage::parquet::build_parquet_schema_tree; -use databend_common_storage::parquet::traverse_parquet_schema_tree; use databend_common_storage::ColumnNode; use databend_common_storage::ColumnNodes; +use databend_common_storage::parquet::build_parquet_schema_tree; +use databend_common_storage::parquet::traverse_parquet_schema_tree; use parquet::arrow::ProjectionMask; use parquet::schema::types::SchemaDescriptor; diff --git a/src/query/catalog/src/plan/pushdown.rs b/src/query/catalog/src/plan/pushdown.rs index 053f89de5c272..b7b5ca448558a 100644 --- a/src/query/catalog/src/plan/pushdown.rs +++ b/src/query/catalog/src/plan/pushdown.rs @@ -17,16 +17,16 @@ use std::collections::HashSet; use std::fmt::Debug; use databend_common_ast::ast::SampleConfig; -use databend_common_expression::types::DataType; -use databend_common_expression::types::F32; use databend_common_expression::ColumnId; use databend_common_expression::DataSchema; use databend_common_expression::RemoteExpr; +use databend_common_expression::SEARCH_MATCHED_COL_NAME; use databend_common_expression::Scalar; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; -use databend_common_expression::SEARCH_MATCHED_COL_NAME; +use databend_common_expression::types::DataType; +use databend_common_expression::types::F32; use databend_storages_common_table_meta::table::ChangeType; use super::AggIndexInfo; diff --git a/src/query/catalog/src/plan/stream_column.rs b/src/query/catalog/src/plan/stream_column.rs index 1e7e202b77046..c590854c4898d 100644 --- a/src/query/catalog/src/plan/stream_column.rs +++ b/src/query/catalog/src/plan/stream_column.rs @@ -18,26 +18,26 @@ use std::sync::Arc; use databend_common_base::base::uuid::Uuid; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::decimal::DecimalScalar; -use databend_common_expression::types::AnyType; -use databend_common_expression::types::DataType; -use databend_common_expression::types::DecimalDataType; -use databend_common_expression::types::DecimalSize; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::UInt64Type; use databend_common_expression::BlockEntry; use databend_common_expression::BlockMetaInfo; use databend_common_expression::BlockMetaInfoDowncast; use databend_common_expression::BlockMetaInfoPtr; use databend_common_expression::ColumnId; use databend_common_expression::FromData; +use databend_common_expression::ORIGIN_BLOCK_ID_COLUMN_ID; +use databend_common_expression::ORIGIN_BLOCK_ROW_NUM_COLUMN_ID; +use databend_common_expression::ORIGIN_VERSION_COLUMN_ID; use databend_common_expression::Scalar; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::Value; -use databend_common_expression::ORIGIN_BLOCK_ID_COLUMN_ID; -use databend_common_expression::ORIGIN_BLOCK_ROW_NUM_COLUMN_ID; -use databend_common_expression::ORIGIN_VERSION_COLUMN_ID; +use databend_common_expression::types::AnyType; +use databend_common_expression::types::DataType; +use databend_common_expression::types::DecimalDataType; +use databend_common_expression::types::DecimalSize; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::UInt64Type; +use databend_common_expression::types::decimal::DecimalScalar; use databend_storages_common_table_meta::meta::try_extract_uuid_str_from_path; use crate::plan::PartInfo; diff --git a/src/query/catalog/src/runtime_filter_info.rs b/src/query/catalog/src/runtime_filter_info.rs index 5b108f96fd46f..3dcc5f26f5f7a 100644 --- a/src/query/catalog/src/runtime_filter_info.rs +++ b/src/query/catalog/src/runtime_filter_info.rs @@ -14,9 +14,9 @@ use std::fmt::Debug; use std::fmt::Formatter; +use std::sync::Arc; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; -use std::sync::Arc; use databend_common_base::base::tokio::sync::watch; use databend_common_base::base::tokio::sync::watch::Receiver; diff --git a/src/query/catalog/src/sbbf.rs b/src/query/catalog/src/sbbf.rs index 368a5f4ac2893..3ae85a46d7619 100644 --- a/src/query/catalog/src/sbbf.rs +++ b/src/query/catalog/src/sbbf.rs @@ -73,12 +73,12 @@ //! [sbbf-paper]: https://arxiv.org/pdf/2101.01719 //! [bf-formulae]: http://tfk.mit.edu/pdf/bloom.pdf -use core::simd::cmp::SimdPartialEq; use core::simd::Simd; +use core::simd::cmp::SimdPartialEq; use std::mem::size_of; +use std::sync::Arc; use std::sync::atomic::AtomicU32; use std::sync::atomic::Ordering; -use std::sync::Arc; use databend_common_base::runtime::Runtime; use databend_common_base::runtime::TrySpawn; diff --git a/src/query/catalog/src/table_args.rs b/src/query/catalog/src/table_args.rs index 4f1c26e1596f6..edf6a32c1dabc 100644 --- a/src/query/catalog/src/table_args.rs +++ b/src/query/catalog/src/table_args.rs @@ -16,8 +16,8 @@ use std::collections::HashMap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::NumberScalar; use databend_common_expression::Scalar; +use databend_common_expression::types::NumberScalar; use log::debug; #[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq)] diff --git a/src/query/catalog/src/table_context.rs b/src/query/catalog/src/table_context.rs index 06f28ab50669b..7a1a4d7776578 100644 --- a/src/query/catalog/src/table_context.rs +++ b/src/query/catalog/src/table_context.rs @@ -285,7 +285,7 @@ pub trait TableContext: Send + Sync { async fn get_connection(&self, name: &str) -> Result; async fn get_table(&self, catalog: &str, database: &str, table: &str) - -> Result>; + -> Result>; async fn get_zero_table(&self) -> Result> { let catalog = self.get_catalog("default").await?; diff --git a/src/query/catalog/tests/it/partitions.rs b/src/query/catalog/tests/it/partitions.rs index df6317ba986ee..cc8832f9049e6 100644 --- a/src/query/catalog/tests/it/partitions.rs +++ b/src/query/catalog/tests/it/partitions.rs @@ -21,13 +21,13 @@ use std::io::Write; use std::sync::Arc; use databend_common_base::base::GlobalUniqName; -use databend_common_catalog::plan::compute_row_id_prefix; -use databend_common_catalog::plan::split_prefix; use databend_common_catalog::plan::PartInfo; use databend_common_catalog::plan::PartInfoPtr; use databend_common_catalog::plan::PartInfoType; use databend_common_catalog::plan::Partitions; use databend_common_catalog::plan::PartitionsShuffleKind; +use databend_common_catalog::plan::compute_row_id_prefix; +use databend_common_catalog::plan::split_prefix; use databend_common_meta_types::NodeInfo; use databend_storages_common_table_meta::meta::NUM_BLOCK_ID_BITS; use goldenfile::Mint; diff --git a/src/query/catalog/tests/it/projection.rs b/src/query/catalog/tests/it/projection.rs index 13763156d975d..e1d1e8c9d3b25 100644 --- a/src/query/catalog/tests/it/projection.rs +++ b/src/query/catalog/tests/it/projection.rs @@ -16,10 +16,10 @@ use std::collections::BTreeMap; use databend_common_catalog::plan::Projection; use databend_common_exception::Result; -use databend_common_expression::types::NumberDataType; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; +use databend_common_expression::types::NumberDataType; use parquet::arrow::ArrowSchemaConverter; #[test] diff --git a/src/query/codegen/src/writes/arithmetics_type.rs b/src/query/codegen/src/writes/arithmetics_type.rs index cfd715e082e70..69225ab9efbc7 100644 --- a/src/query/codegen/src/writes/arithmetics_type.rs +++ b/src/query/codegen/src/writes/arithmetics_type.rs @@ -267,9 +267,5 @@ fn sum_coercion(a: NumberDataType) -> NumberDataType { } const fn next_bit_width(width: u8) -> u8 { - if width < 64 { - width * 2 - } else { - 64 - } + if width < 64 { width * 2 } else { 64 } } diff --git a/src/query/config/src/config.rs b/src/query/config/src/config.rs index b31992d393f02..df1a033ecc380 100644 --- a/src/query/config/src/config.rs +++ b/src/query/config/src/config.rs @@ -23,8 +23,8 @@ use clap::ArgAction; use clap::Args; use clap::Subcommand; use clap::ValueEnum; -use databend_common_base::base::mask_string; use databend_common_base::base::OrderedFloat; +use databend_common_base::base::mask_string; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_meta_app::principal::UserSettingValue; @@ -44,6 +44,7 @@ use databend_common_meta_app::storage::StorageWebhdfsConfig as InnerStorageWebhd use databend_common_meta_app::tenant::Tenant; use databend_common_meta_app::tenant::TenantQuota; use databend_common_storage::StorageConfig as InnerStorageConfig; +use databend_common_tracing::CONFIG_DEFAULT_LOG_LEVEL; use databend_common_tracing::Config as InnerLogConfig; use databend_common_tracing::FileConfig as InnerFileLogConfig; use databend_common_tracing::HistoryConfig as InnerHistoryConfig; @@ -58,7 +59,6 @@ use databend_common_tracing::QueryLogConfig as InnerQueryLogConfig; use databend_common_tracing::StderrConfig as InnerStderrLogConfig; use databend_common_tracing::StructLogConfig as InnerStructLogConfig; use databend_common_tracing::TracingConfig as InnerTracingConfig; -use databend_common_tracing::CONFIG_DEFAULT_LOG_LEVEL; use serde::Deserialize; use serde::Serialize; use serde_with::with_prefix; diff --git a/src/query/config/src/inner.rs b/src/query/config/src/inner.rs index ebd0847d60945..5d17517fb0329 100644 --- a/src/query/config/src/inner.rs +++ b/src/query/config/src/inner.rs @@ -21,10 +21,10 @@ use std::path::PathBuf; use std::str::FromStr; use std::time::Duration; -use databend_common_base::base::mask_string; use databend_common_base::base::BuildInfoRef; use databend_common_base::base::GlobalUniqName; use databend_common_base::base::OrderedFloat; +use databend_common_base::base::mask_string; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_grpc::RpcClientConf; diff --git a/src/query/config/src/lib.rs b/src/query/config/src/lib.rs index fdcaa8b64622a..07efa5a0bc065 100644 --- a/src/query/config/src/lib.rs +++ b/src/query/config/src/lib.rs @@ -13,7 +13,7 @@ // limitations under the License. #![allow(clippy::uninlined_format_args)] -#![feature(no_sanitize)] +#![allow(clippy::derivable_impls)] mod builtin; /// Config mods provide config support. diff --git a/src/query/config/src/mask.rs b/src/query/config/src/mask.rs index 869e788a36c91..5bfe26f62d13f 100644 --- a/src/query/config/src/mask.rs +++ b/src/query/config/src/mask.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::Config; +use crate::StorageConfig; use crate::config::AzblobStorageConfig; use crate::config::CosStorageConfig; use crate::config::FsStorageConfig; @@ -24,8 +26,6 @@ use crate::config::QueryConfig; use crate::config::S3StorageConfig; use crate::config::SpillConfig; use crate::config::WebhdfsStorageConfig; -use crate::Config; -use crate::StorageConfig; fn mask_sensitive_field(field: &str) -> String { if field.is_empty() { diff --git a/src/query/config/src/toml.rs b/src/query/config/src/toml.rs index 652f64aceb248..5d3a6a9073751 100644 --- a/src/query/config/src/toml.rs +++ b/src/query/config/src/toml.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use anyhow::anyhow; use anyhow::Result; +use anyhow::anyhow; use serde::de::DeserializeOwned; use serfig::Parser; diff --git a/src/query/datavalues/src/data_field.rs b/src/query/datavalues/src/data_field.rs index e353af6d2562c..25ecb69f1c030 100644 --- a/src/query/datavalues/src/data_field.rs +++ b/src/query/datavalues/src/data_field.rs @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::TypeID; use crate::remove_nullable; use crate::types::data_type::DataType; use crate::types::data_type::DataTypeImpl; use crate::wrap_nullable; -use crate::TypeID; #[derive(serde::Serialize, serde::Deserialize, Eq, PartialEq, Clone)] pub struct DataField { diff --git a/src/query/datavalues/src/prelude.rs b/src/query/datavalues/src/prelude.rs index e26cfcbcc6712..57fc4caa09603 100644 --- a/src/query/datavalues/src/prelude.rs +++ b/src/query/datavalues/src/prelude.rs @@ -14,11 +14,11 @@ //! Everything you need to get started with this crate. -pub use crate::data_value::*; -pub use crate::types::*; // common structs pub use crate::DataField; pub use crate::DataSchema; pub use crate::DataSchemaRef; pub use crate::DataSchemaRefExt; pub use crate::DataValue; +pub use crate::data_value::*; +pub use crate::types::*; diff --git a/src/query/datavalues/src/types/eq.rs b/src/query/datavalues/src/types/eq.rs index af881fba127fd..e2f037d8afa44 100644 --- a/src/query/datavalues/src/types/eq.rs +++ b/src/query/datavalues/src/types/eq.rs @@ -14,11 +14,11 @@ use std::sync::Arc; +use super::DataType; +use super::DataTypeImpl; use super::type_array::ArrayType; use super::type_nullable::NullableType; use super::type_struct::StructType; -use super::DataType; -use super::DataTypeImpl; impl Eq for DataTypeImpl {} diff --git a/src/query/datavalues/src/types/type_primitive.rs b/src/query/datavalues/src/types/type_primitive.rs index 0ee5fb257a02c..391245d527442 100644 --- a/src/query/datavalues/src/types/type_primitive.rs +++ b/src/query/datavalues/src/types/type_primitive.rs @@ -26,7 +26,7 @@ pub struct PrimitiveDataType { + ($ty:ident, $tname:ident, $name: expr_2021, $sql_name:expr_2021, $alias: expr_2021) => { impl PrimitiveDataType<$ty> { pub fn new_impl() -> DataTypeImpl { DataTypeImpl::$tname(Self { _t: PhantomData }) diff --git a/src/query/ee/src/attach_table/handler.rs b/src/query/ee/src/attach_table/handler.rs index 8a662fe7f8e03..66c7f49dec670 100644 --- a/src/query/ee/src/attach_table/handler.rs +++ b/src/query/ee/src/attach_table/handler.rs @@ -25,9 +25,9 @@ use databend_common_meta_app::schema::TableStatistics; use databend_common_sql::plans::CreateTablePlan; use databend_common_storage::check_operator; use databend_common_storage::init_operator; +use databend_common_storages_fuse::FUSE_OPT_KEY_ATTACH_COLUMN_IDS; use databend_common_storages_fuse::io::MetaReaders; use databend_common_storages_fuse::operations::load_last_snapshot_hint; -use databend_common_storages_fuse::FUSE_OPT_KEY_ATTACH_COLUMN_IDS; use databend_enterprise_attach_table::AttachTableHandler; use databend_enterprise_attach_table::AttachTableHandlerWrapper; use databend_storages_common_cache::LoadParams; diff --git a/src/query/ee/src/data_mask/data_mask_handler.rs b/src/query/ee/src/data_mask/data_mask_handler.rs index a1052154b8acc..358b4f818b5a7 100644 --- a/src/query/ee/src/data_mask/data_mask_handler.rs +++ b/src/query/ee/src/data_mask/data_mask_handler.rs @@ -18,12 +18,12 @@ use databend_common_base::base::GlobalInstance; use databend_common_exception::Result; use databend_common_meta_api::DatamaskApi; use databend_common_meta_app::app_error::AppError; -use databend_common_meta_app::data_mask::data_mask_name_ident::Resource; use databend_common_meta_app::data_mask::CreateDatamaskReply; use databend_common_meta_app::data_mask::CreateDatamaskReq; use databend_common_meta_app::data_mask::DataMaskNameIdent; use databend_common_meta_app::data_mask::DatamaskMeta; use databend_common_meta_app::data_mask::DropDatamaskReq; +use databend_common_meta_app::data_mask::data_mask_name_ident::Resource; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_app::tenant_key::errors::ExistError; use databend_common_meta_store::MetaStore; diff --git a/src/query/ee/src/fail_safe/handler.rs b/src/query/ee/src/fail_safe/handler.rs index 94d8d43e6808e..87cc74dd05b02 100644 --- a/src/query/ee/src/fail_safe/handler.rs +++ b/src/query/ee/src/fail_safe/handler.rs @@ -21,8 +21,8 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::TableSchema; use databend_common_meta_app::schema::TableInfo; -use databend_common_storages_fuse::io::MetaReaders; use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::io::MetaReaders; use databend_enterprise_fail_safe::FailSafeHandler; use databend_enterprise_fail_safe::FailSafeHandlerWrapper; use databend_storages_common_cache::LoadParams; diff --git a/src/query/ee/src/hilbert_clustering/handler.rs b/src/query/ee/src/hilbert_clustering/handler.rs index cebfbadc5947e..406a6e6bd6a29 100644 --- a/src/query/ee/src/hilbert_clustering/handler.rs +++ b/src/query/ee/src/hilbert_clustering/handler.rs @@ -22,12 +22,12 @@ use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; use databend_common_io::constants::DEFAULT_BLOCK_BUFFER_SIZE; use databend_common_io::constants::DEFAULT_BLOCK_PER_SEGMENT; -use databend_common_storages_fuse::pruning::create_segment_location_vector; -use databend_common_storages_fuse::statistics::reducers::merge_statistics_mut; -use databend_common_storages_fuse::FuseTable; -use databend_common_storages_fuse::SegmentLocation; use databend_common_storages_fuse::FUSE_OPT_KEY_BLOCK_IN_MEM_SIZE_THRESHOLD; use databend_common_storages_fuse::FUSE_OPT_KEY_BLOCK_PER_SEGMENT; +use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::SegmentLocation; +use databend_common_storages_fuse::pruning::create_segment_location_vector; +use databend_common_storages_fuse::statistics::reducers::merge_statistics_mut; use databend_enterprise_hilbert_clustering::HilbertClusteringHandler; use databend_enterprise_hilbert_clustering::HilbertClusteringHandlerWrapper; use databend_storages_common_table_meta::meta::ClusterStatistics; diff --git a/src/query/ee/src/lib.rs b/src/query/ee/src/lib.rs index 00467de13c8c3..5c283bdb761cb 100644 --- a/src/query/ee/src/lib.rs +++ b/src/query/ee/src/lib.rs @@ -12,6 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![allow( + clippy::collapsible_if, + clippy::let_and_return, + clippy::manual_is_multiple_of +)] + pub mod attach_table; pub mod data_mask; pub mod enterprise_services; diff --git a/src/query/ee/src/license/license_mgr.rs b/src/query/ee/src/license/license_mgr.rs index bc14920406c02..cd267c219bc4e 100644 --- a/src/query/ee/src/license/license_mgr.rs +++ b/src/query/ee/src/license/license_mgr.rs @@ -16,20 +16,20 @@ use std::sync::Arc; use dashmap::DashMap; use databend_common_base::base::GlobalInstance; -use databend_common_exception::exception::ErrorCode; use databend_common_exception::Result; use databend_common_exception::ToErrorCode; +use databend_common_exception::exception::ErrorCode; use databend_common_license::license::Feature; use databend_common_license::license::LicenseInfo; use databend_common_license::license::VerifyResult; use databend_common_license::license_manager::LicenseManager; use databend_common_license::license_manager::LicenseManagerSwitch; use databend_common_version::DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY; +use jwt_simple::JWTError; use jwt_simple::algorithms::ES256PublicKey; use jwt_simple::claims::JWTClaims; use jwt_simple::prelude::Clock; use jwt_simple::prelude::ECDSAP256PublicKeyLike; -use jwt_simple::JWTError; use log::warn; const LICENSE_PUBLIC_KEY: &str = r#"-----BEGIN PUBLIC KEY----- @@ -50,10 +50,10 @@ pub struct RealLicenseManager { impl RealLicenseManager { fn parse_license_impl(&self, raw: &str) -> Result> { for public_key in &self.public_keys { - let public_key = ES256PublicKey::from_pem(public_key) - .map_err_to_code(ErrorCode::LicenseKeyParseError, || { - "[LicenseManager] Public key load failed" - })?; + let public_key = ES256PublicKey::from_pem(public_key).map_err_to_code( + ErrorCode::LicenseKeyParseError, + || "[LicenseManager] Public key load failed", + )?; return match public_key.verify_token::(raw, None) { Ok(v) => Ok(v), diff --git a/src/query/ee/src/row_access_policy/row_access_policy_handler.rs b/src/query/ee/src/row_access_policy/row_access_policy_handler.rs index 53601bf3d6fc0..ad1d1cac50d1d 100644 --- a/src/query/ee/src/row_access_policy/row_access_policy_handler.rs +++ b/src/query/ee/src/row_access_policy/row_access_policy_handler.rs @@ -18,13 +18,13 @@ use databend_common_base::base::GlobalInstance; use databend_common_exception::Result; use databend_common_meta_api::RowAccessPolicyApi; use databend_common_meta_app::app_error::AppError; -use databend_common_meta_app::row_access_policy::row_access_policy_name_ident::Resource; use databend_common_meta_app::row_access_policy::CreateRowAccessPolicyReply; use databend_common_meta_app::row_access_policy::CreateRowAccessPolicyReq; use databend_common_meta_app::row_access_policy::DropRowAccessPolicyReq; use databend_common_meta_app::row_access_policy::RowAccessPolicyId; use databend_common_meta_app::row_access_policy::RowAccessPolicyMeta; use databend_common_meta_app::row_access_policy::RowAccessPolicyNameIdent; +use databend_common_meta_app::row_access_policy::row_access_policy_name_ident::Resource; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_app::tenant_key::errors::ExistError; use databend_common_meta_store::MetaStore; diff --git a/src/query/ee/src/storages/fuse/operations/handler.rs b/src/query/ee/src/storages/fuse/operations/handler.rs index 48e53d0e2122a..9f26ccaf22112 100644 --- a/src/query/ee/src/storages/fuse/operations/handler.rs +++ b/src/query/ee/src/storages/fuse/operations/handler.rs @@ -19,10 +19,10 @@ use databend_common_catalog::table::Table; use databend_common_catalog::table_context::AbortChecker; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_enterprise_vacuum_handler::vacuum_handler::VacuumDropTablesResult; -use databend_enterprise_vacuum_handler::vacuum_handler::VacuumTempOptions; use databend_enterprise_vacuum_handler::VacuumHandler; use databend_enterprise_vacuum_handler::VacuumHandlerWrapper; +use databend_enterprise_vacuum_handler::vacuum_handler::VacuumDropTablesResult; +use databend_enterprise_vacuum_handler::vacuum_handler::VacuumTempOptions; use crate::storages::fuse::do_vacuum; use crate::storages::fuse::operations::vacuum_table_v2::do_vacuum2; diff --git a/src/query/ee/src/storages/fuse/operations/vacuum_table.rs b/src/query/ee/src/storages/fuse/operations/vacuum_table.rs index 4e4f4114a16f0..e4d9af0c3037a 100644 --- a/src/query/ee/src/storages/fuse/operations/vacuum_table.rs +++ b/src/query/ee/src/storages/fuse/operations/vacuum_table.rs @@ -22,12 +22,12 @@ use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::RetentionPolicy; use databend_common_storages_fuse::io::MetaReaders; use databend_common_storages_fuse::io::SnapshotLiteExtended; use databend_common_storages_fuse::io::SnapshotsIO; use databend_common_storages_fuse::io::TableMetaLocationGenerator; -use databend_common_storages_fuse::FuseTable; -use databend_common_storages_fuse::RetentionPolicy; use databend_storages_common_cache::LoadParams; use databend_storages_common_table_meta::meta::SegmentInfo; diff --git a/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs b/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs index 27ad1abe8fcc0..c3c659c720724 100644 --- a/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs +++ b/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs @@ -27,28 +27,27 @@ use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_meta_app::schema::least_visible_time_ident::LeastVisibleTimeIdent; use databend_common_meta_app::schema::LeastVisibleTime; use databend_common_meta_app::schema::ListIndexesByIdReq; +use databend_common_meta_app::schema::least_visible_time_ident::LeastVisibleTimeIdent; +use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::RetentionPolicy; use databend_common_storages_fuse::io::MetaReaders; use databend_common_storages_fuse::io::SegmentsIO; use databend_common_storages_fuse::io::TableMetaLocationGenerator; -use databend_common_storages_fuse::FuseTable; -use databend_common_storages_fuse::RetentionPolicy; use databend_storages_common_cache::CacheAccessor; use databend_storages_common_cache::CacheManager; use databend_storages_common_cache::LoadParams; use databend_storages_common_io::Files; -use databend_storages_common_table_meta::meta::uuid_from_date_time; use databend_storages_common_table_meta::meta::CompactSegmentInfo; use databend_storages_common_table_meta::meta::TableSnapshot; use databend_storages_common_table_meta::meta::VACUUM2_OBJECT_KEY_PREFIX; +use databend_storages_common_table_meta::meta::uuid_from_date_time; use futures_util::TryStreamExt; use log::info; use opendal::Entry; use opendal::ErrorKind; use opendal::Operator; -use opendal::Scheme; use uuid::Version; /// An assumption of the maximum duration from the time the first block is written to the time the @@ -486,7 +485,7 @@ async fn list_until_prefix( let dal = fuse_table.get_operator_ref(); match dal.info().scheme() { - Scheme::Fs => fs_list_until_prefix(dal, path, until, need_one_more, gc_root_meta_ts).await, + "fs" => fs_list_until_prefix(dal, path, until, need_one_more, gc_root_meta_ts).await, _ => general_list_until_prefix(dal, path, until, need_one_more, gc_root_meta_ts).await, } } @@ -587,6 +586,9 @@ async fn is_gc_candidate_segment_block( })? }; + let last_modified = + DateTime::from_timestamp_nanos(last_modified.into_inner().as_nanosecond() as i64); + Ok(last_modified + ASSUMPTION_MAX_TXN_DURATION < gc_root_meta_ts) } @@ -674,12 +676,15 @@ async fn select_gc_root( let gc_root = read_snapshot_from_location(fuse_table, &gc_root_path).await; let gc_root_meta_ts = match dal.stat(&gc_root_path).await { - Ok(v) => v.last_modified().ok_or_else(|| { - ErrorCode::StorageOther(format!( - "Failed to get `last_modified` metadata of the gc root object '{}'", - gc_root_path - )) - })?, + Ok(v) => v + .last_modified() + .ok_or_else(|| { + ErrorCode::StorageOther(format!( + "Failed to get `last_modified` metadata of the gc root object '{}'", + gc_root_path + )) + }) + .map(|v| DateTime::from_timestamp_nanos(v.into_inner().as_nanosecond() as i64))?, Err(e) => { return if e.kind() == ErrorKind::NotFound { // Concurrent vacuum, ignore it @@ -711,8 +716,13 @@ async fn select_gc_root( gc_root_path )) })?, - Some(v) => v, + Some(v) => v }; + + let last_modified = DateTime::from_timestamp_nanos( + last_modified.into_inner().as_nanosecond() as i64, + ); + if last_modified + ASSUMPTION_MAX_TXN_DURATION < gc_root_meta_ts { gc_candidates.push(path.to_owned()); } diff --git a/src/query/ee/src/storages/fuse/operations/vacuum_temporary_files.rs b/src/query/ee/src/storages/fuse/operations/vacuum_temporary_files.rs index 7458db51ee331..b595a2bb2c326 100644 --- a/src/query/ee/src/storages/fuse/operations/vacuum_temporary_files.rs +++ b/src/query/ee/src/storages/fuse/operations/vacuum_temporary_files.rs @@ -105,7 +105,7 @@ async fn vacuum_by_duration( let meta = meta.unwrap(); if let Some(modified) = meta.last_modified() { - if timestamp - modified.timestamp_millis() < expire_time { + if timestamp - modified.into_inner().as_millisecond() < expire_time { continue; } } diff --git a/src/query/ee/src/storages/fuse/operations/virtual_columns.rs b/src/query/ee/src/storages/fuse/operations/virtual_columns.rs index bb857055e3ed6..25c8e5c24ed4d 100644 --- a/src/query/ee/src/storages/fuse/operations/virtual_columns.rs +++ b/src/query/ee/src/storages/fuse/operations/virtual_columns.rs @@ -23,13 +23,13 @@ use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::local_block_meta_serde; use databend_common_expression::BlockMetaInfo; use databend_common_expression::BlockMetaInfoDowncast; use databend_common_expression::ComputedExpr; use databend_common_expression::DataBlock; use databend_common_expression::TableDataType; use databend_common_expression::TableSchema; +use databend_common_expression::local_block_meta_serde; use databend_common_metrics::storage::metrics_inc_block_virtual_column_write_bytes; use databend_common_metrics::storage::metrics_inc_block_virtual_column_write_milliseconds; use databend_common_metrics::storage::metrics_inc_block_virtual_column_write_nums; @@ -40,20 +40,20 @@ use databend_common_pipeline_transforms::processors::AsyncTransform; use databend_common_pipeline_transforms::processors::TransformPipelineHelper; use databend_common_sql::executor::physical_plans::MutationKind; use databend_common_sql::plans::RefreshSelection; -use databend_common_storages_fuse::io::read::read_segment_stats; -use databend_common_storages_fuse::io::write_data; +use databend_common_storages_fuse::FuseStorageFormat; +use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::io::BlockReader; use databend_common_storages_fuse::io::MetaReaders; use databend_common_storages_fuse::io::VirtualColumnBuilder; use databend_common_storages_fuse::io::WriteSettings; +use databend_common_storages_fuse::io::read::read_segment_stats; +use databend_common_storages_fuse::io::write_data; use databend_common_storages_fuse::operations::BlockMetaIndex; use databend_common_storages_fuse::operations::CommitSink; use databend_common_storages_fuse::operations::MutationGenerator; use databend_common_storages_fuse::operations::MutationLogEntry; use databend_common_storages_fuse::operations::MutationLogs; use databend_common_storages_fuse::operations::TableMutationAggregator; -use databend_common_storages_fuse::FuseStorageFormat; -use databend_common_storages_fuse::FuseTable; use databend_storages_common_cache::LoadParams; use databend_storages_common_io::ReadSettings; use databend_storages_common_table_meta::meta::BlockHLLState; diff --git a/src/query/ee/src/test_kits/setup.rs b/src/query/ee/src/test_kits/setup.rs index 9e512d8330981..079db7708aa73 100644 --- a/src/query/ee/src/test_kits/setup.rs +++ b/src/query/ee/src/test_kits/setup.rs @@ -16,8 +16,8 @@ use databend_common_config::InnerConfig; use databend_common_exception::Result; use databend_common_tracing::set_panic_hook; use databend_common_version::BUILD_INFO; -use databend_query::clusters::ClusterDiscovery; use databend_query::GlobalServices; +use databend_query::clusters::ClusterDiscovery; use log::info; use crate::test_kits::mock_services::MockServices; @@ -28,7 +28,7 @@ impl TestFixture { pub async fn setup(config: &InnerConfig, public_key: String) -> Result<()> { let version = &BUILD_INFO; set_panic_hook(version.commit_detail.clone()); - std::env::set_var("UNIT_TEST", "TRUE"); + unsafe { std::env::set_var("UNIT_TEST", "TRUE") }; #[cfg(debug_assertions)] { diff --git a/src/query/ee/tests/it/license/license_mgr.rs b/src/query/ee/tests/it/license/license_mgr.rs index 256c903909190..af6d2be7514c2 100644 --- a/src/query/ee/tests/it/license/license_mgr.rs +++ b/src/query/ee/tests/it/license/license_mgr.rs @@ -55,13 +55,17 @@ async fn test_parse_license() -> databend_common_exception::Result<()> { let parsed = license_mgr.parse_license(token.as_str()); assert!(parsed.is_ok()); - assert!(license_mgr - .check_enterprise_enabled(token.clone(), Feature::Test) - .is_ok()); + assert!( + license_mgr + .check_enterprise_enabled(token.clone(), Feature::Test) + .is_ok() + ); // test cache hit - assert!(license_mgr - .check_enterprise_enabled(token, Feature::Test) - .is_ok()); + assert!( + license_mgr + .check_enterprise_enabled(token, Feature::Test) + .is_ok() + ); // test expired token let mut claims = Claims::with_custom_claims( @@ -72,9 +76,11 @@ async fn test_parse_license() -> databend_common_exception::Result<()> { let token = key_pair.sign(claims)?; let parsed = license_mgr.parse_license(token.as_str()); assert!(parsed.is_err()); - assert!(license_mgr - .check_enterprise_enabled(token, Feature::Test) - .is_err()); + assert!( + license_mgr + .check_enterprise_enabled(token, Feature::Test) + .is_err() + ); Ok(()) } @@ -106,29 +112,41 @@ async fn test_license_features() -> databend_common_exception::Result<()> { let parsed = license_mgr.parse_license(token.as_str()); assert!(parsed.is_ok()); - assert!(license_mgr - .check_enterprise_enabled(token.clone(), Feature::ComputedColumn) - .is_err()); + assert!( + license_mgr + .check_enterprise_enabled(token.clone(), Feature::ComputedColumn) + .is_err() + ); - assert!(license_mgr - .check_enterprise_enabled(token.clone(), Feature::LicenseInfo) - .is_ok()); + assert!( + license_mgr + .check_enterprise_enabled(token.clone(), Feature::LicenseInfo) + .is_ok() + ); - assert!(license_mgr - .check_enterprise_enabled(token.clone(), Feature::VirtualColumn) - .is_err()); + assert!( + license_mgr + .check_enterprise_enabled(token.clone(), Feature::VirtualColumn) + .is_err() + ); - assert!(license_mgr - .check_enterprise_enabled(token.clone(), Feature::Test) - .is_ok()); + assert!( + license_mgr + .check_enterprise_enabled(token.clone(), Feature::Test) + .is_ok() + ); - assert!(license_mgr - .check_enterprise_enabled(token.clone(), Feature::Vacuum) - .is_ok()); + assert!( + license_mgr + .check_enterprise_enabled(token.clone(), Feature::Vacuum) + .is_ok() + ); - assert!(license_mgr - .check_enterprise_enabled(token, Feature::Stream) - .is_ok()); + assert!( + license_mgr + .check_enterprise_enabled(token, Feature::Stream) + .is_ok() + ); // test expired token let mut claims = Claims::with_custom_claims( @@ -143,9 +161,11 @@ async fn test_license_features() -> databend_common_exception::Result<()> { let token = key_pair.sign(claims)?; let parsed = license_mgr.parse_license(token.as_str()); assert!(parsed.is_err()); - assert!(license_mgr - .check_enterprise_enabled(token, Feature::Test) - .is_err()); + assert!( + license_mgr + .check_enterprise_enabled(token, Feature::Test) + .is_err() + ); Ok(()) } diff --git a/src/query/ee/tests/it/storages/fuse/operations/vacuum.rs b/src/query/ee/tests/it/storages/fuse/operations/vacuum.rs index eec3b1edbed7d..e6b54943945ff 100644 --- a/src/query/ee/tests/it/storages/fuse/operations/vacuum.rs +++ b/src/query/ee/tests/it/storages/fuse/operations/vacuum.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![allow(clippy::let_and_return)] use std::fmt::Debug; use std::sync::Arc; use std::time::Duration; @@ -50,13 +51,13 @@ use databend_enterprise_vacuum_handler::vacuum_handler::VacuumTempOptions; use databend_query::test_kits::*; use databend_storages_common_io::Files; use databend_storages_common_table_meta::table::OPT_KEY_DATABASE_ID; +use opendal::EntryMode; +use opendal::Metadata; +use opendal::OperatorBuilder; use opendal::raw::Access; use opendal::raw::AccessorInfo; use opendal::raw::OpStat; use opendal::raw::RpStat; -use opendal::EntryMode; -use opendal::Metadata; -use opendal::OperatorBuilder; #[tokio::test(flavor = "multi_thread")] async fn test_fuse_do_vacuum_drop_tables() -> Result<()> { @@ -226,13 +227,13 @@ mod test_accessor { use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; - use opendal::raw::oio; - use opendal::raw::oio::Entry; use opendal::raw::MaybeSend; use opendal::raw::OpDelete; use opendal::raw::OpList; use opendal::raw::RpDelete; use opendal::raw::RpList; + use opendal::raw::oio; + use opendal::raw::oio::Entry; use super::*; @@ -297,17 +298,15 @@ mod test_accessor { } impl oio::Delete for MockDeleter { - fn delete(&mut self, _path: &str, _args: OpDelete) -> opendal::Result<()> { + async fn delete(&mut self, _path: &str, _args: OpDelete) -> opendal::Result<()> { self.size += 1; Ok(()) } - async fn flush(&mut self) -> opendal::Result { + async fn close(&mut self) -> opendal::Result<()> { self.hit_batch.store(true, Ordering::Release); - - let n = self.size; self.size = 0; - Ok(n) + Ok(()) } } @@ -889,10 +888,12 @@ async fn test_vacuum_drop_create_or_replace_impl(vacuum_stmts: &[&str]) -> Resul // db1.t1 should still be accessible fixture.execute_command("select * from db1.t1").await?; // db2.t1 should not exist - assert!(fixture - .execute_command("select * from db2.t1") - .await - .is_err()); + assert!( + fixture + .execute_command("select * from db2.t1") + .await + .is_err() + ); Ok(()) } diff --git a/src/query/ee/tests/it/storages/fuse/operations/virtual_columns.rs b/src/query/ee/tests/it/storages/fuse/operations/virtual_columns.rs index 34ccd78a482db..ad16dbd75172a 100644 --- a/src/query/ee/tests/it/storages/fuse/operations/virtual_columns.rs +++ b/src/query/ee/tests/it/storages/fuse/operations/virtual_columns.rs @@ -15,14 +15,14 @@ use databend_common_base::base::tokio; use databend_common_exception::Result; use databend_common_storage::read_parquet_schema_async_rs; -use databend_common_storages_fuse::io::MetaReaders; use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::TableContext; +use databend_common_storages_fuse::io::MetaReaders; use databend_enterprise_query::storages::fuse::operations::virtual_columns::do_refresh_virtual_column; use databend_enterprise_query::test_kits::context::EESetup; +use databend_query::pipelines::PipelineBuildResult; use databend_query::pipelines::executor::ExecutorSettings; use databend_query::pipelines::executor::PipelineCompleteExecutor; -use databend_query::pipelines::PipelineBuildResult; use databend_query::test_kits::*; use databend_storages_common_cache::LoadParams; diff --git a/src/query/ee/tests/it/storages/fuse/operations/virtual_columns_builder.rs b/src/query/ee/tests/it/storages/fuse/operations/virtual_columns_builder.rs index 5ecf847ca4d18..19c05c8828388 100644 --- a/src/query/ee/tests/it/storages/fuse/operations/virtual_columns_builder.rs +++ b/src/query/ee/tests/it/storages/fuse/operations/virtual_columns_builder.rs @@ -16,14 +16,14 @@ use std::str::FromStr; use databend_common_base::base::tokio; use databend_common_exception::Result; -use databend_common_expression::types::Int32Type; -use databend_common_expression::types::VariantType; use databend_common_expression::ColumnId; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::VariantDataType; -use databend_common_storages_fuse::io::VirtualColumnBuilder; +use databend_common_expression::types::Int32Type; +use databend_common_expression::types::VariantType; use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::io::VirtualColumnBuilder; use databend_enterprise_query::test_kits::context::EESetup; use databend_query::test_kits::*; use databend_storages_common_table_meta::meta::DraftVirtualColumnMeta; diff --git a/src/query/ee/tests/it/stream/stream_create.rs b/src/query/ee/tests/it/stream/stream_create.rs index fa15f1b794e1a..b8bb9c8bd4bcb 100644 --- a/src/query/ee/tests/it/stream/stream_create.rs +++ b/src/query/ee/tests/it/stream/stream_create.rs @@ -17,8 +17,8 @@ use chrono::Utc; use databend_common_base::base::tokio; use databend_common_exception::Result; use databend_enterprise_query::test_kits::context::EESetup; -use databend_query::test_kits::generate_snapshots; use databend_query::test_kits::TestFixture; +use databend_query::test_kits::generate_snapshots; #[tokio::test(flavor = "multi_thread")] async fn test_stream_create() -> Result<()> { diff --git a/src/query/ee_features/attach_table/src/lib.rs b/src/query/ee_features/attach_table/src/lib.rs index 77f2680307324..1c77ce9f85dd9 100644 --- a/src/query/ee_features/attach_table/src/lib.rs +++ b/src/query/ee_features/attach_table/src/lib.rs @@ -14,6 +14,6 @@ mod handler; -pub use handler::get_attach_table_handler; pub use handler::AttachTableHandler; pub use handler::AttachTableHandlerWrapper; +pub use handler::get_attach_table_handler; diff --git a/src/query/ee_features/data_mask/src/data_mask_handler.rs b/src/query/ee_features/data_mask/src/data_mask_handler.rs index c5369275cecce..e7f3bf7494b05 100644 --- a/src/query/ee_features/data_mask/src/data_mask_handler.rs +++ b/src/query/ee_features/data_mask/src/data_mask_handler.rs @@ -30,11 +30,11 @@ use std::sync::Arc; use databend_common_base::base::GlobalInstance; use databend_common_exception::Result; -use databend_common_meta_app::data_mask::data_mask_name_ident::Resource; use databend_common_meta_app::data_mask::CreateDatamaskReply; use databend_common_meta_app::data_mask::CreateDatamaskReq; use databend_common_meta_app::data_mask::DatamaskMeta; use databend_common_meta_app::data_mask::DropDatamaskReq; +use databend_common_meta_app::data_mask::data_mask_name_ident::Resource; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_app::tenant_key::errors::ExistError; use databend_common_meta_store::MetaStore; diff --git a/src/query/ee_features/data_mask/src/lib.rs b/src/query/ee_features/data_mask/src/lib.rs index 7b553cc7a42f8..be982832e4e17 100644 --- a/src/query/ee_features/data_mask/src/lib.rs +++ b/src/query/ee_features/data_mask/src/lib.rs @@ -14,6 +14,6 @@ pub mod data_mask_handler; -pub use data_mask_handler::get_datamask_handler; pub use data_mask_handler::DatamaskHandler; pub use data_mask_handler::DatamaskHandlerWrapper; +pub use data_mask_handler::get_datamask_handler; diff --git a/src/query/ee_features/row_access_policy/src/lib.rs b/src/query/ee_features/row_access_policy/src/lib.rs index eeb49b240b3f4..283f66b746f09 100644 --- a/src/query/ee_features/row_access_policy/src/lib.rs +++ b/src/query/ee_features/row_access_policy/src/lib.rs @@ -14,6 +14,6 @@ pub mod row_access_policy_handler; -pub use row_access_policy_handler::get_row_access_policy_handler; pub use row_access_policy_handler::RowAccessPolicyHandler; pub use row_access_policy_handler::RowAccessPolicyHandlerWrapper; +pub use row_access_policy_handler::get_row_access_policy_handler; diff --git a/src/query/ee_features/row_access_policy/src/row_access_policy_handler.rs b/src/query/ee_features/row_access_policy/src/row_access_policy_handler.rs index 6ad0b541871bb..fd9b019e6a208 100644 --- a/src/query/ee_features/row_access_policy/src/row_access_policy_handler.rs +++ b/src/query/ee_features/row_access_policy/src/row_access_policy_handler.rs @@ -30,12 +30,12 @@ use std::sync::Arc; use databend_common_base::base::GlobalInstance; use databend_common_exception::Result; -use databend_common_meta_app::row_access_policy::row_access_policy_name_ident::Resource; use databend_common_meta_app::row_access_policy::CreateRowAccessPolicyReply; use databend_common_meta_app::row_access_policy::CreateRowAccessPolicyReq; use databend_common_meta_app::row_access_policy::DropRowAccessPolicyReq; use databend_common_meta_app::row_access_policy::RowAccessPolicyId; use databend_common_meta_app::row_access_policy::RowAccessPolicyMeta; +use databend_common_meta_app::row_access_policy::row_access_policy_name_ident::Resource; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_app::tenant_key::errors::ExistError; use databend_common_meta_store::MetaStore; diff --git a/src/query/ee_features/storage_encryption/src/lib.rs b/src/query/ee_features/storage_encryption/src/lib.rs index a3baf8afa1fb4..7d0ce2b4e67c6 100644 --- a/src/query/ee_features/storage_encryption/src/lib.rs +++ b/src/query/ee_features/storage_encryption/src/lib.rs @@ -14,6 +14,6 @@ mod handler; -pub use handler::get_storage_encryption_handler; pub use handler::StorageEncryptionHandler; pub use handler::StorageEncryptionHandlerWrapper; +pub use handler::get_storage_encryption_handler; diff --git a/src/query/ee_features/stream_handler/src/lib.rs b/src/query/ee_features/stream_handler/src/lib.rs index a2f6773e2f2ff..1c6132561a993 100644 --- a/src/query/ee_features/stream_handler/src/lib.rs +++ b/src/query/ee_features/stream_handler/src/lib.rs @@ -14,6 +14,6 @@ mod handler; -pub use handler::get_stream_handler; pub use handler::StreamHandler; pub use handler::StreamHandlerWrapper; +pub use handler::get_stream_handler; diff --git a/src/query/ee_features/vacuum_handler/src/lib.rs b/src/query/ee_features/vacuum_handler/src/lib.rs index 7a0f2be30ada0..fe0402c525590 100644 --- a/src/query/ee_features/vacuum_handler/src/lib.rs +++ b/src/query/ee_features/vacuum_handler/src/lib.rs @@ -14,6 +14,6 @@ pub mod vacuum_handler; -pub use vacuum_handler::get_vacuum_handler; pub use vacuum_handler::VacuumHandler; pub use vacuum_handler::VacuumHandlerWrapper; +pub use vacuum_handler::get_vacuum_handler; diff --git a/src/query/ee_features/virtual_column/src/lib.rs b/src/query/ee_features/virtual_column/src/lib.rs index 77f585fb96c74..df2a42f742b33 100644 --- a/src/query/ee_features/virtual_column/src/lib.rs +++ b/src/query/ee_features/virtual_column/src/lib.rs @@ -14,6 +14,6 @@ pub mod virtual_column; -pub use virtual_column::get_virtual_column_handler; pub use virtual_column::VirtualColumnHandler; pub use virtual_column::VirtualColumnHandlerWrapper; +pub use virtual_column::get_virtual_column_handler; diff --git a/src/query/expression/benches/bench.rs b/src/query/expression/benches/bench.rs index fc2abc16ee70e..d8cd80c859304 100644 --- a/src/query/expression/benches/bench.rs +++ b/src/query/expression/benches/bench.rs @@ -17,18 +17,18 @@ use arrow_buffer::ScalarBuffer; use databend_common_base::vec_ext::VecExt; use databend_common_column::bitmap::Bitmap; use databend_common_column::buffer::Buffer; +use databend_common_expression::Column; +use databend_common_expression::DataBlock; +use databend_common_expression::FromData; use databend_common_expression::arrow::deserialize_column; use databend_common_expression::arrow::serialize_column; use databend_common_expression::types::BinaryType; use databend_common_expression::types::DecimalType; use databend_common_expression::types::ReturnType; use databend_common_expression::types::StringType; -use databend_common_expression::Column; -use databend_common_expression::DataBlock; -use databend_common_expression::FromData; -use rand::rngs::StdRng; use rand::Rng; use rand::SeedableRng; +use rand::rngs::StdRng; fn main() { // Run registered benchmarks. diff --git a/src/query/expression/src/aggregate/aggregate_function.rs b/src/query/expression/src/aggregate/aggregate_function.rs index ed57ff82c4467..05a751287d580 100755 --- a/src/query/expression/src/aggregate/aggregate_function.rs +++ b/src/query/expression/src/aggregate/aggregate_function.rs @@ -22,13 +22,13 @@ use super::AggrState; use super::AggrStateLoc; use super::AggrStateRegistry; use super::StateAddr; -use crate::types::DataType; use crate::BlockEntry; use crate::ColumnBuilder; use crate::ProjectedBlock; use crate::Scalar; use crate::StateSerdeItem; use crate::StateSerdeType; +use crate::types::DataType; pub type AggregateFunctionRef = Arc; diff --git a/src/query/expression/src/aggregate/aggregate_function_state.rs b/src/query/expression/src/aggregate/aggregate_function_state.rs index 057996de1629d..8801dab035bb1 100644 --- a/src/query/expression/src/aggregate/aggregate_function_state.rs +++ b/src/query/expression/src/aggregate/aggregate_function_state.rs @@ -20,9 +20,9 @@ use databend_common_exception::Result; use enum_as_inner::EnumAsInner; use super::AggregateFunctionRef; -use crate::types::binary::BinaryColumnBuilder; -use crate::types::DataType; use crate::ColumnBuilder; +use crate::types::DataType; +use crate::types::binary::BinaryColumnBuilder; #[derive(Clone, Copy, Debug)] pub struct StateAddr(*mut u8); diff --git a/src/query/expression/src/aggregate/aggregate_hashtable.rs b/src/query/expression/src/aggregate/aggregate_hashtable.rs index 71563593d6909..15dbdd8ae4bbe 100644 --- a/src/query/expression/src/aggregate/aggregate_hashtable.rs +++ b/src/query/expression/src/aggregate/aggregate_hashtable.rs @@ -14,29 +14,29 @@ // A new AggregateHashtable which inspired by duckdb's https://duckdb.org/2022/03/07/aggregate-hashtable.html -use std::sync::atomic::Ordering; use std::sync::Arc; +use std::sync::atomic::Ordering; use bumpalo::Bump; use databend_common_exception::Result; +use super::BATCH_SIZE; +use super::Entry; +use super::HashTableConfig; +use super::LOAD_FACTOR; +use super::MAX_PAGE_SIZE; +use super::Payload; use super::group_hash_entries; use super::hash_index::AdapterImpl; use super::hash_index::HashIndex; use super::partitioned_payload::PartitionedPayload; use super::payload_flush::PayloadFlushState; use super::probe_state::ProbeState; -use super::Entry; -use super::HashTableConfig; -use super::Payload; -use super::BATCH_SIZE; -use super::LOAD_FACTOR; -use super::MAX_PAGE_SIZE; -use crate::types::DataType; use crate::AggregateFunctionRef; use crate::BlockEntry; use crate::ColumnBuilder; use crate::ProjectedBlock; +use crate::types::DataType; pub struct AggregateHashTable { pub payload: PartitionedPayload, diff --git a/src/query/expression/src/aggregate/group_hash.rs b/src/query/expression/src/aggregate/group_hash.rs index 3c93f7b65b6ac..090aae48b88d4 100644 --- a/src/query/expression/src/aggregate/group_hash.rs +++ b/src/query/expression/src/aggregate/group_hash.rs @@ -19,18 +19,18 @@ use databend_common_column::types::Index; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use crate::types::decimal::Decimal; -use crate::types::*; -use crate::visitor::ValueVisitor; -use crate::with_decimal_mapped_type; -use crate::with_number_mapped_type; -use crate::with_number_type; use crate::BlockEntry; use crate::Column; use crate::ProjectedBlock; use crate::Scalar; use crate::ScalarRef; use crate::Value; +use crate::types::decimal::Decimal; +use crate::types::*; +use crate::visitor::ValueVisitor; +use crate::with_decimal_mapped_type; +use crate::with_number_mapped_type; +use crate::with_number_type; const NULL_HASH_VAL: u64 = 0xd1cefa08eb382d69; @@ -630,7 +630,11 @@ mod tests { use databend_common_column::types::timestamp_tz; use super::*; - use crate::types::geography::Geography; + use crate::BlockEntry; + use crate::DataBlock; + use crate::FromData; + use crate::ProjectedBlock; + use crate::Value; use crate::types::ArgType; use crate::types::DecimalSize; use crate::types::NullableColumn; @@ -638,11 +642,7 @@ mod tests { use crate::types::OpaqueScalar; use crate::types::VectorDataType; use crate::types::VectorScalar; - use crate::BlockEntry; - use crate::DataBlock; - use crate::FromData; - use crate::ProjectedBlock; - use crate::Value; + use crate::types::geography::Geography; fn merge_hash_slice(ls: &[u64]) -> u64 { ls.iter().cloned().reduce(merge_hash).unwrap() diff --git a/src/query/expression/src/aggregate/hash_index.rs b/src/query/expression/src/aggregate/hash_index.rs index 95de04d0f7932..b8032982b828d 100644 --- a/src/query/expression/src/aggregate/hash_index.rs +++ b/src/query/expression/src/aggregate/hash_index.rs @@ -14,11 +14,11 @@ use std::fmt::Debug; -use super::payload_row::CompareState; +use super::LOAD_FACTOR; use super::PartitionedPayload; use super::ProbeState; use super::RowPtr; -use super::LOAD_FACTOR; +use super::payload_row::CompareState; use crate::ProjectedBlock; pub(super) struct HashIndex { diff --git a/src/query/expression/src/aggregate/mod.rs b/src/query/expression/src/aggregate/mod.rs index f19400517329b..1766205bdc46c 100644 --- a/src/query/expression/src/aggregate/mod.rs +++ b/src/query/expression/src/aggregate/mod.rs @@ -27,9 +27,9 @@ mod payload_row; mod probe_state; mod row_ptr; +use std::sync::Arc; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; -use std::sync::Arc; pub use aggregate_function::*; pub use aggregate_function_state::*; diff --git a/src/query/expression/src/aggregate/partitioned_payload.rs b/src/query/expression/src/aggregate/partitioned_payload.rs index 8f3c7689db41c..7f85d5db54552 100644 --- a/src/query/expression/src/aggregate/partitioned_payload.rs +++ b/src/query/expression/src/aggregate/partitioned_payload.rs @@ -20,13 +20,13 @@ use itertools::Itertools; use super::payload::Payload; use super::probe_state::ProbeState; use super::row_ptr::RowLayout; -use crate::get_states_layout; -use crate::types::DataType; use crate::AggregateFunctionRef; +use crate::BATCH_SIZE; use crate::PayloadFlushState; use crate::ProjectedBlock; use crate::StatesLayout; -use crate::BATCH_SIZE; +use crate::get_states_layout; +use crate::types::DataType; #[derive(Debug, Clone, Copy)] struct PartitionMask { diff --git a/src/query/expression/src/aggregate/payload.rs b/src/query/expression/src/aggregate/payload.rs index 42c6d92015b66..fe46ee997fbf7 100644 --- a/src/query/expression/src/aggregate/payload.rs +++ b/src/query/expression/src/aggregate/payload.rs @@ -20,25 +20,25 @@ use databend_common_base::runtime::drop_guard; use log::info; use strength_reduce::StrengthReducedU64; +use super::RowID; use super::payload_row::rowformat_size; use super::payload_row::serialize_column_to_rowformat; use super::payload_row::serialize_const_column_to_rowformat; use super::row_ptr::RowLayout; use super::row_ptr::RowPtr; -use super::RowID; -use crate::types::DataType; use crate::AggrState; use crate::AggregateFunctionRef; +use crate::BATCH_SIZE; use crate::BlockEntry; use crate::Column; use crate::ColumnBuilder; use crate::DataBlock; +use crate::MAX_PAGE_SIZE; use crate::PayloadFlushState; use crate::ProjectedBlock; use crate::StateAddr; use crate::StatesLayout; -use crate::BATCH_SIZE; -use crate::MAX_PAGE_SIZE; +use crate::types::DataType; // payload layout // [VALIDITY][GROUPS][HASH][STATE_ADDRS] diff --git a/src/query/expression/src/aggregate/payload_flush.rs b/src/query/expression/src/aggregate/payload_flush.rs index d456a343c12f2..72c18e3695ac7 100644 --- a/src/query/expression/src/aggregate/payload_flush.rs +++ b/src/query/expression/src/aggregate/payload_flush.rs @@ -19,14 +19,13 @@ use super::partitioned_payload::PartitionedPayload; use super::payload::Payload; use super::probe_state::ProbeState; use super::row_ptr::RowPtr; -use crate::types::binary::BinaryColumn; -use crate::types::binary::BinaryColumnBuilder; -use crate::types::decimal::Decimal; -use crate::types::decimal::DecimalType; -use crate::types::i256; -use crate::types::nullable::NullableColumn; -use crate::types::string::StringColumn; -use crate::types::string::StringColumnBuilder; +use crate::BATCH_SIZE; +use crate::BlockEntry; +use crate::Column; +use crate::ColumnBuilder; +use crate::DataBlock; +use crate::Scalar; +use crate::StateAddr; use crate::types::AccessType; use crate::types::ArgType; use crate::types::BooleanType; @@ -38,14 +37,15 @@ use crate::types::NumberDataType; use crate::types::NumberType; use crate::types::ReturnType; use crate::types::TimestampType; +use crate::types::binary::BinaryColumn; +use crate::types::binary::BinaryColumnBuilder; +use crate::types::decimal::Decimal; +use crate::types::decimal::DecimalType; +use crate::types::i256; +use crate::types::nullable::NullableColumn; +use crate::types::string::StringColumn; +use crate::types::string::StringColumnBuilder; use crate::with_number_mapped_type; -use crate::BlockEntry; -use crate::Column; -use crate::ColumnBuilder; -use crate::DataBlock; -use crate::Scalar; -use crate::StateAddr; -use crate::BATCH_SIZE; pub struct PayloadFlushState { pub probe_state: Box, diff --git a/src/query/expression/src/aggregate/payload_row.rs b/src/query/expression/src/aggregate/payload_row.rs index 7c00477de9b2d..a9cf316f15b13 100644 --- a/src/query/expression/src/aggregate/payload_row.rs +++ b/src/query/expression/src/aggregate/payload_row.rs @@ -21,9 +21,11 @@ use databend_common_io::prelude::bincode_serialize_into_buf; use super::RowID; use super::RowLayout; use super::RowPtr; -use crate::types::decimal::Decimal; -use crate::types::decimal::DecimalColumn; -use crate::types::i256; +use crate::BATCH_SIZE; +use crate::BlockEntry; +use crate::Column; +use crate::ProjectedBlock; +use crate::Scalar; use crate::types::AccessType; use crate::types::AnyType; use crate::types::BinaryType; @@ -37,13 +39,11 @@ use crate::types::NumberColumn; use crate::types::NumberScalar; use crate::types::NumberType; use crate::types::TimestampType; +use crate::types::decimal::Decimal; +use crate::types::decimal::DecimalColumn; +use crate::types::i256; use crate::with_decimal_mapped_type; use crate::with_number_mapped_type; -use crate::BlockEntry; -use crate::Column; -use crate::ProjectedBlock; -use crate::Scalar; -use crate::BATCH_SIZE; pub(super) fn rowformat_size(data_type: &DataType) -> usize { match data_type { @@ -116,49 +116,63 @@ pub(super) unsafe fn serialize_column_to_rowformat( let val: u8 = if v.null_count() == 0 { 1 } else { 0 }; // faster path for row in select_vector { - address[*row].write_u8(offset, val); + unsafe { + address[*row].write_u8(offset, val); + } } } else { for row in select_vector { - address[*row].write_u8(offset, v.get_bit(row.to_usize()) as u8); + unsafe { + address[*row].write_u8(offset, v.get_bit(row.to_usize()) as u8); + } } } } Column::Binary(v) | Column::Bitmap(v) | Column::Variant(v) | Column::Geometry(v) => { for row in select_vector { - let data = arena.alloc_slice_copy(v.index_unchecked(row.to_usize())); - address[*row].write_bytes(offset, data); + let data = arena.alloc_slice_copy(unsafe { v.index_unchecked(row.to_usize()) }); + unsafe { + address[*row].write_bytes(offset, data); + } } } Column::String(v) => { for row in select_vector { - let data = arena.alloc_str(v.index_unchecked(row.to_usize())); - address[*row].write_bytes(offset, data.as_bytes()); + let data = arena.alloc_str(unsafe { v.index_unchecked(row.to_usize()) }); + unsafe { + address[*row].write_bytes(offset, data.as_bytes()); + } } } Column::Timestamp(buffer) => { for row in select_vector { - address[*row].write(offset, &buffer[*row]); + unsafe { + address[*row].write(offset, &buffer[*row]); + } } } Column::Date(buffer) => { for row in select_vector { - address[*row].write(offset, &buffer[*row]); + unsafe { + address[*row].write(offset, &buffer[*row]); + } } } - Column::Nullable(c) => { + Column::Nullable(c) => unsafe { serialize_column_to_rowformat(arena, &c.column, select_vector, address, offset, scratch) - } + }, // for complex column other => { for row in select_vector { - let s = other.index_unchecked(row.to_usize()).to_owned(); + let s = unsafe { other.index_unchecked(row.to_usize()) }.to_owned(); scratch.clear(); bincode_serialize_into_buf(scratch, &s).unwrap(); let data = arena.alloc_slice_copy(scratch); - address[*row].write_bytes(offset, data); + unsafe { + address[*row].write_bytes(offset, data); + } } } } @@ -173,82 +187,86 @@ pub(super) unsafe fn serialize_const_column_to_rowformat( offset: usize, scratch: &mut Vec, ) { - match scalar { - Scalar::Null => { - if let Some(box data_type) = data_type.as_nullable() { - serialize_const_column_to_rowformat( - arena, - &Scalar::default_value(data_type), - data_type, - select_vector, - address, - offset, - scratch, - ) - } - } - Scalar::EmptyArray | Scalar::EmptyMap => (), - Scalar::Number(number_scalar) => with_number_mapped_type!(|NUM_TYPE| match number_scalar { - NumberScalar::NUM_TYPE(value) => { - for row in select_vector { - address[*row].write(offset, value); + unsafe { + match scalar { + Scalar::Null => { + if let Some(box data_type) = data_type.as_nullable() { + serialize_const_column_to_rowformat( + arena, + &Scalar::default_value(data_type), + data_type, + select_vector, + address, + offset, + scratch, + ) } } - }), - Scalar::Decimal(decimal_scalar) => { - let size = decimal_scalar.size(); - with_decimal_mapped_type!(|T| match size.data_kind() { - DecimalDataKind::T => { - let value: T = decimal_scalar.as_decimal(); - for row in select_vector { - address[*row].write(offset, &value); + Scalar::EmptyArray | Scalar::EmptyMap => (), + Scalar::Number(number_scalar) => { + with_number_mapped_type!(|NUM_TYPE| match number_scalar { + NumberScalar::NUM_TYPE(value) => { + for row in select_vector { + address[*row].write(offset, value); + } } + }) + } + Scalar::Decimal(decimal_scalar) => { + let size = decimal_scalar.size(); + with_decimal_mapped_type!(|T| match size.data_kind() { + DecimalDataKind::T => { + let value: T = decimal_scalar.as_decimal(); + for row in select_vector { + address[*row].write(offset, &value); + } + } + }) + } + Scalar::Boolean(value) => { + let value = if *value { 1 } else { 0 }; + for row in select_vector { + address[*row].write_u8(offset, value); } - }) - } - Scalar::Boolean(value) => { - let value = if *value { 1 } else { 0 }; - for row in select_vector { - address[*row].write_u8(offset, value); } - } - Scalar::Timestamp(value) => { - for row in select_vector { - address[*row].write(offset, value); + Scalar::Timestamp(value) => { + for row in select_vector { + address[*row].write(offset, value); + } } - } - Scalar::Date(value) => { - for row in select_vector { - address[*row].write(offset, value); + Scalar::Date(value) => { + for row in select_vector { + address[*row].write(offset, value); + } } - } - Scalar::Interval(value) => { - for row in select_vector { - address[*row].write(offset, value); + Scalar::Interval(value) => { + for row in select_vector { + address[*row].write(offset, value); + } } - } - Scalar::String(value) => { - let data = arena.alloc_str(value); - let bytes = data.as_bytes(); - for row in select_vector { - address[*row].write_bytes(offset, bytes); + Scalar::String(value) => { + let data = arena.alloc_str(value); + let bytes = data.as_bytes(); + for row in select_vector { + address[*row].write_bytes(offset, bytes); + } } - } - Scalar::Binary(value) - | Scalar::Bitmap(value) - | Scalar::Variant(value) - | Scalar::Geometry(value) => { - let data = arena.alloc_slice_copy(value); - for row in select_vector { - address[*row].write_bytes(offset, data); + Scalar::Binary(value) + | Scalar::Bitmap(value) + | Scalar::Variant(value) + | Scalar::Geometry(value) => { + let data = arena.alloc_slice_copy(value); + for row in select_vector { + address[*row].write_bytes(offset, data); + } } - } - other => { - scratch.clear(); - bincode_serialize_into_buf(scratch, other).unwrap(); - let data = arena.alloc_slice_copy(scratch); - for row in select_vector { - address[*row].write_bytes(offset, data); + other => { + scratch.clear(); + bincode_serialize_into_buf(scratch, other).unwrap(); + let data = arena.alloc_slice_copy(scratch); + for row in select_vector { + address[*row].write_bytes(offset, data); + } } } } @@ -262,9 +280,11 @@ unsafe fn serialize_fixed_size_column_to_rowformat( ) where T: AccessType, { - for row in select_vector { - let val = T::index_column_unchecked_scalar(column, row.to_usize()); - address[*row].write(offset, &val); + unsafe { + for row in select_vector { + let val = T::index_column_unchecked_scalar(column, row.to_usize()); + address[*row].write(offset, &val); + } } } diff --git a/src/query/expression/src/aggregate/probe_state.rs b/src/query/expression/src/aggregate/probe_state.rs index 60d9c9a67f727..7b9b04243fb26 100644 --- a/src/query/expression/src/aggregate/probe_state.rs +++ b/src/query/expression/src/aggregate/probe_state.rs @@ -15,9 +15,9 @@ use std::ops::Index; use std::ops::IndexMut; -use super::row_ptr::RowPtr; -use super::StateAddr; use super::BATCH_SIZE; +use super::StateAddr; +use super::row_ptr::RowPtr; pub type SelectVector = [RowID; BATCH_SIZE]; diff --git a/src/query/expression/src/aggregate/row_ptr.rs b/src/query/expression/src/aggregate/row_ptr.rs index c88cd203c97e7..c3e1f4b83c7f3 100644 --- a/src/query/expression/src/aggregate/row_ptr.rs +++ b/src/query/expression/src/aggregate/row_ptr.rs @@ -13,9 +13,9 @@ // limitations under the License. use super::RowID; -use crate::types::StringColumn; use crate::StateAddr; use crate::StatesLayout; +use crate::types::StringColumn; /// A wrapper around raw pointer that provides safe and convenient methods /// for accessing row data in the aggregate hash table. @@ -36,31 +36,39 @@ impl RowPtr { } pub(super) unsafe fn read(&self, offset: usize) -> T { - core::ptr::read_unaligned(self.0.add(offset).cast::().cast_const()) + unsafe { core::ptr::read_unaligned(self.0.add(offset).cast::().cast_const()) } } pub(super) unsafe fn write(&mut self, offset: usize, value: &T) { - core::ptr::copy_nonoverlapping( - value as *const T as *const u8, - self.0.add(offset), - size_of::(), - ); + unsafe { + core::ptr::copy_nonoverlapping( + value as *const T as *const u8, + self.0.add(offset), + size_of::(), + ); + } } pub(super) unsafe fn write_bytes(&mut self, offset: usize, value: &[u8]) { - self.write(offset, &(value.len() as u32)); - self.write(offset + 4, &(value.as_ptr() as u64)); + unsafe { + self.write(offset, &(value.len() as u32)); + self.write(offset + 4, &(value.as_ptr() as u64)); + } } pub(super) unsafe fn read_bytes(&self, offset: usize) -> &[u8] { - let len = self.read::(offset) as usize; - let data_ptr = self.read::(offset + 4) as *const u8; - std::slice::from_raw_parts(data_ptr, len) + unsafe { + let len = self.read::(offset) as usize; + let data_ptr = self.read::(offset + 4) as *const u8; + std::slice::from_raw_parts(data_ptr, len) + } } pub(super) unsafe fn is_bytes_eq(&self, offset: usize, other: &[u8]) -> bool { - let scalar = self.read_bytes(offset); - scalar.len() == other.len() && databend_common_hashtable::fast_memcmp(scalar, other) + unsafe { + let scalar = self.read_bytes(offset); + scalar.len() == other.len() && databend_common_hashtable::fast_memcmp(scalar, other) + } } pub(super) unsafe fn eq_string_view( @@ -69,26 +77,30 @@ impl RowPtr { str_view: &StringColumn, row: RowID, ) -> bool { - let row = row.to_usize(); - let v = str_view.views().get_unchecked(row); - let len = self.read::(offset); - if v.length != len { - return false; + unsafe { + let row = row.to_usize(); + let v = str_view.views().get_unchecked(row); + let len = self.read::(offset); + if v.length != len { + return false; + } + let scalar = { + let data_ptr = self.read::(offset + size_of::()) as *const u8; + std::slice::from_raw_parts(data_ptr, len as _) + }; + let other = v.get_slice_unchecked(str_view.data_buffers()); + databend_common_hashtable::fast_memcmp(scalar, other) } - let scalar = { - let data_ptr = self.read::(offset + size_of::()) as *const u8; - std::slice::from_raw_parts(data_ptr, len as _) - }; - let other = v.get_slice_unchecked(str_view.data_buffers()); - databend_common_hashtable::fast_memcmp(scalar, other) } pub(super) unsafe fn read_bool(&self, offset: usize) -> bool { - self.read::(offset) != 0 + unsafe { self.read::(offset) != 0 } } pub(super) unsafe fn write_u8(&mut self, offset: usize, value: u8) { - self.write::(offset, &value); + unsafe { + self.write::(offset, &value); + } } pub(super) fn hash(&self, layout: &RowLayout) -> u64 { diff --git a/src/query/expression/src/block.rs b/src/query/expression/src/block.rs index 64eeb2849cb2d..bd0fdd7be4cfa 100644 --- a/src/query/expression/src/block.rs +++ b/src/query/expression/src/block.rs @@ -22,13 +22,6 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use enum_as_inner::EnumAsInner; -use crate::schema::DataSchema; -use crate::types::AccessType; -use crate::types::AnyType; -use crate::types::ArgType; -use crate::types::BooleanType; -use crate::types::DataType; -use crate::types::ValueType; use crate::Column; use crate::ColumnBuilder; use crate::ColumnSet; @@ -39,6 +32,13 @@ use crate::Scalar; use crate::ScalarRef; use crate::TableSchemaRef; use crate::Value; +use crate::schema::DataSchema; +use crate::types::AccessType; +use crate::types::AnyType; +use crate::types::ArgType; +use crate::types::BooleanType; +use crate::types::DataType; +use crate::types::ValueType; pub type SendableDataBlockStream = std::pin::Pin> + Send>>; @@ -188,20 +188,22 @@ impl BlockEntry { /// # Safety /// /// Calling this method with an out-of-bounds index is *[undefined behavior]* - pub unsafe fn index_unchecked(&self, index: usize) -> ScalarRef { - match self { - BlockEntry::Const(scalar, _, _n) => { - #[cfg(debug_assertions)] - if index >= *_n { - panic!( - "index out of bounds: the len is {:?} but the index is {}", - _n, index - ) - } + pub unsafe fn index_unchecked(&self, index: usize) -> ScalarRef<'_> { + unsafe { + match self { + BlockEntry::Const(scalar, _, _n) => { + #[cfg(debug_assertions)] + if index >= *_n { + panic!( + "index out of bounds: the len is {:?} but the index is {}", + _n, index + ) + } - scalar.as_ref() + scalar.as_ref() + } + BlockEntry::Column(column) => column.index_unchecked(index), } - BlockEntry::Column(column) => column.index_unchecked(index), } } @@ -274,7 +276,7 @@ impl ColumnView { self.len() == 0 } - pub fn iter(&self) -> ColumnViewIter { + pub fn iter(&self) -> ColumnViewIter<'_, T> { match self { ColumnView::Const(scalar, num_rows) => { ColumnViewIter::Const(T::to_scalar_ref(scalar), *num_rows) @@ -300,10 +302,12 @@ impl ColumnView { /// /// Calling this method with an out-of-bounds index is *[undefined behavior]* pub unsafe fn index_unchecked(&self, i: usize) -> T::ScalarRef<'_> { - debug_assert!(i < self.len()); - match self { - ColumnView::Const(scalar, _) => T::to_scalar_ref(scalar), - ColumnView::Column(column) => T::index_column_unchecked(column, i), + unsafe { + debug_assert!(i < self.len()); + match self { + ColumnView::Const(scalar, _) => T::to_scalar_ref(scalar), + ColumnView::Column(column) => T::index_column_unchecked(column, i), + } } } } @@ -450,10 +454,10 @@ impl DataBlock { c.check_valid()?; if c.len() != num_rows { return Err(ErrorCode::Internal(format!( - "DataBlock corrupted, column length mismatch, col rows: {}, num_rows: {num_rows}, datatype: {}", - c.len(), - c.data_type() - ))); + "DataBlock corrupted, column length mismatch, col rows: {}, num_rows: {num_rows}, datatype: {}", + c.len(), + c.data_type() + ))); } } } diff --git a/src/query/expression/src/constant_folder.rs b/src/query/expression/src/constant_folder.rs index 3e2647b6d56c7..2b278d9d9393b 100644 --- a/src/query/expression/src/constant_folder.rs +++ b/src/query/expression/src/constant_folder.rs @@ -17,6 +17,15 @@ use std::collections::HashMap; use databend_common_ast::Span; use log::error; +use crate::ColumnBuilder; +use crate::ColumnIndex; +use crate::EvalContext; +use crate::FunctionContext; +use crate::FunctionDomain; +use crate::FunctionEval; +use crate::FunctionRegistry; +use crate::Scalar; +use crate::Value; use crate::block::DataBlock; use crate::evaluator::Evaluator; use crate::expression::Cast; @@ -28,19 +37,10 @@ use crate::expression::LambdaFunctionCall; use crate::property::Domain; use crate::type_check::check_function; use crate::type_check::get_simple_cast_function; +use crate::types::DataType; use crate::types::boolean::BooleanDomain; use crate::types::nullable::NullableDomain; use crate::types::number::NumberScalar; -use crate::types::DataType; -use crate::ColumnBuilder; -use crate::ColumnIndex; -use crate::EvalContext; -use crate::FunctionContext; -use crate::FunctionDomain; -use crate::FunctionEval; -use crate::FunctionRegistry; -use crate::Scalar; -use crate::Value; const MAX_FUNCTION_ARGS_TO_FOLD: usize = 4096; diff --git a/src/query/expression/src/converts/arrow/from.rs b/src/query/expression/src/converts/arrow/from.rs index 6902f7f1cf1f9..d42661c64a027 100644 --- a/src/query/expression/src/converts/arrow/from.rs +++ b/src/query/expression/src/converts/arrow/from.rs @@ -40,7 +40,15 @@ use super::ARROW_EXT_TYPE_TIMESTAMP_TIMEZONE; use super::ARROW_EXT_TYPE_VARIANT; use super::ARROW_EXT_TYPE_VECTOR; use super::EXTENSION_KEY; -use crate::types::opaque::OpaqueColumn; +use crate::Column; +use crate::DataBlock; +use crate::DataField; +use crate::DataSchema; +use crate::Scalar; +use crate::TableDataType; +use crate::TableField; +use crate::TableSchema; +use crate::Value; use crate::types::AnyType; use crate::types::ArrayColumn; use crate::types::DataType; @@ -53,15 +61,7 @@ use crate::types::NumberColumn; use crate::types::NumberDataType; use crate::types::VectorColumn; use crate::types::VectorDataType; -use crate::Column; -use crate::DataBlock; -use crate::DataField; -use crate::DataSchema; -use crate::Scalar; -use crate::TableDataType; -use crate::TableField; -use crate::TableSchema; -use crate::Value; +use crate::types::opaque::OpaqueColumn; impl TryFrom<&Field> for DataField { type Error = ErrorCode; diff --git a/src/query/expression/src/converts/arrow/to.rs b/src/query/expression/src/converts/arrow/to.rs index d6b8644e251be..901f6d8e6bc44 100644 --- a/src/query/expression/src/converts/arrow/to.rs +++ b/src/query/expression/src/converts/arrow/to.rs @@ -40,6 +40,14 @@ use super::ARROW_EXT_TYPE_TIMESTAMP_TIMEZONE; use super::ARROW_EXT_TYPE_VARIANT; use super::ARROW_EXT_TYPE_VECTOR; use super::EXTENSION_KEY; +use crate::Column; +use crate::ColumnId; +use crate::DataBlock; +use crate::DataField; +use crate::DataSchema; +use crate::TableDataType; +use crate::TableField; +use crate::TableSchema; use crate::infer_table_schema; use crate::schema::is_internal_column_id; use crate::types::DataType; @@ -50,14 +58,6 @@ use crate::types::NumberDataType; use crate::types::VectorColumn; use crate::types::VectorDataType; use crate::with_number_type; -use crate::Column; -use crate::ColumnId; -use crate::DataBlock; -use crate::DataField; -use crate::DataSchema; -use crate::TableDataType; -use crate::TableField; -use crate::TableSchema; impl From<&DataSchema> for Schema { fn from(schema: &DataSchema) -> Self { diff --git a/src/query/expression/src/converts/datavalues/from.rs b/src/query/expression/src/converts/datavalues/from.rs index c04a12af61055..98dcb433e6a5a 100644 --- a/src/query/expression/src/converts/datavalues/from.rs +++ b/src/query/expression/src/converts/datavalues/from.rs @@ -15,13 +15,13 @@ use databend_common_datavalues::DataTypeImpl; use databend_common_datavalues::DataValue; -use crate::types::number::NumberScalar; -use crate::types::*; use crate::ColumnBuilder; use crate::Scalar; use crate::TableDataType; use crate::TableField; use crate::TableSchema; +use crate::types::number::NumberScalar; +use crate::types::*; pub fn can_convert(datatype: &DataTypeImpl) -> bool { !matches!( diff --git a/src/query/expression/src/converts/meta/bincode.rs b/src/query/expression/src/converts/meta/bincode.rs index 37b316499d6e1..33499ab26d841 100644 --- a/src/query/expression/src/converts/meta/bincode.rs +++ b/src/query/expression/src/converts/meta/bincode.rs @@ -24,6 +24,8 @@ use serde::Deserializer; use serde::Serialize; use serde::Serializer; +use crate::Column; +use crate::Scalar; use crate::types::array::ArrayColumn; use crate::types::binary::BinaryColumn; use crate::types::decimal::DecimalColumn; @@ -32,8 +34,6 @@ use crate::types::nullable::NullableColumn; use crate::types::number::NumberColumn; use crate::types::number::NumberScalar; use crate::types::*; -use crate::Column; -use crate::Scalar; #[derive(Clone, Serialize, Deserialize)] pub enum LegacyScalar { @@ -54,6 +54,7 @@ pub enum LegacyScalar { Variant(Vec), } +#[allow(unused, dead_code)] #[derive(Clone, EnumAsInner)] pub enum LegacyColumn { Null { len: usize }, diff --git a/src/query/expression/src/converts/meta/index_scalar.rs b/src/query/expression/src/converts/meta/index_scalar.rs index 2c90037f5f7f6..1255a8600a6c9 100644 --- a/src/query/expression/src/converts/meta/index_scalar.rs +++ b/src/query/expression/src/converts/meta/index_scalar.rs @@ -23,11 +23,11 @@ use enum_as_inner::EnumAsInner; use serde::Deserialize; use serde::Serialize; +use crate::Scalar; +use crate::types::DecimalSize; use crate::types::decimal::DecimalScalar; use crate::types::i256; use crate::types::number::NumberScalar; -use crate::types::DecimalSize; -use crate::Scalar; #[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq, EnumAsInner)] pub enum IndexDecimalScalar { diff --git a/src/query/expression/src/evaluator.rs b/src/query/expression/src/evaluator.rs index 48fa973602a70..a36d072b38a80 100644 --- a/src/query/expression/src/evaluator.rs +++ b/src/query/expression/src/evaluator.rs @@ -23,21 +23,23 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use itertools::Itertools; +use crate::BlockEntry; +use crate::FunctionContext; +use crate::FunctionEval; +use crate::FunctionRegistry; +use crate::RemoteExpr; +use crate::ScalarRef; use crate::block::DataBlock; use crate::expr::*; use crate::expression::Expr; use crate::function::EvalContext; use crate::type_check::check_function; use crate::type_check::get_simple_cast_function; -use crate::types::any::AnyType; -use crate::types::array::ArrayColumn; -use crate::types::boolean; -use crate::types::nullable::NullableColumn; -use crate::types::string::StringColumnBuilder; use crate::types::BooleanType; use crate::types::DataType; use crate::types::DecimalColumn; use crate::types::DecimalDataType; +use crate::types::F32; use crate::types::NullableType; use crate::types::NumberScalar; use crate::types::ReturnType; @@ -47,18 +49,16 @@ use crate::types::VariantType; use crate::types::VectorColumn; use crate::types::VectorDataType; use crate::types::VectorScalar; -use crate::types::F32; +use crate::types::any::AnyType; +use crate::types::array::ArrayColumn; +use crate::types::boolean; +use crate::types::nullable::NullableColumn; +use crate::types::string::StringColumnBuilder; use crate::values::Column; use crate::values::ColumnBuilder; use crate::values::Scalar; use crate::values::Value; use crate::visitor::ValueVisitor; -use crate::BlockEntry; -use crate::FunctionContext; -use crate::FunctionEval; -use crate::FunctionRegistry; -use crate::RemoteExpr; -use crate::ScalarRef; pub struct EvaluateOptions<'a> { pub selection: Option<&'a [u32]>, @@ -235,13 +235,14 @@ impl<'a> Evaluator<'a> { .iter() .map(|expr| self.partial_run(expr, validity.clone(), options)) .collect::>>()?; - assert!(args - .iter() - .filter_map(|val| match val { - Value::Column(col) => Some(col.len()), - Value::Scalar(_) => None, - }) - .all_equal()); + assert!( + args.iter() + .filter_map(|val| match val { + Value::Column(col) => Some(col.len()), + Value::Scalar(_) => None, + }) + .all_equal() + ); self.run_lambda(name, args, data_types, lambda_expr, return_type)? } @@ -296,13 +297,14 @@ impl<'a> Evaluator<'a> { .map(|expr| self.partial_run(expr, validity.clone(), &mut child_option)) .collect::>>()?; - assert!(args - .iter() - .filter_map(|val| match val { - Value::Column(col) => Some(col.len()), - Value::Scalar(_) => None, - }) - .all_equal()); + assert!( + args.iter() + .filter_map(|val| match val { + Value::Column(col) => Some(col.len()), + Value::Scalar(_) => None, + }) + .all_equal() + ); let errors = if child_suppress_error { child_option.errors.take() @@ -1335,15 +1337,17 @@ impl<'a> Evaluator<'a> { let else_result = self.partial_run(&args[args.len() - 1], Some(validity), options)?; // Assert that all the arguments have the same length. - assert!(conds - .iter() - .chain(results.iter()) - .chain([&else_result]) - .filter_map(|val| match val { - Value::Column(col) => Some(col.len()), - Value::Scalar(_) => None, - }) - .all_equal()); + assert!( + conds + .iter() + .chain(results.iter()) + .chain([&else_result]) + .filter_map(|val| match val { + Value::Column(col) => Some(col.len()), + Value::Scalar(_) => None, + }) + .all_equal() + ); // Pick the results from the result branches depending on the condition. let mut output_builder = ColumnBuilder::with_capacity(&generics[0], len.unwrap_or(1)); @@ -1866,13 +1870,15 @@ impl<'a> Evaluator<'a> { .iter() .map(|expr| self.get_select_child(expr, options)) .collect::>>()?; - assert!(children - .iter() - .filter_map(|val| match &val.0 { - Value::Column(col) => Some(col.len()), - Value::Scalar(_) => None, - }) - .all_equal()); + assert!( + children + .iter() + .filter_map(|val| match &val.0 { + Value::Column(col) => Some(col.len()), + Value::Scalar(_) => None, + }) + .all_equal() + ); Ok(children) } @@ -1977,13 +1983,14 @@ impl<'a> Evaluator<'a> { .iter() .map(|expr| self.get_select_child(expr, &mut child_option)) .collect::>>()?; - assert!(args - .iter() - .filter_map(|val| match &val.0 { - Value::Column(col) => Some(col.len()), - Value::Scalar(_) => None, - }) - .all_equal()); + assert!( + args.iter() + .filter_map(|val| match &val.0 { + Value::Column(col) => Some(col.len()), + Value::Scalar(_) => None, + }) + .all_equal() + ); let args = args.into_iter().map(|(val, _)| val).collect::>(); @@ -2035,13 +2042,14 @@ impl<'a> Evaluator<'a> { .iter() .map(|expr| self.partial_run(expr, None, &mut EvaluateOptions::default())) .collect::>>()?; - assert!(args - .iter() - .filter_map(|val| match val { - Value::Column(col) => Some(col.len()), - Value::Scalar(_) => None, - }) - .all_equal()); + assert!( + args.iter() + .filter_map(|val| match val { + Value::Column(col) => Some(col.len()), + Value::Scalar(_) => None, + }) + .all_equal() + ); Ok(( self.run_lambda(name, args, data_types, lambda_expr, return_type)?, diff --git a/src/query/expression/src/expression.rs b/src/query/expression/src/expression.rs index 6cd5a24a699cd..464057f7a4d94 100644 --- a/src/query/expression/src/expression.rs +++ b/src/query/expression/src/expression.rs @@ -489,6 +489,7 @@ pub fn visit_expr>( /// /// The remote node will recover the `Arc` pointer within `FunctionCall` by looking /// up the function registry with the `FunctionID`. +#[allow(unused, dead_code)] #[derive(Debug, Clone, Educe, Serialize, Deserialize, EnumAsInner)] #[educe(PartialEq(bound(false)), Eq, Hash(bound(false)))] pub enum RemoteExpr { @@ -821,12 +822,16 @@ impl Expr { ) -> Result>, Self::Error> { if call.function.signature.name == self.name { match call.args.as_slice() { - [Expr::ColumnRef(ColumnRef { id, .. }), Expr::Constant(Constant { scalar, .. })] => - { + [ + Expr::ColumnRef(ColumnRef { id, .. }), + Expr::Constant(Constant { scalar, .. }), + ] => { (self.visitor)(id, scalar, true); } - [Expr::Constant(Constant { scalar, .. }), Expr::ColumnRef(ColumnRef { id, .. })] => - { + [ + Expr::Constant(Constant { scalar, .. }), + Expr::ColumnRef(ColumnRef { id, .. }), + ] => { (self.visitor)(id, scalar, false); } _ => {} @@ -869,12 +874,16 @@ impl Expr { ) -> Result>, Self::Error> { if call.function.signature.name == self.name { match call.args.as_slice() { - [Expr::ColumnRef(ColumnRef { id, .. }), Expr::Constant(Constant { scalar, .. })] => - { + [ + Expr::ColumnRef(ColumnRef { id, .. }), + Expr::Constant(Constant { scalar, .. }), + ] => { return Ok((self.visitor)(id, scalar, call)); } - [Expr::Constant(Constant { scalar, .. }), Expr::ColumnRef(ColumnRef { id, .. })] => - { + [ + Expr::Constant(Constant { scalar, .. }), + Expr::ColumnRef(ColumnRef { id, .. }), + ] => { return Ok((self.visitor)(id, scalar, call)); } _ => {} diff --git a/src/query/expression/src/filter/filter_executor.rs b/src/query/expression/src/filter/filter_executor.rs index 8acb1860cdbd3..31f879bd77709 100644 --- a/src/query/expression/src/filter/filter_executor.rs +++ b/src/query/expression/src/filter/filter_executor.rs @@ -17,17 +17,17 @@ use core::ops::Range; use databend_common_column::bitmap::MutableBitmap; use databend_common_exception::Result; -use crate::filter::SelectExpr; -use crate::filter::Selector; -use crate::types::BooleanType; use crate::ColumnSet; use crate::DataBlock; use crate::Evaluator; use crate::Expr; use crate::FunctionContext; use crate::FunctionRegistry; -use crate::SelectExprBuilder; use crate::SELECTIVITY_THRESHOLD; +use crate::SelectExprBuilder; +use crate::filter::SelectExpr; +use crate::filter::Selector; +use crate::types::BooleanType; // FilterExecutor is used to filter `DataBlock` by `SelectExpr`. pub struct FilterExecutor { diff --git a/src/query/expression/src/filter/mod.rs b/src/query/expression/src/filter/mod.rs index c4072ac38a384..13ef4f04260d2 100644 --- a/src/query/expression/src/filter/mod.rs +++ b/src/query/expression/src/filter/mod.rs @@ -23,9 +23,9 @@ mod selector; mod volnitsky; pub use filter_executor::FilterExecutor; +pub use like::LikePattern; pub use like::generate_like_pattern; pub use like::is_like_pattern_escape; -pub use like::LikePattern; pub use select_expr::SelectExpr; pub use select_expr::SelectExprBuilder; pub use select_op::SelectOp; diff --git a/src/query/expression/src/filter/select.rs b/src/query/expression/src/filter/select.rs index 29727beb3fd45..5de5e1f76272e 100644 --- a/src/query/expression/src/filter/select.rs +++ b/src/query/expression/src/filter/select.rs @@ -16,24 +16,23 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use super::SelectionBuffers; +use crate::Selector; +use crate::Value; use crate::arrow::and_validities; use crate::filter::SelectOp; -use crate::types::nullable::NullableColumn; -use crate::types::number::*; -use crate::types::timestamp_tz::TimestampTzType; use crate::types::AnyType; use crate::types::BooleanType; use crate::types::DataType; use crate::types::DateType; -use crate::types::Decimal128As256Type; +use crate::types::Decimal64As128Type; +use crate::types::Decimal64As256Type; +use crate::types::Decimal64Type; use crate::types::Decimal128As64Type; +use crate::types::Decimal128As256Type; use crate::types::Decimal128Type; -use crate::types::Decimal256As128Type; use crate::types::Decimal256As64Type; +use crate::types::Decimal256As128Type; use crate::types::Decimal256Type; -use crate::types::Decimal64As128Type; -use crate::types::Decimal64As256Type; -use crate::types::Decimal64Type; use crate::types::DecimalDataKind; use crate::types::DecimalDataType; use crate::types::EmptyArrayType; @@ -42,9 +41,10 @@ use crate::types::NumberType; use crate::types::StringType; use crate::types::TimestampType; use crate::types::VariantType; +use crate::types::nullable::NullableColumn; +use crate::types::number::*; +use crate::types::timestamp_tz::TimestampTzType; use crate::with_number_mapped_type; -use crate::Selector; -use crate::Value; impl Selector<'_> { // Select indices by comparing two `Value`. diff --git a/src/query/expression/src/filter/select_expr.rs b/src/query/expression/src/filter/select_expr.rs index e426f2a3f8964..4a7edf4cb3d57 100644 --- a/src/query/expression/src/filter/select_expr.rs +++ b/src/query/expression/src/filter/select_expr.rs @@ -17,17 +17,17 @@ use std::sync::Arc; use itertools::Itertools; -use crate::expr::*; -use crate::filter::select_expr_permutation::FilterPermutation; -use crate::filter::SelectOp; -use crate::generate_like_pattern; -use crate::type_check::convert_escape_pattern; -use crate::types::DataType; use crate::Expr; use crate::Function; use crate::FunctionID; use crate::LikePattern; use crate::Scalar; +use crate::expr::*; +use crate::filter::SelectOp; +use crate::filter::select_expr_permutation::FilterPermutation; +use crate::generate_like_pattern; +use crate::type_check::convert_escape_pattern; +use crate::types::DataType; /// The `SelectExpr` is used to represent the predicates expression. #[derive(Clone, Debug)] @@ -176,31 +176,31 @@ impl SelectExprBuilder { _ => None, }); let can_reorder = Self::can_reorder(column); - if matches!(column_data_type, DataType::String | DataType::Nullable(box DataType::String)) - && let Scalar::String(like_str) = scalar - { - let pattern = option_escape - .and_then(|escape| { - escape - .chars() - .next() - .map(|escape| convert_escape_pattern(like_str, escape)) - }) - .unwrap_or(like_str.to_string()) - .into(); - let like_pattern: LikePattern<'static> = - generate_like_pattern(Cow::Owned(pattern), 0); + let is_string = matches!(column_data_type, DataType::String | DataType::Nullable(box DataType::String)); - SelectExprBuildResult::new(SelectExpr::Like(( - column.clone(), - Arc::new(like_pattern), - not, - ))) - .can_reorder(can_reorder) - } else { - SelectExprBuildResult::new(SelectExpr::Others(expr.clone())) - .can_push_down_not(false) + match (scalar, is_string) { + (Scalar::String(like_str), true) => { + let pattern = option_escape + .and_then(|escape| { + escape.chars().next().map(|escape| { + convert_escape_pattern(like_str, escape) + }) + }) + .unwrap_or(like_str.to_string()) + .into(); + let like_pattern: LikePattern<'static> = + generate_like_pattern(Cow::Owned(pattern), 0); + + SelectExprBuildResult::new(SelectExpr::Like(( + column.clone(), + Arc::new(like_pattern), + not, + ))) .can_reorder(can_reorder) + } + _ => SelectExprBuildResult::new(SelectExpr::Others(expr.clone())) + .can_push_down_not(false) + .can_reorder(can_reorder), } } "is_true" => self.build_select_expr(&args[0], not), diff --git a/src/query/expression/src/filter/select_value/mod.rs b/src/query/expression/src/filter/select_value/mod.rs index d0b811a40bb51..85e385fe8490e 100644 --- a/src/query/expression/src/filter/select_value/mod.rs +++ b/src/query/expression/src/filter/select_value/mod.rs @@ -16,14 +16,14 @@ use databend_common_column::bitmap::Bitmap; use databend_common_exception::Result; use super::SelectionBuffers; -use crate::types::string::StringColumn; -use crate::types::AccessType; -use crate::types::AnyType; -use crate::with_mapped_cmp_method; use crate::LikePattern; use crate::SelectOp; use crate::Selector; use crate::Value; +use crate::types::AccessType; +use crate::types::AnyType; +use crate::types::string::StringColumn; +use crate::with_mapped_cmp_method; mod select_column; mod select_column_scalar; diff --git a/src/query/expression/src/filter/select_value/select_column_scalar.rs b/src/query/expression/src/filter/select_value/select_column_scalar.rs index 32183748422ff..322c7eaa3ab23 100644 --- a/src/query/expression/src/filter/select_value/select_column_scalar.rs +++ b/src/query/expression/src/filter/select_value/select_column_scalar.rs @@ -16,11 +16,11 @@ use databend_common_column::bitmap::Bitmap; use databend_common_exception::Result; use super::SelectionBuffers; +use crate::LikePattern; use crate::filter::SelectStrategy; use crate::filter::Selector; -use crate::types::string::StringColumn; use crate::types::AccessType; -use crate::LikePattern; +use crate::types::string::StringColumn; impl<'a> Selector<'a> { // Select indices by comparing scalar and column. diff --git a/src/query/expression/src/filter/selector.rs b/src/query/expression/src/filter/selector.rs index 153cf5f77bf19..8a3ad13d2b9ec 100644 --- a/src/query/expression/src/filter/selector.rs +++ b/src/query/expression/src/filter/selector.rs @@ -19,12 +19,6 @@ use databend_common_exception::Result; use itertools::Itertools; use super::SelectionBuffers; -use crate::expr::*; -use crate::filter::select_expr_permutation::FilterPermutation; -use crate::filter::SelectExpr; -use crate::filter::SelectOp; -use crate::types::AnyType; -use crate::types::DataType; use crate::Column; use crate::EvalContext; use crate::EvaluateOptions; @@ -33,6 +27,12 @@ use crate::Expr; use crate::LikePattern; use crate::Scalar; use crate::Value; +use crate::expr::*; +use crate::filter::SelectExpr; +use crate::filter::SelectOp; +use crate::filter::select_expr_permutation::FilterPermutation; +use crate::types::AnyType; +use crate::types::DataType; // SelectStrategy is used to determine the iteration strategy of the index. // (1) True: iterate true index in `true_selection`. @@ -505,13 +505,14 @@ impl<'a> Selector<'a> { .iter() .map(|expr| self.evaluator.partial_run(expr, None, &mut eval_options)) .collect::>>()?; - assert!(args - .iter() - .filter_map(|val| match val { - Value::Column(col) => Some(col.len()), - Value::Scalar(_) => None, - }) - .all_equal()); + assert!( + args.iter() + .filter_map(|val| match val { + Value::Column(col) => Some(col.len()), + Value::Scalar(_) => None, + }) + .all_equal() + ); let mut ctx = EvalContext { generics, num_rows: self.evaluator.data_block().num_rows(), @@ -592,13 +593,14 @@ impl<'a> Selector<'a> { .iter() .map(|expr| self.evaluator.partial_run(expr, None, &mut eval_options)) .collect::>>()?; - assert!(args - .iter() - .filter_map(|val| match val { - Value::Column(col) => Some(col.len()), - Value::Scalar(_) => None, - }) - .all_equal()); + assert!( + args.iter() + .filter_map(|val| match val { + Value::Column(col) => Some(col.len()), + Value::Scalar(_) => None, + }) + .all_equal() + ); let result = self.evaluator .run_lambda(name, args, data_types, lambda_expr, return_type)?; diff --git a/src/query/expression/src/function.rs b/src/query/expression/src/function.rs index 52cbd7811bc56..48ba9624ee396 100755 --- a/src/query/expression/src/function.rs +++ b/src/query/expression/src/function.rs @@ -28,11 +28,16 @@ use databend_common_exception::Result; use databend_common_io::GeometryDataType; use enum_as_inner::EnumAsInner; use itertools::Itertools; -use jiff::tz::TimeZone; use jiff::Zoned; +use jiff::tz::TimeZone; use serde::Deserialize; use serde::Serialize; +use crate::Column; +use crate::ColumnIndex; +use crate::Expr; +use crate::FunctionDomain; +use crate::Scalar; use crate::property::Domain; use crate::property::FunctionProperty; use crate::type_check::try_unify_signature; @@ -40,11 +45,6 @@ use crate::types::nullable::NullableColumn; use crate::types::nullable::NullableDomain; use crate::types::*; use crate::values::Value; -use crate::Column; -use crate::ColumnIndex; -use crate::Expr; -use crate::FunctionDomain; -use crate::Scalar; pub type AutoCastRules<'a> = &'a [(DataType, DataType)]; pub type DynamicCastRules = Vec bool + Send + Sync>>; @@ -128,6 +128,7 @@ pub struct FunctionSignature { } #[derive(EnumAsInner)] +#[allow(unused, dead_code)] #[allow(clippy::type_complexity)] pub enum FunctionEval { /// Scalar function that returns a single value. @@ -246,11 +247,12 @@ pub struct FunctionRegistry { impl Function { pub fn passthrough_nullable(self) -> Self { - debug_assert!(self - .signature - .args_type - .iter() - .all(|ty| !ty.is_nullable_or_null())); + debug_assert!( + self.signature + .args_type + .iter() + .all(|ty| !ty.is_nullable_or_null()) + ); let (calc_domain, eval) = self.eval.into_scalar().unwrap(); @@ -716,9 +718,9 @@ impl EvalContext<'_> { format!("{error} while evaluating function `{func_name}({args})` in expr `{expr_name}`") } else { format!( - "{error} while evaluating function `{func_name}({params})({args})` in expr `{expr_name}`", - params = params.iter().join(", ") - ) + "{error} while evaluating function `{func_name}({params})({args})` in expr `{expr_name}`", + params = params.iter().join(", ") + ) }; Err(ErrorCode::BadArguments(err_msg).set_span(span)) diff --git a/src/query/expression/src/kernels/concat.rs b/src/query/expression/src/kernels/concat.rs index 679c5792e39a3..9882d68de347b 100644 --- a/src/query/expression/src/kernels/concat.rs +++ b/src/query/expression/src/kernels/concat.rs @@ -22,16 +22,11 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use itertools::Itertools; -use crate::types::array::ArrayColumnBuilder; -use crate::types::decimal::Decimal; -use crate::types::decimal::DecimalColumn; -use crate::types::i256; -use crate::types::map::KvColumnBuilder; -use crate::types::nullable::NullableColumn; -use crate::types::number::NumberColumn; -use crate::types::opaque::OpaqueType; -use crate::types::timestamp_tz::TimestampTzType; -use crate::types::vector::VectorColumnBuilder; +use crate::BlockEntry; +use crate::Column; +use crate::ColumnBuilder; +use crate::DataBlock; +use crate::Value; use crate::types::AccessType; use crate::types::AnyType; use crate::types::ArgType; @@ -46,15 +41,20 @@ use crate::types::TimestampType; use crate::types::ValueType; use crate::types::VectorColumn; use crate::types::VectorDataType; +use crate::types::array::ArrayColumnBuilder; +use crate::types::decimal::Decimal; +use crate::types::decimal::DecimalColumn; +use crate::types::i256; +use crate::types::map::KvColumnBuilder; +use crate::types::nullable::NullableColumn; +use crate::types::number::NumberColumn; +use crate::types::opaque::OpaqueType; +use crate::types::timestamp_tz::TimestampTzType; +use crate::types::vector::VectorColumnBuilder; use crate::with_decimal_mapped_type; use crate::with_number_mapped_type; use crate::with_opaque_size; use crate::with_vector_number_type; -use crate::BlockEntry; -use crate::Column; -use crate::ColumnBuilder; -use crate::DataBlock; -use crate::Value; impl DataBlock { pub fn concat(blocks: &[DataBlock]) -> Result { diff --git a/src/query/expression/src/kernels/filter.rs b/src/query/expression/src/kernels/filter.rs index 55e9a83e81852..28c2f3dedc68b 100644 --- a/src/query/expression/src/kernels/filter.rs +++ b/src/query/expression/src/kernels/filter.rs @@ -13,24 +13,24 @@ // limitations under the License. use binary::BinaryColumnBuilder; -use databend_common_column::bitmap::utils::SlicesIterator; use databend_common_column::bitmap::Bitmap; use databend_common_column::bitmap::MutableBitmap; use databend_common_column::bitmap::TrueIdxIter; +use databend_common_column::bitmap::utils::SlicesIterator; use databend_common_column::buffer::Buffer; use databend_common_exception::Result; use string::StringColumnBuilder; -use crate::types::binary::BinaryColumn; -use crate::types::nullable::NullableColumn; -use crate::types::string::StringColumn; -use crate::types::*; -use crate::visitor::ValueVisitor; use crate::BlockEntry; use crate::Column; use crate::ColumnBuilder; use crate::DataBlock; use crate::Value; +use crate::types::binary::BinaryColumn; +use crate::types::nullable::NullableColumn; +use crate::types::string::StringColumn; +use crate::types::*; +use crate::visitor::ValueVisitor; impl DataBlock { pub fn filter_with_bitmap(self, bitmap: &Bitmap) -> Result { diff --git a/src/query/expression/src/kernels/group_by.rs b/src/query/expression/src/kernels/group_by.rs index 6ca4f8db673cb..ae37a57a20598 100644 --- a/src/query/expression/src/kernels/group_by.rs +++ b/src/query/expression/src/kernels/group_by.rs @@ -14,17 +14,17 @@ use databend_common_exception::Result; +use super::group_by_hash::HashMethodKeysU8; use super::group_by_hash::HashMethodKeysU16; use super::group_by_hash::HashMethodKeysU32; use super::group_by_hash::HashMethodKeysU64; -use super::group_by_hash::HashMethodKeysU8; use super::group_by_hash::HashMethodKind; use super::group_by_hash::HashMethodSerializer; use super::group_by_hash::HashMethodSingleBinary; -use crate::types::DataType; use crate::DataBlock; use crate::HashMethodKeysU128; use crate::HashMethodKeysU256; +use crate::types::DataType; impl DataBlock { pub fn choose_hash_method(block: &DataBlock, indices: &[usize]) -> Result { diff --git a/src/query/expression/src/kernels/group_by_hash/method.rs b/src/query/expression/src/kernels/group_by_hash/method.rs index 8ee90e3e07774..146a9ac30d82e 100644 --- a/src/query/expression/src/kernels/group_by_hash/method.rs +++ b/src/query/expression/src/kernels/group_by_hash/method.rs @@ -23,18 +23,18 @@ use databend_common_hashtable::FastHash; use either::Either; use ethnum::u256; -use crate::types::binary::BinaryColumn; -use crate::types::StringColumn; use crate::Column; -use crate::HashMethodKeysU128; +use crate::HashMethodKeysU8; use crate::HashMethodKeysU16; -use crate::HashMethodKeysU256; use crate::HashMethodKeysU32; use crate::HashMethodKeysU64; -use crate::HashMethodKeysU8; +use crate::HashMethodKeysU128; +use crate::HashMethodKeysU256; use crate::HashMethodSerializer; use crate::HashMethodSingleBinary; use crate::ProjectedBlock; +use crate::types::StringColumn; +use crate::types::binary::BinaryColumn; #[derive(Debug, Clone)] pub enum KeysState { diff --git a/src/query/expression/src/kernels/group_by_hash/method_fixed_keys.rs b/src/query/expression/src/kernels/group_by_hash/method_fixed_keys.rs index e81a62fa96177..1db545da8e368 100644 --- a/src/query/expression/src/kernels/group_by_hash/method_fixed_keys.rs +++ b/src/query/expression/src/kernels/group_by_hash/method_fixed_keys.rs @@ -23,22 +23,22 @@ use databend_common_hashtable::FastHash; use ethnum::u256; use micromarshal::Marshal; -use crate::types::decimal::DecimalColumn; -use crate::types::i256; -use crate::types::number::Number; -use crate::types::number::NumberColumn; +use crate::Column; +use crate::HashMethod; +use crate::KeyAccessor; +use crate::KeysState; +use crate::ProjectedBlock; use crate::types::AccessType; use crate::types::ArgType; use crate::types::DecimalDataKind; use crate::types::DecimalView; use crate::types::NumberType; +use crate::types::decimal::DecimalColumn; +use crate::types::i256; +use crate::types::number::Number; +use crate::types::number::NumberColumn; use crate::with_decimal_mapped_type; use crate::with_number_mapped_type; -use crate::Column; -use crate::HashMethod; -use crate::KeyAccessor; -use crate::KeysState; -use crate::ProjectedBlock; pub type HashMethodKeysU8 = HashMethodFixedKeys; pub type HashMethodKeysU16 = HashMethodFixedKeys; @@ -556,7 +556,7 @@ impl KeyAccessor for PrimitiveKeyAccessor { /// # Safety /// Calling this method with an out-of-bounds index is *[undefined behavior]*. unsafe fn key_unchecked(&self, index: usize) -> &Self::Key { - self.data.get_unchecked(index) + unsafe { self.data.get_unchecked(index) } } fn len(&self) -> usize { diff --git a/src/query/expression/src/kernels/group_by_hash/method_serializer.rs b/src/query/expression/src/kernels/group_by_hash/method_serializer.rs index 7ae271d68f9e2..ce94111fbd9ca 100644 --- a/src/query/expression/src/kernels/group_by_hash/method_serializer.rs +++ b/src/query/expression/src/kernels/group_by_hash/method_serializer.rs @@ -16,12 +16,12 @@ use databend_common_exception::Result; use databend_common_hashtable::hash_join_fast_string_hash; use super::utils::serialize_group_columns; -use crate::types::binary::BinaryColumnIter; use crate::Column; use crate::HashMethod; use crate::KeyAccessor; use crate::KeysState; use crate::ProjectedBlock; +use crate::types::binary::BinaryColumnIter; #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct HashMethodSerializer {} diff --git a/src/query/expression/src/kernels/group_by_hash/method_single_string.rs b/src/query/expression/src/kernels/group_by_hash/method_single_string.rs index 61fad6c199584..5ea9890de5cfd 100644 --- a/src/query/expression/src/kernels/group_by_hash/method_single_string.rs +++ b/src/query/expression/src/kernels/group_by_hash/method_single_string.rs @@ -15,13 +15,13 @@ use databend_common_exception::Result; use databend_common_hashtable::hash_join_fast_string_hash; -use crate::types::binary::BinaryColumnIter; -use crate::types::BinaryColumn; use crate::Column; use crate::HashMethod; use crate::KeyAccessor; use crate::KeysState; use crate::ProjectedBlock; +use crate::types::BinaryColumn; +use crate::types::binary::BinaryColumnIter; #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct HashMethodSingleBinary {} @@ -78,8 +78,10 @@ impl KeyAccessor for BinaryColumn { /// # Safety /// Calling this method with an out-of-bounds index is *[undefined behavior]*. unsafe fn key_unchecked(&self, index: usize) -> &Self::Key { - debug_assert!(index + 1 < self.offsets().len()); - self.index_unchecked(index) + unsafe { + debug_assert!(index + 1 < self.offsets().len()); + self.index_unchecked(index) + } } fn len(&self) -> usize { diff --git a/src/query/expression/src/kernels/group_by_hash/utils.rs b/src/query/expression/src/kernels/group_by_hash/utils.rs index c730d79e948ba..da5ce4809e70f 100644 --- a/src/query/expression/src/kernels/group_by_hash/utils.rs +++ b/src/query/expression/src/kernels/group_by_hash/utils.rs @@ -15,16 +15,16 @@ use databend_common_base::vec_ext::VecExt; use databend_common_base::vec_ext::VecU8Ext; +use crate::Column; +use crate::ProjectedBlock; +use crate::types::BinaryColumn; +use crate::types::NumberColumn; use crate::types::binary::BinaryColumnBuilder; use crate::types::decimal::DecimalColumn; use crate::types::vector::VectorScalarRef; -use crate::types::BinaryColumn; -use crate::types::NumberColumn; use crate::with_decimal_type; use crate::with_number_mapped_type; use crate::with_vector_number_type; -use crate::Column; -use crate::ProjectedBlock; /// The serialize_size is equal to the number of bytes required by serialization. pub fn serialize_group_columns( @@ -52,81 +52,83 @@ pub fn serialize_group_columns( /// /// * The size of the memory pointed by `row_space` is equal to the number of bytes required by serialization. pub unsafe fn serialize_column_binary(column: &Column, row: usize, row_space: &mut Vec) { - match column { - Column::Null { .. } | Column::EmptyArray { .. } | Column::EmptyMap { .. } => {} - Column::Number(v) => with_number_mapped_type!(|NUM_TYPE| match v { - NumberColumn::NUM_TYPE(v) => { - row_space.store_value_uncheckd(&v[row]); - } - }), - Column::Decimal(v) => { - with_decimal_type!(|DECIMAL_TYPE| match v { - DecimalColumn::DECIMAL_TYPE(v, _) => { + unsafe { + match column { + Column::Null { .. } | Column::EmptyArray { .. } | Column::EmptyMap { .. } => {} + Column::Number(v) => with_number_mapped_type!(|NUM_TYPE| match v { + NumberColumn::NUM_TYPE(v) => { row_space.store_value_uncheckd(&v[row]); } - }) - } - Column::Boolean(v) => row_space.push_unchecked(v.get_bit(row) as u8), - Column::Binary(v) | Column::Bitmap(v) | Column::Variant(v) | Column::Geometry(v) => { - let value = unsafe { v.index_unchecked(row) }; - let len = value.len(); + }), + Column::Decimal(v) => { + with_decimal_type!(|DECIMAL_TYPE| match v { + DecimalColumn::DECIMAL_TYPE(v, _) => { + row_space.store_value_uncheckd(&v[row]); + } + }) + } + Column::Boolean(v) => row_space.push_unchecked(v.get_bit(row) as u8), + Column::Binary(v) | Column::Bitmap(v) | Column::Variant(v) | Column::Geometry(v) => { + let value = v.index_unchecked(row); + let len = value.len(); - row_space.store_value_uncheckd(&(len as u64)); - row_space.extend_from_slice_unchecked(value); - } - Column::Geography(v) => { - let value = unsafe { v.index_unchecked(row) }; - let value = borsh::to_vec(&value.0).unwrap(); - let len = value.len(); + row_space.store_value_uncheckd(&(len as u64)); + row_space.extend_from_slice_unchecked(value); + } + Column::Geography(v) => { + let value = v.index_unchecked(row); + let value = borsh::to_vec(&value.0).unwrap(); + let len = value.len(); - row_space.store_value_uncheckd(&(len as u64)); - row_space.extend_from_slice_unchecked(&value); - } - Column::String(v) => { - let value = unsafe { v.index_unchecked(row) }; - let len = value.len(); + row_space.store_value_uncheckd(&(len as u64)); + row_space.extend_from_slice_unchecked(&value); + } + Column::String(v) => { + let value = v.index_unchecked(row); + let len = value.len(); - row_space.store_value_uncheckd(&(len as u64)); - row_space.extend_from_slice_unchecked(value.as_bytes()); - } - Column::Opaque(_v) => { - unimplemented!() - } - Column::Timestamp(v) => row_space.store_value_uncheckd(&v[row]), - Column::TimestampTz(v) => row_space.store_value_uncheckd(&v[row]), - Column::Date(v) => row_space.store_value_uncheckd(&v[row]), - Column::Interval(v) => row_space.store_value_uncheckd(&v[row]), - Column::Array(array) | Column::Map(array) => { - let data = array.index(row).unwrap(); - row_space.store_value_uncheckd(&(data.len() as u64)); + row_space.store_value_uncheckd(&(len as u64)); + row_space.extend_from_slice_unchecked(value.as_bytes()); + } + Column::Opaque(_v) => { + unimplemented!() + } + Column::Timestamp(v) => row_space.store_value_uncheckd(&v[row]), + Column::TimestampTz(v) => row_space.store_value_uncheckd(&v[row]), + Column::Date(v) => row_space.store_value_uncheckd(&v[row]), + Column::Interval(v) => row_space.store_value_uncheckd(&v[row]), + Column::Array(array) | Column::Map(array) => { + let data = array.index(row).unwrap(); + row_space.store_value_uncheckd(&(data.len() as u64)); - for i in 0..data.len() { - serialize_column_binary(&data, i, row_space); + for i in 0..data.len() { + serialize_column_binary(&data, i, row_space); + } } - } - Column::Nullable(c) => { - let valid = c.validity.get_bit(row); + Column::Nullable(c) => { + let valid = c.validity.get_bit(row); - row_space.push_unchecked(valid as u8); + row_space.push_unchecked(valid as u8); - if valid { - serialize_column_binary(&c.column, row, row_space); + if valid { + serialize_column_binary(&c.column, row, row_space); + } } - } - Column::Tuple(fields) => { - for inner_col in fields.iter() { - serialize_column_binary(inner_col, row, row_space); + Column::Tuple(fields) => { + for inner_col in fields.iter() { + serialize_column_binary(inner_col, row, row_space); + } } - } - Column::Vector(col) => { - let scalar = col.index_unchecked(row); - with_vector_number_type!(|NUM_TYPE| match scalar { - VectorScalarRef::NUM_TYPE(vals) => { - for val in vals { - row_space.store_value_uncheckd(val); + Column::Vector(col) => { + let scalar = col.index_unchecked(row); + with_vector_number_type!(|NUM_TYPE| match scalar { + VectorScalarRef::NUM_TYPE(vals) => { + for val in vals { + row_space.store_value_uncheckd(val); + } } - } - }) + }) + } } } } diff --git a/src/query/expression/src/kernels/sort.rs b/src/query/expression/src/kernels/sort.rs index 095bea0ae505a..df2ca0fe9a4ac 100644 --- a/src/query/expression/src/kernels/sort.rs +++ b/src/query/expression/src/kernels/sort.rs @@ -14,14 +14,14 @@ use databend_common_exception::Result; -use crate::types::DataType; -use crate::visitor::ValueVisitor; use crate::Column; use crate::ColumnBuilder; use crate::DataBlock; use crate::Scalar; use crate::SortCompare; use crate::Value; +use crate::types::DataType; +use crate::visitor::ValueVisitor; #[derive(Clone, Debug)] pub struct SortColumnDescription { diff --git a/src/query/expression/src/kernels/sort_compare.rs b/src/query/expression/src/kernels/sort_compare.rs index 67232150511d7..0229db36de1f3 100644 --- a/src/query/expression/src/kernels/sort_compare.rs +++ b/src/query/expression/src/kernels/sort_compare.rs @@ -20,6 +20,8 @@ use databend_common_column::buffer::Buffer; use databend_common_exception::Result; use memchr::memchr; +use crate::LimitType; +use crate::SortColumnDescription; use crate::types::AnyType; use crate::types::DataType; use crate::types::NullableColumn; @@ -27,8 +29,6 @@ use crate::types::Number; use crate::types::StringColumn; use crate::types::ValueType; use crate::visitor::ValueVisitor; -use crate::LimitType; -use crate::SortColumnDescription; pub struct SortCompare { rows: usize, diff --git a/src/query/expression/src/kernels/stream_partition.rs b/src/query/expression/src/kernels/stream_partition.rs index 15a7b7e51f007..7b9af52186240 100644 --- a/src/query/expression/src/kernels/stream_partition.rs +++ b/src/query/expression/src/kernels/stream_partition.rs @@ -19,10 +19,10 @@ use databend_common_column::bitmap::MutableBitmap; use databend_common_column::buffer::Buffer; use databend_common_column::types::Index; -use crate::types::array::ArrayColumnBuilder; -use crate::types::binary::BinaryColumnBuilder; -use crate::types::nullable::NullableColumnBuilder; -use crate::types::string::StringColumnBuilder; +use crate::BlockEntry; +use crate::Column; +use crate::ColumnBuilder; +use crate::DataBlock; use crate::types::AnyType; use crate::types::ArrayColumn; use crate::types::BinaryColumn; @@ -36,12 +36,12 @@ use crate::types::OpaqueColumnBuilder; use crate::types::StringColumn; use crate::types::VectorColumn; use crate::types::VectorColumnBuilder; +use crate::types::array::ArrayColumnBuilder; +use crate::types::binary::BinaryColumnBuilder; +use crate::types::nullable::NullableColumnBuilder; +use crate::types::string::StringColumnBuilder; use crate::with_decimal_type; use crate::with_number_mapped_type; -use crate::BlockEntry; -use crate::Column; -use crate::ColumnBuilder; -use crate::DataBlock; struct PartitionBlockBuilder { num_rows: usize, @@ -264,7 +264,8 @@ pub fn copy_column(indices: &[I], from: &Column, to: &mut ColumnBuilde _ => unreachable!( "ColumnBuilder::with_capacity for Array type should return ColumnBuilder::Array, \ but got different variant. data_type: {}, capacity: {}", - from.data_type(), capacity + from.data_type(), + capacity ), } } @@ -298,9 +299,12 @@ pub fn copy_column(indices: &[I], from: &Column, to: &mut ColumnBuilde copy_nullable(&mut builder, column, indices); *to = ColumnBuilder::Nullable(builder); } - _ => unreachable!("ColumnBuilder::with_capacity for Nullable type should return ColumnBuilder::Nullable, \ + _ => unreachable!( + "ColumnBuilder::with_capacity for Nullable type should return ColumnBuilder::Nullable, \ but got different variant. data_type: {}, capacity: {}", - from.data_type(), capacity), + from.data_type(), + capacity + ), } } _ => unreachable!( @@ -330,9 +334,12 @@ pub fn copy_column(indices: &[I], from: &Column, to: &mut ColumnBuilde copy_array(&mut builder, column, indices); *to = ColumnBuilder::Map(builder); } - _ => unreachable!("ColumnBuilder::with_capacity for Map type should return ColumnBuilder::Map, \ + _ => unreachable!( + "ColumnBuilder::with_capacity for Map type should return ColumnBuilder::Map, \ but got different variant. data_type: {}, capacity: {}", - from.data_type(), capacity), + from.data_type(), + capacity + ), } } _ => unreachable!( diff --git a/src/query/expression/src/kernels/take.rs b/src/query/expression/src/kernels/take.rs index 5d71c4924d091..177779ed7d43a 100644 --- a/src/query/expression/src/kernels/take.rs +++ b/src/query/expression/src/kernels/take.rs @@ -20,6 +20,12 @@ use databend_common_column::buffer::Buffer; use databend_common_exception::Result; use string::StringColumnBuilder; +use crate::BlockEntry; +use crate::Column; +use crate::ColumnBuilder; +use crate::DataBlock; +use crate::SELECTIVITY_THRESHOLD; +use crate::Value; use crate::types::binary::BinaryColumn; use crate::types::date::CoreDate; use crate::types::nullable::NullableColumn; @@ -29,12 +35,6 @@ use crate::types::timestamp::CoreTimestamp; use crate::types::*; use crate::visitor::ValueVisitor; use crate::with_number_mapped_type; -use crate::BlockEntry; -use crate::Column; -use crate::ColumnBuilder; -use crate::DataBlock; -use crate::Value; -use crate::SELECTIVITY_THRESHOLD; pub const BIT_MASK: [u8; 8] = [1, 2, 4, 8, 16, 32, 64, 128]; diff --git a/src/query/expression/src/kernels/take_chunks.rs b/src/query/expression/src/kernels/take_chunks.rs index 19e953ab2b9c3..c9a1f19b8e2c8 100644 --- a/src/query/expression/src/kernels/take_chunks.rs +++ b/src/query/expression/src/kernels/take_chunks.rs @@ -21,6 +21,13 @@ use databend_common_hashtable::RowPtr; use itertools::Itertools; use string::StringColumnBuilder; +use crate::BlockEntry; +use crate::Column; +use crate::ColumnBuilder; +use crate::ColumnVec; +use crate::DataBlock; +use crate::Scalar; +use crate::Value; use crate::kernels::take::BIT_MASK; use crate::types::array::ArrayColumnBuilder; use crate::types::binary::BinaryColumn; @@ -46,13 +53,6 @@ use crate::with_opaque_mapped_type; use crate::with_opaque_size; use crate::with_opaque_type; use crate::with_vector_number_type; -use crate::BlockEntry; -use crate::Column; -use crate::ColumnBuilder; -use crate::ColumnVec; -use crate::DataBlock; -use crate::Scalar; -use crate::Value; // Block idx, row idx in the block, repeat times pub type BlockRowIndex = (u32, u32, usize); diff --git a/src/query/expression/src/kernels/take_compact.rs b/src/query/expression/src/kernels/take_compact.rs index 75a7ec87c9b9a..d4c15e5d76c39 100644 --- a/src/query/expression/src/kernels/take_compact.rs +++ b/src/query/expression/src/kernels/take_compact.rs @@ -17,15 +17,15 @@ use databend_common_base::vec_ext::VecExt; use databend_common_column::buffer::Buffer; use databend_common_exception::Result; +use crate::BlockEntry; +use crate::ColumnBuilder; +use crate::DataBlock; +use crate::Value; use crate::types::binary::BinaryColumn; use crate::types::nullable::NullableColumn; use crate::types::string::StringColumn; use crate::types::*; use crate::visitor::ValueVisitor; -use crate::BlockEntry; -use crate::ColumnBuilder; -use crate::DataBlock; -use crate::Value; impl DataBlock { pub fn take_compacted_indices(&self, indices: &[(u32, u32)], num_rows: usize) -> Result { diff --git a/src/query/expression/src/kernels/take_ranges.rs b/src/query/expression/src/kernels/take_ranges.rs index e11ae845560a6..a56fb77135bc2 100644 --- a/src/query/expression/src/kernels/take_ranges.rs +++ b/src/query/expression/src/kernels/take_ranges.rs @@ -21,15 +21,15 @@ use databend_common_column::bitmap::MutableBitmap; use databend_common_column::buffer::Buffer; use databend_common_exception::Result; +use crate::BlockEntry; +use crate::ColumnBuilder; +use crate::DataBlock; +use crate::Value; use crate::types::binary::BinaryColumn; use crate::types::nullable::NullableColumn; use crate::types::string::StringColumn; use crate::types::*; use crate::visitor::ValueVisitor; -use crate::BlockEntry; -use crate::ColumnBuilder; -use crate::DataBlock; -use crate::Value; impl DataBlock { // Generate a new `DataBlock` by the specified indices ranges. diff --git a/src/query/expression/src/kernels/topk.rs b/src/query/expression/src/kernels/topk.rs index 32cab5f3b688e..04f713fca16b3 100644 --- a/src/query/expression/src/kernels/topk.rs +++ b/src/query/expression/src/kernels/topk.rs @@ -21,10 +21,10 @@ use std::ptr; use databend_common_base::runtime::drop_guard; use databend_common_column::bitmap::MutableBitmap; -use crate::types::*; -use crate::with_number_mapped_type; use crate::Column; use crate::Scalar; +use crate::types::*; +use crate::with_number_mapped_type; #[derive(Clone)] pub struct TopKSorter { @@ -202,11 +202,7 @@ impl TopKSorter { } fn ordering(&self) -> Ordering { - if self.asc { - Less - } else { - Less.reverse() - } + if self.asc { Less } else { Less.reverse() } } } diff --git a/src/query/expression/src/lib.rs b/src/query/expression/src/lib.rs index a104ccaa90fd2..a1fa3bb8398de 100755 --- a/src/query/expression/src/lib.rs +++ b/src/query/expression/src/lib.rs @@ -17,8 +17,14 @@ #![allow(clippy::missing_transmute_annotations)] #![allow(clippy::arc_with_non_send_sync)] #![allow(internal_features)] +#![allow(unused)] // FIXME: we should avoid this by implementing Ord correctly. #![allow(clippy::non_canonical_partial_ord_impl)] +#![allow(clippy::collapsible_if)] +#![allow(clippy::manual_is_multiple_of)] +#![allow(clippy::let_and_return)] +#![allow(clippy::cloned_ref_to_slice_refs)] +#![allow(clippy::useless_vec)] #![allow(incomplete_features)] #![feature(fmt_internals)] #![feature(const_try)] @@ -31,21 +37,16 @@ #![feature(anonymous_lifetime_in_impl_trait)] #![feature(generic_const_exprs)] #![feature(trait_alias)] -#![feature(vec_into_raw_parts)] #![feature(iterator_try_collect)] #![feature(core_intrinsics)] #![feature(trusted_len)] #![feature(iter_order_by)] #![feature(int_roundings)] #![feature(try_blocks)] -#![feature(let_chains)] #![feature(alloc_layout_extra)] #![feature(debug_closure_helpers)] #![feature(never_type)] -#![feature(slice_as_array)] -#![feature(new_zeroed_alloc)] #![feature(iter_map_windows)] - #[allow(dead_code)] mod block; diff --git a/src/query/expression/src/projected_block.rs b/src/query/expression/src/projected_block.rs index 37a5711e82b84..c88740d2cef10 100644 --- a/src/query/expression/src/projected_block.rs +++ b/src/query/expression/src/projected_block.rs @@ -63,7 +63,7 @@ impl<'a> ProjectedBlock<'a> { } } - pub fn iter(&self) -> EntriesIter { + pub fn iter(&self) -> EntriesIter<'_> { match &self.map { Some(map) => EntriesIter { iter: 0..map.len(), @@ -154,9 +154,9 @@ impl<'a> From<&'a Vec> for ProjectedBlock<'a> { #[cfg(test)] mod tests { use super::ProjectedBlock; - use crate::types::*; use crate::DataBlock; use crate::FromData; + use crate::types::*; #[test] fn test_input_columns() { diff --git a/src/query/expression/src/property.rs b/src/query/expression/src/property.rs index 52220607b5d19..de7c2ee38f4f8 100644 --- a/src/query/expression/src/property.rs +++ b/src/query/expression/src/property.rs @@ -16,19 +16,8 @@ use databend_common_column::types::months_days_micros; use databend_common_column::types::timestamp_tz; use enum_as_inner::EnumAsInner; -use crate::types::boolean::BooleanDomain; -use crate::types::decimal::Decimal; -use crate::types::decimal::DecimalDomain; -use crate::types::decimal::DecimalScalar; -use crate::types::i256; -use crate::types::nullable::NullableDomain; -use crate::types::number::NumberDomain; -use crate::types::number::NumberScalar; -use crate::types::number::SimpleDomain; -use crate::types::number::F32; -use crate::types::number::F64; -use crate::types::string::StringDomain; -use crate::types::timestamp_tz::TimestampTzType; +use crate::ColumnBuilder; +use crate::Scalar; use crate::types::AccessType; use crate::types::AnyType; use crate::types::ArgType; @@ -42,11 +31,22 @@ use crate::types::NumberDataType; use crate::types::NumberType; use crate::types::StringType; use crate::types::TimestampType; +use crate::types::boolean::BooleanDomain; +use crate::types::decimal::Decimal; +use crate::types::decimal::DecimalDomain; +use crate::types::decimal::DecimalScalar; +use crate::types::i256; +use crate::types::nullable::NullableDomain; +use crate::types::number::F32; +use crate::types::number::F64; +use crate::types::number::NumberDomain; +use crate::types::number::NumberScalar; +use crate::types::number::SimpleDomain; +use crate::types::string::StringDomain; +use crate::types::timestamp_tz::TimestampTzType; use crate::with_decimal_mapped_type; use crate::with_decimal_type; use crate::with_number_type; -use crate::ColumnBuilder; -use crate::Scalar; #[derive(Debug, Clone)] pub struct FunctionProperty { diff --git a/src/query/expression/src/register.rs b/src/query/expression/src/register.rs index 02ba968c98f04..2c102518bf39b 100755 --- a/src/query/expression/src/register.rs +++ b/src/query/expression/src/register.rs @@ -17,11 +17,6 @@ #![allow(unused_parens)] #![allow(unused_variables)] #![allow(clippy::redundant_closure)] -use crate::property::Domain; -use crate::register_vectorize::*; -use crate::types::nullable::NullableDomain; -use crate::types::*; -use crate::values::Value; use crate::EvalContext; use crate::Function; use crate::FunctionContext; @@ -29,6 +24,11 @@ use crate::FunctionDomain; use crate::FunctionEval; use crate::FunctionRegistry; use crate::FunctionSignature; +use crate::property::Domain; +use crate::register_vectorize::*; +use crate::types::nullable::NullableDomain; +use crate::types::*; +use crate::values::Value; impl FunctionRegistry { pub fn register_1_arg( diff --git a/src/query/expression/src/register_comparison.rs b/src/query/expression/src/register_comparison.rs index b452aeca53262..c051e0abb613d 100755 --- a/src/query/expression/src/register_comparison.rs +++ b/src/query/expression/src/register_comparison.rs @@ -15,12 +15,12 @@ #![allow(unused_parens)] #![allow(unused_variables)] #![allow(clippy::redundant_closure)] -use crate::types::*; -use crate::values::Value; use crate::EvalContext; use crate::FunctionContext; use crate::FunctionDomain; use crate::FunctionRegistry; +use crate::types::*; +use crate::values::Value; impl FunctionRegistry { pub fn register_comparison_2_arg( diff --git a/src/query/expression/src/register_vectorize.rs b/src/query/expression/src/register_vectorize.rs index 9583dbbd2d595..b7503bc6248a4 100755 --- a/src/query/expression/src/register_vectorize.rs +++ b/src/query/expression/src/register_vectorize.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::EvalContext; use crate::types::nullable::NullableColumn; use crate::types::*; use crate::values::Value; -use crate::EvalContext; pub fn vectorize_1_arg( func: impl Fn(I1::ScalarRef<'_>, &mut EvalContext) -> O::Scalar + Copy + Send + Sync, @@ -36,9 +36,9 @@ pub fn vectorize_1_arg( pub fn vectorize_2_arg( func: impl Fn(I1::ScalarRef<'_>, I2::ScalarRef<'_>, &mut EvalContext) -> O::Scalar - + Copy - + Send - + Sync, + + Copy + + Send + + Sync, ) -> impl Fn(Value, Value, &mut EvalContext) -> Value + Copy + Send + Sync { move |arg1, arg2, ctx| match (arg1, arg2) { (Value::Scalar(arg1), Value::Scalar(arg2)) => { @@ -75,10 +75,15 @@ pub fn vectorize_2_arg( } pub fn vectorize_3_arg( - func: impl Fn(I1::ScalarRef<'_>, I2::ScalarRef<'_>, I3::ScalarRef<'_>, &mut EvalContext) -> O::Scalar - + Copy - + Send - + Sync, + func: impl Fn( + I1::ScalarRef<'_>, + I2::ScalarRef<'_>, + I3::ScalarRef<'_>, + &mut EvalContext, + ) -> O::Scalar + + Copy + + Send + + Sync, ) -> impl Fn(Value, Value, Value, &mut EvalContext) -> Value + Copy + Send + Sync { move |arg1, arg2, arg3, ctx| { let generics = ctx.generics.to_vec(); @@ -110,15 +115,15 @@ pub fn vectorize_4_arg< O: ReturnType, >( func: impl Fn( - I1::ScalarRef<'_>, - I2::ScalarRef<'_>, - I3::ScalarRef<'_>, - I4::ScalarRef<'_>, - &mut EvalContext, - ) -> O::Scalar - + Copy - + Send - + Sync, + I1::ScalarRef<'_>, + I2::ScalarRef<'_>, + I3::ScalarRef<'_>, + I4::ScalarRef<'_>, + &mut EvalContext, + ) -> O::Scalar + + Copy + + Send + + Sync, ) -> impl Fn(Value, Value, Value, Value, &mut EvalContext) -> Value + Copy + Send + Sync { move |arg1, arg2, arg3, arg4, ctx| { @@ -170,9 +175,9 @@ pub fn vectorize_with_builder_1_arg( pub fn vectorize_with_builder_2_arg( func: impl Fn(I1::ScalarRef<'_>, I2::ScalarRef<'_>, &mut O::ColumnBuilder, &mut EvalContext) - + Copy - + Send - + Sync, + + Copy + + Send + + Sync, ) -> impl Fn(Value, Value, &mut EvalContext) -> Value + Copy + Send + Sync { move |arg1, arg2, ctx| { let generics = ctx.generics.to_vec(); @@ -201,14 +206,14 @@ pub fn vectorize_with_builder_3_arg< O: ReturnType, >( func: impl Fn( - I1::ScalarRef<'_>, - I2::ScalarRef<'_>, - I3::ScalarRef<'_>, - &mut O::ColumnBuilder, - &mut EvalContext, - ) + Copy - + Send - + Sync, + I1::ScalarRef<'_>, + I2::ScalarRef<'_>, + I3::ScalarRef<'_>, + &mut O::ColumnBuilder, + &mut EvalContext, + ) + Copy + + Send + + Sync, ) -> impl Fn(Value, Value, Value, &mut EvalContext) -> Value + Copy + Send + Sync { move |arg1, arg2, arg3, ctx| { let generics = ctx.generics.to_vec(); @@ -240,15 +245,15 @@ pub fn vectorize_with_builder_4_arg< O: ReturnType, >( func: impl Fn( - I1::ScalarRef<'_>, - I2::ScalarRef<'_>, - I3::ScalarRef<'_>, - I4::ScalarRef<'_>, - &mut O::ColumnBuilder, - &mut EvalContext, - ) + Copy - + Send - + Sync, + I1::ScalarRef<'_>, + I2::ScalarRef<'_>, + I3::ScalarRef<'_>, + I4::ScalarRef<'_>, + &mut O::ColumnBuilder, + &mut EvalContext, + ) + Copy + + Send + + Sync, ) -> impl Fn(Value, Value, Value, Value, &mut EvalContext) -> Value + Copy + Send + Sync { move |arg1, arg2, arg3, arg4, ctx| { @@ -279,9 +284,9 @@ pub fn vectorize_with_builder_4_arg< pub fn passthrough_nullable_1_arg( func: impl for<'a> Fn(Value, &mut EvalContext) -> Value + Copy + Send + Sync, ) -> impl for<'a> Fn(Value>, &mut EvalContext) -> Value> - + Copy - + Send - + Sync { ++ Copy ++ Send ++ Sync { move |arg1, ctx| { let mut args_validity = arg1.validity(ctx.num_rows); if let Some(validity) = ctx.validity.as_ref() { @@ -311,9 +316,9 @@ pub fn passthrough_nullable_2_arg Value>, &mut EvalContext, ) -> Value> - + Copy - + Send - + Sync { ++ Copy ++ Send ++ Sync { move |arg1, arg2, ctx| { let mut args_validity = arg1.validity(ctx.num_rows); args_validity = &args_validity & &arg2.validity(ctx.num_rows); @@ -340,18 +345,18 @@ pub fn passthrough_nullable_2_arg pub fn passthrough_nullable_3_arg( func: impl for<'a> Fn(Value, Value, Value, &mut EvalContext) -> Value - + Copy - + Send - + Sync, + + Copy + + Send + + Sync, ) -> impl for<'a> Fn( Value>, Value>, Value>, &mut EvalContext, ) -> Value> - + Copy - + Send - + Sync { ++ Copy ++ Send ++ Sync { move |arg1, arg2, arg3, ctx| { let mut args_validity = arg1.validity(ctx.num_rows); args_validity = &args_validity & &arg2.validity(ctx.num_rows); @@ -384,9 +389,9 @@ pub fn passthrough_nullable_4_arg< O: ReturnType, >( func: impl for<'a> Fn(Value, Value, Value, Value, &mut EvalContext) -> Value - + Copy - + Send - + Sync, + + Copy + + Send + + Sync, ) -> impl for<'a> Fn( Value>, Value>, @@ -394,9 +399,9 @@ pub fn passthrough_nullable_4_arg< Value>, &mut EvalContext, ) -> Value> - + Copy - + Send - + Sync { ++ Copy ++ Send ++ Sync { move |arg1, arg2, arg3, arg4, ctx| { let mut args_validity = arg1.validity(ctx.num_rows); args_validity = &args_validity & &arg2.validity(ctx.num_rows); @@ -425,9 +430,9 @@ pub fn passthrough_nullable_4_arg< pub fn combine_nullable_1_arg( func: impl for<'a> Fn(Value, &mut EvalContext) -> Value> + Copy + Send + Sync, ) -> impl for<'a> Fn(Value>, &mut EvalContext) -> Value> - + Copy - + Send - + Sync { ++ Copy ++ Send ++ Sync { move |arg1, ctx| { let mut args_validity = arg1.validity(ctx.num_rows); if let Some(validity) = ctx.validity.as_ref() { @@ -453,17 +458,17 @@ pub fn combine_nullable_1_arg( pub fn combine_nullable_2_arg( func: impl for<'a> Fn(Value, Value, &mut EvalContext) -> Value> - + Copy - + Send - + Sync, + + Copy + + Send + + Sync, ) -> impl for<'a> Fn( Value>, Value>, &mut EvalContext, ) -> Value> - + Copy - + Send - + Sync { ++ Copy ++ Send ++ Sync { move |arg1, arg2, ctx| { let mut args_validity = arg1.validity(ctx.num_rows); args_validity = &args_validity & &arg2.validity(ctx.num_rows); @@ -490,18 +495,18 @@ pub fn combine_nullable_2_arg( pub fn combine_nullable_3_arg( func: impl for<'a> Fn(Value, Value, Value, &mut EvalContext) -> Value> - + Copy - + Send - + Sync, + + Copy + + Send + + Sync, ) -> impl for<'a> Fn( Value>, Value>, Value>, &mut EvalContext, ) -> Value> - + Copy - + Send - + Sync { ++ Copy ++ Send ++ Sync { move |arg1, arg2, arg3, ctx| { let mut args_validity = arg1.validity(ctx.num_rows); args_validity = &args_validity & &arg2.validity(ctx.num_rows); @@ -535,15 +540,15 @@ pub fn combine_nullable_4_arg< O: ReturnType, >( func: impl for<'a> Fn( - Value, - Value, - Value, - Value, - &mut EvalContext, - ) -> Value> - + Copy - + Send - + Sync, + Value, + Value, + Value, + Value, + &mut EvalContext, + ) -> Value> + + Copy + + Send + + Sync, ) -> impl for<'a> Fn( Value>, Value>, @@ -551,9 +556,9 @@ pub fn combine_nullable_4_arg< Value>, &mut EvalContext, ) -> Value> - + Copy - + Send - + Sync { ++ Copy ++ Send ++ Sync { move |arg1, arg2, arg3, arg4, ctx| { let mut args_validity = arg1.validity(ctx.num_rows); args_validity = &args_validity & &arg2.validity(ctx.num_rows); diff --git a/src/query/expression/src/row_encoding.rs b/src/query/expression/src/row_encoding.rs index 71e80470802dc..fc4c27c251dca 100644 --- a/src/query/expression/src/row_encoding.rs +++ b/src/query/expression/src/row_encoding.rs @@ -15,9 +15,9 @@ use databend_common_column::types::months_days_micros; use databend_common_column::types::timestamp_tz; -use crate::types::i256; use crate::types::F32; use crate::types::F64; +use crate::types::i256; pub trait FixedLengthEncoding: Copy { // 1 for null byte diff --git a/src/query/expression/src/sampler/fixed_rate_sampler.rs b/src/query/expression/src/sampler/fixed_rate_sampler.rs index 44f4e159ff35a..57a9dcd77eba4 100644 --- a/src/query/expression/src/sampler/fixed_rate_sampler.rs +++ b/src/query/expression/src/sampler/fixed_rate_sampler.rs @@ -179,8 +179,8 @@ mod rate_sampling { #[cfg(test)] mod tests { - use rand::rngs::StdRng; use rand::SeedableRng; + use rand::rngs::StdRng; use super::*; use crate::types::Int32Type; diff --git a/src/query/expression/src/sampler/fixed_size_sampler.rs b/src/query/expression/src/sampler/fixed_size_sampler.rs index dd7500d40759b..649944d7626e6 100644 --- a/src/query/expression/src/sampler/fixed_size_sampler.rs +++ b/src/query/expression/src/sampler/fixed_size_sampler.rs @@ -210,8 +210,8 @@ mod reservoir_sampling { #[cfg(test)] mod tests { - use rand::rngs::StdRng; use rand::SeedableRng; + use rand::rngs::StdRng; use super::*; @@ -244,8 +244,8 @@ mod reservoir_sampling { #[cfg(test)] mod tests { - use rand::rngs::StdRng; use rand::SeedableRng; + use rand::rngs::StdRng; use super::*; diff --git a/src/query/expression/src/schema.rs b/src/query/expression/src/schema.rs index 11bc9b66f9f78..5abf8978f13ef 100644 --- a/src/query/expression/src/schema.rs +++ b/src/query/expression/src/schema.rs @@ -21,21 +21,21 @@ use std::sync::LazyLock; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_frozen_api::frozen_api; use databend_common_frozen_api::FrozenAPI; +use databend_common_frozen_api::frozen_api; use itertools::Itertools; use serde::Deserialize; use serde::Serialize; +use crate::BlockMetaInfo; +use crate::BlockMetaInfoDowncast; +use crate::Scalar; use crate::display::display_tuple_field_name; -use crate::types::decimal::DecimalDataType; use crate::types::DataType; use crate::types::DecimalDataKind; use crate::types::NumberDataType; use crate::types::VectorDataType; -use crate::BlockMetaInfo; -use crate::BlockMetaInfoDowncast; -use crate::Scalar; +use crate::types::decimal::DecimalDataType; // Column id of TableField pub type ColumnId = u32; diff --git a/src/query/expression/src/type_check.rs b/src/query/expression/src/type_check.rs index 2164c977276d6..a0f12f91fa952 100755 --- a/src/query/expression/src/type_check.rs +++ b/src/query/expression/src/type_check.rs @@ -21,25 +21,25 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use itertools::Itertools; +use crate::AutoCastRules; +use crate::ColumnIndex; +use crate::ConstantFolder; +use crate::DynamicCastRules; +use crate::ExprVisitor; +use crate::FunctionContext; +use crate::Scalar; use crate::cast_scalar; use crate::expr::*; use crate::expression::Expr; use crate::expression::RawExpr; use crate::function::FunctionRegistry; use crate::function::FunctionSignature; -use crate::types::decimal::DecimalSize; -use crate::types::i256; use crate::types::DataType; use crate::types::Decimal; use crate::types::Number; +use crate::types::decimal::DecimalSize; +use crate::types::i256; use crate::visit_expr; -use crate::AutoCastRules; -use crate::ColumnIndex; -use crate::ConstantFolder; -use crate::DynamicCastRules; -use crate::ExprVisitor; -use crate::FunctionContext; -use crate::Scalar; #[recursive::recursive] pub fn check( @@ -97,16 +97,22 @@ pub fn check( // This may hurt the bloom filter, we should try cast to literal as the datatype of column if name == "eq" && args_expr.len() == 2 { match args_expr.as_mut_slice() { - [e, Expr::Constant(Constant { - span, - scalar, - data_type, - })] - | [Expr::Constant(Constant { - span, - scalar, - data_type, - }), e] => { + [ + e, + Expr::Constant(Constant { + span, + scalar, + data_type, + }), + ] + | [ + Expr::Constant(Constant { + span, + scalar, + data_type, + }), + e, + ] => { let src_ty = data_type.remove_nullable(); let dest_ty = e.data_type().remove_nullable(); @@ -358,11 +364,7 @@ pub fn check_function( .zip(args_not_const.iter().copied()) .map(|(expr, not_const)| { // smaller score win - if not_const && expr.is_cast() { - 1 - } else { - 0 - } + if not_const && expr.is_cast() { 1 } else { 0 } }) .sum::() } else { diff --git a/src/query/expression/src/types.rs b/src/query/expression/src/types.rs index b7f4606364105..d2ed974f2e2a6 100755 --- a/src/query/expression/src/types.rs +++ b/src/query/expression/src/types.rs @@ -57,6 +57,8 @@ use enum_as_inner::EnumAsInner; use serde::Deserialize; use serde::Serialize; +pub use self::OpaqueColumn; +pub use self::OpaqueColumnBuilder; pub use self::any::AnyType; pub use self::array::ArrayColumn; pub use self::array::ArrayType; @@ -96,8 +98,8 @@ pub use self::vector::VectorScalar; pub use self::vector::VectorScalarRef; pub use self::vector::VectorType; use self::zero_size_type::*; -pub use self::OpaqueColumn; -pub use self::OpaqueColumnBuilder; +use crate::ColumnBuilder; +use crate::ScalarRef; use crate::property::Domain; use crate::types::date::DATE_MAX; use crate::types::date::DATE_MIN; @@ -105,8 +107,6 @@ use crate::types::timestamp::TIMESTAMP_MAX; use crate::types::timestamp::TIMESTAMP_MIN; use crate::values::Column; use crate::values::Scalar; -use crate::ColumnBuilder; -use crate::ScalarRef; pub type GenericMap = [DataType]; @@ -597,7 +597,7 @@ pub trait AccessType: Debug + Clone + PartialEq + Sized + 'static { /// /// Calling this method with an out-of-bounds index is *[undefined behavior]* unsafe fn index_column_unchecked_scalar(col: &Self::Column, index: usize) -> Self::Scalar { - Self::to_owned_scalar(Self::index_column_unchecked(col, index)) + unsafe { Self::to_owned_scalar(Self::index_column_unchecked(col, index)) } } fn slice_column(col: &Self::Column, range: Range) -> Self::Column; diff --git a/src/query/expression/src/types/any.rs b/src/query/expression/src/types/any.rs index 85cef0a5c6700..d42139718c4fd 100755 --- a/src/query/expression/src/types/any.rs +++ b/src/query/expression/src/types/any.rs @@ -21,12 +21,12 @@ use super::AccessType; use super::BuilderMut; use super::DataType; use super::ValueType; -use crate::property::Domain; -use crate::values::Column; -use crate::values::Scalar; use crate::ColumnBuilder; use crate::ColumnIterator; use crate::ScalarRef; +use crate::property::Domain; +use crate::values::Column; +use crate::values::Scalar; #[derive(Debug, Clone, PartialEq, Eq)] pub struct AnyType; diff --git a/src/query/expression/src/types/array.rs b/src/query/expression/src/types/array.rs index dabfce64bebad..ff98730d6d639 100755 --- a/src/query/expression/src/types/array.rs +++ b/src/query/expression/src/types/array.rs @@ -13,8 +13,8 @@ // limitations under the License. use std::cmp::Ordering; -use std::iter::once; use std::iter::TrustedLen; +use std::iter::once; use std::marker::PhantomData; use std::ops::Range; @@ -22,9 +22,6 @@ use databend_common_column::buffer::Buffer; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use super::column_type_error; -use super::domain_type_error; -use super::scalar_type_error; use super::AccessType; use super::AnyType; use super::ArgType; @@ -36,8 +33,11 @@ use super::ReturnType; use super::Scalar; use super::ScalarRef; use super::ValueType; -use crate::property::Domain; +use super::column_type_error; +use super::domain_type_error; +use super::scalar_type_error; use crate::ColumnBuilder; +use crate::property::Domain; #[derive(Debug, Clone, PartialEq, Eq)] pub struct ArrayType(PhantomData); @@ -89,7 +89,7 @@ impl AccessType for ArrayType { #[inline(always)] unsafe fn index_column_unchecked(col: &Self::Column, index: usize) -> Self::ScalarRef<'_> { - col.index_unchecked(index) + unsafe { col.index_unchecked(index) } } fn slice_column(col: &Self::Column, range: Range) -> Self::Column { @@ -262,7 +262,7 @@ impl ArrayColumn { } } - pub fn iter(&self) -> ArrayIterator { + pub fn iter(&self) -> ArrayIterator<'_, T> { ArrayIterator { values: &self.values, offsets: self.offsets.windows(2), diff --git a/src/query/expression/src/types/binary.rs b/src/query/expression/src/types/binary.rs index 0be4eb05efcc3..79cf3652ba20d 100644 --- a/src/query/expression/src/types/binary.rs +++ b/src/query/expression/src/types/binary.rs @@ -17,9 +17,6 @@ use std::ops::Range; use databend_common_exception::Result; -use super::column_type_error; -use super::domain_type_error; -use super::scalar_type_error; use super::AccessType; use super::ArgType; use super::BuilderMut; @@ -28,10 +25,13 @@ use super::GenericMap; use super::ReturnType; use super::ScalarRef; use super::ValueType; +use super::column_type_error; +use super::domain_type_error; +use super::scalar_type_error; +use crate::ColumnBuilder; use crate::property::Domain; use crate::values::Column; use crate::values::Scalar; -use crate::ColumnBuilder; pub type BinaryColumn = databend_common_column::binary::BinaryColumn; pub type BinaryColumnBuilder = databend_common_column::binary::BinaryColumnBuilder; @@ -85,7 +85,7 @@ impl AccessType for BinaryType { } unsafe fn index_column_unchecked(col: &Self::Column, index: usize) -> Self::ScalarRef<'_> { - col.index_unchecked(index) + unsafe { col.index_unchecked(index) } } fn slice_column(col: &Self::Column, range: Range) -> Self::Column { diff --git a/src/query/expression/src/types/bitmap.rs b/src/query/expression/src/types/bitmap.rs index c86020c508031..900673b6fd40d 100644 --- a/src/query/expression/src/types/bitmap.rs +++ b/src/query/expression/src/types/bitmap.rs @@ -17,12 +17,6 @@ use std::ops::Range; use databend_common_exception::Result; -use super::binary::BinaryColumn; -use super::binary::BinaryColumnBuilder; -use super::binary::BinaryColumnIter; -use super::column_type_error; -use super::domain_type_error; -use super::scalar_type_error; use super::AccessType; use super::ArgType; use super::BuilderMut; @@ -30,11 +24,17 @@ use super::DataType; use super::GenericMap; use super::ReturnType; use super::ValueType; +use super::binary::BinaryColumn; +use super::binary::BinaryColumnBuilder; +use super::binary::BinaryColumnIter; +use super::column_type_error; +use super::domain_type_error; +use super::scalar_type_error; +use crate::ColumnBuilder; +use crate::ScalarRef; use crate::property::Domain; use crate::values::Column; use crate::values::Scalar; -use crate::ColumnBuilder; -use crate::ScalarRef; #[derive(Debug, Clone, PartialEq, Eq)] pub struct BitmapType; @@ -85,7 +85,7 @@ impl AccessType for BitmapType { #[inline(always)] unsafe fn index_column_unchecked(col: &Self::Column, index: usize) -> Self::ScalarRef<'_> { - col.index_unchecked(index) + unsafe { col.index_unchecked(index) } } fn slice_column(col: &Self::Column, range: Range) -> Self::Column { diff --git a/src/query/expression/src/types/boolean.rs b/src/query/expression/src/types/boolean.rs index 96c21e9584a5b..72fb603f74562 100644 --- a/src/query/expression/src/types/boolean.rs +++ b/src/query/expression/src/types/boolean.rs @@ -19,9 +19,6 @@ use std::ops::Range; pub use databend_common_column::bitmap::*; use databend_common_exception::Result; -use super::column_type_error; -use super::domain_type_error; -use super::scalar_type_error; use super::AccessType; use super::ArgType; use super::BuilderMut; @@ -29,13 +26,16 @@ use super::DataType; use super::GenericMap; use super::ReturnType; use super::ValueType; +use super::column_type_error; +use super::domain_type_error; +use super::scalar_type_error; +use crate::ColumnBuilder; +use crate::ColumnView; +use crate::ScalarRef; use crate::property::Domain; use crate::utils::arrow::bitmap_into_mut; use crate::values::Column; use crate::values::Scalar; -use crate::ColumnBuilder; -use crate::ColumnView; -use crate::ScalarRef; #[derive(Debug, Clone, PartialEq, Eq)] pub struct BooleanType; @@ -86,9 +86,11 @@ impl AccessType for BooleanType { #[inline(always)] unsafe fn index_column_unchecked(col: &Self::Column, index: usize) -> Self::ScalarRef<'_> { - debug_assert!(index < col.len()); + unsafe { + debug_assert!(index < col.len()); - col.get_bit_unchecked(index) + col.get_bit_unchecked(index) + } } fn slice_column(col: &Self::Column, range: Range) -> Self::Column { diff --git a/src/query/expression/src/types/compute_view.rs b/src/query/expression/src/types/compute_view.rs index 95686ca883a00..18b977f22fca8 100644 --- a/src/query/expression/src/types/compute_view.rs +++ b/src/query/expression/src/types/compute_view.rs @@ -95,8 +95,10 @@ where } unsafe fn index_column_unchecked(col: &Self::Column, index: usize) -> Self::ScalarRef<'_> { - let scalar = F::index_column_unchecked(col, index); - C::compute(scalar) + unsafe { + let scalar = F::index_column_unchecked(col, index); + C::compute(scalar) + } } fn slice_column(col: &Self::Column, range: Range) -> Self::Column { diff --git a/src/query/expression/src/types/date.rs b/src/query/expression/src/types/date.rs index caf7d6213ba33..560f671563c68 100644 --- a/src/query/expression/src/types/date.rs +++ b/src/query/expression/src/types/date.rs @@ -25,17 +25,17 @@ use jiff::fmt::strtime; use jiff::tz::TimeZone; use num_traits::AsPrimitive; -use super::number::SimpleDomain; use super::ArgType; use super::DataType; use super::SimpleType; use super::SimpleValueType; +use super::number::SimpleDomain; +use crate::ColumnBuilder; +use crate::ScalarRef; use crate::date_helper::DateConverter; use crate::property::Domain; use crate::values::Column; use crate::values::Scalar; -use crate::ColumnBuilder; -use crate::ScalarRef; pub const DATE_FORMAT: &str = "%Y-%m-%d"; /// Minimum valid date, represented by the day offset from 1970-01-01. diff --git a/src/query/expression/src/types/decimal.rs b/src/query/expression/src/types/decimal.rs index 22b7d2fddddc6..50a2b5de7ed8d 100644 --- a/src/query/expression/src/types/decimal.rs +++ b/src/query/expression/src/types/decimal.rs @@ -38,8 +38,8 @@ use databend_common_exception::Result; use databend_common_io::display_decimal_128; use databend_common_io::display_decimal_256; use enum_as_inner::EnumAsInner; -use ethnum::u256; use ethnum::AsI256; +use ethnum::u256; use itertools::Itertools; use micromarshal::Marshal; use num_bigint::BigInt; @@ -49,29 +49,29 @@ use num_traits::ToPrimitive; use serde::Deserialize; use serde::Serialize; -use super::column_type_error; -use super::compute_view::Compute; -use super::compute_view::ComputeView; -use super::domain_type_error; -use super::scalar_type_error; use super::AccessType; use super::AnyType; use super::DataType; +use super::F64; use super::NumberType; use super::SimpleDomain; use super::SimpleType; use super::SimpleValueType; use super::ValueType; -use super::F64; -use crate::utils::arrow::buffer_into_mut; -use crate::with_decimal_mapped_type; -use crate::with_decimal_type; +use super::column_type_error; +use super::compute_view::Compute; +use super::compute_view::ComputeView; +use super::domain_type_error; +use super::scalar_type_error; use crate::Column; use crate::ColumnBuilder; use crate::Domain; use crate::Scalar; use crate::ScalarRef; use crate::Value; +use crate::utils::arrow::buffer_into_mut; +use crate::with_decimal_mapped_type; +use crate::with_decimal_type; #[derive(Debug, Clone, PartialEq, Eq)] pub struct CoreDecimal(PhantomData); @@ -130,7 +130,7 @@ impl AccessType for CoreDecimal { } unsafe fn index_column_unchecked(col: &Self::Column, index: usize) -> Self::ScalarRef<'_> { - *col.get_unchecked(index) + unsafe { *col.get_unchecked(index) } } fn slice_column(col: &Self::Column, range: Range) -> Self::Column { @@ -270,7 +270,7 @@ impl AccessType for CoreScalarDecimal { } unsafe fn index_column_unchecked(col: &Self::Column, index: usize) -> Self::ScalarRef<'_> { - *col.get_unchecked(index) + unsafe { *col.get_unchecked(index) } } fn slice_column(col: &Self::Column, range: Range) -> Self::Column { @@ -709,7 +709,7 @@ impl Decimal for i64 { fn e(n: u8) -> Self { const L: usize = i64::MAX_PRECISION as usize + 1; const TAB: [i64; L] = { - const fn gen() -> [i64; L] { + const fn generate_array() -> [i64; L] { let mut arr = [0; L]; let mut i = 0; loop { @@ -721,7 +721,7 @@ impl Decimal for i64 { } arr } - gen() + generate_array() }; TAB.get(n as usize) .copied() @@ -974,7 +974,7 @@ impl Decimal for i128 { fn e(n: u8) -> Self { const L: usize = i128::MAX_PRECISION as usize + 1; const TAB: [i128; L] = { - const fn gen() -> [i128; L] { + const fn generate_array() -> [i128; L] { let mut arr = [0; L]; let mut i = 0; loop { @@ -986,7 +986,7 @@ impl Decimal for i128 { } arr } - gen() + generate_array() }; TAB.get(n as usize) .copied() @@ -1066,7 +1066,7 @@ impl Decimal for i128 { /// `MIN_DECIMAL_FOR_EACH_PRECISION[p]` holds the minimum `i128` value that can /// be stored in a [arrow_schema::DataType::Decimal128] value of precision `p` const MIN_DECIMAL_FOR_EACH_PRECISION: [i128; 38] = { - const fn gen() -> [i128; 38] { + const fn generate_array() -> [i128; 38] { let mut arr = [0; 38]; let mut i = 0; loop { @@ -1078,7 +1078,7 @@ impl Decimal for i128 { } arr } - gen() + generate_array() }; MIN_DECIMAL_FOR_EACH_PRECISION[to_precision as usize - 1] @@ -1088,7 +1088,7 @@ impl Decimal for i128 { /// `MAX_DECIMAL_FOR_EACH_PRECISION[p]` holds the maximum `i128` value that can /// be stored in [arrow_schema::DataType::Decimal128] value of precision `p` const MAX_DECIMAL_FOR_EACH_PRECISION: [i128; 38] = { - const fn gen() -> [i128; 38] { + const fn generate_array() -> [i128; 38] { let mut arr = [0; 38]; let mut i = 0; loop { @@ -1100,7 +1100,7 @@ impl Decimal for i128 { } arr } - gen() + generate_array() }; MAX_DECIMAL_FOR_EACH_PRECISION[to_precision as usize - 1] } @@ -3008,7 +3008,7 @@ impl TryFrom for i256 { impl BorshSerialize for i256 { fn serialize(&self, writer: &mut W) -> borsh::io::Result<()> { - BorshSerialize::serialize(&self.0 .0, writer) + BorshSerialize::serialize(&self.0.0, writer) } } diff --git a/src/query/expression/src/types/empty_array.rs b/src/query/expression/src/types/empty_array.rs index d865c24976dc9..5ac7d0eea6c0e 100644 --- a/src/query/expression/src/types/empty_array.rs +++ b/src/query/expression/src/types/empty_array.rs @@ -16,11 +16,11 @@ use super::ArgType; use super::DataType; use super::ZeroSizeType; use super::ZeroSizeValueType; +use crate::ColumnBuilder; +use crate::ScalarRef; use crate::property::Domain; use crate::values::Column; use crate::values::Scalar; -use crate::ColumnBuilder; -use crate::ScalarRef; #[derive(Debug, Clone, PartialEq, Eq)] pub struct CoreEmptyArray; diff --git a/src/query/expression/src/types/empty_map.rs b/src/query/expression/src/types/empty_map.rs index 88a965a4492c2..e968932cb5fb0 100644 --- a/src/query/expression/src/types/empty_map.rs +++ b/src/query/expression/src/types/empty_map.rs @@ -16,11 +16,11 @@ use super::ArgType; use super::DataType; use super::ZeroSizeType; use super::ZeroSizeValueType; +use crate::ColumnBuilder; +use crate::ScalarRef; use crate::property::Domain; use crate::values::Column; use crate::values::Scalar; -use crate::ColumnBuilder; -use crate::ScalarRef; #[derive(Debug, Clone, PartialEq, Eq)] pub struct CoreEmptyMap; diff --git a/src/query/expression/src/types/geography.rs b/src/query/expression/src/types/geography.rs index f8d6b5b350b8d..a4bf30f5db11f 100644 --- a/src/query/expression/src/types/geography.rs +++ b/src/query/expression/src/types/geography.rs @@ -21,20 +21,14 @@ use borsh::BorshDeserialize; use borsh::BorshSerialize; use databend_common_exception::Result; use databend_common_io::geography::*; +pub use databend_common_io::wkb::WkbInfo; use databend_common_io::wkb::make_point; use databend_common_io::wkb::read_wkb_header; -pub use databend_common_io::wkb::WkbInfo; -use geozero::wkb::Ewkb; use geozero::ToWkt; +use geozero::wkb::Ewkb; use serde::Deserialize; use serde::Serialize; -use super::binary::BinaryColumn; -use super::binary::BinaryColumnBuilder; -use super::binary::BinaryColumnIter; -use super::column_type_error; -use super::domain_type_error; -use super::scalar_type_error; use super::AccessType; use super::ArgType; use super::BuilderMut; @@ -42,11 +36,17 @@ use super::DataType; use super::GenericMap; use super::ReturnType; use super::ValueType; +use super::binary::BinaryColumn; +use super::binary::BinaryColumnBuilder; +use super::binary::BinaryColumnIter; +use super::column_type_error; +use super::domain_type_error; +use super::scalar_type_error; +use crate::ColumnBuilder; use crate::property::Domain; use crate::values::Column; use crate::values::Scalar; use crate::values::ScalarRef; -use crate::ColumnBuilder; #[derive( Clone, @@ -160,7 +160,7 @@ impl AccessType for GeographyType { #[inline(always)] unsafe fn index_column_unchecked(col: &Self::Column, index: usize) -> Self::ScalarRef<'_> { - col.index_unchecked(index) + unsafe { col.index_unchecked(index) } } fn slice_column(col: &Self::Column, range: Range) -> Self::Column { @@ -301,7 +301,7 @@ impl GeographyColumn { self.0.memory_size() } - pub fn index(&self, index: usize) -> Option { + pub fn index(&self, index: usize) -> Option> { self.0.index(index).map(GeographyRef) } @@ -310,7 +310,7 @@ impl GeographyColumn { /// Calling this method with an out-of-bounds index is *[undefined behavior]* #[inline] pub unsafe fn index_unchecked(&self, index: usize) -> GeographyRef<'_> { - GeographyRef(self.0.index_unchecked(index)) + unsafe { GeographyRef(self.0.index_unchecked(index)) } } pub fn slice(&self, range: Range) -> Self { diff --git a/src/query/expression/src/types/geometry.rs b/src/query/expression/src/types/geometry.rs index 7cfab7024a896..8c6008a28e3fb 100644 --- a/src/query/expression/src/types/geometry.rs +++ b/src/query/expression/src/types/geometry.rs @@ -20,12 +20,6 @@ use geozero::wkb::FromWkb; use geozero::wkb::WkbDialect; use geozero::wkt::Ewkt; -use super::binary::BinaryColumn; -use super::binary::BinaryColumnBuilder; -use super::binary::BinaryColumnIter; -use super::column_type_error; -use super::domain_type_error; -use super::scalar_type_error; use super::AccessType; use super::ArgType; use super::BuilderMut; @@ -35,9 +29,15 @@ use super::ReturnType; use super::Scalar; use super::ScalarRef; use super::ValueType; +use super::binary::BinaryColumn; +use super::binary::BinaryColumnBuilder; +use super::binary::BinaryColumnIter; +use super::column_type_error; +use super::domain_type_error; +use super::scalar_type_error; +use crate::ColumnBuilder; use crate::property::Domain; use crate::values::Column; -use crate::ColumnBuilder; #[derive(Debug, Clone, PartialEq, Eq)] pub struct GeometryType; @@ -88,7 +88,7 @@ impl AccessType for GeometryType { #[inline(always)] unsafe fn index_column_unchecked(col: &Self::Column, index: usize) -> Self::ScalarRef<'_> { - col.index_unchecked(index) + unsafe { col.index_unchecked(index) } } fn slice_column(col: &Self::Column, range: Range) -> Self::Column { diff --git a/src/query/expression/src/types/interval.rs b/src/query/expression/src/types/interval.rs index d990e45f4a7f6..ed9ace18e9170 100644 --- a/src/query/expression/src/types/interval.rs +++ b/src/query/expression/src/types/interval.rs @@ -19,16 +19,16 @@ use databend_common_column::buffer::Buffer; use databend_common_column::types::months_days_micros; use databend_common_io::Interval; -use super::number::SimpleDomain; use super::ArgType; use super::DataType; use super::SimpleType; use super::SimpleValueType; +use super::number::SimpleDomain; +use crate::ColumnBuilder; +use crate::ScalarRef; use crate::property::Domain; use crate::values::Column; use crate::values::Scalar; -use crate::ColumnBuilder; -use crate::ScalarRef; #[derive(Debug, Clone, PartialEq, Eq)] pub struct CoreInterval; diff --git a/src/query/expression/src/types/map.rs b/src/query/expression/src/types/map.rs index 4fb851f866117..76d28d702176f 100755 --- a/src/query/expression/src/types/map.rs +++ b/src/query/expression/src/types/map.rs @@ -19,11 +19,6 @@ use std::ops::Range; use databend_common_exception::Result; -use super::array::ArrayColumn; -use super::array::ArrayColumnBuilderMut; -use super::column_type_error; -use super::domain_type_error; -use super::scalar_type_error; use super::AccessType; use super::ArgType; use super::ArrayType; @@ -34,9 +29,14 @@ use super::ReturnType; use super::Scalar; use super::ScalarRef; use super::ValueType; +use super::array::ArrayColumn; +use super::array::ArrayColumnBuilderMut; +use super::column_type_error; +use super::domain_type_error; +use super::scalar_type_error; +use crate::ColumnBuilder; use crate::property::Domain; use crate::values::Column; -use crate::ColumnBuilder; // Structurally equals to `Tuple(K, V)` #[derive(Debug, Clone, PartialEq, Eq)] @@ -96,7 +96,7 @@ impl AccessType for KvPair { } unsafe fn index_column_unchecked(col: &Self::Column, index: usize) -> Self::ScalarRef<'_> { - col.index_unchecked(index) + unsafe { col.index_unchecked(index) } } fn slice_column(col: &Self::Column, range: Range) -> Self::Column { @@ -269,10 +269,12 @@ impl KvColumn { /// /// Calling this method with an out-of-bounds index is *[undefined behavior]* pub unsafe fn index_unchecked(&self, index: usize) -> (K::ScalarRef<'_>, V::ScalarRef<'_>) { - ( - K::index_column_unchecked(&self.keys, index), - V::index_column_unchecked(&self.values, index), - ) + unsafe { + ( + K::index_column_unchecked(&self.keys, index), + V::index_column_unchecked(&self.values, index), + ) + } } fn slice(&self, range: Range) -> Self { @@ -282,7 +284,7 @@ impl KvColumn { } } - pub fn iter(&self) -> KvIterator { + pub fn iter(&self) -> KvIterator<'_, K, V> { KvIterator { keys: K::iter_column(&self.keys), values: V::iter_column(&self.values), @@ -486,7 +488,7 @@ impl AccessType for MapType { #[inline(always)] unsafe fn index_column_unchecked(col: &Self::Column, index: usize) -> Self::ScalarRef<'_> { - MapInternal::::index_column_unchecked(col, index) + unsafe { MapInternal::::index_column_unchecked(col, index) } } fn slice_column(col: &Self::Column, range: Range) -> Self::Column { diff --git a/src/query/expression/src/types/null.rs b/src/query/expression/src/types/null.rs index 094c114778882..af0b7754b2775 100644 --- a/src/query/expression/src/types/null.rs +++ b/src/query/expression/src/types/null.rs @@ -12,16 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::nullable::NullableDomain; use super::ArgType; use super::DataType; use super::ZeroSizeType; use super::ZeroSizeValueType; +use super::nullable::NullableDomain; +use crate::ColumnBuilder; +use crate::ScalarRef; use crate::property::Domain; use crate::values::Column; use crate::values::Scalar; -use crate::ColumnBuilder; -use crate::ScalarRef; #[derive(Debug, PartialEq, Clone)] pub struct CoreNull; diff --git a/src/query/expression/src/types/nullable.rs b/src/query/expression/src/types/nullable.rs index 5861f81d25769..01850ea4cca64 100755 --- a/src/query/expression/src/types/nullable.rs +++ b/src/query/expression/src/types/nullable.rs @@ -21,8 +21,6 @@ use databend_common_column::bitmap::Bitmap; use databend_common_column::bitmap::MutableBitmap; use databend_common_exception::Result; -use super::column_type_error; -use super::domain_type_error; use super::AccessType; use super::AnyType; use super::ArgType; @@ -31,13 +29,15 @@ use super::DataType; use super::GenericMap; use super::ReturnType; use super::ValueType; +use super::column_type_error; +use super::domain_type_error; +use crate::ColumnBuilder; +use crate::ColumnVec; +use crate::ScalarRef; use crate::property::Domain; use crate::utils::arrow::bitmap_into_mut; use crate::values::Column; use crate::values::Scalar; -use crate::ColumnBuilder; -use crate::ColumnVec; -use crate::ScalarRef; #[derive(Debug, Clone, PartialEq, Eq)] pub struct NullableType(PhantomData); @@ -101,7 +101,7 @@ impl AccessType for NullableType { #[inline(always)] unsafe fn index_column_unchecked(col: &Self::Column, index: usize) -> Self::ScalarRef<'_> { - col.index_unchecked(index) + unsafe { col.index_unchecked(index) } } fn slice_column(col: &Self::Column, range: Range) -> Self::Column { @@ -135,7 +135,7 @@ impl AccessType for NullableType { } unsafe fn index_column_unchecked_scalar(col: &Self::Column, index: usize) -> Self::Scalar { - Self::to_owned_scalar(Self::index_column_unchecked(col, index)) + unsafe { Self::to_owned_scalar(Self::index_column_unchecked(col, index)) } } fn equal(left: Self::ScalarRef<'_>, right: Self::ScalarRef<'_>) -> bool { @@ -321,15 +321,17 @@ impl NullableColumn { /// /// Calling this method with an out-of-bounds index is *[undefined behavior]* pub unsafe fn index_unchecked(&self, index: usize) -> Option> { - // we need to check the validity firstly - // cause `self.validity.get_bit_unchecked` may check the index from buffer address with `true` result - if index < self.validity.len() { - match self.validity.get_bit_unchecked(index) { - true => Some(T::index_column_unchecked(&self.column, index)), - false => None, + unsafe { + // we need to check the validity firstly + // cause `self.validity.get_bit_unchecked` may check the index from buffer address with `true` result + if index < self.validity.len() { + match self.validity.get_bit_unchecked(index) { + true => Some(T::index_column_unchecked(&self.column, index)), + false => None, + } + } else { + None } - } else { - None } } @@ -343,7 +345,7 @@ impl NullableColumn { } } - pub fn iter(&self) -> NullableIterator { + pub fn iter(&self) -> NullableIterator<'_, T> { NullableIterator { iter: T::iter_column(&self.column), validity: self.validity.iter(), @@ -417,11 +419,7 @@ impl<'a, T: AccessType> Iterator for NullableIterator<'a, T> { .zip(self.validity.next()) .map( |(scalar, is_not_null)| { - if is_not_null { - Some(scalar) - } else { - None - } + if is_not_null { Some(scalar) } else { None } }, ) } diff --git a/src/query/expression/src/types/number.rs b/src/query/expression/src/types/number.rs index 5845b63a7d4d5..eaa626a5b4f8b 100644 --- a/src/query/expression/src/types/number.rs +++ b/src/query/expression/src/types/number.rs @@ -27,22 +27,22 @@ use databend_common_exception::Result; use enum_as_inner::EnumAsInner; use itertools::Itertools; use lexical_core::ToLexicalWithOptions; -use num_traits::float::FloatCore; use num_traits::NumCast; +use num_traits::float::FloatCore; use serde::Deserialize; use serde::Serialize; -use super::decimal::DecimalSize; use super::ArgType; use super::DataType; use super::SimpleType; use super::SimpleValueType; +use super::decimal::DecimalSize; +use crate::ColumnBuilder; +use crate::ScalarRef; use crate::property::Domain; use crate::utils::arrow::buffer_into_mut; use crate::values::Column; use crate::values::Scalar; -use crate::ColumnBuilder; -use crate::ScalarRef; pub type F32 = OrderedFloat; pub type F64 = OrderedFloat; diff --git a/src/query/expression/src/types/opaque.rs b/src/query/expression/src/types/opaque.rs index 02d9145564e43..1af689cce93da 100644 --- a/src/query/expression/src/types/opaque.rs +++ b/src/query/expression/src/types/opaque.rs @@ -20,13 +20,13 @@ use databend_common_column::buffer::Buffer; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use super::column_type_error; -use super::domain_type_error; -use super::scalar_type_error; use super::AccessType; use super::BuilderMut; use super::DataType; use super::ValueType; +use super::column_type_error; +use super::domain_type_error; +use super::scalar_type_error; use crate::Column; use crate::ColumnBuilder; use crate::Domain; @@ -159,7 +159,7 @@ impl AccessType for OpaqueType { } unsafe fn index_column_unchecked(col: &Self::Column, index: usize) -> Self::ScalarRef<'_> { - col.get_unchecked(index) + unsafe { col.get_unchecked(index) } } fn slice_column(col: &Self::Column, range: std::ops::Range) -> Self::Column { diff --git a/src/query/expression/src/types/simple_type.rs b/src/query/expression/src/types/simple_type.rs index 4034af4d202cc..d0b35233f18f4 100644 --- a/src/query/expression/src/types/simple_type.rs +++ b/src/query/expression/src/types/simple_type.rs @@ -20,9 +20,6 @@ use std::ops::Range; use databend_common_column::buffer::Buffer; use databend_common_exception::Result; -use super::column_type_error; -use super::domain_type_error; -use super::scalar_type_error; use super::AccessType; use super::BuilderMut; use super::DataType; @@ -30,11 +27,14 @@ use super::GenericMap; use super::ReturnType; use super::Scalar; use super::ValueType; -use crate::arrow::buffer_into_mut; +use super::column_type_error; +use super::domain_type_error; +use super::scalar_type_error; use crate::Column; use crate::ColumnBuilder; use crate::Domain; use crate::ScalarRef; +use crate::arrow::buffer_into_mut; pub trait SimpleType: Debug + Clone + PartialEq + Sized + 'static { type Scalar: Debug + Clone + Copy + PartialEq + Eq + Default + Send + 'static; @@ -124,8 +124,10 @@ impl AccessType for SimpleValueType { buffer: &Buffer, index: usize, ) -> Self::ScalarRef<'_> { - debug_assert!(index < buffer.len()); - *buffer.get_unchecked(index) + unsafe { + debug_assert!(index < buffer.len()); + *buffer.get_unchecked(index) + } } fn slice_column(buffer: &Buffer, range: Range) -> Buffer { @@ -140,7 +142,7 @@ impl AccessType for SimpleValueType { col: &Buffer, index: usize, ) -> Self::Scalar { - Self::to_owned_scalar(Self::index_column_unchecked(col, index)) + unsafe { Self::to_owned_scalar(Self::index_column_unchecked(col, index)) } } fn scalar_memory_size(_: &Self::ScalarRef<'_>) -> usize { diff --git a/src/query/expression/src/types/string.rs b/src/query/expression/src/types/string.rs index 13c3da57204c6..27df842368f7d 100644 --- a/src/query/expression/src/types/string.rs +++ b/src/query/expression/src/types/string.rs @@ -20,10 +20,6 @@ use databend_common_column::binview::BinaryViewColumnIter; use databend_common_column::binview::Utf8ViewColumn; use databend_common_exception::Result; -use super::binary::BinaryColumn; -use super::column_type_error; -use super::domain_type_error; -use super::scalar_type_error; use super::AccessType; use super::ArgType; use super::BuilderMut; @@ -32,10 +28,14 @@ use super::DataType; use super::GenericMap; use super::ReturnType; use super::ValueType; +use super::binary::BinaryColumn; +use super::column_type_error; +use super::domain_type_error; +use super::scalar_type_error; +use crate::ScalarRef; use crate::property::Domain; use crate::values::Column; use crate::values::Scalar; -use crate::ScalarRef; #[derive(Debug, Clone, PartialEq, Eq)] pub struct StringType; @@ -85,7 +85,7 @@ impl AccessType for StringType { #[inline] unsafe fn index_column_unchecked(col: &Self::Column, index: usize) -> Self::ScalarRef<'_> { - col.value_unchecked(index) + unsafe { col.value_unchecked(index) } } fn slice_column(col: &Self::Column, range: Range) -> Self::Column { @@ -349,7 +349,7 @@ impl StringColumnBuilder { /// /// Calling this method with an out-of-bounds index is *[undefined behavior]* pub unsafe fn index_unchecked(&self, row: usize) -> &str { - self.data.value_unchecked(row) + unsafe { self.data.value_unchecked(row) } } pub fn push_repeat(&mut self, item: &str, n: usize) { diff --git a/src/query/expression/src/types/timestamp.rs b/src/query/expression/src/types/timestamp.rs index 082a700916483..bfc2d9cf4c989 100644 --- a/src/query/expression/src/types/timestamp.rs +++ b/src/query/expression/src/types/timestamp.rs @@ -21,21 +21,21 @@ use databend_common_exception::ErrorCode; use databend_common_io::cursor_ext::BufferReadDateTimeExt; use databend_common_io::cursor_ext::DateTimeResType; use databend_common_io::cursor_ext::ReadBytesExt; +use jiff::Zoned; use jiff::fmt::strtime; use jiff::tz::TimeZone; -use jiff::Zoned; -use super::number::SimpleDomain; use super::ArgType; use super::DataType; use super::SimpleType; use super::SimpleValueType; +use super::number::SimpleDomain; +use crate::ColumnBuilder; +use crate::ScalarRef; use crate::property::Domain; use crate::utils::date_helper::DateConverter; use crate::values::Column; use crate::values::Scalar; -use crate::ColumnBuilder; -use crate::ScalarRef; pub const TIMESTAMP_FORMAT: &str = "%Y-%m-%d %H:%M:%S%.6f"; /// Minimum valid timestamp `0001-01-01 00:00:00.000000`, represented by the microsecs offset from 1970-01-01. diff --git a/src/query/expression/src/types/timestamp_tz.rs b/src/query/expression/src/types/timestamp_tz.rs index 2102be500d698..c492513a696ef 100644 --- a/src/query/expression/src/types/timestamp_tz.rs +++ b/src/query/expression/src/types/timestamp_tz.rs @@ -25,13 +25,13 @@ use jiff::fmt; use jiff::tz; use jiff::tz::TimeZone; +use super::ArgType; +use super::DataType; +use super::SimpleDomain; use super::simple_type::SimpleType; use super::simple_type::SimpleValueType; use super::timestamp::TIMESTAMP_MAX; use super::timestamp::TIMESTAMP_MIN; -use super::ArgType; -use super::DataType; -use super::SimpleDomain; use crate::Column; use crate::ColumnBuilder; use crate::Domain; diff --git a/src/query/expression/src/types/tuple.rs b/src/query/expression/src/types/tuple.rs index a3b013e0d8aea..1269d7464ef80 100644 --- a/src/query/expression/src/types/tuple.rs +++ b/src/query/expression/src/types/tuple.rs @@ -19,11 +19,11 @@ use std::marker::PhantomData; use databend_common_exception::Result; +use super::AccessType; +use super::AnyType; use super::column_type_error; use super::domain_type_error; use super::scalar_type_error; -use super::AccessType; -use super::AnyType; use crate::Column; use crate::Domain; use crate::ScalarRef; @@ -96,7 +96,7 @@ where A: AccessType } unsafe fn index_column_unchecked(col: &Self::Column, index: usize) -> Self::ScalarRef<'_> { - A::index_column_unchecked(col, index) + unsafe { A::index_column_unchecked(col, index) } } fn slice_column(col: &Self::Column, range: std::ops::Range) -> Self::Column { @@ -174,10 +174,12 @@ where } unsafe fn index_column_unchecked((a, b): &Self::Column, index: usize) -> Self::ScalarRef<'_> { - ( - A::index_column_unchecked(a, index), - B::index_column_unchecked(b, index), - ) + unsafe { + ( + A::index_column_unchecked(a, index), + B::index_column_unchecked(b, index), + ) + } } fn slice_column((a, b): &Self::Column, range: std::ops::Range) -> Self::Column { @@ -290,11 +292,13 @@ where (a, b, c): &Self::Column, index: usize, ) -> Self::ScalarRef<'_> { - ( - A::index_column_unchecked(a, index), - B::index_column_unchecked(b, index), - C::index_column_unchecked(c, index), - ) + unsafe { + ( + A::index_column_unchecked(a, index), + B::index_column_unchecked(b, index), + C::index_column_unchecked(c, index), + ) + } } fn slice_column((a, b, c): &Self::Column, range: std::ops::Range) -> Self::Column { @@ -424,12 +428,14 @@ where (a, b, c, d): &Self::Column, index: usize, ) -> Self::ScalarRef<'_> { - ( - A::index_column_unchecked(a, index), - B::index_column_unchecked(b, index), - C::index_column_unchecked(c, index), - D::index_column_unchecked(d, index), - ) + unsafe { + ( + A::index_column_unchecked(a, index), + B::index_column_unchecked(b, index), + C::index_column_unchecked(c, index), + D::index_column_unchecked(d, index), + ) + } } fn slice_column((a, b, c, d): &Self::Column, range: std::ops::Range) -> Self::Column { diff --git a/src/query/expression/src/types/variant.rs b/src/query/expression/src/types/variant.rs index 6c630fe1fce4e..183305bcf4d56 100644 --- a/src/query/expression/src/types/variant.rs +++ b/src/query/expression/src/types/variant.rs @@ -18,23 +18,13 @@ use std::ops::Range; use databend_common_exception::Result; use databend_common_io::deserialize_bitmap; -use geozero::wkb::Ewkb; use geozero::ToJson; +use geozero::wkb::Ewkb; use jiff::tz::TimeZone; use jsonb::OwnedJsonb; use jsonb::RawJsonb; use jsonb::Value; -use super::binary::BinaryColumn; -use super::binary::BinaryColumnBuilder; -use super::binary::BinaryColumnIter; -use super::column_type_error; -use super::date::date_to_string; -use super::domain_type_error; -use super::map::KvPair; -use super::number::NumberScalar; -use super::scalar_type_error; -use super::timestamp::timestamp_to_string; use super::AccessType; use super::AnyType; use super::ArgType; @@ -45,13 +35,23 @@ use super::GenericMap; use super::ReturnType; use super::ValueType; use super::VectorScalarRef; +use super::binary::BinaryColumn; +use super::binary::BinaryColumnBuilder; +use super::binary::BinaryColumnIter; +use super::column_type_error; +use super::date::date_to_string; +use super::domain_type_error; +use super::map::KvPair; +use super::number::NumberScalar; +use super::scalar_type_error; +use super::timestamp::timestamp_to_string; +use crate::ColumnBuilder; +use crate::TableDataType; use crate::property::Domain; use crate::values::Column; use crate::values::Scalar; use crate::values::ScalarRef; use crate::with_vector_number_type; -use crate::ColumnBuilder; -use crate::TableDataType; /// JSONB bytes representation of `null`. pub const JSONB_NULL: &[u8] = &[0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]; @@ -105,7 +105,7 @@ impl AccessType for VariantType { #[inline(always)] unsafe fn index_column_unchecked(col: &Self::Column, index: usize) -> Self::ScalarRef<'_> { - col.index_unchecked(index) + unsafe { col.index_unchecked(index) } } fn slice_column(col: &Self::Column, range: Range) -> Self::Column { diff --git a/src/query/expression/src/types/vector.rs b/src/query/expression/src/types/vector.rs index f286681ba888f..0d883c696f9ec 100644 --- a/src/query/expression/src/types/vector.rs +++ b/src/query/expression/src/types/vector.rs @@ -24,22 +24,22 @@ use enum_as_inner::EnumAsInner; use serde::Deserialize; use serde::Serialize; -use super::column_type_error; -use super::domain_type_error; -use super::scalar_type_error; use super::AccessType; use super::BuilderMut; use super::DataType; +use super::F32; use super::NumberColumn; use super::NumberDataType; use super::ValueType; -use super::F32; -use crate::property::Domain; -use crate::values::Column; -use crate::values::Scalar; +use super::column_type_error; +use super::domain_type_error; +use super::scalar_type_error; use crate::ColumnBuilder; use crate::ScalarRef; use crate::TableDataType; +use crate::property::Domain; +use crate::values::Column; +use crate::values::Scalar; #[derive(Debug, Clone, PartialEq, Eq)] pub struct VectorType; @@ -92,7 +92,7 @@ impl AccessType for VectorType { col: &'a Self::Column, index: usize, ) -> Self::ScalarRef<'a> { - col.index_unchecked(index) + unsafe { col.index_unchecked(index) } } fn slice_column(col: &Self::Column, range: Range) -> Self::Column { @@ -445,7 +445,7 @@ impl VectorColumn { }) } - pub fn iter(&self) -> VectorIterator { + pub fn iter(&self) -> VectorIterator<'_> { VectorIterator { column: self, index: 0, diff --git a/src/query/expression/src/types/zero_size_type.rs b/src/query/expression/src/types/zero_size_type.rs index e1c135d566115..ec6171731e1f7 100644 --- a/src/query/expression/src/types/zero_size_type.rs +++ b/src/query/expression/src/types/zero_size_type.rs @@ -19,9 +19,6 @@ use std::ops::Range; use databend_common_exception::Result; -use super::column_type_error; -use super::domain_type_error; -use super::scalar_type_error; use super::AccessType; use super::BuilderMut; use super::DataType; @@ -29,6 +26,9 @@ use super::GenericMap; use super::ReturnType; use super::Scalar; use super::ValueType; +use super::column_type_error; +use super::domain_type_error; +use super::scalar_type_error; use crate::Column; use crate::ColumnBuilder; use crate::Domain; @@ -79,11 +79,7 @@ impl AccessType for ZeroSizeValueType { } fn index_column(len: &usize, index: usize) -> Option<()> { - if index < *len { - Some(()) - } else { - None - } + if index < *len { Some(()) } else { None } } unsafe fn index_column_unchecked(_: &usize, _: usize) {} diff --git a/src/query/expression/src/utils/arithmetics_type.rs b/src/query/expression/src/utils/arithmetics_type.rs index 5b6ee66b477f6..0e6ab1f5a78bb 100644 --- a/src/query/expression/src/utils/arithmetics_type.rs +++ b/src/query/expression/src/utils/arithmetics_type.rs @@ -14,9 +14,9 @@ // This code is generated by src/query/codegen/src/writes/arithmetics_type.rs. DO NOT EDIT. -use crate::types::number::Number; use crate::types::number::F32; use crate::types::number::F64; +use crate::types::number::Number; pub trait ResultTypeOfBinary: Sized { type AddMul: Number; diff --git a/src/query/expression/src/utils/arrow.rs b/src/query/expression/src/utils/arrow.rs index 92c92c2d2fc3d..6d5e3dc660760 100644 --- a/src/query/expression/src/utils/arrow.rs +++ b/src/query/expression/src/utils/arrow.rs @@ -19,10 +19,10 @@ use std::io::Write; use std::sync::Arc; use arrow_array::RecordBatch; +use arrow_ipc::CompressionType; use arrow_ipc::reader::FileReaderBuilder; use arrow_ipc::writer::FileWriter; use arrow_ipc::writer::IpcWriteOptions; -use arrow_ipc::CompressionType; use arrow_schema::Schema; use databend_common_column::bitmap::Bitmap; use databend_common_column::bitmap::MutableBitmap; diff --git a/src/query/expression/src/utils/block_debug.rs b/src/query/expression/src/utils/block_debug.rs index f978151d805d3..fe63cfa6f4ef6 100644 --- a/src/query/expression/src/utils/block_debug.rs +++ b/src/query/expression/src/utils/block_debug.rs @@ -18,8 +18,8 @@ use comfy_table::Cell; use comfy_table::CellAlignment; use comfy_table::Table; use databend_common_exception::Result; -use terminal_size::terminal_size; use terminal_size::Width; +use terminal_size::terminal_size; use unicode_segmentation::UnicodeSegmentation; use crate::DataBlock; diff --git a/src/query/expression/src/utils/column_from.rs b/src/query/expression/src/utils/column_from.rs index 4014f008f670f..c88372983bb46 100755 --- a/src/query/expression/src/utils/column_from.rs +++ b/src/query/expression/src/utils/column_from.rs @@ -17,13 +17,13 @@ use std::iter::Iterator; use databend_common_column::bitmap::MutableBitmap; use itertools::Itertools; +use crate::Column; use crate::types::decimal::*; use crate::types::geometry::GeometryType; use crate::types::nullable::NullableColumn; use crate::types::number::*; use crate::types::timestamp_tz::TimestampTzType; use crate::types::*; -use crate::Column; pub trait FromData { fn from_data(_: Vec) -> Column; diff --git a/src/query/expression/src/utils/date_helper.rs b/src/query/expression/src/utils/date_helper.rs index a1cbe8cd65435..0a18e49250e58 100644 --- a/src/query/expression/src/utils/date_helper.rs +++ b/src/query/expression/src/utils/date_helper.rs @@ -16,27 +16,27 @@ use std::sync::LazyLock; use databend_common_column::types::timestamp_tz; use databend_common_exception::Result; +use databend_common_timezone::DateTimeComponents; use databend_common_timezone::fast_components_from_timestamp; use databend_common_timezone::fast_utc_from_local; -use databend_common_timezone::DateTimeComponents; -use jiff::civil::date; -use jiff::civil::datetime; -use jiff::civil::Date; -use jiff::civil::DateTime; -use jiff::civil::Time; -use jiff::civil::Weekday; -use jiff::tz::TimeZone; use jiff::SignedDuration; use jiff::SpanRelativeTo; use jiff::Timestamp; use jiff::ToSpan; use jiff::Unit; use jiff::Zoned; +use jiff::civil::Date; +use jiff::civil::DateTime; +use jiff::civil::Time; +use jiff::civil::Weekday; +use jiff::civil::date; +use jiff::civil::datetime; +use jiff::tz::TimeZone; use num_traits::AsPrimitive; use crate::types::date::clamp_date; -use crate::types::timestamp::clamp_timestamp; use crate::types::timestamp::MICROS_PER_SEC; +use crate::types::timestamp::clamp_timestamp; // jiff's `Timestamp` only accepts UTC seconds in // [-377705023201, 253402207200] so that any ±25:59:59 offset still diff --git a/src/query/expression/src/utils/display.rs b/src/query/expression/src/utils/display.rs index c150775f152de..6a649b3ed8379 100755 --- a/src/query/expression/src/utils/display.rs +++ b/src/query/expression/src/utils/display.rs @@ -19,8 +19,8 @@ use std::fmt::Write; use comfy_table::Cell; use comfy_table::Table; -use databend_common_ast::ast::quote::display_ident; use databend_common_ast::ast::quote::QuotedString; +use databend_common_ast::ast::quote::display_ident; use databend_common_ast::parser::Dialect; use databend_common_column::binary::BinaryColumn; use databend_common_io::deserialize_bitmap; @@ -36,6 +36,12 @@ use num_traits::FromPrimitive; use rust_decimal::Decimal; use rust_decimal::RoundingStrategy; +use crate::Column; +use crate::ColumnIndex; +use crate::ExprVisitor; +use crate::FunctionEval; +use crate::TableDataType; +use crate::VariantDataType; use crate::block::DataBlock; use crate::expr::*; use crate::expression::Expr; @@ -44,6 +50,13 @@ use crate::function::Function; use crate::function::FunctionSignature; use crate::property::Domain; use crate::property::FunctionProperty; +use crate::types::AccessType; +use crate::types::AnyType; +use crate::types::DataType; +use crate::types::DecimalSize; +use crate::types::NumberClass; +use crate::types::ValueType; +use crate::types::VectorScalarRef; use crate::types::boolean::BooleanDomain; use crate::types::date::date_to_string; use crate::types::decimal::DecimalColumn; @@ -63,13 +76,6 @@ use crate::types::opaque::OpaqueScalarRef; use crate::types::string::StringDomain; use crate::types::timestamp::timestamp_to_string; use crate::types::vector::VectorDataType; -use crate::types::AccessType; -use crate::types::AnyType; -use crate::types::DataType; -use crate::types::DecimalSize; -use crate::types::NumberClass; -use crate::types::ValueType; -use crate::types::VectorScalarRef; use crate::values::Scalar; use crate::values::ScalarRef; use crate::values::Value; @@ -77,12 +83,6 @@ use crate::visit_expr; use crate::with_integer_mapped_type; use crate::with_opaque_type; use crate::with_vector_number_type; -use crate::Column; -use crate::ColumnIndex; -use crate::ExprVisitor; -use crate::FunctionEval; -use crate::TableDataType; -use crate::VariantDataType; const FLOAT_NUM_FRAC_DIGITS: u32 = 10; @@ -1026,51 +1026,23 @@ impl Expr { Expr::FunctionCall(FunctionCall { function, args, id, .. }) => match (function.signature.name.as_str(), args.as_slice()) { - ("and", [ref lhs, ref rhs]) => { - write_binary_op("AND", lhs, rhs, 10, min_precedence) - } - ("or", [ref lhs, ref rhs]) => { - write_binary_op("OR", lhs, rhs, 5, min_precedence) - } - ("not", [ref expr]) => write_unary_op("NOT", expr, 15, min_precedence), - ("gte", [ref lhs, ref rhs]) => { - write_binary_op(">=", lhs, rhs, 20, min_precedence) - } - ("gt", [ref lhs, ref rhs]) => { - write_binary_op(">", lhs, rhs, 20, min_precedence) - } - ("lte", [ref lhs, ref rhs]) => { - write_binary_op("<=", lhs, rhs, 20, min_precedence) - } - ("lt", [ref lhs, ref rhs]) => { - write_binary_op("<", lhs, rhs, 20, min_precedence) - } - ("eq", [ref lhs, ref rhs]) => { - write_binary_op("=", lhs, rhs, 20, min_precedence) - } - ("noteq", [ref lhs, ref rhs]) => { - write_binary_op("<>", lhs, rhs, 20, min_precedence) - } - ("plus", [ref expr]) => write_unary_op("+", expr, 50, min_precedence), - ("minus", [ref expr]) => write_unary_op("-", expr, 50, min_precedence), - ("plus", [ref lhs, ref rhs]) => { - write_binary_op("+", lhs, rhs, 30, min_precedence) - } - ("minus", [ref lhs, ref rhs]) => { - write_binary_op("-", lhs, rhs, 30, min_precedence) - } - ("multiply", [ref lhs, ref rhs]) => { - write_binary_op("*", lhs, rhs, 40, min_precedence) - } - ("divide", [ref lhs, ref rhs]) => { - write_binary_op("/", lhs, rhs, 40, min_precedence) - } - ("div", [ref lhs, ref rhs]) => { - write_binary_op("DIV", lhs, rhs, 40, min_precedence) - } - ("modulo", [ref lhs, ref rhs]) => { - write_binary_op("%", lhs, rhs, 40, min_precedence) - } + ("and", [lhs, rhs]) => write_binary_op("AND", lhs, rhs, 10, min_precedence), + ("or", [lhs, rhs]) => write_binary_op("OR", lhs, rhs, 5, min_precedence), + ("not", [expr]) => write_unary_op("NOT", expr, 15, min_precedence), + ("gte", [lhs, rhs]) => write_binary_op(">=", lhs, rhs, 20, min_precedence), + ("gt", [lhs, rhs]) => write_binary_op(">", lhs, rhs, 20, min_precedence), + ("lte", [lhs, rhs]) => write_binary_op("<=", lhs, rhs, 20, min_precedence), + ("lt", [lhs, rhs]) => write_binary_op("<", lhs, rhs, 20, min_precedence), + ("eq", [lhs, rhs]) => write_binary_op("=", lhs, rhs, 20, min_precedence), + ("noteq", [lhs, rhs]) => write_binary_op("<>", lhs, rhs, 20, min_precedence), + ("plus", [expr]) => write_unary_op("+", expr, 50, min_precedence), + ("minus", [expr]) => write_unary_op("-", expr, 50, min_precedence), + ("plus", [lhs, rhs]) => write_binary_op("+", lhs, rhs, 30, min_precedence), + ("minus", [lhs, rhs]) => write_binary_op("-", lhs, rhs, 30, min_precedence), + ("multiply", [lhs, rhs]) => write_binary_op("*", lhs, rhs, 40, min_precedence), + ("divide", [lhs, rhs]) => write_binary_op("/", lhs, rhs, 40, min_precedence), + ("div", [lhs, rhs]) => write_binary_op("DIV", lhs, rhs, 40, min_precedence), + ("modulo", [lhs, rhs]) => write_binary_op("%", lhs, rhs, 40, min_precedence), _ => { let mut s = String::new(); s += &function.signature.name; diff --git a/src/query/expression/src/utils/filter_helper.rs b/src/query/expression/src/utils/filter_helper.rs index f177ea0e2f2ef..16a2216be092f 100644 --- a/src/query/expression/src/utils/filter_helper.rs +++ b/src/query/expression/src/utils/filter_helper.rs @@ -16,8 +16,6 @@ use std::collections::HashSet; use databend_common_column::bitmap::MutableBitmap; -use crate::arrow::bitmap_into_mut; -use crate::types::BooleanType; use crate::ColumnIndex; use crate::Constant; use crate::ConstantFolder; @@ -26,6 +24,8 @@ use crate::FunctionContext; use crate::FunctionRegistry; use crate::Scalar; use crate::Value; +use crate::arrow::bitmap_into_mut; +use crate::types::BooleanType; pub struct FilterHelpers; diff --git a/src/query/expression/src/utils/mod.rs b/src/query/expression/src/utils/mod.rs index 80f242043411c..b06a708884a3f 100644 --- a/src/query/expression/src/utils/mod.rs +++ b/src/query/expression/src/utils/mod.rs @@ -31,18 +31,6 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; pub use self::column_from::*; -use crate::types::decimal::DecimalScalar; -use crate::types::i256; -use crate::types::AnyType; -use crate::types::DataType; -use crate::types::Decimal; -use crate::types::DecimalDataKind; -use crate::types::DecimalDataType; -use crate::types::DecimalSize; -use crate::types::NumberDataType; -use crate::types::NumberScalar; -use crate::types::F32; -use crate::types::F64; use crate::BlockEntry; use crate::Column; use crate::DataBlock; @@ -52,6 +40,18 @@ use crate::FunctionRegistry; use crate::RawExpr; use crate::Scalar; use crate::Value; +use crate::types::AnyType; +use crate::types::DataType; +use crate::types::Decimal; +use crate::types::DecimalDataKind; +use crate::types::DecimalDataType; +use crate::types::DecimalSize; +use crate::types::F32; +use crate::types::F64; +use crate::types::NumberDataType; +use crate::types::NumberScalar; +use crate::types::decimal::DecimalScalar; +use crate::types::i256; /// A convenient shortcut to evaluate a scalar function. pub fn eval_function( diff --git a/src/query/expression/src/utils/serialize.rs b/src/query/expression/src/utils/serialize.rs index a92f6e3e568b1..4db037a2cab13 100644 --- a/src/query/expression/src/utils/serialize.rs +++ b/src/query/expression/src/utils/serialize.rs @@ -15,8 +15,8 @@ use std::cmp::Ordering; use std::result::Result; -use jiff::civil::Date; use jiff::Unit; +use jiff::civil::Date; use crate::types::decimal::Decimal; use crate::types::decimal::DecimalSize; diff --git a/src/query/expression/src/utils/udf_client.rs b/src/query/expression/src/utils/udf_client.rs index e5847e14c7bc5..510ca61b60739 100644 --- a/src/query/expression/src/utils/udf_client.rs +++ b/src/query/expression/src/utils/udf_client.rs @@ -19,11 +19,11 @@ use std::time::Duration; use std::time::Instant; use arrow_array::RecordBatch; +use arrow_flight::FlightDescriptor; use arrow_flight::decode::FlightRecordBatchStream; use arrow_flight::encode::FlightDataEncoderBuilder; use arrow_flight::error::FlightError; use arrow_flight::flight_service_client::FlightServiceClient; -use arrow_flight::FlightDescriptor; use arrow_schema::ArrowError; use arrow_select::concat::concat_batches; use databend_common_base::headers::HEADER_FUNCTION; @@ -40,28 +40,28 @@ use databend_common_metrics::external_server::record_request_external_batch_rows use databend_common_metrics::external_server::record_request_external_duration; use databend_common_metrics::external_server::record_running_requests_external_finish; use databend_common_metrics::external_server::record_running_requests_external_start; -use futures::stream; use futures::StreamExt; use futures::TryStreamExt; +use futures::stream; use hyper_util::client::legacy::connect::HttpConnector; use itertools::Itertools; +use tonic::Request; +use tonic::Status; use tonic::metadata::KeyAndValueRef; use tonic::metadata::MetadataKey; use tonic::metadata::MetadataMap; use tonic::metadata::MetadataValue; -use tonic::transport::channel::Channel; use tonic::transport::ClientTlsConfig; use tonic::transport::Endpoint; -use tonic::Request; -use tonic::Status; +use tonic::transport::channel::Channel; -use crate::types::DataType; -use crate::variant_transform::contains_variant; -use crate::variant_transform::transform_variant; use crate::BlockEntry; use crate::DataBlock; use crate::DataField; use crate::DataSchema; +use crate::types::DataType; +use crate::variant_transform::contains_variant; +use crate::variant_transform::transform_variant; const UDF_TCP_KEEP_ALIVE_SEC: u64 = 30; const UDF_HTTP2_KEEP_ALIVE_INTERVAL_SEC: u64 = 60; @@ -288,9 +288,7 @@ impl UDFFlightClient { if expect_return_type[0] != return_type { return Err(ErrorCode::UDFSchemaMismatch(format!( "UDF return type mismatch on UDF function {}, expected return type: {}, actual return type: {}", - func_name, - expect_return_type[0], - return_type + func_name, expect_return_type[0], return_type ))); } diff --git a/src/query/expression/src/utils/variant_transform.rs b/src/query/expression/src/utils/variant_transform.rs index a61c3a862cd5f..1f8b63ed5555f 100644 --- a/src/query/expression/src/utils/variant_transform.rs +++ b/src/query/expression/src/utils/variant_transform.rs @@ -14,16 +14,16 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use jsonb::parse_owned_jsonb; use jsonb::RawJsonb; +use jsonb::parse_owned_jsonb; +use crate::ColumnBuilder; +use crate::ScalarRef; use crate::types::AnyType; use crate::types::DataType; use crate::values::Column; use crate::values::Scalar; use crate::values::Value; -use crate::ColumnBuilder; -use crate::ScalarRef; pub fn contains_variant(data_type: &DataType) -> bool { match data_type { diff --git a/src/query/expression/src/values.rs b/src/query/expression/src/values.rs index 6d72495197ee0..eb1b8cb8c9aba 100755 --- a/src/query/expression/src/values.rs +++ b/src/query/expression/src/values.rs @@ -32,10 +32,10 @@ use databend_common_column::types::months_days_micros; use databend_common_column::types::timestamp_tz; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_frozen_api::frozen_api; use databend_common_frozen_api::FrozenAPI; -use databend_common_io::prelude::BinaryRead; +use databend_common_frozen_api::frozen_api; use databend_common_io::HybridBitmap; +use databend_common_io::prelude::BinaryRead; use enum_as_inner::EnumAsInner; use geo::Geometry; use geo::Point; @@ -43,11 +43,11 @@ use geozero::CoordDimensions; use geozero::ToWkb; use itertools::Itertools; use jsonb::RawJsonb; -use serde::de::Visitor; use serde::Deserialize; use serde::Deserializer; use serde::Serialize; use serde::Serializer; +use serde::de::Visitor; use string::StringColumnBuilder; use crate::property::Domain; @@ -66,19 +66,19 @@ use crate::types::decimal::DecimalScalar; use crate::types::geography::Geography; use crate::types::geography::GeographyColumn; use crate::types::geography::GeographyRef; -use crate::types::geometry::compare_geometry; use crate::types::geometry::GeometryType; +use crate::types::geometry::compare_geometry; use crate::types::i256; use crate::types::nullable::NullableColumn; use crate::types::nullable::NullableColumnBuilder; use crate::types::nullable::NullableColumnVec; use crate::types::nullable::NullableDomain; +use crate::types::number::F32; +use crate::types::number::F64; use crate::types::number::NumberColumn; use crate::types::number::NumberColumnBuilder; use crate::types::number::NumberScalar; use crate::types::number::SimpleDomain; -use crate::types::number::F32; -use crate::types::number::F64; use crate::types::opaque::OpaqueColumn; use crate::types::opaque::OpaqueColumnBuilder; use crate::types::opaque::OpaqueColumnVec; @@ -87,20 +87,20 @@ use crate::types::opaque::OpaqueScalarRef; use crate::types::opaque::OpaqueType; use crate::types::string::StringColumn; use crate::types::string::StringDomain; -use crate::types::timestamp::clamp_timestamp; use crate::types::timestamp::TIMESTAMP_MAX; use crate::types::timestamp::TIMESTAMP_MIN; +use crate::types::timestamp::clamp_timestamp; use crate::types::timestamp_tz::TimestampTzType; use crate::types::variant::JSONB_NULL; use crate::types::vector::VectorColumn; use crate::types::vector::VectorColumnBuilder; use crate::types::*; +use crate::utils::FromData; use crate::utils::arrow::append_bitmap; use crate::utils::arrow::bitmap_into_mut; use crate::utils::arrow::buffer_into_mut; use crate::utils::arrow::deserialize_column; use crate::utils::arrow::serialize_column; -use crate::utils::FromData; use crate::values::decimal::DecimalColumnVec; use crate::values::map::KvPair; use crate::with_decimal_mapped_type; @@ -181,6 +181,7 @@ pub enum ScalarRef<'a> { Opaque(OpaqueScalarRef<'a>), } +#[allow(unused, dead_code)] #[derive(Clone, EnumAsInner)] pub enum Column { Null { len: usize }, @@ -252,6 +253,7 @@ pub enum ColumnVec { Opaque(OpaqueColumnVec), } +#[allow(unused, dead_code)] #[derive(Debug, Clone, EnumAsInner)] pub enum ColumnBuilder { Null { len: usize }, @@ -315,9 +317,11 @@ impl Value { /// /// Calling this method with an out-of-bounds index is *[undefined behavior]* pub unsafe fn index_unchecked(&self, index: usize) -> T::ScalarRef<'_> { - match self { - Value::Scalar(scalar) => T::to_scalar_ref(scalar), - Value::Column(c) => T::index_column_unchecked(c, index), + unsafe { + match self { + Value::Scalar(scalar) => T::to_scalar_ref(scalar), + Value::Column(c) => T::index_column_unchecked(c, index), + } } } } @@ -421,7 +425,7 @@ impl Value { } impl Scalar { - pub fn as_ref(&self) -> ScalarRef { + pub fn as_ref(&self) -> ScalarRef<'_> { match self { Scalar::Null => ScalarRef::Null, Scalar::EmptyArray => ScalarRef::EmptyArray, @@ -1175,7 +1179,7 @@ impl Column { } } - pub fn index(&self, index: usize) -> Option { + pub fn index(&self, index: usize) -> Option> { match self { Column::Null { .. } => Some(ScalarRef::Null), Column::EmptyArray { .. } => Some(ScalarRef::EmptyArray), @@ -1210,35 +1214,37 @@ impl Column { /// # Safety /// /// Calling this method with an out-of-bounds index is *[undefined behavior]* - pub unsafe fn index_unchecked(&self, index: usize) -> ScalarRef { - match self { - Column::Null { .. } => ScalarRef::Null, - Column::EmptyArray { .. } => ScalarRef::EmptyArray, - Column::EmptyMap { .. } => ScalarRef::EmptyMap, - Column::Number(col) => ScalarRef::Number(col.index_unchecked(index)), - Column::Decimal(col) => ScalarRef::Decimal(col.index_unchecked(index)), - Column::Boolean(col) => ScalarRef::Boolean(col.get_bit_unchecked(index)), - Column::Binary(col) => ScalarRef::Binary(col.index_unchecked(index)), - Column::String(col) => ScalarRef::String(col.index_unchecked(index)), - Column::Timestamp(col) => ScalarRef::Timestamp(*col.get_unchecked(index)), - Column::TimestampTz(col) => ScalarRef::TimestampTz(*col.get_unchecked(index)), - Column::Date(col) => ScalarRef::Date(*col.get_unchecked(index)), - Column::Interval(col) => ScalarRef::Interval(*col.get_unchecked(index)), - Column::Array(col) => ScalarRef::Array(col.index_unchecked(index)), - Column::Map(col) => ScalarRef::Map(col.index_unchecked(index)), - Column::Bitmap(col) => ScalarRef::Bitmap(col.index_unchecked(index)), - Column::Nullable(col) => col.index_unchecked(index).unwrap_or(ScalarRef::Null), - Column::Tuple(fields) => ScalarRef::Tuple( - fields - .iter() - .map(|field| field.index_unchecked(index)) - .collect::>(), - ), - Column::Variant(col) => ScalarRef::Variant(col.index_unchecked(index)), - Column::Geometry(col) => ScalarRef::Geometry(col.index_unchecked(index)), - Column::Geography(col) => ScalarRef::Geography(col.index_unchecked(index)), - Column::Vector(col) => ScalarRef::Vector(col.index_unchecked(index)), - Column::Opaque(col) => ScalarRef::Opaque(col.index_unchecked(index)), + pub unsafe fn index_unchecked(&self, index: usize) -> ScalarRef<'_> { + unsafe { + match self { + Column::Null { .. } => ScalarRef::Null, + Column::EmptyArray { .. } => ScalarRef::EmptyArray, + Column::EmptyMap { .. } => ScalarRef::EmptyMap, + Column::Number(col) => ScalarRef::Number(col.index_unchecked(index)), + Column::Decimal(col) => ScalarRef::Decimal(col.index_unchecked(index)), + Column::Boolean(col) => ScalarRef::Boolean(col.get_bit_unchecked(index)), + Column::Binary(col) => ScalarRef::Binary(col.index_unchecked(index)), + Column::String(col) => ScalarRef::String(col.index_unchecked(index)), + Column::Timestamp(col) => ScalarRef::Timestamp(*col.get_unchecked(index)), + Column::TimestampTz(col) => ScalarRef::TimestampTz(*col.get_unchecked(index)), + Column::Date(col) => ScalarRef::Date(*col.get_unchecked(index)), + Column::Interval(col) => ScalarRef::Interval(*col.get_unchecked(index)), + Column::Array(col) => ScalarRef::Array(col.index_unchecked(index)), + Column::Map(col) => ScalarRef::Map(col.index_unchecked(index)), + Column::Bitmap(col) => ScalarRef::Bitmap(col.index_unchecked(index)), + Column::Nullable(col) => col.index_unchecked(index).unwrap_or(ScalarRef::Null), + Column::Tuple(fields) => ScalarRef::Tuple( + fields + .iter() + .map(|field| field.index_unchecked(index)) + .collect::>(), + ), + Column::Variant(col) => ScalarRef::Variant(col.index_unchecked(index)), + Column::Geometry(col) => ScalarRef::Geometry(col.index_unchecked(index)), + Column::Geography(col) => ScalarRef::Geography(col.index_unchecked(index)), + Column::Vector(col) => ScalarRef::Vector(col.index_unchecked(index)), + Column::Opaque(col) => ScalarRef::Opaque(col.index_unchecked(index)), + } } } @@ -1304,7 +1310,7 @@ impl Column { } } - pub fn iter(&self) -> ColumnIterator { + pub fn iter(&self) -> ColumnIterator<'_> { ColumnIterator { column: self, index: 0, @@ -1491,10 +1497,10 @@ impl Column { } pub fn random(ty: &DataType, len: usize, options: Option) -> Self { - use rand::distributions::Alphanumeric; - use rand::rngs::SmallRng; use rand::Rng; use rand::SeedableRng; + use rand::distributions::Alphanumeric; + use rand::rngs::SmallRng; let mut rng = match &options { Some(RandomOptions { seed: Some(seed), .. @@ -1552,7 +1558,7 @@ impl Column { with_number_mapped_type!(|NUM_TYPE| match num_ty { NumberDataType::NUM_TYPE => { NumberType::::from_data( - (0..len).map(|_| rng.gen::()).collect_vec(), + (0..len).map(|_| rng.r#gen::()).collect_vec(), ) } }) @@ -1561,7 +1567,7 @@ impl Column { with_decimal_mapped_type!(|DECIMAL| match DecimalDataType::from(*size) { DecimalDataType::DECIMAL(size) => { let values = (0..len) - .map(|_| DECIMAL::from(rng.gen::())) + .map(|_| DECIMAL::from(rng.r#gen::())) .collect::>(); ::upcast_column(values.into(), size) } @@ -1641,7 +1647,7 @@ impl Column { DataType::Bitmap => BitmapType::from_data( (0..len) .map(|_| { - let data: [u64; 4] = rng.gen(); + let data: [u64; 4] = rng.r#gen(); let rb = HybridBitmap::from_iter(data.iter()); let mut buf = vec![]; rb.serialize_into(&mut buf) @@ -1668,8 +1674,8 @@ impl Column { DataType::Geometry => { let mut data = Vec::with_capacity(len); (0..len).for_each(|_| { - let x = rng.gen::(); - let y = rng.gen::(); + let x = rng.r#gen::(); + let y = rng.r#gen::(); let val = Point::new(x, y); data.push( Geometry::from(val) @@ -1696,14 +1702,14 @@ impl Column { match vector_ty { VectorDataType::Int8(dimension) => { for _ in 0..len { - let value = (0..*dimension).map(|_| rng.gen::()).collect_vec(); + let value = (0..*dimension).map(|_| rng.r#gen::()).collect_vec(); let scalar = VectorScalarRef::Int8(&value); builder.push(&scalar); } } VectorDataType::Float32(dimension) => { for _ in 0..len { - let value = (0..*dimension).map(|_| rng.gen::()).collect_vec(); + let value = (0..*dimension).map(|_| rng.r#gen::()).collect_vec(); let scalar = VectorScalarRef::Float32(&value); builder.push(&scalar); } @@ -1719,7 +1725,7 @@ impl Column { for _ in 0..len { let mut value = [0u64; N]; for i in 0..N { - value[i] = rng.gen::(); + value[i] = rng.r#gen::(); } builder.push(value); } diff --git a/src/query/expression/tests/it/arrow.rs b/src/query/expression/tests/it/arrow.rs index 8bcf38f0ee17e..662d2e44a9e84 100644 --- a/src/query/expression/tests/it/arrow.rs +++ b/src/query/expression/tests/it/arrow.rs @@ -18,8 +18,8 @@ use arrow_array::ArrayRef; use arrow_array::DictionaryArray; use arrow_array::Int32Array; use arrow_array::StringArray; -use databend_common_expression::types::DataType; use databend_common_expression::Column; +use databend_common_expression::types::DataType; #[test] fn test_from_arrow_rs() { diff --git a/src/query/expression/tests/it/block.rs b/src/query/expression/tests/it/block.rs index 53927027de212..3ef3c9362bef3 100644 --- a/src/query/expression/tests/it/block.rs +++ b/src/query/expression/tests/it/block.rs @@ -13,9 +13,14 @@ // limitations under the License. use databend_common_column::buffer::Buffer; +use databend_common_expression::BlockEntry; +use databend_common_expression::Column; +use databend_common_expression::DataBlock; +use databend_common_expression::DataField; +use databend_common_expression::DataSchemaRefExt; +use databend_common_expression::FromData; +use databend_common_expression::Scalar; use databend_common_expression::block_debug::box_render; -use databend_common_expression::types::number::NumberScalar; -use databend_common_expression::types::string::StringColumnBuilder; use databend_common_expression::types::AccessType; use databend_common_expression::types::ArrayColumn; use databend_common_expression::types::DataType; @@ -23,13 +28,8 @@ use databend_common_expression::types::Int32Type; use databend_common_expression::types::Int64Type; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::StringType; -use databend_common_expression::BlockEntry; -use databend_common_expression::Column; -use databend_common_expression::DataBlock; -use databend_common_expression::DataField; -use databend_common_expression::DataSchemaRefExt; -use databend_common_expression::FromData; -use databend_common_expression::Scalar; +use databend_common_expression::types::number::NumberScalar; +use databend_common_expression::types::string::StringColumnBuilder; #[test] fn test_split_block() { diff --git a/src/query/expression/tests/it/decimal.rs b/src/query/expression/tests/it/decimal.rs index 3121430b2891f..0097ecbe7a212 100644 --- a/src/query/expression/tests/it/decimal.rs +++ b/src/query/expression/tests/it/decimal.rs @@ -16,11 +16,11 @@ use databend_common_exception::Result; use databend_common_expression::serialize::read_decimal; use databend_common_expression::serialize::read_decimal_with_size; use databend_common_expression::type_check::common_super_type; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; use databend_common_expression::types::decimal::Decimal; use databend_common_expression::types::decimal::DecimalSize; use databend_common_expression::types::i256; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; use num_bigint::BigInt; use pretty_assertions::assert_eq; diff --git a/src/query/expression/tests/it/display.rs b/src/query/expression/tests/it/display.rs index 76beef1d94f56..9eef31cd11bb6 100644 --- a/src/query/expression/tests/it/display.rs +++ b/src/query/expression/tests/it/display.rs @@ -12,13 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_expression::type_check; -use databend_common_expression::types::*; use databend_common_expression::FunctionDomain; use databend_common_expression::FunctionRegistry; use databend_common_expression::RawExpr; use databend_common_expression::Scalar; use databend_common_expression::Value; +use databend_common_expression::type_check; +use databend_common_expression::types::*; #[test] fn test_fmt_expr() { @@ -54,5 +54,8 @@ fn test_fmt_expr() { "test_fn(CAST(aaa AS String), CAST(aaa AS String))", format!("{expr}") ); - assert_eq!("test_fn(CAST(aaa (#1) AS String), CAST(aaa (#1) AS String))",format!("{}",expr.fmt_with_options(true))); + assert_eq!( + "test_fn(CAST(aaa (#1) AS String), CAST(aaa (#1) AS String))", + format!("{}", expr.fmt_with_options(true)) + ); } diff --git a/src/query/expression/tests/it/fill_field_default_value.rs b/src/query/expression/tests/it/fill_field_default_value.rs index c332c38e29577..44b27d5a8470d 100644 --- a/src/query/expression/tests/it/fill_field_default_value.rs +++ b/src/query/expression/tests/it/fill_field_default_value.rs @@ -12,12 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![allow(clippy::useless_vec)] + use std::collections::HashSet; use databend_common_exception::Result; -use databend_common_expression::types::number::*; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::number::*; use databend_common_expression::*; use goldenfile::Mint; diff --git a/src/query/expression/tests/it/group_by.rs b/src/query/expression/tests/it/group_by.rs index 4229f07c406ca..e424fa502ab8d 100644 --- a/src/query/expression/tests/it/group_by.rs +++ b/src/query/expression/tests/it/group_by.rs @@ -13,10 +13,10 @@ // limitations under the License. use databend_common_exception::Result; -use databend_common_expression::types::decimal::*; -use databend_common_expression::types::number::*; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::StringType; +use databend_common_expression::types::decimal::*; +use databend_common_expression::types::number::*; use databend_common_expression::*; use ethnum::u256; diff --git a/src/query/expression/tests/it/hilbert.rs b/src/query/expression/tests/it/hilbert.rs index 7a1ebb4e2bdb1..c4802a2f6ed16 100644 --- a/src/query/expression/tests/it/hilbert.rs +++ b/src/query/expression/tests/it/hilbert.rs @@ -13,10 +13,10 @@ // limitations under the License. use databend_common_exception::Result; +use databend_common_expression::FixedLengthEncoding; use databend_common_expression::hilbert_decompress; use databend_common_expression::hilbert_decompress_state_list; use databend_common_expression::hilbert_index; -use databend_common_expression::FixedLengthEncoding; #[test] fn test_hilbert() -> Result<()> { diff --git a/src/query/expression/tests/it/kernel.rs b/src/query/expression/tests/it/kernel.rs index 57927fa5dba26..46e276f66cbb9 100644 --- a/src/query/expression/tests/it/kernel.rs +++ b/src/query/expression/tests/it/kernel.rs @@ -16,8 +16,16 @@ use core::ops::Range; use databend_common_base::base::OrderedFloat; use databend_common_column::bitmap::Bitmap; +use databend_common_expression::BlockEntry; +use databend_common_expression::Column; +use databend_common_expression::ColumnBuilder; +use databend_common_expression::DataBlock; +use databend_common_expression::FilterVisitor; +use databend_common_expression::FromData; +use databend_common_expression::IterationStrategy; +use databend_common_expression::Scalar; +use databend_common_expression::ScalarRef; use databend_common_expression::block_debug::assert_block_value_eq; -use databend_common_expression::types::number::*; use databend_common_expression::types::AccessType; use databend_common_expression::types::AnyType; use databend_common_expression::types::DataType; @@ -28,22 +36,14 @@ use databend_common_expression::types::NullableType; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::StringType; use databend_common_expression::types::ValueType; +use databend_common_expression::types::number::*; use databend_common_expression::visitor::ValueVisitor; -use databend_common_expression::BlockEntry; -use databend_common_expression::Column; -use databend_common_expression::ColumnBuilder; -use databend_common_expression::DataBlock; -use databend_common_expression::FilterVisitor; -use databend_common_expression::FromData; -use databend_common_expression::IterationStrategy; -use databend_common_expression::Scalar; -use databend_common_expression::ScalarRef; use goldenfile::Mint; +use crate::DataTypeFilter; use crate::common::*; use crate::get_all_test_data_types; use crate::rand_block_for_all_types; -use crate::DataTypeFilter; #[test] pub fn test_pass() { @@ -242,9 +242,9 @@ pub fn build_range_selection(selection: &[u32], count: usize) -> Vec> /// This test covers take.rs, take_chunks.rs, take_compact.rs, take_ranges.rs, filter.rs, concat.rs. #[test] pub fn test_take_and_filter_and_concat() -> databend_common_exception::Result<()> { - use databend_common_expression::types::DataType; use databend_common_expression::Column; use databend_common_expression::DataBlock; + use databend_common_expression::types::DataType; use databend_common_hashtable::RowPtr; use itertools::Itertools; use rand::Rng; @@ -360,9 +360,9 @@ pub fn test_take_and_filter_and_concat() -> databend_common_exception::Result<() #[test] pub fn test_concat_scalar() -> databend_common_exception::Result<()> { - use databend_common_expression::types::DataType; use databend_common_expression::DataBlock; use databend_common_expression::Scalar; + use databend_common_expression::types::DataType; let ty = DataType::Number(NumberDataType::UInt8); let scalar = Scalar::Number(NumberScalar::UInt8(1)); @@ -441,9 +441,9 @@ pub fn test_take_compact() -> databend_common_exception::Result<()> { /// B.slice(0, l) == B.slice(l, l) == A #[test] pub fn test_filters() -> databend_common_exception::Result<()> { - use databend_common_expression::types::DataType; use databend_common_expression::Column; use databend_common_expression::DataBlock; + use databend_common_expression::types::DataType; use rand::Rng; let mut rng = rand::thread_rng(); diff --git a/src/query/expression/tests/it/main.rs b/src/query/expression/tests/it/main.rs index 3085cd9320307..c0f1d278ac24d 100644 --- a/src/query/expression/tests/it/main.rs +++ b/src/query/expression/tests/it/main.rs @@ -15,11 +15,11 @@ #![feature(box_patterns)] #![feature(try_blocks)] +use databend_common_expression::Column; +use databend_common_expression::DataBlock; use databend_common_expression::types::DataType; use databend_common_expression::types::DecimalSize; use databend_common_expression::types::NumberDataType; -use databend_common_expression::Column; -use databend_common_expression::DataBlock; extern crate core; diff --git a/src/query/expression/tests/it/meta_scalar.rs b/src/query/expression/tests/it/meta_scalar.rs index 9a9905e77b563..ace1addc70a30 100644 --- a/src/query/expression/tests/it/meta_scalar.rs +++ b/src/query/expression/tests/it/meta_scalar.rs @@ -12,16 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_expression::Column; +use databend_common_expression::Scalar; use databend_common_expression::converts::meta::IndexScalar; use databend_common_expression::converts::meta::LegacyColumn; use databend_common_expression::converts::meta::LegacyScalar; -use databend_common_expression::Column; -use databend_common_expression::Scalar; use databend_common_io::prelude::bincode_deserialize_from_slice; use databend_common_io::prelude::bincode_serialize_into_buf; -use crate::rand_block_for_all_types; use crate::DataTypeFilter; +use crate::rand_block_for_all_types; #[test] pub fn test_legacy_converts() -> databend_common_exception::Result<()> { diff --git a/src/query/expression/tests/it/schema.rs b/src/query/expression/tests/it/schema.rs index 7c2af0649a1e2..a19408618938a 100644 --- a/src/query/expression/tests/it/schema.rs +++ b/src/query/expression/tests/it/schema.rs @@ -12,16 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![allow(clippy::cloned_ref_to_slice_refs)] +#![allow(clippy::useless_vec)] + use std::collections::BTreeMap; use databend_common_exception::Result; -use databend_common_expression::create_test_complex_schema; -use databend_common_expression::types::NumberDataType; use databend_common_expression::ColumnId; use databend_common_expression::Scalar; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; +use databend_common_expression::create_test_complex_schema; +use databend_common_expression::types::NumberDataType; use pretty_assertions::assert_eq; #[test] @@ -642,9 +645,9 @@ fn test_leaf_columns_of() -> Result<()> { #[test] fn test_geography_as_arrow() { + use databend_common_expression::Column; use databend_common_expression::types::binary::BinaryColumnBuilder; use databend_common_expression::types::geography::GeographyColumn; - use databend_common_expression::Column; use databend_common_io::wkb::make_point; let mut builder = BinaryColumnBuilder::with_capacity(3, 0); diff --git a/src/query/expression/tests/it/serde.rs b/src/query/expression/tests/it/serde.rs index 59934888268d3..69f9a90250f34 100644 --- a/src/query/expression/tests/it/serde.rs +++ b/src/query/expression/tests/it/serde.rs @@ -17,14 +17,14 @@ use std::vec; use borsh::BorshDeserialize; use borsh::BorshSerialize; use databend_common_exception::Result; -use databend_common_expression::arrow::deserialize_column; -use databend_common_expression::arrow::serialize_column; -use databend_common_expression::types::DataType; -use databend_common_expression::types::StringType; use databend_common_expression::Column; use databend_common_expression::FromData; use databend_common_expression::RemoteExpr; use databend_common_expression::Scalar; +use databend_common_expression::arrow::deserialize_column; +use databend_common_expression::arrow::serialize_column; +use databend_common_expression::types::DataType; +use databend_common_expression::types::StringType; use databend_common_io::prelude::bincode_deserialize_from_slice; use databend_common_io::prelude::bincode_serialize_into_buf; use databend_common_io::prelude::borsh_deserialize_from_slice; diff --git a/src/query/expression/tests/it/sort.rs b/src/query/expression/tests/it/sort.rs index 2f62605033608..85836c0806c10 100644 --- a/src/query/expression/tests/it/sort.rs +++ b/src/query/expression/tests/it/sort.rs @@ -15,16 +15,16 @@ use std::vec; use databend_common_exception::Result; -use databend_common_expression::types::decimal::*; -use databend_common_expression::types::number::*; -use databend_common_expression::types::StringType; use databend_common_expression::Column; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::SortColumnDescription; +use databend_common_expression::types::StringType; +use databend_common_expression::types::decimal::*; +use databend_common_expression::types::number::*; -use crate::rand_block_for_all_types; use crate::DataTypeFilter; +use crate::rand_block_for_all_types; #[test] fn test_block_sort() -> Result<()> { @@ -198,8 +198,8 @@ fn sort_concat() { // Sort(Sort A || Sort B) = Sort (A || B) use databend_common_expression::DataBlock; use itertools::Itertools; - use rand::seq::SliceRandom; use rand::Rng; + use rand::seq::SliceRandom; let mut rng = rand::thread_rng(); let num_blocks = 100; diff --git a/src/query/expression/tests/it/types.rs b/src/query/expression/tests/it/types.rs index 1c997b1e12671..df48cff10bae4 100644 --- a/src/query/expression/tests/it/types.rs +++ b/src/query/expression/tests/it/types.rs @@ -13,18 +13,18 @@ // limitations under the License. use arrow_schema::Schema; +use databend_common_expression::DataField; +use databend_common_expression::DataSchema; use databend_common_expression::arrow::deserialize_column; use databend_common_expression::arrow::serialize_column; use databend_common_expression::types::timestamp::timestamp_to_string; -use databend_common_expression::DataField; -use databend_common_expression::DataSchema; use jiff::fmt::strtime::BrokenDownTime; use jiff::tz; use jiff::tz::TimeZone; +use crate::DataTypeFilter; use crate::get_all_test_data_types; use crate::rand_block_for_all_types; -use crate::DataTypeFilter; #[test] fn test_timestamp_to_string_formats() { diff --git a/src/query/formats/src/binary.rs b/src/query/formats/src/binary.rs index b23215142ad1d..5c1f9f8a8deab 100644 --- a/src/query/formats/src/binary.rs +++ b/src/query/formats/src/binary.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use base64::engine::general_purpose; use base64::Engine as _; +use base64::engine::general_purpose; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_meta_app::principal::BinaryFormat; diff --git a/src/query/formats/src/column_from_json.rs b/src/query/formats/src/column_from_json.rs index 036ab2022cbe2..c28ac52f21ab1 100644 --- a/src/query/formats/src/column_from_json.rs +++ b/src/query/formats/src/column_from_json.rs @@ -15,15 +15,15 @@ use chrono_tz::UTC; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::Column; use databend_common_expression::ColumnBuilder; +use databend_common_expression::types::DataType; use databend_common_io::GeometryDataType; use jiff::tz::TimeZone; use serde_json::Value; -use crate::field_decoder::FieldJsonAstDecoder; use crate::FileFormatOptionsExt; +use crate::field_decoder::FieldJsonAstDecoder; fn default_json_options() -> FileFormatOptionsExt { FileFormatOptionsExt { @@ -48,7 +48,7 @@ pub fn column_from_json_value(data_type: &DataType, json: Value) -> Result { return Err(ErrorCode::BadArguments(format!( "from_json! expects a json array to describe column values, got {other:?}" - ))) + ))); } }; @@ -71,8 +71,8 @@ macro_rules! column_from_json { #[cfg(test)] mod tests { - use databend_common_expression::types::*; use databend_common_expression::FromData; + use databend_common_expression::types::*; #[test] fn test_from_json_macro_strings() { diff --git a/src/query/formats/src/field_decoder/common.rs b/src/query/formats/src/field_decoder/common.rs index b4fe9a6be082a..b93712127100d 100644 --- a/src/query/formats/src/field_decoder/common.rs +++ b/src/query/formats/src/field_decoder/common.rs @@ -20,10 +20,10 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::types::timestamp::clamp_timestamp; use databend_common_expression::types::timestamp_tz::string_to_timestamp_tz; -use databend_common_io::cursor_ext::read_num_text_exact; use databend_common_io::cursor_ext::BufferReadDateTimeExt; use databend_common_io::cursor_ext::DateTimeResType; use databend_common_io::cursor_ext::ReadBytesExt; +use databend_common_io::cursor_ext::read_num_text_exact; use databend_functions_scalar_datetime::datetime::int64_to_timestamp; use crate::InputCommonSettings; diff --git a/src/query/formats/src/field_decoder/fast_values.rs b/src/query/formats/src/field_decoder/fast_values.rs index f03182b552cb1..11c20975b5138 100644 --- a/src/query/formats/src/field_decoder/fast_values.rs +++ b/src/query/formats/src/field_decoder/fast_values.rs @@ -27,8 +27,14 @@ use databend_common_column::types::timestamp_tz; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_exception::ToErrorCode; +use databend_common_expression::ColumnBuilder; +use databend_common_expression::Scalar; use databend_common_expression::serialize::read_decimal_with_size; use databend_common_expression::serialize::uniform_date; +use databend_common_expression::types::AnyType; +use databend_common_expression::types::MutableBitmap; +use databend_common_expression::types::NumberColumnBuilder; +use databend_common_expression::types::VectorScalarRef; use databend_common_expression::types::array::ArrayColumnBuilder; use databend_common_expression::types::binary::BinaryColumnBuilder; use databend_common_expression::types::date::clamp_date; @@ -39,14 +45,9 @@ use databend_common_expression::types::nullable::NullableColumnBuilder; use databend_common_expression::types::number::Number; use databend_common_expression::types::string::StringColumnBuilder; use databend_common_expression::types::vector::VectorColumnBuilder; -use databend_common_expression::types::AnyType; -use databend_common_expression::types::MutableBitmap; -use databend_common_expression::types::NumberColumnBuilder; -use databend_common_expression::types::VectorScalarRef; use databend_common_expression::with_decimal_type; use databend_common_expression::with_number_mapped_type; -use databend_common_expression::ColumnBuilder; -use databend_common_expression::Scalar; +use databend_common_io::Interval; use databend_common_io::constants::FALSE_BYTES_LOWER; use databend_common_io::constants::NAN_BYTES_LOWER; use databend_common_io::constants::NULL_BYTES_UPPER; @@ -60,15 +61,14 @@ use databend_common_io::geography::geography_from_ewkt_bytes; use databend_common_io::parse_bitmap; use databend_common_io::parse_bytes_to_ewkb; use databend_common_io::prelude::FormatSettings; -use databend_common_io::Interval; use jsonb::parse_owned_jsonb_with_buf; use lexical_core::FromLexical; use num_traits::NumCast; -use crate::field_decoder::common::read_timestamp; -use crate::field_decoder::common::read_timestamp_tz; use crate::FieldDecoder; use crate::InputCommonSettings; +use crate::field_decoder::common::read_timestamp; +use crate::field_decoder::common::read_timestamp_tz; #[derive(Clone)] pub struct FastFieldDecoderValues { @@ -767,11 +767,11 @@ mod test { use databend_common_exception::ErrorCode; use databend_common_exception::Result; - use databend_common_expression::types::DataType; - use databend_common_expression::types::NumberDataType; use databend_common_expression::ColumnBuilder; use databend_common_expression::DataBlock; use databend_common_expression::Scalar; + use databend_common_expression::types::DataType; + use databend_common_expression::types::NumberDataType; use databend_common_io::prelude::FormatSettings; use goldenfile::Mint; diff --git a/src/query/formats/src/field_decoder/json_ast.rs b/src/query/formats/src/field_decoder/json_ast.rs index 4f549e1547f1a..5e94fa5067014 100644 --- a/src/query/formats/src/field_decoder/json_ast.rs +++ b/src/query/formats/src/field_decoder/json_ast.rs @@ -19,8 +19,14 @@ use databend_common_column::types::months_days_micros; use databend_common_column::types::timestamp_tz; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::ColumnBuilder; use databend_common_expression::serialize::read_decimal_from_json; use databend_common_expression::serialize::uniform_date; +use databend_common_expression::types::AnyType; +use databend_common_expression::types::MutableBitmap; +use databend_common_expression::types::NumberColumnBuilder; +use databend_common_expression::types::VectorColumnBuilder; +use databend_common_expression::types::VectorScalarRef; use databend_common_expression::types::array::ArrayColumnBuilder; use databend_common_expression::types::binary::BinaryColumnBuilder; use databend_common_expression::types::date::clamp_date; @@ -32,21 +38,15 @@ use databend_common_expression::types::number::Number; use databend_common_expression::types::string::StringColumnBuilder; use databend_common_expression::types::timestamp::clamp_timestamp; use databend_common_expression::types::timestamp_tz::string_to_timestamp_tz; -use databend_common_expression::types::AnyType; -use databend_common_expression::types::MutableBitmap; -use databend_common_expression::types::NumberColumnBuilder; -use databend_common_expression::types::VectorColumnBuilder; -use databend_common_expression::types::VectorScalarRef; use databend_common_expression::with_decimal_type; use databend_common_expression::with_number_mapped_type; -use databend_common_expression::ColumnBuilder; +use databend_common_io::HybridBitmap; +use databend_common_io::Interval; use databend_common_io::cursor_ext::BufferReadDateTimeExt; use databend_common_io::cursor_ext::DateTimeResType; use databend_common_io::geography::geography_from_ewkt; use databend_common_io::geometry_from_ewkt; use databend_common_io::parse_bitmap; -use databend_common_io::HybridBitmap; -use databend_common_io::Interval; use databend_functions_scalar_datetime::datetime::int64_to_timestamp; use jiff::tz::TimeZone; use lexical_core::FromLexical; diff --git a/src/query/formats/src/field_decoder/nested.rs b/src/query/formats/src/field_decoder/nested.rs index 021bc48951cb9..f41e3c9cb266e 100644 --- a/src/query/formats/src/field_decoder/nested.rs +++ b/src/query/formats/src/field_decoder/nested.rs @@ -22,8 +22,13 @@ use databend_common_column::types::timestamp_tz; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_exception::ToErrorCode; +use databend_common_expression::ColumnBuilder; use databend_common_expression::serialize::read_decimal_with_size; use databend_common_expression::serialize::uniform_date; +use databend_common_expression::types::AnyType; +use databend_common_expression::types::MutableBitmap; +use databend_common_expression::types::NumberColumnBuilder; +use databend_common_expression::types::VectorScalarRef; use databend_common_expression::types::array::ArrayColumnBuilder; use databend_common_expression::types::binary::BinaryColumnBuilder; use databend_common_expression::types::date::clamp_date; @@ -34,13 +39,9 @@ use databend_common_expression::types::nullable::NullableColumnBuilder; use databend_common_expression::types::number::Number; use databend_common_expression::types::string::StringColumnBuilder; use databend_common_expression::types::vector::VectorColumnBuilder; -use databend_common_expression::types::AnyType; -use databend_common_expression::types::MutableBitmap; -use databend_common_expression::types::NumberColumnBuilder; -use databend_common_expression::types::VectorScalarRef; use databend_common_expression::with_decimal_type; use databend_common_expression::with_number_mapped_type; -use databend_common_expression::ColumnBuilder; +use databend_common_io::Interval; use databend_common_io::constants::FALSE_BYTES_LOWER; use databend_common_io::constants::NULL_BYTES_LOWER; use databend_common_io::constants::NULL_BYTES_UPPER; @@ -53,15 +54,14 @@ use databend_common_io::cursor_ext::ReadNumberExt; use databend_common_io::geography::geography_from_ewkt_bytes; use databend_common_io::parse_bitmap; use databend_common_io::parse_bytes_to_ewkb; -use databend_common_io::Interval; use jsonb::parse_owned_jsonb_with_buf; use lexical_core::FromLexical; +use crate::FileFormatOptionsExt; +use crate::InputCommonSettings; use crate::binary::decode_binary; use crate::field_decoder::common::read_timestamp; use crate::field_decoder::common::read_timestamp_tz; -use crate::FileFormatOptionsExt; -use crate::InputCommonSettings; #[derive(Clone)] pub struct NestedValues { diff --git a/src/query/formats/src/field_decoder/separated_text.rs b/src/query/formats/src/field_decoder/separated_text.rs index 8268f7cd1d541..e348e3da21d7a 100644 --- a/src/query/formats/src/field_decoder/separated_text.rs +++ b/src/query/formats/src/field_decoder/separated_text.rs @@ -21,8 +21,13 @@ use databend_common_column::types::timestamp_tz; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_exception::ToErrorCode; +use databend_common_expression::ColumnBuilder; use databend_common_expression::serialize::read_decimal_with_size; use databend_common_expression::serialize::uniform_date; +use databend_common_expression::types::AnyType; +use databend_common_expression::types::MutableBitmap; +use databend_common_expression::types::Number; +use databend_common_expression::types::NumberColumnBuilder; use databend_common_expression::types::array::ArrayColumnBuilder; use databend_common_expression::types::binary::BinaryColumnBuilder; use databend_common_expression::types::date::clamp_date; @@ -31,38 +36,33 @@ use databend_common_expression::types::decimal::DecimalColumnBuilder; use databend_common_expression::types::decimal::DecimalSize; use databend_common_expression::types::nullable::NullableColumnBuilder; use databend_common_expression::types::vector::VectorColumnBuilder; -use databend_common_expression::types::AnyType; -use databend_common_expression::types::MutableBitmap; -use databend_common_expression::types::Number; -use databend_common_expression::types::NumberColumnBuilder; use databend_common_expression::with_decimal_type; use databend_common_expression::with_number_mapped_type; -use databend_common_expression::ColumnBuilder; +use databend_common_io::Interval; use databend_common_io::constants::FALSE_BYTES_LOWER; use databend_common_io::constants::FALSE_BYTES_NUM; use databend_common_io::constants::NULL_BYTES_ESCAPE; use databend_common_io::constants::TRUE_BYTES_LOWER; use databend_common_io::constants::TRUE_BYTES_NUM; +use databend_common_io::cursor_ext::BufferReadDateTimeExt; use databend_common_io::cursor_ext::collect_number; use databend_common_io::cursor_ext::read_num_text_exact; -use databend_common_io::cursor_ext::BufferReadDateTimeExt; use databend_common_io::geography::geography_from_ewkt_bytes; use databend_common_io::parse_bitmap; use databend_common_io::parse_bytes_to_ewkb; -use databend_common_io::Interval; use databend_common_meta_app::principal::CsvFileFormatParams; use databend_common_meta_app::principal::TsvFileFormatParams; use jsonb::parse_owned_jsonb_with_buf; use lexical_core::FromLexical; use num_traits::NumCast; -use crate::binary::decode_binary; -use crate::field_decoder::common::read_timestamp; -use crate::field_decoder::common::read_timestamp_tz; -use crate::field_decoder::FieldDecoder; use crate::FileFormatOptionsExt; use crate::InputCommonSettings; use crate::NestedValues; +use crate::binary::decode_binary; +use crate::field_decoder::FieldDecoder; +use crate::field_decoder::common::read_timestamp; +use crate::field_decoder::common::read_timestamp_tz; #[derive(Clone)] pub struct SeparatedTextDecoder { diff --git a/src/query/formats/src/field_encoder/csv.rs b/src/query/formats/src/field_encoder/csv.rs index a7ef079bd9ca3..4d36ca96d41a6 100644 --- a/src/query/formats/src/field_encoder/csv.rs +++ b/src/query/formats/src/field_encoder/csv.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_expression::Column; +use databend_common_expression::types::AnyType; use databend_common_expression::types::nullable::NullableColumn; use databend_common_expression::types::opaque::OpaqueColumn; -use databend_common_expression::types::AnyType; -use databend_common_expression::Column; use databend_common_io::constants::FALSE_BYTES_LOWER; use databend_common_io::constants::FALSE_BYTES_NUM; use databend_common_io::constants::INF_BYTES_LONG; @@ -24,14 +24,14 @@ use databend_common_io::constants::TRUE_BYTES_LOWER; use databend_common_io::constants::TRUE_BYTES_NUM; use databend_common_meta_app::principal::CsvFileFormatParams; use databend_common_meta_app::principal::TsvFileFormatParams; -use geozero::wkb::Ewkb; use geozero::ToWkt; +use geozero::wkb::Ewkb; -use crate::binary::encode_binary; -use crate::field_encoder::write_tsv_escaped_string; -use crate::field_encoder::FieldEncoderValues; use crate::FileFormatOptionsExt; use crate::OutputCommonSettings; +use crate::binary::encode_binary; +use crate::field_encoder::FieldEncoderValues; +use crate::field_encoder::write_tsv_escaped_string; pub enum StringFormatter { Csv { quote_char: u8 }, diff --git a/src/query/formats/src/field_encoder/json.rs b/src/query/formats/src/field_encoder/json.rs index 5168e47fd04a8..aafddd1518db2 100644 --- a/src/query/formats/src/field_encoder/json.rs +++ b/src/query/formats/src/field_encoder/json.rs @@ -12,22 +12,22 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_expression::Column; +use databend_common_expression::types::AnyType; use databend_common_expression::types::array::ArrayColumn; use databend_common_expression::types::nullable::NullableColumn; use databend_common_expression::types::opaque::OpaqueColumn; -use databend_common_expression::types::AnyType; -use databend_common_expression::Column; use databend_common_io::constants::FALSE_BYTES_LOWER; use databend_common_io::constants::NULL_BYTES_LOWER; use databend_common_io::constants::TRUE_BYTES_LOWER; -use geozero::wkb::Ewkb; use geozero::ToJson; +use geozero::wkb::Ewkb; use jsonb::RawJsonb; -use crate::field_encoder::helpers::write_json_string; -use crate::field_encoder::FieldEncoderValues; use crate::FileFormatOptionsExt; use crate::OutputCommonSettings; +use crate::field_encoder::FieldEncoderValues; +use crate::field_encoder::helpers::write_json_string; pub struct FieldEncoderJSON { pub simple: FieldEncoderValues, diff --git a/src/query/formats/src/field_encoder/mod.rs b/src/query/formats/src/field_encoder/mod.rs index a93f7c02d659f..cf46f0505bd94 100644 --- a/src/query/formats/src/field_encoder/mod.rs +++ b/src/query/formats/src/field_encoder/mod.rs @@ -17,8 +17,8 @@ pub mod helpers; mod json; mod values; -pub use csv::write_csv_string; pub use csv::FieldEncoderCSV; +pub use csv::write_csv_string; pub use helpers::write_tsv_escaped_string; pub use json::FieldEncoderJSON; pub use values::FieldEncoderValues; diff --git a/src/query/formats/src/field_encoder/values.rs b/src/query/formats/src/field_encoder/values.rs index 9c5979d0bcd49..86c16216cca4b 100644 --- a/src/query/formats/src/field_encoder/values.rs +++ b/src/query/formats/src/field_encoder/values.rs @@ -16,6 +16,14 @@ use chrono_tz::Tz; use databend_common_base::base::OrderedFloat; use databend_common_column::types::months_days_micros; use databend_common_column::types::timestamp_tz; +use databend_common_expression::Column; +use databend_common_expression::types::AnyType; +use databend_common_expression::types::BinaryColumn; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::Buffer; +use databend_common_expression::types::NumberColumn; +use databend_common_expression::types::VectorColumn; +use databend_common_expression::types::VectorScalarRef; use databend_common_expression::types::array::ArrayColumn; use databend_common_expression::types::date::date_to_string; use databend_common_expression::types::decimal::DecimalColumn; @@ -25,14 +33,7 @@ use databend_common_expression::types::nullable::NullableColumn; use databend_common_expression::types::opaque::OpaqueColumn; use databend_common_expression::types::string::StringColumn; use databend_common_expression::types::timestamp::timestamp_to_string; -use databend_common_expression::types::AnyType; -use databend_common_expression::types::BinaryColumn; -use databend_common_expression::types::Bitmap; -use databend_common_expression::types::Buffer; -use databend_common_expression::types::NumberColumn; -use databend_common_expression::types::VectorColumn; -use databend_common_expression::types::VectorScalarRef; -use databend_common_expression::Column; +use databend_common_io::GeometryDataType; use databend_common_io::constants::FALSE_BYTES_NUM; use databend_common_io::constants::INF_BYTES_LONG; use databend_common_io::constants::INF_BYTES_LOWER; @@ -46,7 +47,6 @@ use databend_common_io::geo_to_ewkt; use databend_common_io::geo_to_json; use databend_common_io::geo_to_wkb; use databend_common_io::geo_to_wkt; -use databend_common_io::GeometryDataType; use geozero::wkb::Ewkb; use jiff::tz::TimeZone; use jsonb::RawJsonb; @@ -54,10 +54,10 @@ use lexical_core::ToLexical; use micromarshal::Marshal; use micromarshal::Unmarshal; -use crate::field_encoder::helpers::write_quoted_string; -use crate::field_encoder::helpers::PrimitiveWithFormat; use crate::FileFormatOptionsExt; use crate::OutputCommonSettings; +use crate::field_encoder::helpers::PrimitiveWithFormat; +use crate::field_encoder::helpers::write_quoted_string; pub struct FieldEncoderValues { pub common_settings: OutputCommonSettings, diff --git a/src/query/formats/src/file_format_type.rs b/src/query/formats/src/file_format_type.rs index ba1388e501497..74224d3a40879 100644 --- a/src/query/formats/src/file_format_type.rs +++ b/src/query/formats/src/file_format_type.rs @@ -22,6 +22,7 @@ use databend_common_meta_app::principal::StageFileFormatType; use databend_common_settings::Settings; use jiff::tz::TimeZone; +use crate::ClickhouseFormatType; use crate::output_format::CSVOutputFormat; use crate::output_format::CSVWithNamesAndTypesOutputFormat; use crate::output_format::CSVWithNamesOutputFormat; @@ -32,7 +33,6 @@ use crate::output_format::ParquetOutputFormat; use crate::output_format::TSVOutputFormat; use crate::output_format::TSVWithNamesAndTypesOutputFormat; use crate::output_format::TSVWithNamesOutputFormat; -use crate::ClickhouseFormatType; pub trait FileFormatTypeExt { fn get_content_type(&self) -> String; diff --git a/src/query/formats/src/lib.rs b/src/query/formats/src/lib.rs index 2bc36066b9479..913d80cfdc875 100644 --- a/src/query/formats/src/lib.rs +++ b/src/query/formats/src/lib.rs @@ -14,6 +14,7 @@ #![feature(box_patterns)] #![feature(cursor_split)] +#![allow(clippy::collapsible_if)] mod binary; mod clickhouse; @@ -29,9 +30,9 @@ pub use clickhouse::ClickhouseFormatType; pub use column_from_json::column_from_json_value; pub use delimiter::RecordDelimiter; pub use field_decoder::*; -pub use file_format_type::parse_timezone; pub use file_format_type::FileFormatOptionsExt; pub use file_format_type::FileFormatTypeExt; +pub use file_format_type::parse_timezone; pub use crate::common_settings::InputCommonSettings; pub use crate::common_settings::OutputCommonSettings; diff --git a/src/query/formats/src/output_format/csv.rs b/src/query/formats/src/output_format/csv.rs index b839c5bc43d79..66bf6823b5a1a 100644 --- a/src/query/formats/src/output_format/csv.rs +++ b/src/query/formats/src/output_format/csv.rs @@ -18,10 +18,10 @@ use databend_common_expression::DataBlock; use databend_common_expression::TableSchemaRef; use databend_common_meta_app::principal::CsvFileFormatParams; -use crate::field_encoder::write_csv_string; +use crate::FileFormatOptionsExt; use crate::field_encoder::FieldEncoderCSV; +use crate::field_encoder::write_csv_string; use crate::output_format::OutputFormat; -use crate::FileFormatOptionsExt; pub type CSVOutputFormat = CSVOutputFormatBase; pub type CSVWithNamesOutputFormat = CSVOutputFormatBase; diff --git a/src/query/formats/src/output_format/json.rs b/src/query/formats/src/output_format/json.rs index 94c6649c900c5..177f217947202 100644 --- a/src/query/formats/src/output_format/json.rs +++ b/src/query/formats/src/output_format/json.rs @@ -12,23 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_expression::date_helper::DateConverter; -use databend_common_expression::types::interval::interval_to_string; -use databend_common_expression::types::number::NumberScalar; -use databend_common_expression::types::VectorScalarRef; use databend_common_expression::DataBlock; use databend_common_expression::ScalarRef; use databend_common_expression::TableSchemaRef; +use databend_common_expression::date_helper::DateConverter; +use databend_common_expression::types::VectorScalarRef; +use databend_common_expression::types::interval::interval_to_string; +use databend_common_expression::types::number::NumberScalar; use databend_common_io::deserialize_bitmap; use databend_common_io::prelude::FormatSettings; -use geozero::wkb::Ewkb; use geozero::ToJson; +use geozero::wkb::Ewkb; use jiff::fmt::strtime; use serde_json::Map as JsonMap; use serde_json::Value as JsonValue; -use crate::output_format::OutputFormat; use crate::FileFormatOptionsExt; +use crate::output_format::OutputFormat; pub struct JSONOutputFormat { schema: TableSchemaRef, diff --git a/src/query/formats/src/output_format/mod.rs b/src/query/formats/src/output_format/mod.rs index 2d892d6671c16..015cf635d679e 100644 --- a/src/query/formats/src/output_format/mod.rs +++ b/src/query/formats/src/output_format/mod.rs @@ -47,14 +47,6 @@ pub trait OutputFormat: Send { #[cfg(test)] mod utils { use databend_common_exception::Result; - use databend_common_expression::types::nullable::NullableColumn; - use databend_common_expression::types::number::Float64Type; - use databend_common_expression::types::number::Int32Type; - use databend_common_expression::types::Bitmap; - use databend_common_expression::types::BooleanType; - use databend_common_expression::types::DateType; - use databend_common_expression::types::NumberDataType; - use databend_common_expression::types::StringType; use databend_common_expression::Column; use databend_common_expression::DataBlock; use databend_common_expression::FromData; @@ -62,6 +54,14 @@ mod utils { use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; + use databend_common_expression::types::Bitmap; + use databend_common_expression::types::BooleanType; + use databend_common_expression::types::DateType; + use databend_common_expression::types::NumberDataType; + use databend_common_expression::types::StringType; + use databend_common_expression::types::nullable::NullableColumn; + use databend_common_expression::types::number::Float64Type; + use databend_common_expression::types::number::Int32Type; use databend_common_meta_app::tenant::Tenant; use databend_common_settings::Settings; diff --git a/src/query/formats/src/output_format/ndjson.rs b/src/query/formats/src/output_format/ndjson.rs index c4eaa87051f4b..af1298410fb7e 100644 --- a/src/query/formats/src/output_format/ndjson.rs +++ b/src/query/formats/src/output_format/ndjson.rs @@ -17,9 +17,9 @@ use databend_common_expression::Column; use databend_common_expression::DataBlock; use databend_common_expression::TableSchemaRef; +use crate::FileFormatOptionsExt; use crate::field_encoder::FieldEncoderJSON; use crate::output_format::OutputFormat; -use crate::FileFormatOptionsExt; pub struct NDJSONOutputFormatBase< const STRINGS: bool, @@ -151,13 +151,13 @@ impl; pub type TSVWithNamesOutputFormat = TSVOutputFormatBase; @@ -127,21 +127,21 @@ mod test { use std::collections::BTreeMap; use databend_common_exception::Result; - use databend_common_expression::types::number::Int32Type; - use databend_common_expression::types::NumberDataType; use databend_common_expression::FromData; use databend_common_expression::TableDataType; use databend_common_expression::TableField; + use databend_common_expression::types::NumberDataType; + use databend_common_expression::types::number::Int32Type; use databend_common_meta_app::principal::FileFormatOptionsReader; use databend_common_meta_app::principal::FileFormatParams; use databend_common_meta_app::tenant::Tenant; use databend_common_settings::Settings; use pretty_assertions::assert_eq; + use crate::FileFormatOptionsExt; use crate::output_format::utils::gen_schema_and_block; use crate::output_format::utils::get_output_format_clickhouse; use crate::output_format::utils::get_simple_block; - use crate::FileFormatOptionsExt; fn test_data_block(is_nullable: bool) -> Result<()> { let (schema, block) = get_simple_block(is_nullable); diff --git a/src/query/functions/benches/bench.rs b/src/query/functions/benches/bench.rs index fa47167fb5fdf..ce1883e46dc4d 100644 --- a/src/query/functions/benches/bench.rs +++ b/src/query/functions/benches/bench.rs @@ -29,12 +29,12 @@ fn main() { // ╰─ 102400 1.82 s │ 1.82 s │ 1.82 s │ 1.82 s │ 1 │ 1 #[divan::bench_group(max_time = 0.5)] mod dummy { - use databend_common_expression::type_check; use databend_common_expression::DataBlock; use databend_common_expression::Evaluator; use databend_common_expression::FunctionContext; - use databend_common_functions::test_utils as parser; + use databend_common_expression::type_check; use databend_common_functions::BUILTIN_FUNCTIONS; + use databend_common_functions::test_utils as parser; #[divan::bench(args = [10240, 102400])] fn parse(bencher: divan::Bencher, n: usize) { @@ -71,14 +71,14 @@ mod dummy { #[divan::bench_group(max_time = 0.5)] mod bitmap { - use databend_common_expression::types::number::UInt64Type; - use databend_common_expression::types::BitmapType; use databend_common_expression::BlockEntry; use databend_common_expression::Column; use databend_common_expression::FromData; + use databend_common_expression::types::BitmapType; + use databend_common_expression::types::number::UInt64Type; use databend_common_functions::aggregates::eval_aggr; - use databend_common_io::deserialize_bitmap; use databend_common_io::HybridBitmap; + use databend_common_io::deserialize_bitmap; fn expected_xor_values(rows: usize) -> Vec { const PERIOD: usize = 15; @@ -326,26 +326,26 @@ mod bitmap { mod datetime_fast_path { use std::sync::LazyLock; - use databend_common_expression::date_helper::DateConverter; - use databend_common_expression::type_check; - use databend_common_expression::types::string::StringColumn; - use databend_common_expression::types::string::StringColumnBuilder; - use databend_common_expression::types::timestamp::microseconds_to_days; - use databend_common_expression::types::timestamp::timestamp_to_string; - use databend_common_expression::types::DataType; use databend_common_expression::BlockEntry; use databend_common_expression::Column; use databend_common_expression::DataBlock; use databend_common_expression::Evaluator; use databend_common_expression::Expr; use databend_common_expression::FunctionContext; - use databend_common_functions::test_utils as parser; + use databend_common_expression::date_helper::DateConverter; + use databend_common_expression::type_check; + use databend_common_expression::types::DataType; + use databend_common_expression::types::string::StringColumn; + use databend_common_expression::types::string::StringColumnBuilder; + use databend_common_expression::types::timestamp::microseconds_to_days; + use databend_common_expression::types::timestamp::timestamp_to_string; use databend_common_functions::BUILTIN_FUNCTIONS; + use databend_common_functions::test_utils as parser; use jiff::civil::date; use jiff::tz::TimeZone; - use rand::rngs::StdRng; use rand::Rng; use rand::SeedableRng; + use rand::rngs::StdRng; const ROWS: usize = 100_000; const SPECIAL_EVERY: usize = 20_000; diff --git a/src/query/functions/src/aggregates/adaptors/aggregate_combinator_distinct.rs b/src/query/functions/src/aggregates/adaptors/aggregate_combinator_distinct.rs index 68b973c615013..8d48d95911f6c 100644 --- a/src/query/functions/src/aggregates/adaptors/aggregate_combinator_distinct.rs +++ b/src/query/functions/src/aggregates/adaptors/aggregate_combinator_distinct.rs @@ -18,11 +18,6 @@ use std::marker::PhantomData; use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::types::number::NumberColumnBuilder; -use databend_common_expression::types::Bitmap; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::with_number_mapped_type; use databend_common_expression::AggrStateRegistry; use databend_common_expression::AggrStateType; use databend_common_expression::BlockEntry; @@ -30,14 +25,12 @@ use databend_common_expression::ColumnBuilder; use databend_common_expression::ProjectedBlock; use databend_common_expression::Scalar; use databend_common_expression::StateSerdeItem; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::number::NumberColumnBuilder; +use databend_common_expression::with_number_mapped_type; -use super::aggregate_distinct_state::AggregateDistinctNumberState; -use super::aggregate_distinct_state::AggregateDistinctState; -use super::aggregate_distinct_state::AggregateDistinctStringState; -use super::aggregate_distinct_state::AggregateUniqStringState; -use super::aggregate_distinct_state::DistinctStateFunc; -use super::aggregate_null_result::AggregateNullResultFunction; -use super::assert_variadic_arguments; use super::AggrState; use super::AggrStateLoc; use super::AggregateCountFunction; @@ -49,6 +42,13 @@ use super::AggregateFunctionFeatures; use super::AggregateFunctionSortDesc; use super::CombinatorDescription; use super::StateAddr; +use super::aggregate_distinct_state::AggregateDistinctNumberState; +use super::aggregate_distinct_state::AggregateDistinctState; +use super::aggregate_distinct_state::AggregateDistinctStringState; +use super::aggregate_distinct_state::AggregateUniqStringState; +use super::aggregate_distinct_state::DistinctStateFunc; +use super::aggregate_null_result::AggregateNullResultFunction; +use super::assert_variadic_arguments; pub struct AggregateDistinctCombinator { name: String, @@ -76,7 +76,7 @@ impl Clone for AggregateDistinctCombinator { impl AggregateDistinctCombinator where State: Send + 'static { - fn get_state(place: AggrState) -> &mut State { + fn get_state(place: AggrState<'_>) -> &mut State { place .addr .next(place.loc[0].into_custom().unwrap().1) @@ -193,10 +193,10 @@ where State: DistinctStateFunc unsafe fn drop_state(&self, place: AggrState) { let state = Self::get_state(place); - std::ptr::drop_in_place(state); + unsafe { std::ptr::drop_in_place(state) }; if self.nested.need_manual_drop_state() { - self.nested.drop_state(place.remove_first_loc()); + unsafe { self.nested.drop_state(place.remove_first_loc()) }; } } diff --git a/src/query/functions/src/aggregates/adaptors/aggregate_combinator_if.rs b/src/query/functions/src/aggregates/adaptors/aggregate_combinator_if.rs index 97fe39e85136f..3b04f8c79ebeb 100644 --- a/src/query/functions/src/aggregates/adaptors/aggregate_combinator_if.rs +++ b/src/query/functions/src/aggregates/adaptors/aggregate_combinator_if.rs @@ -17,10 +17,6 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::AccessType; -use databend_common_expression::types::Bitmap; -use databend_common_expression::types::BooleanType; -use databend_common_expression::types::DataType; use databend_common_expression::AggrStateRegistry; use databend_common_expression::BlockEntry; use databend_common_expression::Column; @@ -29,6 +25,10 @@ use databend_common_expression::ColumnView; use databend_common_expression::ProjectedBlock; use databend_common_expression::Scalar; use databend_common_expression::StateSerdeItem; +use databend_common_expression::types::AccessType; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::BooleanType; +use databend_common_expression::types::DataType; use super::AggrState; use super::AggrStateLoc; @@ -233,7 +233,7 @@ impl AggregateFunction for AggregateIfCombinator { } unsafe fn drop_state(&self, place: AggrState) { - self.nested.drop_state(place); + unsafe { self.nested.drop_state(place) }; } fn get_if_condition(&self, entries: ProjectedBlock) -> Option { diff --git a/src/query/functions/src/aggregates/adaptors/aggregate_combinator_state.rs b/src/query/functions/src/aggregates/adaptors/aggregate_combinator_state.rs index 9a18492641528..659f25e57d983 100644 --- a/src/query/functions/src/aggregates/adaptors/aggregate_combinator_state.rs +++ b/src/query/functions/src/aggregates/adaptors/aggregate_combinator_state.rs @@ -16,14 +16,14 @@ use std::fmt; use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::types::Bitmap; -use databend_common_expression::types::DataType; use databend_common_expression::AggrStateRegistry; use databend_common_expression::BlockEntry; use databend_common_expression::ColumnBuilder; use databend_common_expression::ProjectedBlock; use databend_common_expression::Scalar; use databend_common_expression::StateSerdeItem; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::DataType; use super::AggrState; use super::AggrStateLoc; @@ -151,7 +151,7 @@ impl AggregateFunction for AggregateStateCombinator { } unsafe fn drop_state(&self, place: AggrState) { - self.nested.drop_state(place); + unsafe { self.nested.drop_state(place) }; } fn get_own_null_adaptor( diff --git a/src/query/functions/src/aggregates/adaptors/aggregate_null_adaptor.rs b/src/query/functions/src/aggregates/adaptors/aggregate_null_adaptor.rs index 6d227428c5ffb..e8385a4eda105 100644 --- a/src/query/functions/src/aggregates/adaptors/aggregate_null_adaptor.rs +++ b/src/query/functions/src/aggregates/adaptors/aggregate_null_adaptor.rs @@ -17,16 +17,16 @@ use std::sync::Arc; use databend_common_base::hints::assume; use databend_common_exception::Result; -use databend_common_expression::types::Bitmap; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::utils::column_merge_validity; use databend_common_expression::BlockEntry; use databend_common_expression::Column; use databend_common_expression::ColumnBuilder; use databend_common_expression::ProjectedBlock; use databend_common_expression::Scalar; use databend_common_expression::StateSerdeItem; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::utils::column_merge_validity; use super::AggrState; use super::AggrStateLoc; @@ -215,7 +215,7 @@ impl AggregateFunction for AggregateNullUnaryAdapto } unsafe fn drop_state(&self, place: AggrState) { - self.0.drop_state(place); + unsafe { self.0.drop_state(place) }; } fn get_if_condition(&self, columns: ProjectedBlock) -> Option { @@ -348,7 +348,7 @@ impl AggregateFunction } unsafe fn drop_state(&self, place: AggrState) { - self.0.drop_state(place); + unsafe { self.0.drop_state(place) }; } fn get_if_condition(&self, columns: ProjectedBlock) -> Option { @@ -638,7 +638,7 @@ impl CommonNullAdaptor { return self.nested.merge_result(place, read_only, builder); } - let ColumnBuilder::Nullable(ref mut inner) = builder else { + let ColumnBuilder::Nullable(inner) = builder else { unreachable!() }; @@ -654,9 +654,9 @@ impl CommonNullAdaptor { unsafe fn drop_state(&self, place: AggrState) { if !NULLABLE_RESULT { - self.nested.drop_state(place) + unsafe { self.nested.drop_state(place) } } else { - self.nested.drop_state(place.remove_last_loc()) + unsafe { self.nested.drop_state(place.remove_last_loc()) } } } diff --git a/src/query/functions/src/aggregates/adaptors/aggregate_ornull_adaptor.rs b/src/query/functions/src/aggregates/adaptors/aggregate_ornull_adaptor.rs index 8176eecfb193f..0232c0bcc3b0c 100644 --- a/src/query/functions/src/aggregates/adaptors/aggregate_ornull_adaptor.rs +++ b/src/query/functions/src/aggregates/adaptors/aggregate_ornull_adaptor.rs @@ -17,8 +17,6 @@ use std::sync::Arc; use databend_common_base::hints::assume; use databend_common_exception::Result; -use databend_common_expression::types::Bitmap; -use databend_common_expression::types::DataType; use databend_common_expression::AggrStateRegistry; use databend_common_expression::AggrStateType; use databend_common_expression::BlockEntry; @@ -27,6 +25,8 @@ use databend_common_expression::ColumnBuilder; use databend_common_expression::ProjectedBlock; use databend_common_expression::Scalar; use databend_common_expression::StateSerdeItem; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::DataType; use super::AggrState; use super::AggrStateLoc; @@ -313,7 +313,7 @@ impl AggregateFunction for AggregateFunctionOrNullAdaptor { } unsafe fn drop_state(&self, place: AggrState) { - self.nested.drop_state(place.remove_last_loc()) + unsafe { self.nested.drop_state(place.remove_last_loc()) } } } diff --git a/src/query/functions/src/aggregates/adaptors/aggregate_sort_adaptor.rs b/src/query/functions/src/aggregates/adaptors/aggregate_sort_adaptor.rs index 722b630ec8f6c..fe063aa2056a9 100644 --- a/src/query/functions/src/aggregates/adaptors/aggregate_sort_adaptor.rs +++ b/src/query/functions/src/aggregates/adaptors/aggregate_sort_adaptor.rs @@ -22,9 +22,6 @@ use borsh::BorshDeserialize; use borsh::BorshSerialize; use databend_common_column::bitmap::Bitmap; use databend_common_exception::Result; -use databend_common_expression::types::AnyType; -use databend_common_expression::types::BinaryType; -use databend_common_expression::types::DataType; use databend_common_expression::AggrState; use databend_common_expression::AggrStateLoc; use databend_common_expression::AggrStateRegistry; @@ -39,13 +36,16 @@ use databend_common_expression::ProjectedBlock; use databend_common_expression::SortColumnDescription; use databend_common_expression::StateAddr; use databend_common_expression::StateSerdeItem; +use databend_common_expression::types::AnyType; +use databend_common_expression::types::BinaryType; +use databend_common_expression::types::DataType; use itertools::Itertools; -use super::batch_merge1; -use super::batch_serialize1; use super::AggregateFunctionSortDesc; use super::SerializeInfo; use super::StateSerde; +use super::batch_merge1; +use super::batch_serialize1; #[derive(Debug, Clone)] pub struct SortAggState { @@ -270,10 +270,10 @@ impl AggregateFunction for AggregateFunctionSortAdaptor { unsafe fn drop_state(&self, place: AggrState) { let state = Self::get_state(place); - std::ptr::drop_in_place(state); + unsafe { std::ptr::drop_in_place(state) }; if self.inner.need_manual_drop_state() { - self.inner.drop_state(place.remove_first_loc()); + unsafe { self.inner.drop_state(place.remove_first_loc()) }; } } @@ -297,7 +297,7 @@ impl AggregateFunctionSortAdaptor { })) } - fn get_state(place: AggrState) -> &mut SortAggState { + fn get_state(place: AggrState<'_>) -> &mut SortAggState { place .addr .next(place.loc[0].into_custom().unwrap().1) diff --git a/src/query/functions/src/aggregates/aggregate_approx_count_distinct.rs b/src/query/functions/src/aggregates/aggregate_approx_count_distinct.rs index cf70334655470..621afd2aafcc3 100644 --- a/src/query/functions/src/aggregates/aggregate_approx_count_distinct.rs +++ b/src/query/functions/src/aggregates/aggregate_approx_count_distinct.rs @@ -18,19 +18,16 @@ use std::sync::Arc; use borsh::BorshDeserialize; use borsh::BorshSerialize; use databend_common_exception::Result; -use databend_common_expression::types::*; -use databend_common_expression::with_number_mapped_type; use databend_common_expression::AggrStateLoc; use databend_common_expression::BlockEntry; use databend_common_expression::ColumnBuilder; use databend_common_expression::Scalar; use databend_common_expression::StateAddr; use databend_common_expression::StateSerdeItem; +use databend_common_expression::types::*; +use databend_common_expression::with_number_mapped_type; use simple_hll::HyperLogLog; -use super::assert_unary_arguments; -use super::batch_merge1; -use super::extract_number_param; use super::AggrState; use super::AggregateFunction; use super::AggregateFunctionDescription; @@ -39,6 +36,9 @@ use super::AggregateUnaryFunction; use super::SerializeInfo; use super::StateSerde; use super::UnaryState; +use super::assert_unary_arguments; +use super::batch_merge1; +use super::extract_number_param; /// Use Hyperloglog to estimate distinct of values type AggregateApproxCountDistinctState = HyperLogLog; diff --git a/src/query/functions/src/aggregates/aggregate_arg_min_max.rs b/src/query/functions/src/aggregates/aggregate_arg_min_max.rs index 5f6adfe1420d4..65f8434c7176b 100644 --- a/src/query/functions/src/aggregates/aggregate_arg_min_max.rs +++ b/src/query/functions/src/aggregates/aggregate_arg_min_max.rs @@ -19,10 +19,6 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::number::*; -use databend_common_expression::types::Bitmap; -use databend_common_expression::types::*; -use databend_common_expression::with_number_mapped_type; use databend_common_expression::AggrStateRegistry; use databend_common_expression::AggrStateType; use databend_common_expression::BlockEntry; @@ -31,7 +27,18 @@ use databend_common_expression::ColumnView; use databend_common_expression::ProjectedBlock; use databend_common_expression::Scalar; use databend_common_expression::StateSerdeItem; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::number::*; +use databend_common_expression::types::*; +use databend_common_expression::with_number_mapped_type; +use super::AggrState; +use super::AggrStateLoc; +use super::AggregateFunction; +use super::AggregateFunctionDescription; +use super::AggregateFunctionRef; +use super::AggregateFunctionSortDesc; +use super::StateAddr; use super::aggregate_scalar_state::ChangeIf; use super::aggregate_scalar_state::CmpAny; use super::aggregate_scalar_state::CmpMax; @@ -43,13 +50,6 @@ use super::assert_binary_arguments; use super::assert_params; use super::batch_merge3; use super::batch_serialize3; -use super::AggrState; -use super::AggrStateLoc; -use super::AggregateFunction; -use super::AggregateFunctionDescription; -use super::AggregateFunctionRef; -use super::AggregateFunctionSortDesc; -use super::StateAddr; use crate::with_compare_mapped_type; use crate::with_simple_no_number_mapped_type; @@ -326,7 +326,7 @@ where unsafe fn drop_state(&self, place: AggrState) { let state = place.get::>(); - std::ptr::drop_in_place(state); + unsafe { std::ptr::drop_in_place(state) }; } } diff --git a/src/query/functions/src/aggregates/aggregate_array_agg.rs b/src/query/functions/src/aggregates/aggregate_array_agg.rs index f375a77449f86..181342987c9e9 100644 --- a/src/query/functions/src/aggregates/aggregate_array_agg.rs +++ b/src/query/functions/src/aggregates/aggregate_array_agg.rs @@ -23,6 +23,19 @@ use borsh::BorshDeserialize; use borsh::BorshSerialize; use databend_common_column::binary::BinaryColumnBuilder; use databend_common_exception::Result; +use databend_common_expression::AggrStateRegistry; +use databend_common_expression::AggrStateType; +use databend_common_expression::BlockEntry; +use databend_common_expression::Column; +use databend_common_expression::ColumnBuilder; +use databend_common_expression::ColumnView; +use databend_common_expression::ProjectedBlock; +use databend_common_expression::Scalar; +use databend_common_expression::ScalarRef; +use databend_common_expression::StateSerdeItem; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::DataType; +use databend_common_expression::types::ValueType; use databend_common_expression::types::date::CoreDate; use databend_common_expression::types::decimal::*; use databend_common_expression::types::empty_array::CoreEmptyArray; @@ -37,28 +50,10 @@ use databend_common_expression::types::string::StringColumnBuilder; use databend_common_expression::types::timestamp::CoreTimestamp; use databend_common_expression::types::zero_size_type::ZeroSizeType; use databend_common_expression::types::zero_size_type::ZeroSizeValueType; -use databend_common_expression::types::Bitmap; -use databend_common_expression::types::DataType; -use databend_common_expression::types::ValueType; use databend_common_expression::types::*; use databend_common_expression::with_decimal_mapped_type; use databend_common_expression::with_number_mapped_type; -use databend_common_expression::AggrStateRegistry; -use databend_common_expression::AggrStateType; -use databend_common_expression::BlockEntry; -use databend_common_expression::Column; -use databend_common_expression::ColumnBuilder; -use databend_common_expression::ColumnView; -use databend_common_expression::ProjectedBlock; -use databend_common_expression::Scalar; -use databend_common_expression::ScalarRef; -use databend_common_expression::StateSerdeItem; -use super::aggregate_scalar_state::ScalarStateFunc; -use super::assert_params; -use super::assert_unary_arguments; -use super::batch_merge1; -use super::batch_serialize1; use super::AggrState; use super::AggrStateLoc; use super::AggregateFunction; @@ -68,6 +63,11 @@ use super::AggregateFunctionSortDesc; use super::SerializeInfo; use super::StateAddr; use super::StateSerde; +use super::aggregate_scalar_state::ScalarStateFunc; +use super::assert_params; +use super::assert_unary_arguments; +use super::batch_merge1; +use super::batch_serialize1; #[derive(Debug)] struct ArrayAggStateAny @@ -699,7 +699,7 @@ where unsafe fn drop_state(&self, place: AggrState) { let state = place.get::(); - std::ptr::drop_in_place(state); + unsafe { std::ptr::drop_in_place(state) }; } } diff --git a/src/query/functions/src/aggregates/aggregate_array_moving.rs b/src/query/functions/src/aggregates/aggregate_array_moving.rs index 6f3664feadb81..72d8352636a9a 100644 --- a/src/query/functions/src/aggregates/aggregate_array_moving.rs +++ b/src/query/functions/src/aggregates/aggregate_array_moving.rs @@ -20,10 +20,6 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::*; -use databend_common_expression::utils::arithmetics_type::ResultTypeOfUnary; -use databend_common_expression::with_decimal_mapped_type; -use databend_common_expression::with_number_mapped_type; use databend_common_expression::AggrStateRegistry; use databend_common_expression::AggrStateType; use databend_common_expression::BlockEntry; @@ -33,13 +29,12 @@ use databend_common_expression::ProjectedBlock; use databend_common_expression::Scalar; use databend_common_expression::ScalarRef; use databend_common_expression::StateSerdeItem; +use databend_common_expression::types::*; +use databend_common_expression::utils::arithmetics_type::ResultTypeOfUnary; +use databend_common_expression::with_decimal_mapped_type; +use databend_common_expression::with_number_mapped_type; use num_traits::AsPrimitive; -use super::assert_unary_arguments; -use super::assert_variadic_params; -use super::batch_merge1; -use super::batch_serialize1; -use super::extract_number_param; use super::AggrState; use super::AggrStateLoc; use super::AggregateFunction; @@ -50,6 +45,11 @@ use super::AggregateFunctionSortDesc; use super::SerializeInfo; use super::StateAddr; use super::StateSerde; +use super::assert_unary_arguments; +use super::assert_variadic_params; +use super::batch_merge1; +use super::batch_serialize1; +use super::extract_number_param; trait SumState: StateSerde + Send + Default + 'static { fn merge(&mut self, other: &Self) -> Result<()>; @@ -632,7 +632,7 @@ where State: SumState unsafe fn drop_state(&self, place: AggrState) { let state = place.get::(); - std::ptr::drop_in_place(state); + unsafe { std::ptr::drop_in_place(state) }; } } @@ -823,7 +823,7 @@ where State: SumState unsafe fn drop_state(&self, place: AggrState) { let state = place.get::(); - std::ptr::drop_in_place(state); + unsafe { std::ptr::drop_in_place(state) }; } } diff --git a/src/query/functions/src/aggregates/aggregate_avg.rs b/src/query/functions/src/aggregates/aggregate_avg.rs index 040744870adae..4a3f9f7ecdb2c 100644 --- a/src/query/functions/src/aggregates/aggregate_avg.rs +++ b/src/query/functions/src/aggregates/aggregate_avg.rs @@ -16,23 +16,19 @@ use std::marker::PhantomData; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::decimal::*; -use databend_common_expression::types::*; -use databend_common_expression::utils::arithmetics_type::ResultTypeOfUnary; -use databend_common_expression::with_decimal_mapped_type; -use databend_common_expression::with_number_mapped_type; use databend_common_expression::AggrStateLoc; use databend_common_expression::BlockEntry; use databend_common_expression::ColumnBuilder; use databend_common_expression::Scalar; use databend_common_expression::StateAddr; use databend_common_expression::StateSerdeItem; +use databend_common_expression::types::decimal::*; +use databend_common_expression::types::*; +use databend_common_expression::utils::arithmetics_type::ResultTypeOfUnary; +use databend_common_expression::with_decimal_mapped_type; +use databend_common_expression::with_number_mapped_type; use num_traits::AsPrimitive; -use super::assert_params; -use super::assert_unary_arguments; -use super::batch_merge2; -use super::batch_serialize2; use super::AggregateFunctionDescription; use super::AggregateFunctionRef; use super::AggregateFunctionSortDesc; @@ -40,6 +36,10 @@ use super::AggregateUnaryFunction; use super::SerializeInfo; use super::StateSerde; use super::UnaryState; +use super::assert_params; +use super::assert_unary_arguments; +use super::batch_merge2; +use super::batch_serialize2; struct NumberAvgState where TSum: ValueType diff --git a/src/query/functions/src/aggregates/aggregate_bitmap.rs b/src/query/functions/src/aggregates/aggregate_bitmap.rs index dcdde29a53dac..f617125701834 100644 --- a/src/query/functions/src/aggregates/aggregate_bitmap.rs +++ b/src/query/functions/src/aggregates/aggregate_bitmap.rs @@ -24,15 +24,6 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::decimal::DecimalType; -use databend_common_expression::types::i256; -use databend_common_expression::types::number::Number; -use databend_common_expression::types::Bitmap; -use databend_common_expression::types::MutableBitmap; -use databend_common_expression::types::*; -use databend_common_expression::with_decimal_mapped_type; -use databend_common_expression::with_number_mapped_type; -use databend_common_expression::with_unsigned_integer_mapped_type; use databend_common_expression::AggrStateRegistry; use databend_common_expression::AggrStateType; use databend_common_expression::BlockEntry; @@ -40,15 +31,19 @@ use databend_common_expression::ColumnBuilder; use databend_common_expression::ProjectedBlock; use databend_common_expression::Scalar; use databend_common_expression::StateSerdeItem; -use databend_common_io::prelude::BinaryWrite; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::MutableBitmap; +use databend_common_expression::types::decimal::DecimalType; +use databend_common_expression::types::i256; +use databend_common_expression::types::number::Number; +use databend_common_expression::types::*; +use databend_common_expression::with_decimal_mapped_type; +use databend_common_expression::with_number_mapped_type; +use databend_common_expression::with_unsigned_integer_mapped_type; use databend_common_io::HybridBitmap; +use databend_common_io::prelude::BinaryWrite; use num_traits::AsPrimitive; -use super::assert_arguments; -use super::assert_params; -use super::assert_unary_arguments; -use super::assert_variadic_params; -use super::extract_number_param; use super::AggrState; use super::AggrStateLoc; use super::AggregateFunction; @@ -56,6 +51,11 @@ use super::AggregateFunctionDescription; use super::AggregateFunctionSortDesc; use super::StateAddr; use super::StateAddrs; +use super::assert_arguments; +use super::assert_params; +use super::assert_unary_arguments; +use super::assert_variadic_params; +use super::extract_number_param; use crate::with_simple_no_number_mapped_type; #[derive(Clone)] @@ -388,7 +388,7 @@ where unsafe fn drop_state(&self, place: AggrState) { let state = place.get::(); - std::ptr::drop_in_place(state); + unsafe { std::ptr::drop_in_place(state) }; } } @@ -575,7 +575,7 @@ where unsafe fn drop_state(&self, place: AggrState) { let state = place.get::(); - std::ptr::drop_in_place(state); + unsafe { std::ptr::drop_in_place(state) }; } } diff --git a/src/query/functions/src/aggregates/aggregate_boolean.rs b/src/query/functions/src/aggregates/aggregate_boolean.rs index 4c0b3a42183ef..df08d4bbf45c3 100644 --- a/src/query/functions/src/aggregates/aggregate_boolean.rs +++ b/src/query/functions/src/aggregates/aggregate_boolean.rs @@ -15,22 +15,19 @@ use boolean::TrueIdxIter; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::Bitmap; -use databend_common_expression::types::BuilderMut; -use databend_common_expression::types::*; use databend_common_expression::AggrStateLoc; use databend_common_expression::AggregateFunctionRef; use databend_common_expression::BlockEntry; use databend_common_expression::ColumnBuilder; use databend_common_expression::ColumnView; +use databend_common_expression::SELECTIVITY_THRESHOLD; use databend_common_expression::Scalar; use databend_common_expression::StateAddr; use databend_common_expression::StateSerdeItem; -use databend_common_expression::SELECTIVITY_THRESHOLD; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::BuilderMut; +use databend_common_expression::types::*; -use super::assert_params; -use super::assert_unary_arguments; -use super::batch_merge1; use super::AggrState; use super::AggregateFunctionDescription; use super::AggregateFunctionSortDesc; @@ -38,6 +35,9 @@ use super::AggregateUnaryFunction; use super::SerializeInfo; use super::StateSerde; use super::UnaryState; +use super::assert_params; +use super::assert_unary_arguments; +use super::batch_merge1; pub struct BooleanState { pub value: bool, diff --git a/src/query/functions/src/aggregates/aggregate_count.rs b/src/query/functions/src/aggregates/aggregate_count.rs index b878a44855fc3..aac7e55192b2f 100644 --- a/src/query/functions/src/aggregates/aggregate_count.rs +++ b/src/query/functions/src/aggregates/aggregate_count.rs @@ -17,15 +17,6 @@ use std::fmt; use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::types::number::NumberColumnBuilder; -use databend_common_expression::types::ArgType; -use databend_common_expression::types::Bitmap; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::UInt64Type; -use databend_common_expression::types::UnaryType; -use databend_common_expression::types::ValueType; -use databend_common_expression::utils::column_merge_validity; use databend_common_expression::AggrStateRegistry; use databend_common_expression::AggrStateType; use databend_common_expression::BlockEntry; @@ -34,15 +25,24 @@ use databend_common_expression::ColumnBuilder; use databend_common_expression::ProjectedBlock; use databend_common_expression::Scalar; use databend_common_expression::StateSerdeItem; +use databend_common_expression::types::ArgType; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::UInt64Type; +use databend_common_expression::types::UnaryType; +use databend_common_expression::types::ValueType; +use databend_common_expression::types::number::NumberColumnBuilder; +use databend_common_expression::utils::column_merge_validity; -use super::assert_params; -use super::assert_variadic_arguments; use super::AggrState; use super::AggrStateLoc; use super::AggregateFunction; use super::AggregateFunctionDescription; use super::AggregateFunctionSortDesc; use super::StateAddr; +use super::assert_params; +use super::assert_variadic_arguments; struct AggregateCountState { count: u64, diff --git a/src/query/functions/src/aggregates/aggregate_covariance.rs b/src/query/functions/src/aggregates/aggregate_covariance.rs index ea99e438f5d1b..40e04fb163b96 100644 --- a/src/query/functions/src/aggregates/aggregate_covariance.rs +++ b/src/query/functions/src/aggregates/aggregate_covariance.rs @@ -21,8 +21,13 @@ use borsh::BorshDeserialize; use borsh::BorshSerialize; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::number::Number; -use databend_common_expression::types::number::F64; +use databend_common_expression::AggrStateRegistry; +use databend_common_expression::AggrStateType; +use databend_common_expression::BlockEntry; +use databend_common_expression::ColumnBuilder; +use databend_common_expression::ProjectedBlock; +use databend_common_expression::Scalar; +use databend_common_expression::StateSerdeItem; use databend_common_expression::types::BinaryType; use databend_common_expression::types::Bitmap; use databend_common_expression::types::DataType; @@ -30,19 +35,11 @@ use databend_common_expression::types::NumberDataType; use databend_common_expression::types::NumberType; use databend_common_expression::types::UnaryType; use databend_common_expression::types::ValueType; +use databend_common_expression::types::number::F64; +use databend_common_expression::types::number::Number; use databend_common_expression::with_number_mapped_type; -use databend_common_expression::AggrStateRegistry; -use databend_common_expression::AggrStateType; -use databend_common_expression::BlockEntry; -use databend_common_expression::ColumnBuilder; -use databend_common_expression::ProjectedBlock; -use databend_common_expression::Scalar; -use databend_common_expression::StateSerdeItem; use num_traits::AsPrimitive; -use super::aggregator_common::assert_binary_arguments; -use super::aggregator_common::assert_params; -use super::borsh_partial_deserialize; use super::AggrState; use super::AggrStateLoc; use super::AggregateFunction; @@ -50,6 +47,9 @@ use super::AggregateFunctionDescription; use super::AggregateFunctionRef; use super::AggregateFunctionSortDesc; use super::StateAddr; +use super::aggregator_common::assert_binary_arguments; +use super::aggregator_common::assert_params; +use super::borsh_partial_deserialize; #[derive(BorshSerialize, BorshDeserialize)] pub struct AggregateCovarianceState { diff --git a/src/query/functions/src/aggregates/aggregate_distinct_state.rs b/src/query/functions/src/aggregates/aggregate_distinct_state.rs index 116f30cd98be6..7583340ee2dc8 100644 --- a/src/query/functions/src/aggregates/aggregate_distinct_state.rs +++ b/src/query/functions/src/aggregates/aggregate_distinct_state.rs @@ -20,8 +20,6 @@ use std::sync::Arc; use borsh::BorshSerialize; use bumpalo::Bump; use databend_common_exception::Result; -use databend_common_expression::types::string::StringColumnBuilder; -use databend_common_expression::types::*; use databend_common_expression::AggrState; use databend_common_expression::AggrStateLoc; use databend_common_expression::BlockEntry; @@ -30,6 +28,8 @@ use databend_common_expression::ColumnBuilder; use databend_common_expression::ProjectedBlock; use databend_common_expression::Scalar; use databend_common_expression::StateSerdeItem; +use databend_common_expression::types::string::StringColumnBuilder; +use databend_common_expression::types::*; use databend_common_hashtable::HashSet as CommonHashSet; use databend_common_hashtable::HashtableKeyable; use databend_common_hashtable::HashtableLike; @@ -39,12 +39,12 @@ use databend_common_io::prelude::*; use siphasher::sip128::Hasher128; use siphasher::sip128::SipHasher24; -use super::batch_merge1; -use super::batch_serialize1; -use super::borsh_partial_deserialize; use super::SerializeInfo; use super::StateAddr; use super::StateSerde; +use super::batch_merge1; +use super::batch_serialize1; +use super::borsh_partial_deserialize; pub(super) trait DistinctStateFunc: Sized + Send + StateSerde + 'static { fn new() -> Self; diff --git a/src/query/functions/src/aggregates/aggregate_function_factory.rs b/src/query/functions/src/aggregates/aggregate_function_factory.rs index 96fde1d6bf569..19ac63beadf73 100644 --- a/src/query/functions/src/aggregates/aggregate_function_factory.rs +++ b/src/query/functions/src/aggregates/aggregate_function_factory.rs @@ -18,8 +18,8 @@ use std::sync::LazyLock; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::Scalar; +use databend_common_expression::types::DataType; use super::AggregateFunctionCombinatorNull; use super::AggregateFunctionOrNullAdaptor; diff --git a/src/query/functions/src/aggregates/aggregate_histogram.rs b/src/query/functions/src/aggregates/aggregate_histogram.rs index 201cfd381f19e..80ace1cd2f40f 100644 --- a/src/query/functions/src/aggregates/aggregate_histogram.rs +++ b/src/query/functions/src/aggregates/aggregate_histogram.rs @@ -21,22 +21,20 @@ use borsh::BorshDeserialize; use borsh::BorshSerialize; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::decimal::*; -use databend_common_expression::types::number::*; -use databend_common_expression::types::*; -use databend_common_expression::with_decimal_mapped_type; -use databend_common_expression::with_number_mapped_type; use databend_common_expression::AggrStateLoc; use databend_common_expression::AggregateFunctionRef; use databend_common_expression::BlockEntry; use databend_common_expression::ColumnBuilder; use databend_common_expression::Scalar; use databend_common_expression::StateAddr; +use databend_common_expression::types::decimal::*; +use databend_common_expression::types::number::*; +use databend_common_expression::types::*; +use databend_common_expression::with_decimal_mapped_type; +use databend_common_expression::with_number_mapped_type; use serde::Deserialize; use serde::Serialize; -use super::assert_variadic_arguments; -use super::batch_merge1; use super::AggrState; use super::AggregateFunctionDescription; use super::AggregateFunctionSortDesc; @@ -45,6 +43,8 @@ use super::SerializeInfo; use super::StateSerde; use super::StateSerdeItem; use super::UnaryState; +use super::assert_variadic_arguments; +use super::batch_merge1; pub struct HistogramData { max_num_buckets: u64, diff --git a/src/query/functions/src/aggregates/aggregate_json_array_agg.rs b/src/query/functions/src/aggregates/aggregate_json_array_agg.rs index bc8a1950e2b9b..506e0843a524f 100644 --- a/src/query/functions/src/aggregates/aggregate_json_array_agg.rs +++ b/src/query/functions/src/aggregates/aggregate_json_array_agg.rs @@ -22,12 +22,6 @@ use borsh::BorshDeserialize; use borsh::BorshSerialize; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::variant::cast_scalar_to_variant; -use databend_common_expression::types::BinaryType; -use databend_common_expression::types::Bitmap; -use databend_common_expression::types::DataType; -use databend_common_expression::types::ValueType; -use databend_common_expression::types::*; use databend_common_expression::AggrStateRegistry; use databend_common_expression::AggrStateType; use databend_common_expression::BlockEntry; @@ -38,14 +32,16 @@ use databend_common_expression::ProjectedBlock; use databend_common_expression::Scalar; use databend_common_expression::ScalarRef; use databend_common_expression::StateSerdeItem; +use databend_common_expression::types::BinaryType; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::DataType; +use databend_common_expression::types::ValueType; +use databend_common_expression::types::variant::cast_scalar_to_variant; +use databend_common_expression::types::*; use jiff::tz::TimeZone; use jsonb::OwnedJsonb; use jsonb::RawJsonb; -use super::aggregate_scalar_state::ScalarStateFunc; -use super::assert_params; -use super::assert_unary_arguments; -use super::batch_merge1; use super::AggrState; use super::AggrStateLoc; use super::AggregateFunction; @@ -54,6 +50,10 @@ use super::AggregateFunctionFeatures; use super::AggregateFunctionSortDesc; use super::StateAddr; use super::StateSerde; +use super::aggregate_scalar_state::ScalarStateFunc; +use super::assert_params; +use super::assert_unary_arguments; +use super::batch_merge1; #[derive(BorshSerialize, BorshDeserialize, Debug)] pub struct JsonArrayAggState @@ -320,7 +320,7 @@ where unsafe fn drop_state(&self, place: AggrState) { let state = place.get::>(); - std::ptr::drop_in_place(state); + unsafe { std::ptr::drop_in_place(state) }; } } diff --git a/src/query/functions/src/aggregates/aggregate_json_object_agg.rs b/src/query/functions/src/aggregates/aggregate_json_object_agg.rs index 3081810bdf4a3..8cc5f96321be5 100644 --- a/src/query/functions/src/aggregates/aggregate_json_object_agg.rs +++ b/src/query/functions/src/aggregates/aggregate_json_object_agg.rs @@ -23,12 +23,6 @@ use borsh::BorshDeserialize; use borsh::BorshSerialize; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::string::StringColumn; -use databend_common_expression::types::variant::cast_scalar_to_variant; -use databend_common_expression::types::Bitmap; -use databend_common_expression::types::DataType; -use databend_common_expression::types::ValueType; -use databend_common_expression::types::*; use databend_common_expression::AggrStateRegistry; use databend_common_expression::AggrStateType; use databend_common_expression::BlockEntry; @@ -38,19 +32,25 @@ use databend_common_expression::ProjectedBlock; use databend_common_expression::Scalar; use databend_common_expression::ScalarRef; use databend_common_expression::StateSerdeItem; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::DataType; +use databend_common_expression::types::ValueType; +use databend_common_expression::types::string::StringColumn; +use databend_common_expression::types::variant::cast_scalar_to_variant; +use databend_common_expression::types::*; use jiff::tz::TimeZone; use jsonb::OwnedJsonb; use jsonb::RawJsonb; -use super::assert_binary_arguments; -use super::assert_params; -use super::borsh_partial_deserialize; use super::AggrState; use super::AggrStateLoc; use super::AggregateFunction; use super::AggregateFunctionDescription; use super::AggregateFunctionSortDesc; use super::StateAddr; +use super::assert_binary_arguments; +use super::assert_params; +use super::borsh_partial_deserialize; use crate::aggregates::AggregateFunctionFeatures; pub(super) trait BinaryScalarStateFunc: @@ -360,7 +360,7 @@ where unsafe fn drop_state(&self, place: AggrState) { let state = place.get::(); - std::ptr::drop_in_place(state); + unsafe { std::ptr::drop_in_place(state) }; } } diff --git a/src/query/functions/src/aggregates/aggregate_kurtosis.rs b/src/query/functions/src/aggregates/aggregate_kurtosis.rs index 9fcfd0cf3b25e..0e7470c285a6f 100644 --- a/src/query/functions/src/aggregates/aggregate_kurtosis.rs +++ b/src/query/functions/src/aggregates/aggregate_kurtosis.rs @@ -16,22 +16,19 @@ use borsh::BorshDeserialize; use borsh::BorshSerialize; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::compute_view::NumberConvertView; -use databend_common_expression::types::number::*; -use databend_common_expression::types::*; -use databend_common_expression::with_decimal_mapped_type; -use databend_common_expression::with_number_mapped_type; use databend_common_expression::AggrStateLoc; use databend_common_expression::BlockEntry; use databend_common_expression::ColumnBuilder; use databend_common_expression::Scalar; use databend_common_expression::StateAddr; use databend_common_expression::StateSerdeItem; +use databend_common_expression::types::compute_view::NumberConvertView; +use databend_common_expression::types::number::*; +use databend_common_expression::types::*; +use databend_common_expression::with_decimal_mapped_type; +use databend_common_expression::with_number_mapped_type; use num_traits::AsPrimitive; -use super::assert_params; -use super::assert_unary_arguments; -use super::batch_merge1; use super::AggrState; use super::AggregateFunctionDescription; use super::AggregateFunctionRef; @@ -40,6 +37,9 @@ use super::AggregateUnaryFunction; use super::SerializeInfo; use super::StateSerde; use super::UnaryState; +use super::assert_params; +use super::assert_unary_arguments; +use super::batch_merge1; #[derive(Default, BorshSerialize, BorshDeserialize)] struct KurtosisState { diff --git a/src/query/functions/src/aggregates/aggregate_markov_tarin.rs b/src/query/functions/src/aggregates/aggregate_markov_tarin.rs index 4294ac1f213b9..2fd790150c465 100644 --- a/src/query/functions/src/aggregates/aggregate_markov_tarin.rs +++ b/src/query/functions/src/aggregates/aggregate_markov_tarin.rs @@ -19,21 +19,11 @@ use std::sync::Arc; use borsh::BorshDeserialize; use borsh::BorshSerialize; -use databend_common_base::obfuscator::consume; use databend_common_base::obfuscator::CodePoint; use databend_common_base::obfuscator::NGramHash; +use databend_common_base::obfuscator::consume; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::ArgType; -use databend_common_expression::types::BinaryType; -use databend_common_expression::types::Bitmap; -use databend_common_expression::types::DataType; -use databend_common_expression::types::MapType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::UInt32Type; -use databend_common_expression::types::UnaryType; -use databend_common_expression::types::ValueType; -use databend_common_expression::types::F64; use databend_common_expression::AggrState; use databend_common_expression::AggrStateRegistry; use databend_common_expression::AggrStateType; @@ -43,14 +33,24 @@ use databend_common_expression::ColumnBuilder; use databend_common_expression::ProjectedBlock; use databend_common_expression::Scalar; use databend_common_expression::StateSerdeItem; +use databend_common_expression::types::ArgType; +use databend_common_expression::types::BinaryType; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::DataType; +use databend_common_expression::types::F64; +use databend_common_expression::types::MapType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::UInt32Type; +use databend_common_expression::types::UnaryType; +use databend_common_expression::types::ValueType; -use super::assert_unary_arguments; -use super::borsh_partial_deserialize; -use super::extract_number_param; use super::AggrStateLoc; use super::AggregateFunction; use super::AggregateFunctionDescription; use super::StateAddr; +use super::assert_unary_arguments; +use super::borsh_partial_deserialize; +use super::extract_number_param; pub struct MarkovTarin { display_name: String, @@ -214,7 +214,7 @@ impl AggregateFunction for MarkovTarin { unsafe fn drop_state(&self, place: AggrState) { let state = place.get::(); - std::ptr::drop_in_place(state); + unsafe { std::ptr::drop_in_place(state) }; } } @@ -371,8 +371,13 @@ pub fn aggregate_markov_train_function_desc() -> AggregateFunctionDescription { ..Default::default() } } - [order, frequency_cutoff, num_buckets_cutoff, frequency_add, frequency_desaturate] => - { + [ + order, + frequency_cutoff, + num_buckets_cutoff, + frequency_add, + frequency_desaturate, + ] => { let order = extract_number_param::(order.clone())? as usize; let frequency_cutoff = extract_number_param(frequency_cutoff.clone())?; let num_buckets_cutoff = @@ -393,7 +398,7 @@ pub fn aggregate_markov_train_function_desc() -> AggregateFunctionDescription { "{} expect to have 0, 1 or 5 params, but got {}", display_name, params.len() - ))) + ))); } }; diff --git a/src/query/functions/src/aggregates/aggregate_min_max_any.rs b/src/query/functions/src/aggregates/aggregate_min_max_any.rs index 60613815259ab..bcdfbbfbb4e55 100644 --- a/src/query/functions/src/aggregates/aggregate_min_max_any.rs +++ b/src/query/functions/src/aggregates/aggregate_min_max_any.rs @@ -20,18 +20,27 @@ use borsh::BorshDeserialize; use borsh::BorshSerialize; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::Bitmap; -use databend_common_expression::types::*; -use databend_common_expression::with_decimal_mapped_type; -use databend_common_expression::with_number_mapped_type; use databend_common_expression::AggrStateLoc; use databend_common_expression::BlockEntry; use databend_common_expression::ColumnBuilder; use databend_common_expression::ColumnView; +use databend_common_expression::SELECTIVITY_THRESHOLD; use databend_common_expression::Scalar; use databend_common_expression::StateAddr; -use databend_common_expression::SELECTIVITY_THRESHOLD; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::*; +use databend_common_expression::with_decimal_mapped_type; +use databend_common_expression::with_number_mapped_type; +use super::AggregateFunction; +use super::AggregateFunctionDescription; +use super::AggregateFunctionFeatures; +use super::AggregateFunctionSortDesc; +use super::AggregateUnaryFunction; +use super::SerializeInfo; +use super::StateSerde; +use super::StateSerdeItem; +use super::UnaryState; use super::aggregate_min_max_any_decimal::MinMaxAnyDecimalState; use super::aggregate_scalar_state::ChangeIf; use super::aggregate_scalar_state::CmpAny; @@ -46,15 +55,6 @@ use super::batch_merge1; use super::batch_merge2; use super::batch_serialize1; use super::batch_serialize2; -use super::AggregateFunction; -use super::AggregateFunctionDescription; -use super::AggregateFunctionFeatures; -use super::AggregateFunctionSortDesc; -use super::AggregateUnaryFunction; -use super::SerializeInfo; -use super::StateSerde; -use super::StateSerdeItem; -use super::UnaryState; use crate::with_compare_mapped_type; use crate::with_simple_no_number_no_string_mapped_type; diff --git a/src/query/functions/src/aggregates/aggregate_min_max_any_decimal.rs b/src/query/functions/src/aggregates/aggregate_min_max_any_decimal.rs index 5df0c5952c414..e350a0fd90e6d 100644 --- a/src/query/functions/src/aggregates/aggregate_min_max_any_decimal.rs +++ b/src/query/functions/src/aggregates/aggregate_min_max_any_decimal.rs @@ -15,22 +15,22 @@ use std::marker::PhantomData; use databend_common_exception::Result; -use databend_common_expression::types::decimal::*; -use databend_common_expression::types::Bitmap; -use databend_common_expression::types::*; use databend_common_expression::AggrStateLoc; use databend_common_expression::BlockEntry; use databend_common_expression::ColumnBuilder; use databend_common_expression::ColumnView; use databend_common_expression::StateAddr; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::decimal::*; +use databend_common_expression::types::*; -use super::aggregate_scalar_state::ChangeIf; -use super::batch_merge1; -use super::batch_serialize1; use super::SerializeInfo; use super::StateSerde; use super::StateSerdeItem; use super::UnaryState; +use super::aggregate_scalar_state::ChangeIf; +use super::batch_merge1; +use super::batch_serialize1; pub struct MinMaxAnyDecimalState where @@ -144,9 +144,11 @@ where C: ChangeIf, { fn serialize_type(_: Option<&dyn SerializeInfo>) -> Vec { - vec![DataType::Decimal(T::Scalar::default_decimal_size()) - .wrap_nullable() - .into()] + vec![ + DataType::Decimal(T::Scalar::default_decimal_size()) + .wrap_nullable() + .into(), + ] } fn batch_serialize( diff --git a/src/query/functions/src/aggregates/aggregate_mode.rs b/src/query/functions/src/aggregates/aggregate_mode.rs index 5b8d568b132e0..d2b9a07d051cd 100644 --- a/src/query/functions/src/aggregates/aggregate_mode.rs +++ b/src/query/functions/src/aggregates/aggregate_mode.rs @@ -12,17 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::hash_map::Entry; use std::collections::HashMap; +use std::collections::hash_map::Entry; use std::hash::Hash; use std::ops::AddAssign; use borsh::BorshDeserialize; use borsh::BorshSerialize; use databend_common_exception::Result; -use databend_common_expression::types::*; -use databend_common_expression::with_decimal_mapped_type; -use databend_common_expression::with_number_mapped_type; use databend_common_expression::AggrState; use databend_common_expression::AggrStateLoc; use databend_common_expression::AggregateFunctionRef; @@ -30,10 +27,10 @@ use databend_common_expression::BlockEntry; use databend_common_expression::ColumnBuilder; use databend_common_expression::Scalar; use databend_common_expression::StateAddr; +use databend_common_expression::types::*; +use databend_common_expression::with_decimal_mapped_type; +use databend_common_expression::with_number_mapped_type; -use super::assert_params; -use super::assert_unary_arguments; -use super::batch_merge1; use super::AggregateFunctionDescription; use super::AggregateFunctionSortDesc; use super::AggregateUnaryFunction; @@ -41,6 +38,9 @@ use super::SerializeInfo; use super::StateSerde; use super::StateSerdeItem; use super::UnaryState; +use super::assert_params; +use super::assert_unary_arguments; +use super::batch_merge1; #[derive(BorshSerialize, BorshDeserialize)] pub struct ModeState diff --git a/src/query/functions/src/aggregates/aggregate_null_result.rs b/src/query/functions/src/aggregates/aggregate_null_result.rs index bfeff76a346d9..cd0a057dd1e9d 100644 --- a/src/query/functions/src/aggregates/aggregate_null_result.rs +++ b/src/query/functions/src/aggregates/aggregate_null_result.rs @@ -17,16 +17,16 @@ use std::fmt; use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::types::AnyType; -use databend_common_expression::types::Bitmap; -use databend_common_expression::types::DataType; -use databend_common_expression::types::ValueType; use databend_common_expression::AggrStateRegistry; use databend_common_expression::AggrStateType; use databend_common_expression::BlockEntry; use databend_common_expression::ColumnBuilder; use databend_common_expression::ProjectedBlock; use databend_common_expression::StateSerdeItem; +use databend_common_expression::types::AnyType; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::DataType; +use databend_common_expression::types::ValueType; use super::AggrState; use super::AggrStateLoc; diff --git a/src/query/functions/src/aggregates/aggregate_quantile_cont.rs b/src/query/functions/src/aggregates/aggregate_quantile_cont.rs index 1eb41cb1692d8..c2b7f18580b27 100644 --- a/src/query/functions/src/aggregates/aggregate_quantile_cont.rs +++ b/src/query/functions/src/aggregates/aggregate_quantile_cont.rs @@ -14,11 +14,6 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::array::ArrayColumnBuilderMut; -use databend_common_expression::types::decimal::Decimal; -use databend_common_expression::types::*; -use databend_common_expression::with_decimal_mapped_type; -use databend_common_expression::with_number_mapped_type; use databend_common_expression::AggrStateLoc; use databend_common_expression::BlockEntry; use databend_common_expression::Column; @@ -26,13 +21,13 @@ use databend_common_expression::ColumnBuilder; use databend_common_expression::Scalar; use databend_common_expression::ScalarRef; use databend_common_expression::StateAddr; +use databend_common_expression::types::array::ArrayColumnBuilderMut; +use databend_common_expression::types::decimal::Decimal; +use databend_common_expression::types::*; +use databend_common_expression::with_decimal_mapped_type; +use databend_common_expression::with_number_mapped_type; use num_traits::AsPrimitive; -use super::assert_params; -use super::assert_unary_arguments; -use super::batch_merge1; -use super::batch_serialize1; -use super::get_levels; use super::AggregateFunctionDescription; use super::AggregateFunctionRef; use super::AggregateFunctionSortDesc; @@ -41,6 +36,11 @@ use super::SerializeInfo; use super::StateSerde; use super::StateSerdeItem; use super::UnaryState; +use super::assert_params; +use super::assert_unary_arguments; +use super::batch_merge1; +use super::batch_serialize1; +use super::get_levels; const MEDIAN: u8 = 0; const QUANTILE_CONT: u8 = 1; @@ -320,10 +320,12 @@ where T::Scalar: Decimal, { fn serialize_type(_: Option<&dyn SerializeInfo>) -> Vec { - vec![DataType::Array(Box::new(DataType::Decimal( - T::Scalar::default_decimal_size(), - ))) - .into()] + vec![ + DataType::Array(Box::new(DataType::Decimal( + T::Scalar::default_decimal_size(), + ))) + .into(), + ] } fn batch_serialize( diff --git a/src/query/functions/src/aggregates/aggregate_quantile_disc.rs b/src/query/functions/src/aggregates/aggregate_quantile_disc.rs index 5ed139697721a..c4e7ffe8bbf14 100644 --- a/src/query/functions/src/aggregates/aggregate_quantile_disc.rs +++ b/src/query/functions/src/aggregates/aggregate_quantile_disc.rs @@ -16,19 +16,16 @@ use borsh::BorshDeserialize; use borsh::BorshSerialize; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::array::ArrayColumnBuilderMut; -use databend_common_expression::types::*; -use databend_common_expression::with_decimal_mapped_type; -use databend_common_expression::with_number_mapped_type; use databend_common_expression::AggrStateLoc; use databend_common_expression::BlockEntry; use databend_common_expression::ColumnBuilder; use databend_common_expression::Scalar; use databend_common_expression::StateAddr; +use databend_common_expression::types::array::ArrayColumnBuilderMut; +use databend_common_expression::types::*; +use databend_common_expression::with_decimal_mapped_type; +use databend_common_expression::with_number_mapped_type; -use super::assert_unary_arguments; -use super::batch_merge1; -use super::get_levels; use super::AggrState; use super::AggregateFunctionDescription; use super::AggregateFunctionRef; @@ -39,6 +36,9 @@ use super::SerializeInfo; use super::StateSerde; use super::StateSerdeItem; use super::UnaryState; +use super::assert_unary_arguments; +use super::batch_merge1; +use super::get_levels; use crate::with_simple_no_number_mapped_type; #[derive(BorshSerialize, BorshDeserialize)] diff --git a/src/query/functions/src/aggregates/aggregate_quantile_tdigest.rs b/src/query/functions/src/aggregates/aggregate_quantile_tdigest.rs index 8e3d804f3da1c..54a231b4d76c9 100644 --- a/src/query/functions/src/aggregates/aggregate_quantile_tdigest.rs +++ b/src/query/functions/src/aggregates/aggregate_quantile_tdigest.rs @@ -23,12 +23,6 @@ use borsh::BorshDeserialize; use borsh::BorshSerialize; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::compute_view::NumberConvertView; -use databend_common_expression::types::number::*; -use databend_common_expression::types::Bitmap; -use databend_common_expression::types::*; -use databend_common_expression::with_decimal_mapped_type; -use databend_common_expression::with_number_mapped_type; use databend_common_expression::AggrStateRegistry; use databend_common_expression::AggrStateType; use databend_common_expression::BlockEntry; @@ -37,12 +31,14 @@ use databend_common_expression::ProjectedBlock; use databend_common_expression::Scalar; use databend_common_expression::ScalarRef; use databend_common_expression::StateSerdeItem; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::compute_view::NumberConvertView; +use databend_common_expression::types::number::*; +use databend_common_expression::types::*; +use databend_common_expression::with_decimal_mapped_type; +use databend_common_expression::with_number_mapped_type; use itertools::Itertools; -use super::assert_params; -use super::assert_unary_arguments; -use super::borsh_partial_deserialize; -use super::get_levels; use super::AggrState; use super::AggrStateLoc; use super::AggregateFunction; @@ -50,6 +46,10 @@ use super::AggregateFunctionDescription; use super::AggregateFunctionRef; use super::AggregateFunctionSortDesc; use super::StateAddr; +use super::assert_params; +use super::assert_unary_arguments; +use super::borsh_partial_deserialize; +use super::get_levels; pub(crate) const MEDIAN: u8 = 0; pub(crate) const QUANTILE: u8 = 1; @@ -432,7 +432,7 @@ where for<'a> T: AccessType = F64> unsafe fn drop_state(&self, place: AggrState) { let state = place.get::(); - std::ptr::drop_in_place(state); + unsafe { std::ptr::drop_in_place(state) }; } } diff --git a/src/query/functions/src/aggregates/aggregate_quantile_tdigest_weighted.rs b/src/query/functions/src/aggregates/aggregate_quantile_tdigest_weighted.rs index 7c3fec6737a30..45f6405abcbc7 100644 --- a/src/query/functions/src/aggregates/aggregate_quantile_tdigest_weighted.rs +++ b/src/query/functions/src/aggregates/aggregate_quantile_tdigest_weighted.rs @@ -21,11 +21,6 @@ use std::sync::Arc; use borsh::BorshSerialize; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::number::*; -use databend_common_expression::types::Bitmap; -use databend_common_expression::types::*; -use databend_common_expression::with_number_mapped_type; -use databend_common_expression::with_unsigned_integer_mapped_type; use databend_common_expression::AggrStateRegistry; use databend_common_expression::AggrStateType; use databend_common_expression::BlockEntry; @@ -33,15 +28,13 @@ use databend_common_expression::ColumnBuilder; use databend_common_expression::ProjectedBlock; use databend_common_expression::Scalar; use databend_common_expression::StateSerdeItem; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::number::*; +use databend_common_expression::types::*; +use databend_common_expression::with_number_mapped_type; +use databend_common_expression::with_unsigned_integer_mapped_type; use num_traits::AsPrimitive; -use super::aggregate_quantile_tdigest::QuantileTDigestState; -use super::aggregate_quantile_tdigest::MEDIAN; -use super::aggregate_quantile_tdigest::QUANTILE; -use super::assert_binary_arguments; -use super::assert_params; -use super::borsh_partial_deserialize; -use super::get_levels; use super::AggrState; use super::AggrStateLoc; use super::AggregateFunction; @@ -49,6 +42,13 @@ use super::AggregateFunctionDescription; use super::AggregateFunctionRef; use super::AggregateFunctionSortDesc; use super::StateAddr; +use super::aggregate_quantile_tdigest::MEDIAN; +use super::aggregate_quantile_tdigest::QUANTILE; +use super::aggregate_quantile_tdigest::QuantileTDigestState; +use super::assert_binary_arguments; +use super::assert_params; +use super::borsh_partial_deserialize; +use super::get_levels; #[derive(Clone)] pub struct AggregateQuantileTDigestWeightedFunction { @@ -212,7 +212,7 @@ where unsafe fn drop_state(&self, place: AggrState) { let state = place.get::(); - std::ptr::drop_in_place(state); + unsafe { std::ptr::drop_in_place(state) }; } } diff --git a/src/query/functions/src/aggregates/aggregate_range_bound.rs b/src/query/functions/src/aggregates/aggregate_range_bound.rs index 750f00fb81917..b1ce447c38100 100644 --- a/src/query/functions/src/aggregates/aggregate_range_bound.rs +++ b/src/query/functions/src/aggregates/aggregate_range_bound.rs @@ -16,13 +16,6 @@ use borsh::BorshDeserialize; use borsh::BorshSerialize; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::compare_columns; -use databend_common_expression::types::array::ArrayColumnBuilderMut; -use databend_common_expression::types::i256; -use databend_common_expression::types::Bitmap; -use databend_common_expression::types::*; -use databend_common_expression::with_decimal_mapped_type; -use databend_common_expression::with_number_mapped_type; use databend_common_expression::AggrStateLoc; use databend_common_expression::AggregateFunctionRef; use databend_common_expression::BlockEntry; @@ -30,14 +23,19 @@ use databend_common_expression::ColumnBuilder; use databend_common_expression::ColumnView; use databend_common_expression::Scalar; use databend_common_expression::StateAddr; +use databend_common_expression::compare_columns; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::array::ArrayColumnBuilderMut; +use databend_common_expression::types::i256; +use databend_common_expression::types::*; +use databend_common_expression::with_decimal_mapped_type; +use databend_common_expression::with_number_mapped_type; +use rand::Rng; +use rand::SeedableRng; use rand::prelude::SliceRandom; use rand::rngs::SmallRng; use rand::thread_rng; -use rand::Rng; -use rand::SeedableRng; -use super::assert_unary_arguments; -use super::batch_merge1; use super::AggrState; use super::AggregateFunctionDescription; use super::AggregateFunctionSortDesc; @@ -46,6 +44,8 @@ use super::SerializeInfo; use super::StateSerde; use super::StateSerdeItem; use super::UnaryState; +use super::assert_unary_arguments; +use super::batch_merge1; use crate::with_simple_no_number_mapped_type; pub struct RangeBoundData { diff --git a/src/query/functions/src/aggregates/aggregate_retention.rs b/src/query/functions/src/aggregates/aggregate_retention.rs index ea0ae4090d892..dcdb665cf9aaf 100644 --- a/src/query/functions/src/aggregates/aggregate_retention.rs +++ b/src/query/functions/src/aggregates/aggregate_retention.rs @@ -18,13 +18,6 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::ArgType; -use databend_common_expression::types::Bitmap; -use databend_common_expression::types::BooleanType; -use databend_common_expression::types::BuilderExt; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::UInt32Type; use databend_common_expression::AggrStateRegistry; use databend_common_expression::AggrStateType; use databend_common_expression::BlockEntry; @@ -32,11 +25,14 @@ use databend_common_expression::ColumnBuilder; use databend_common_expression::ProjectedBlock; use databend_common_expression::Scalar; use databend_common_expression::StateSerdeItem; +use databend_common_expression::types::ArgType; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::BooleanType; +use databend_common_expression::types::BuilderExt; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::UInt32Type; -use super::assert_params; -use super::assert_variadic_arguments; -use super::batch_merge1; -use super::batch_serialize1; use super::AggrState; use super::AggrStateLoc; use super::AggregateFunction; @@ -44,6 +40,10 @@ use super::AggregateFunctionDescription; use super::AggregateFunctionRef; use super::AggregateFunctionSortDesc; use super::StateAddr; +use super::assert_params; +use super::assert_variadic_arguments; +use super::batch_merge1; +use super::batch_serialize1; struct AggregateRetentionState { events: u32, diff --git a/src/query/functions/src/aggregates/aggregate_scalar_state.rs b/src/query/functions/src/aggregates/aggregate_scalar_state.rs index 8c56aaf837ee9..722fe58a93e42 100644 --- a/src/query/functions/src/aggregates/aggregate_scalar_state.rs +++ b/src/query/functions/src/aggregates/aggregate_scalar_state.rs @@ -15,11 +15,11 @@ use std::cmp::Ordering; use databend_common_exception::Result; +use databend_common_expression::ColumnBuilder; +use databend_common_expression::ColumnView; use databend_common_expression::types::AccessType; use databend_common_expression::types::Bitmap; use databend_common_expression::types::ValueType; -use databend_common_expression::ColumnBuilder; -use databend_common_expression::ColumnView; use super::StateSerde; diff --git a/src/query/functions/src/aggregates/aggregate_skewness.rs b/src/query/functions/src/aggregates/aggregate_skewness.rs index 33732d03b8f79..2e736b846b99d 100644 --- a/src/query/functions/src/aggregates/aggregate_skewness.rs +++ b/src/query/functions/src/aggregates/aggregate_skewness.rs @@ -16,12 +16,6 @@ use borsh::BorshDeserialize; use borsh::BorshSerialize; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::compute_view::NumberConvertView; -use databend_common_expression::types::number::*; -use databend_common_expression::types::BuilderMut; -use databend_common_expression::types::*; -use databend_common_expression::with_decimal_mapped_type; -use databend_common_expression::with_number_mapped_type; use databend_common_expression::AggrStateLoc; use databend_common_expression::AggregateFunctionRef; use databend_common_expression::BlockEntry; @@ -29,18 +23,24 @@ use databend_common_expression::ColumnBuilder; use databend_common_expression::Scalar; use databend_common_expression::StateAddr; use databend_common_expression::StateSerdeItem; +use databend_common_expression::types::BuilderMut; +use databend_common_expression::types::compute_view::NumberConvertView; +use databend_common_expression::types::number::*; +use databend_common_expression::types::*; +use databend_common_expression::with_decimal_mapped_type; +use databend_common_expression::with_number_mapped_type; use num_traits::AsPrimitive; -use super::aggregate_unary::AggregateUnaryFunction; -use super::aggregate_unary::UnaryState; -use super::assert_params; -use super::assert_unary_arguments; -use super::batch_merge1; use super::AggrState; use super::AggregateFunctionDescription; use super::AggregateFunctionSortDesc; use super::SerializeInfo; use super::StateSerde; +use super::aggregate_unary::AggregateUnaryFunction; +use super::aggregate_unary::UnaryState; +use super::assert_params; +use super::assert_unary_arguments; +use super::batch_merge1; #[derive(Default, BorshSerialize, BorshDeserialize)] pub struct SkewnessStateV2 { diff --git a/src/query/functions/src/aggregates/aggregate_st_collect.rs b/src/query/functions/src/aggregates/aggregate_st_collect.rs index 1cdb80236452e..7c52066ef965b 100644 --- a/src/query/functions/src/aggregates/aggregate_st_collect.rs +++ b/src/query/functions/src/aggregates/aggregate_st_collect.rs @@ -20,13 +20,6 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::ArgType; -use databend_common_expression::types::ArrayType; -use databend_common_expression::types::Bitmap; -use databend_common_expression::types::DataType; -use databend_common_expression::types::GeometryType; -use databend_common_expression::types::NullableType; -use databend_common_expression::types::ValueType; use databend_common_expression::AggrStateRegistry; use databend_common_expression::AggrStateType; use databend_common_expression::BlockEntry; @@ -37,6 +30,13 @@ use databend_common_expression::ProjectedBlock; use databend_common_expression::Scalar; use databend_common_expression::ScalarRef; use databend_common_expression::StateSerdeItem; +use databend_common_expression::types::ArgType; +use databend_common_expression::types::ArrayType; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::DataType; +use databend_common_expression::types::GeometryType; +use databend_common_expression::types::NullableType; +use databend_common_expression::types::ValueType; use databend_common_io::ewkb_to_geo; use databend_common_io::geo_to_ewkb; use geo::Geometry; @@ -49,11 +49,6 @@ use geo::Point; use geo::Polygon; use geozero::wkb::Ewkb; -use super::aggregate_scalar_state::ScalarStateFunc; -use super::assert_params; -use super::assert_unary_arguments; -use super::batch_merge1; -use super::batch_serialize1; use super::AggrState; use super::AggrStateLoc; use super::AggregateFunction; @@ -62,6 +57,11 @@ use super::AggregateFunctionFeatures; use super::AggregateFunctionSortDesc; use super::StateAddr; use super::StateSerde; +use super::aggregate_scalar_state::ScalarStateFunc; +use super::assert_params; +use super::assert_unary_arguments; +use super::batch_merge1; +use super::batch_serialize1; #[derive(Debug)] pub struct StCollectState { @@ -384,7 +384,7 @@ where unsafe fn drop_state(&self, place: AggrState) { let state = place.get::(); - std::ptr::drop_in_place(state); + unsafe { std::ptr::drop_in_place(state) }; } } diff --git a/src/query/functions/src/aggregates/aggregate_stddev.rs b/src/query/functions/src/aggregates/aggregate_stddev.rs index 96d133e84f9f5..a8424da573c7b 100644 --- a/src/query/functions/src/aggregates/aggregate_stddev.rs +++ b/src/query/functions/src/aggregates/aggregate_stddev.rs @@ -19,30 +19,27 @@ use borsh::BorshSerialize; use databend_common_column::bitmap::Bitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::compute_view::NumberConvertView; -use databend_common_expression::types::i256; -use databend_common_expression::types::nullable::NullableColumnBuilderMut; +use databend_common_expression::AggrStateLoc; +use databend_common_expression::BlockEntry; +use databend_common_expression::ColumnBuilder; +use databend_common_expression::Scalar; +use databend_common_expression::StateAddr; +use databend_common_expression::StateSerdeItem; use databend_common_expression::types::AccessType; use databend_common_expression::types::BinaryType; use databend_common_expression::types::DataType; use databend_common_expression::types::DecimalDataKind; use databend_common_expression::types::DecimalF64View; +use databend_common_expression::types::F64; use databend_common_expression::types::Float64Type; use databend_common_expression::types::NullableType; use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::F64; +use databend_common_expression::types::compute_view::NumberConvertView; +use databend_common_expression::types::i256; +use databend_common_expression::types::nullable::NullableColumnBuilderMut; use databend_common_expression::with_decimal_mapped_type; use databend_common_expression::with_number_mapped_type; -use databend_common_expression::AggrStateLoc; -use databend_common_expression::BlockEntry; -use databend_common_expression::ColumnBuilder; -use databend_common_expression::Scalar; -use databend_common_expression::StateAddr; -use databend_common_expression::StateSerdeItem; -use super::aggregator_common::assert_params; -use super::aggregator_common::assert_unary_arguments; -use super::batch_merge1; use super::AggrState; use super::AggregateFunction; use super::AggregateFunctionDescription; @@ -51,6 +48,9 @@ use super::AggregateUnaryFunction; use super::SerializeInfo; use super::StateSerde; use super::UnaryState; +use super::aggregator_common::assert_params; +use super::aggregator_common::assert_unary_arguments; +use super::batch_merge1; const STD_POP: u8 = 0; const STD_SAMP: u8 = 1; diff --git a/src/query/functions/src/aggregates/aggregate_string_agg.rs b/src/query/functions/src/aggregates/aggregate_string_agg.rs index 68b8c8f8aec6a..6271d9cc32b18 100644 --- a/src/query/functions/src/aggregates/aggregate_string_agg.rs +++ b/src/query/functions/src/aggregates/aggregate_string_agg.rs @@ -17,6 +17,11 @@ use std::fmt::Write; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::AggregateFunctionRef; +use databend_common_expression::BlockEntry; +use databend_common_expression::ColumnBuilder; +use databend_common_expression::Scalar; +use databend_common_expression::StateSerdeItem; use databend_common_expression::display::scalar_ref_to_string; use databend_common_expression::types::AccessType; use databend_common_expression::types::AnyType; @@ -29,15 +34,7 @@ use databend_common_expression::types::NumberDataType; use databend_common_expression::types::NumberType; use databend_common_expression::types::StringType; use databend_common_expression::with_number_mapped_type; -use databend_common_expression::AggregateFunctionRef; -use databend_common_expression::BlockEntry; -use databend_common_expression::ColumnBuilder; -use databend_common_expression::Scalar; -use databend_common_expression::StateSerdeItem; -use super::assert_variadic_arguments; -use super::batch_merge1; -use super::batch_serialize1; use super::AggregateFunctionDescription; use super::AggregateFunctionFeatures; use super::AggregateFunctionSortDesc; @@ -45,6 +42,9 @@ use super::AggregateUnaryFunction; use super::SerializeInfo; use super::StateSerde; use super::UnaryState; +use super::assert_variadic_arguments; +use super::batch_merge1; +use super::batch_serialize1; #[derive(Default)] struct StringAggState { diff --git a/src/query/functions/src/aggregates/aggregate_sum.rs b/src/query/functions/src/aggregates/aggregate_sum.rs index 953fcbfe318ee..a6c08665d63c3 100644 --- a/src/query/functions/src/aggregates/aggregate_sum.rs +++ b/src/query/functions/src/aggregates/aggregate_sum.rs @@ -16,37 +16,37 @@ use boolean::TrueIdxIter; use databend_common_column::types::months_days_micros; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::decimal::*; -use databend_common_expression::types::number::*; +use databend_common_expression::AggregateFunctionRef; +use databend_common_expression::BlockEntry; +use databend_common_expression::ColumnBuilder; +use databend_common_expression::ColumnView; +use databend_common_expression::SELECTIVITY_THRESHOLD; +use databend_common_expression::Scalar; +use databend_common_expression::StateAddr; +use databend_common_expression::StateSerdeItem; use databend_common_expression::types::Bitmap; use databend_common_expression::types::Buffer; use databend_common_expression::types::BuilderMut; use databend_common_expression::types::DecimalDataKind; +use databend_common_expression::types::decimal::*; +use databend_common_expression::types::number::*; use databend_common_expression::types::*; use databend_common_expression::utils::arithmetics_type::ResultTypeOfUnary; use databend_common_expression::with_decimal_mapped_type; use databend_common_expression::with_number_mapped_type; -use databend_common_expression::AggregateFunctionRef; -use databend_common_expression::BlockEntry; -use databend_common_expression::ColumnBuilder; -use databend_common_expression::ColumnView; -use databend_common_expression::Scalar; -use databend_common_expression::StateAddr; -use databend_common_expression::StateSerdeItem; -use databend_common_expression::SELECTIVITY_THRESHOLD; use num_traits::AsPrimitive; -use super::aggregate_unary::UnaryState; -use super::assert_params; -use super::assert_unary_arguments; -use super::batch_merge1; -use super::batch_serialize1; use super::AggrStateLoc; use super::AggregateFunctionDescription; use super::AggregateFunctionSortDesc; use super::AggregateUnaryFunction; use super::SerializeInfo; use super::StateSerde; +use super::aggregate_unary::UnaryState; +use super::assert_params; +use super::assert_unary_arguments; +use super::batch_merge1; +use super::batch_serialize1; pub struct NumberSumState where N: ArgType diff --git a/src/query/functions/src/aggregates/aggregate_sum_zero.rs b/src/query/functions/src/aggregates/aggregate_sum_zero.rs index 0da923f8886f9..f1503f5b90aef 100644 --- a/src/query/functions/src/aggregates/aggregate_sum_zero.rs +++ b/src/query/functions/src/aggregates/aggregate_sum_zero.rs @@ -18,7 +18,13 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::number::NumberColumnBuilder; +use databend_common_expression::AggrStateRegistry; +use databend_common_expression::AggrStateType; +use databend_common_expression::BlockEntry; +use databend_common_expression::ColumnBuilder; +use databend_common_expression::ProjectedBlock; +use databend_common_expression::Scalar; +use databend_common_expression::StateSerdeItem; use databend_common_expression::types::ArgType; use databend_common_expression::types::Bitmap; use databend_common_expression::types::DataType; @@ -27,21 +33,15 @@ use databend_common_expression::types::NumberDataType; use databend_common_expression::types::UInt64Type; use databend_common_expression::types::UnaryType; use databend_common_expression::types::ValueType; -use databend_common_expression::AggrStateRegistry; -use databend_common_expression::AggrStateType; -use databend_common_expression::BlockEntry; -use databend_common_expression::ColumnBuilder; -use databend_common_expression::ProjectedBlock; -use databend_common_expression::Scalar; -use databend_common_expression::StateSerdeItem; +use databend_common_expression::types::number::NumberColumnBuilder; -use super::assert_params; use super::AggrState; use super::AggrStateLoc; use super::AggregateFunction; use super::AggregateFunctionDescription; use super::AggregateFunctionSortDesc; use super::StateAddr; +use super::assert_params; use crate::aggregates::assert_unary_arguments; struct AggregateSumZeroState { diff --git a/src/query/functions/src/aggregates/aggregate_unary.rs b/src/query/functions/src/aggregates/aggregate_unary.rs index f7d5264f15ec9..986f7c20fecc0 100644 --- a/src/query/functions/src/aggregates/aggregate_unary.rs +++ b/src/query/functions/src/aggregates/aggregate_unary.rs @@ -19,10 +19,6 @@ use std::marker::PhantomData; use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::types::AccessType; -use databend_common_expression::types::Bitmap; -use databend_common_expression::types::DataType; -use databend_common_expression::types::ValueType; use databend_common_expression::AggrStateRegistry; use databend_common_expression::AggrStateType; use databend_common_expression::AggregateFunction; @@ -33,6 +29,10 @@ use databend_common_expression::ColumnView; use databend_common_expression::ProjectedBlock; use databend_common_expression::StateAddr; use databend_common_expression::StateSerdeItem; +use databend_common_expression::types::AccessType; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::DataType; +use databend_common_expression::types::ValueType; use super::AggrState; use super::AggrStateLoc; @@ -280,6 +280,6 @@ where unsafe fn drop_state(&self, place: AggrState) { let state = place.get::(); - std::ptr::drop_in_place(state); + unsafe { std::ptr::drop_in_place(state) }; } } diff --git a/src/query/functions/src/aggregates/aggregate_window_funnel.rs b/src/query/functions/src/aggregates/aggregate_window_funnel.rs index 178caaf7e13d1..7e16e632bb75f 100644 --- a/src/query/functions/src/aggregates/aggregate_window_funnel.rs +++ b/src/query/functions/src/aggregates/aggregate_window_funnel.rs @@ -23,8 +23,13 @@ use borsh::BorshDeserialize; use borsh::BorshSerialize; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::number::Number; -use databend_common_expression::types::number::UInt8Type; +use databend_common_expression::AggrStateRegistry; +use databend_common_expression::AggrStateType; +use databend_common_expression::BlockEntry; +use databend_common_expression::ColumnBuilder; +use databend_common_expression::ProjectedBlock; +use databend_common_expression::Scalar; +use databend_common_expression::StateSerdeItem; use databend_common_expression::types::ArgType; use databend_common_expression::types::BinaryType; use databend_common_expression::types::Bitmap; @@ -36,20 +41,11 @@ use databend_common_expression::types::NumberType; use databend_common_expression::types::TimestampType; use databend_common_expression::types::UnaryType; use databend_common_expression::types::ValueType; +use databend_common_expression::types::number::Number; +use databend_common_expression::types::number::UInt8Type; use databend_common_expression::with_integer_mapped_type; -use databend_common_expression::AggrStateRegistry; -use databend_common_expression::AggrStateType; -use databend_common_expression::BlockEntry; -use databend_common_expression::ColumnBuilder; -use databend_common_expression::ProjectedBlock; -use databend_common_expression::Scalar; -use databend_common_expression::StateSerdeItem; use num_traits::AsPrimitive; -use super::assert_unary_params; -use super::assert_variadic_arguments; -use super::borsh_partial_deserialize; -use super::extract_number_param; use super::AggrState; use super::AggrStateLoc; use super::AggregateFunction; @@ -58,6 +54,10 @@ use super::AggregateFunctionRef; use super::AggregateFunctionSortDesc; use super::AggregateNullVariadicAdaptor; use super::StateAddr; +use super::assert_unary_params; +use super::assert_variadic_arguments; +use super::borsh_partial_deserialize; +use super::extract_number_param; #[derive(BorshSerialize, BorshDeserialize)] struct AggregateWindowFunnelState { @@ -346,7 +346,7 @@ where unsafe fn drop_state(&self, place: AggrState) { let state = place.get::>(); - std::ptr::drop_in_place(state); + unsafe { std::ptr::drop_in_place(state) }; } fn get_own_null_adaptor( diff --git a/src/query/functions/src/aggregates/aggregator.rs b/src/query/functions/src/aggregates/aggregator.rs index 512341bab3cf4..3be08a27384db 100644 --- a/src/query/functions/src/aggregates/aggregator.rs +++ b/src/query/functions/src/aggregates/aggregator.rs @@ -12,6 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +use super::AggregateCountFunction; +use super::AggregateFunctionFactory; +use super::AggregateIfCombinator; +use super::AggregateStateCombinator; use super::adaptors::*; use super::aggregate_approx_count_distinct::aggregate_approx_count_distinct_function_desc; use super::aggregate_arg_min_max::aggregate_arg_max_function_desc; @@ -58,10 +62,6 @@ use super::aggregate_string_agg::aggregate_string_agg_function_desc; use super::aggregate_sum::aggregate_sum_function_desc; use super::aggregate_sum_zero::AggregateSumZeroFunction; use super::aggregate_window_funnel::aggregate_window_funnel_function_desc; -use super::AggregateCountFunction; -use super::AggregateFunctionFactory; -use super::AggregateIfCombinator; -use super::AggregateStateCombinator; pub struct Aggregators; diff --git a/src/query/functions/src/aggregates/aggregator_common.rs b/src/query/functions/src/aggregates/aggregator_common.rs index 041d1250ec8db..ad63b09855d1f 100644 --- a/src/query/functions/src/aggregates/aggregator_common.rs +++ b/src/query/functions/src/aggregates/aggregator_common.rs @@ -20,31 +20,31 @@ use databend_common_base::runtime::drop_guard; use databend_common_column::bitmap::Bitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::AggrStateLoc; +use databend_common_expression::BlockEntry; +use databend_common_expression::Column; +use databend_common_expression::ColumnBuilder; +use databend_common_expression::Constant; +use databend_common_expression::FunctionContext; +use databend_common_expression::Scalar; +use databend_common_expression::StateAddr; use databend_common_expression::type_check::check_number; use databend_common_expression::types::AccessType; use databend_common_expression::types::BuilderExt; use databend_common_expression::types::DataType; +use databend_common_expression::types::F64; use databend_common_expression::types::Number; use databend_common_expression::types::PairType; use databend_common_expression::types::TernaryType; use databend_common_expression::types::UnaryType; use databend_common_expression::types::ValueType; -use databend_common_expression::types::F64; -use databend_common_expression::AggrStateLoc; -use databend_common_expression::BlockEntry; -use databend_common_expression::Column; -use databend_common_expression::ColumnBuilder; -use databend_common_expression::Constant; -use databend_common_expression::FunctionContext; -use databend_common_expression::Scalar; -use databend_common_expression::StateAddr; -use super::get_states_layout; use super::AggrState; use super::AggregateFunctionFactory; use super::AggregateFunctionRef; use super::AggregateFunctionSortDesc; use super::StatesLayout; +use super::get_states_layout; use crate::BUILTIN_FUNCTIONS; pub(super) fn assert_unary_params(name: D, actual: usize) -> Result<()> { diff --git a/src/query/functions/src/aggregates/mod.rs b/src/query/functions/src/aggregates/mod.rs index e565471040417..8eb2175ae0706 100644 --- a/src/query/functions/src/aggregates/mod.rs +++ b/src/query/functions/src/aggregates/mod.rs @@ -26,10 +26,10 @@ use aggregator::Aggregators; pub use aggregator_common::*; use databend_common_column::bitmap::Bitmap; use databend_common_exception::Result; -use databend_common_expression::aggregate as aggregate_function; -use databend_common_expression::types::DataType; use databend_common_expression::BlockEntry; use databend_common_expression::ColumnBuilder; +use databend_common_expression::aggregate as aggregate_function; +use databend_common_expression::types::DataType; trait StateSerde { fn serialize_type(info: Option<&dyn SerializeInfo>) -> Vec; diff --git a/src/query/functions/src/cast_rules.rs b/src/query/functions/src/cast_rules.rs index 21544b52d0793..5c01e11b8f789 100644 --- a/src/query/functions/src/cast_rules.rs +++ b/src/query/functions/src/cast_rules.rs @@ -12,13 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_expression::AutoCastRules; +use databend_common_expression::FunctionRegistry; use databend_common_expression::type_check::ALL_SIMPLE_CAST_FUNCTIONS; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; use databend_common_expression::types::ALL_INTEGER_TYPES; use databend_common_expression::types::ALL_NUMERICS_TYPES; -use databend_common_expression::AutoCastRules; -use databend_common_expression::FunctionRegistry; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; use crate::scalars::ALL_COMP_FUNC_NAMES; use crate::scalars::ALL_STRING_FUNC_NAMES; diff --git a/src/query/functions/src/lib.rs b/src/query/functions/src/lib.rs index 50b4eb9e6e586..53957e0793caa 100644 --- a/src/query/functions/src/lib.rs +++ b/src/query/functions/src/lib.rs @@ -17,6 +17,10 @@ #![feature(try_blocks)] #![feature(downcast_unchecked)] #![feature(associated_type_defaults)] +#![allow(clippy::let_and_return)] +#![allow(clippy::collapsible_if)] +#![allow(clippy::uninlined_format_args)] +#![allow(clippy::manual_is_multiple_of)] use aggregates::AggregateFunctionFactory; use ctor::ctor; diff --git a/src/query/functions/src/scalars/arithmetic/Cargo.toml b/src/query/functions/src/scalars/arithmetic/Cargo.toml index 8b78af97ee69b..5f73048ef4f87 100644 --- a/src/query/functions/src/scalars/arithmetic/Cargo.toml +++ b/src/query/functions/src/scalars/arithmetic/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "databend-functions-scalar-arithmetic" version = "0.1.0" -edition = "2021" +edition = "2024" [dependencies] databend-common-expression = { workspace = true } diff --git a/src/query/functions/src/scalars/arithmetic/src/arithmetic.rs b/src/query/functions/src/scalars/arithmetic/src/arithmetic.rs index 001ed493072b9..637cf09006ab9 100644 --- a/src/query/functions/src/scalars/arithmetic/src/arithmetic.rs +++ b/src/query/functions/src/scalars/arithmetic/src/arithmetic.rs @@ -19,24 +19,26 @@ use std::ops::BitOr; use std::ops::BitXor; use std::str::FromStr; +use databend_common_expression::FunctionDomain; +use databend_common_expression::FunctionRegistry; use databend_common_expression::serialize::read_decimal_with_size; -use databend_common_expression::types::nullable::NullableColumn; -use databend_common_expression::types::nullable::NullableDomain; -use databend_common_expression::types::number::Number; -use databend_common_expression::types::number::NumberType; -use databend_common_expression::types::number::F64; -use databend_common_expression::types::string::StringColumnBuilder; +use databend_common_expression::types::ALL_INTEGER_TYPES; +use databend_common_expression::types::ALL_NUMBER_CLASSES; +use databend_common_expression::types::ALL_NUMERICS_TYPES; +use databend_common_expression::types::ALL_UNSIGNED_INTEGER_TYPES; use databend_common_expression::types::Bitmap; +use databend_common_expression::types::F32; use databend_common_expression::types::NullableType; use databend_common_expression::types::NumberClass; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::SimpleDomain; use databend_common_expression::types::StringType; -use databend_common_expression::types::ALL_INTEGER_TYPES; -use databend_common_expression::types::ALL_NUMBER_CLASSES; -use databend_common_expression::types::ALL_NUMERICS_TYPES; -use databend_common_expression::types::ALL_UNSIGNED_INTEGER_TYPES; -use databend_common_expression::types::F32; +use databend_common_expression::types::nullable::NullableColumn; +use databend_common_expression::types::nullable::NullableDomain; +use databend_common_expression::types::number::F64; +use databend_common_expression::types::number::Number; +use databend_common_expression::types::number::NumberType; +use databend_common_expression::types::string::StringColumnBuilder; use databend_common_expression::utils::arithmetics_type::ResultTypeOfUnary; use databend_common_expression::values::Value; use databend_common_expression::vectorize_1_arg; @@ -45,8 +47,6 @@ use databend_common_expression::with_integer_mapped_type; use databend_common_expression::with_number_mapped_type; use databend_common_expression::with_number_mapped_type_without_64; use databend_common_expression::with_unsigned_integer_mapped_type; -use databend_common_expression::FunctionDomain; -use databend_common_expression::FunctionRegistry; use databend_functions_scalar_decimal::register_decimal_to_float; use databend_functions_scalar_decimal::register_decimal_to_int; use databend_functions_scalar_decimal::register_decimal_to_string; diff --git a/src/query/functions/src/scalars/array.rs b/src/query/functions/src/scalars/array.rs index 76e0ba64112fe..8f092170c2fbd 100644 --- a/src/query/functions/src/scalars/array.rs +++ b/src/query/functions/src/scalars/array.rs @@ -20,16 +20,29 @@ use bumpalo::Bump; use databend_common_base::runtime::drop_guard; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::aggregate::get_states_layout; +use databend_common_expression::BlockEntry; +use databend_common_expression::Column; +use databend_common_expression::ColumnBuilder; +use databend_common_expression::DataBlock; +use databend_common_expression::Domain; +use databend_common_expression::EvalContext; +use databend_common_expression::Function; +use databend_common_expression::FunctionDomain; +use databend_common_expression::FunctionEval; +use databend_common_expression::FunctionFactory; +use databend_common_expression::FunctionRegistry; +use databend_common_expression::FunctionSignature; +use databend_common_expression::Scalar; +use databend_common_expression::ScalarRef; +use databend_common_expression::SimpleDomainCmp; +use databend_common_expression::SortColumnDescription; +use databend_common_expression::Value; use databend_common_expression::aggregate::AggrState; use databend_common_expression::aggregate::AggregateFunctionRef; use databend_common_expression::aggregate::StateAddr; use databend_common_expression::aggregate::StatesLayout; -use databend_common_expression::types::array::ArrayColumnBuilder; -use databend_common_expression::types::boolean::BooleanDomain; -use databend_common_expression::types::nullable::NullableDomain; -use databend_common_expression::types::number::SimpleDomain; -use databend_common_expression::types::number::UInt64Type; +use databend_common_expression::aggregate::get_states_layout; +use databend_common_expression::types::ALL_NUMERICS_TYPES; use databend_common_expression::types::AccessType; use databend_common_expression::types::AnyType; use databend_common_expression::types::ArgType; @@ -50,30 +63,17 @@ use databend_common_expression::types::ReturnType; use databend_common_expression::types::StringType; use databend_common_expression::types::TimestampType; use databend_common_expression::types::VariantType; -use databend_common_expression::types::ALL_NUMERICS_TYPES; +use databend_common_expression::types::array::ArrayColumnBuilder; +use databend_common_expression::types::boolean::BooleanDomain; +use databend_common_expression::types::nullable::NullableDomain; +use databend_common_expression::types::number::SimpleDomain; +use databend_common_expression::types::number::UInt64Type; use databend_common_expression::vectorize_1_arg; use databend_common_expression::vectorize_2_arg; use databend_common_expression::vectorize_with_builder_1_arg; use databend_common_expression::vectorize_with_builder_2_arg; use databend_common_expression::vectorize_with_builder_3_arg; use databend_common_expression::with_number_mapped_type; -use databend_common_expression::BlockEntry; -use databend_common_expression::Column; -use databend_common_expression::ColumnBuilder; -use databend_common_expression::DataBlock; -use databend_common_expression::Domain; -use databend_common_expression::EvalContext; -use databend_common_expression::Function; -use databend_common_expression::FunctionDomain; -use databend_common_expression::FunctionEval; -use databend_common_expression::FunctionFactory; -use databend_common_expression::FunctionRegistry; -use databend_common_expression::FunctionSignature; -use databend_common_expression::Scalar; -use databend_common_expression::ScalarRef; -use databend_common_expression::SimpleDomainCmp; -use databend_common_expression::SortColumnDescription; -use databend_common_expression::Value; use databend_common_hashtable::HashtableKeyable; use databend_common_hashtable::KeysRef; use databend_common_hashtable::StackHashMap; @@ -1181,7 +1181,7 @@ impl<'a> ArrayAggEvaluator<'a> { } } - fn state(&self) -> AggrState { + fn state(&self) -> AggrState<'_> { AggrState::new(self.addr, &self.state_layout.states_loc[0]) } @@ -1232,7 +1232,7 @@ impl ArrayAggDesc { }) } - fn create_evaluator(&self) -> ArrayAggEvaluator { + fn create_evaluator(&self) -> ArrayAggEvaluator<'_> { ArrayAggEvaluator::new(&self.func, &self.state_layout) } } diff --git a/src/query/functions/src/scalars/binary.rs b/src/query/functions/src/scalars/binary.rs index a9eb7ceea7731..8d1e426fc17e5 100644 --- a/src/query/functions/src/scalars/binary.rs +++ b/src/query/functions/src/scalars/binary.rs @@ -15,12 +15,11 @@ use std::borrow::Cow; use std::io::Write; +use databend_common_expression::EvalContext; +use databend_common_expression::FunctionDomain; +use databend_common_expression::FunctionRegistry; +use databend_common_expression::Value; use databend_common_expression::error_to_null; -use databend_common_expression::types::binary::BinaryColumn; -use databend_common_expression::types::binary::BinaryColumnBuilder; -use databend_common_expression::types::nullable::NullableColumn; -use databend_common_expression::types::string::StringColumn; -use databend_common_expression::types::string::StringColumnBuilder; use databend_common_expression::types::BinaryType; use databend_common_expression::types::Bitmap; use databend_common_expression::types::BitmapType; @@ -29,11 +28,12 @@ use databend_common_expression::types::GeometryType; use databend_common_expression::types::NumberType; use databend_common_expression::types::StringType; use databend_common_expression::types::VariantType; +use databend_common_expression::types::binary::BinaryColumn; +use databend_common_expression::types::binary::BinaryColumnBuilder; +use databend_common_expression::types::nullable::NullableColumn; +use databend_common_expression::types::string::StringColumn; +use databend_common_expression::types::string::StringColumnBuilder; use databend_common_expression::vectorize_1_arg; -use databend_common_expression::EvalContext; -use databend_common_expression::FunctionDomain; -use databend_common_expression::FunctionRegistry; -use databend_common_expression::Value; pub fn register(registry: &mut FunctionRegistry) { registry.register_aliases("to_hex", &["hex", "hex_encode"]); diff --git a/src/query/functions/src/scalars/bitmap.rs b/src/query/functions/src/scalars/bitmap.rs index d5aafbc93737c..c49e1983139f5 100644 --- a/src/query/functions/src/scalars/bitmap.rs +++ b/src/query/functions/src/scalars/bitmap.rs @@ -17,8 +17,11 @@ use std::ops::BitOr; use std::ops::BitXor; use std::ops::Sub; -use databend_common_expression::types::binary::BinaryColumnBuilder; -use databend_common_expression::types::bitmap::BitmapType; +use databend_common_expression::EvalContext; +use databend_common_expression::FunctionDomain; +use databend_common_expression::FunctionRegistry; +use databend_common_expression::types::ALL_SIGNED_INTEGER_TYPES; +use databend_common_expression::types::ALL_UNSIGNED_INTEGER_TYPES; use databend_common_expression::types::ArrayType; use databend_common_expression::types::BooleanType; use databend_common_expression::types::NullableType; @@ -26,19 +29,16 @@ use databend_common_expression::types::NumberDataType; use databend_common_expression::types::NumberType; use databend_common_expression::types::StringType; use databend_common_expression::types::UInt64Type; -use databend_common_expression::types::ALL_SIGNED_INTEGER_TYPES; -use databend_common_expression::types::ALL_UNSIGNED_INTEGER_TYPES; +use databend_common_expression::types::binary::BinaryColumnBuilder; +use databend_common_expression::types::bitmap::BitmapType; use databend_common_expression::vectorize_with_builder_1_arg; use databend_common_expression::vectorize_with_builder_2_arg; use databend_common_expression::vectorize_with_builder_3_arg; use databend_common_expression::with_signed_integer_mapped_type; use databend_common_expression::with_unsigned_integer_mapped_type; -use databend_common_expression::EvalContext; -use databend_common_expression::FunctionDomain; -use databend_common_expression::FunctionRegistry; +use databend_common_io::HybridBitmap; use databend_common_io::deserialize_bitmap; use databend_common_io::parse_bitmap; -use databend_common_io::HybridBitmap; use itertools::join; pub fn register(registry: &mut FunctionRegistry) { diff --git a/src/query/functions/src/scalars/boolean.rs b/src/query/functions/src/scalars/boolean.rs index b8a81d59cd735..2391f7c8b5901 100644 --- a/src/query/functions/src/scalars/boolean.rs +++ b/src/query/functions/src/scalars/boolean.rs @@ -18,11 +18,17 @@ use std::sync::Arc; use databend_common_base::base::OrderedFloat; +use databend_common_expression::EvalContext; +use databend_common_expression::Function; +use databend_common_expression::FunctionDomain; +use databend_common_expression::FunctionEval; +use databend_common_expression::FunctionFactory; +use databend_common_expression::FunctionRegistry; +use databend_common_expression::FunctionSignature; +use databend_common_expression::Value; use databend_common_expression::error_to_null; -use databend_common_expression::types::boolean; -use databend_common_expression::types::boolean::BooleanDomain; -use databend_common_expression::types::nullable::NullableColumn; -use databend_common_expression::types::nullable::NullableDomain; +use databend_common_expression::types::ALL_FLOAT_TYPES; +use databend_common_expression::types::ALL_INTEGER_TYPES; use databend_common_expression::types::BooleanType; use databend_common_expression::types::DataType; use databend_common_expression::types::NullableType; @@ -30,20 +36,14 @@ use databend_common_expression::types::NumberDataType; use databend_common_expression::types::NumberType; use databend_common_expression::types::SimpleDomain; use databend_common_expression::types::StringType; -use databend_common_expression::types::ALL_FLOAT_TYPES; -use databend_common_expression::types::ALL_INTEGER_TYPES; +use databend_common_expression::types::boolean; +use databend_common_expression::types::boolean::BooleanDomain; +use databend_common_expression::types::nullable::NullableColumn; +use databend_common_expression::types::nullable::NullableDomain; use databend_common_expression::vectorize_2_arg; use databend_common_expression::vectorize_with_builder_1_arg; use databend_common_expression::with_float_mapped_type; use databend_common_expression::with_integer_mapped_type; -use databend_common_expression::EvalContext; -use databend_common_expression::Function; -use databend_common_expression::FunctionDomain; -use databend_common_expression::FunctionEval; -use databend_common_expression::FunctionFactory; -use databend_common_expression::FunctionRegistry; -use databend_common_expression::FunctionSignature; -use databend_common_expression::Value; pub fn register(registry: &mut FunctionRegistry) { let fn_filters_factory = |func_name: &'static str| { diff --git a/src/query/functions/src/scalars/comparison.rs b/src/query/functions/src/scalars/comparison.rs index d14fd8a8e65d8..2abedf203bec3 100644 --- a/src/query/functions/src/scalars/comparison.rs +++ b/src/query/functions/src/scalars/comparison.rs @@ -16,11 +16,23 @@ use std::cmp::Ordering; use std::collections::HashMap; use std::sync::Arc; +use databend_common_expression::Column; +use databend_common_expression::EvalContext; +use databend_common_expression::Function; +use databend_common_expression::FunctionDomain; +use databend_common_expression::FunctionEval; +use databend_common_expression::FunctionFactory; +use databend_common_expression::FunctionRegistry; +use databend_common_expression::FunctionSignature; +use databend_common_expression::LikePattern; +use databend_common_expression::Scalar; +use databend_common_expression::ScalarRef; +use databend_common_expression::SimpleDomainCmp; use databend_common_expression::generate_like_pattern; use databend_common_expression::type_check; -use databend_common_expression::types::boolean::BooleanDomain; -use databend_common_expression::types::string::StringDomain; -use databend_common_expression::types::timestamp_tz::TimestampTzType; +use databend_common_expression::types::ALL_FLOAT_TYPES; +use databend_common_expression::types::ALL_INTEGER_TYPES; +use databend_common_expression::types::ALL_NUMBER_CLASSES; use databend_common_expression::types::AccessType; use databend_common_expression::types::AnyType; use databend_common_expression::types::ArgType; @@ -31,6 +43,7 @@ use databend_common_expression::types::BooleanType; use databend_common_expression::types::DataType; use databend_common_expression::types::DateType; use databend_common_expression::types::EmptyArrayType; +use databend_common_expression::types::F64; use databend_common_expression::types::GenericType; use databend_common_expression::types::IntervalType; use databend_common_expression::types::MutableBitmap; @@ -44,27 +57,14 @@ use databend_common_expression::types::StringType; use databend_common_expression::types::TimestampType; use databend_common_expression::types::ValueType; use databend_common_expression::types::VariantType; -use databend_common_expression::types::ALL_FLOAT_TYPES; -use databend_common_expression::types::ALL_INTEGER_TYPES; -use databend_common_expression::types::ALL_NUMBER_CLASSES; -use databend_common_expression::types::F64; +use databend_common_expression::types::boolean::BooleanDomain; +use databend_common_expression::types::string::StringDomain; +use databend_common_expression::types::timestamp_tz::TimestampTzType; use databend_common_expression::values::Value; use databend_common_expression::vectorize_with_builder_2_arg; use databend_common_expression::with_float_mapped_type; use databend_common_expression::with_integer_mapped_type; use databend_common_expression::with_number_mapped_type; -use databend_common_expression::Column; -use databend_common_expression::EvalContext; -use databend_common_expression::Function; -use databend_common_expression::FunctionDomain; -use databend_common_expression::FunctionEval; -use databend_common_expression::FunctionFactory; -use databend_common_expression::FunctionRegistry; -use databend_common_expression::FunctionSignature; -use databend_common_expression::LikePattern; -use databend_common_expression::Scalar; -use databend_common_expression::ScalarRef; -use databend_common_expression::SimpleDomainCmp; use databend_common_io::deserialize_bitmap; use databend_functions_scalar_decimal::register_decimal_compare; use jsonb::RawJsonb; @@ -816,18 +816,10 @@ fn register_tuple_cmp(registry: &mut FunctionRegistry) { } register_tuple_cmp_op(registry, "eq", true, |lhs, rhs| { - if lhs != rhs { - Some(false) - } else { - None - } + if lhs != rhs { Some(false) } else { None } }); register_tuple_cmp_op(registry, "noteq", false, |lhs, rhs| { - if lhs != rhs { - Some(true) - } else { - None - } + if lhs != rhs { Some(true) } else { None } }); register_tuple_cmp_op(registry, "gt", false, |lhs, rhs| { match lhs.partial_cmp(&rhs) { @@ -1087,8 +1079,8 @@ fn variant_vectorize_like_jsonb() -> impl Fn( Value, &mut EvalContext, ) -> Value - + Copy - + Sized { ++ Copy ++ Sized { variant_vectorize_like(|val, pattern_type| match pattern_type { LikePattern::OrdinalStr(_) | LikePattern::StartOfPercent(_) @@ -1125,7 +1117,7 @@ fn vectorize_like( Value, &mut EvalContext, ) -> Value - + Copy { ++ Copy { move |arg1, arg2, arg3, _ctx| { let Value::Scalar(escape) = arg3 else { unreachable!() @@ -1187,7 +1179,7 @@ fn variant_vectorize_like( Value, &mut EvalContext, ) -> Value - + Copy { ++ Copy { move |arg1, arg2, arg3, _ctx| { let Value::Scalar(escape) = arg3 else { unreachable!() @@ -1245,13 +1237,13 @@ fn convert_escape_pattern(escape: &str, arg2: String) -> String { fn vectorize_regexp( func: impl Fn( - &str, - &str, - &mut MutableBitmap, - &mut EvalContext, - &mut HashMap, - &mut HashMap, String>, - ) + Copy, + &str, + &str, + &mut MutableBitmap, + &mut EvalContext, + &mut HashMap, + &mut HashMap, String>, + ) + Copy, ) -> impl Fn(Value, Value, &mut EvalContext) -> Value + Copy { move |arg1, arg2, ctx| { let mut map = HashMap::new(); diff --git a/src/query/functions/src/scalars/control.rs b/src/query/functions/src/scalars/control.rs index 33870f94b903d..9199f76cd7dad 100644 --- a/src/query/functions/src/scalars/control.rs +++ b/src/query/functions/src/scalars/control.rs @@ -14,14 +14,6 @@ use std::sync::Arc; -use databend_common_expression::types::boolean::BooleanDomain; -use databend_common_expression::types::nullable::NullableColumn; -use databend_common_expression::types::nullable::NullableDomain; -use databend_common_expression::types::BooleanType; -use databend_common_expression::types::DataType; -use databend_common_expression::types::GenericType; -use databend_common_expression::types::NullType; -use databend_common_expression::types::NullableType; use databend_common_expression::Domain; use databend_common_expression::Function; use databend_common_expression::FunctionDomain; @@ -30,6 +22,14 @@ use databend_common_expression::FunctionFactory; use databend_common_expression::FunctionRegistry; use databend_common_expression::FunctionSignature; use databend_common_expression::Value; +use databend_common_expression::types::BooleanType; +use databend_common_expression::types::DataType; +use databend_common_expression::types::GenericType; +use databend_common_expression::types::NullType; +use databend_common_expression::types::NullableType; +use databend_common_expression::types::boolean::BooleanDomain; +use databend_common_expression::types::nullable::NullableColumn; +use databend_common_expression::types::nullable::NullableDomain; pub fn register(registry: &mut FunctionRegistry) { let factory = FunctionFactory::Closure(Box::new(|_, args_type: &[DataType]| { diff --git a/src/query/functions/src/scalars/decimal/Cargo.toml b/src/query/functions/src/scalars/decimal/Cargo.toml index 57e08934f770c..34172a150e228 100644 --- a/src/query/functions/src/scalars/decimal/Cargo.toml +++ b/src/query/functions/src/scalars/decimal/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "databend-functions-scalar-decimal" version = "0.1.0" -edition = "2021" +edition = "2024" [dependencies] databend-common-base = { workspace = true } diff --git a/src/query/functions/src/scalars/decimal/src/arithmetic.rs b/src/query/functions/src/scalars/decimal/src/arithmetic.rs index 359da58b16d47..5ee81e4e20f6d 100644 --- a/src/query/functions/src/scalars/decimal/src/arithmetic.rs +++ b/src/query/functions/src/scalars/decimal/src/arithmetic.rs @@ -15,15 +15,6 @@ use std::ops::*; use std::sync::Arc; -use databend_common_expression::types::compute_view::Compute; -use databend_common_expression::types::compute_view::ComputeView; -use databend_common_expression::types::decimal::*; -use databend_common_expression::types::SimpleDomain; -use databend_common_expression::types::*; -use databend_common_expression::vectorize_1_arg; -use databend_common_expression::vectorize_2_arg; -use databend_common_expression::vectorize_with_builder_2_arg; -use databend_common_expression::with_decimal_mapped_type; use databend_common_expression::Domain; use databend_common_expression::EvalContext; use databend_common_expression::Function; @@ -33,6 +24,15 @@ use databend_common_expression::FunctionFactory; use databend_common_expression::FunctionRegistry; use databend_common_expression::FunctionSignature; use databend_common_expression::Value; +use databend_common_expression::types::SimpleDomain; +use databend_common_expression::types::compute_view::Compute; +use databend_common_expression::types::compute_view::ComputeView; +use databend_common_expression::types::decimal::*; +use databend_common_expression::types::*; +use databend_common_expression::vectorize_1_arg; +use databend_common_expression::vectorize_2_arg; +use databend_common_expression::vectorize_with_builder_2_arg; +use databend_common_expression::with_decimal_mapped_type; use super::convert_to_decimal_domain; use crate::decimal_to_decimal_fast; diff --git a/src/query/functions/src/scalars/decimal/src/cast.rs b/src/query/functions/src/scalars/decimal/src/cast.rs index 7bdaac6aac8d7..dd37b51fb76db 100644 --- a/src/query/functions/src/scalars/decimal/src/cast.rs +++ b/src/query/functions/src/scalars/decimal/src/cast.rs @@ -19,17 +19,6 @@ use std::ops::Mul; use std::sync::Arc; use databend_common_base::base::OrderedFloat; -use databend_common_expression::serialize::read_decimal_with_size; -use databend_common_expression::types::compute_view::Compute; -use databend_common_expression::types::decimal::*; -use databend_common_expression::types::i256; -use databend_common_expression::types::*; -use databend_common_expression::vectorize_1_arg; -use databend_common_expression::vectorize_with_builder_1_arg; -use databend_common_expression::with_decimal_mapped_type; -use databend_common_expression::with_decimal_type; -use databend_common_expression::with_integer_mapped_type; -use databend_common_expression::with_number_mapped_type; use databend_common_expression::BlockEntry; use databend_common_expression::DataBlock; use databend_common_expression::Domain; @@ -44,6 +33,17 @@ use databend_common_expression::FunctionRegistry; use databend_common_expression::FunctionSignature; use databend_common_expression::Scalar; use databend_common_expression::Value; +use databend_common_expression::serialize::read_decimal_with_size; +use databend_common_expression::types::compute_view::Compute; +use databend_common_expression::types::decimal::*; +use databend_common_expression::types::i256; +use databend_common_expression::types::*; +use databend_common_expression::vectorize_1_arg; +use databend_common_expression::vectorize_with_builder_1_arg; +use databend_common_expression::with_decimal_mapped_type; +use databend_common_expression::with_decimal_type; +use databend_common_expression::with_integer_mapped_type; +use databend_common_expression::with_number_mapped_type; use num_traits::AsPrimitive; use crate::cast_from_jsonb::variant_to_decimal; @@ -506,11 +506,7 @@ where DataType::Boolean => { let input = input.try_downcast().unwrap(); vectorize_1_arg::>(|a: bool, _| { - if a { - T::e(size.scale()) - } else { - T::zero() - } + if a { T::e(size.scale()) } else { T::zero() } })(input, ctx) } diff --git a/src/query/functions/src/scalars/decimal/src/cast_from_jsonb.rs b/src/query/functions/src/scalars/decimal/src/cast_from_jsonb.rs index 70db00db7a001..5e32d1378f644 100644 --- a/src/query/functions/src/scalars/decimal/src/cast_from_jsonb.rs +++ b/src/query/functions/src/scalars/decimal/src/cast_from_jsonb.rs @@ -12,20 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_expression::EvalContext; +use databend_common_expression::Value; use databend_common_expression::serialize::read_decimal_with_size; -use databend_common_expression::types::i256; -use databend_common_expression::types::nullable::NullableColumnBuilder; use databend_common_expression::types::Decimal; use databend_common_expression::types::DecimalDataType; use databend_common_expression::types::DecimalSize; use databend_common_expression::types::DecimalType; +use databend_common_expression::types::F64; use databend_common_expression::types::MutableBitmap; use databend_common_expression::types::NullableType; use databend_common_expression::types::Number; use databend_common_expression::types::VariantType; -use databend_common_expression::types::F64; -use databend_common_expression::EvalContext; -use databend_common_expression::Value; +use databend_common_expression::types::i256; +use databend_common_expression::types::nullable::NullableColumnBuilder; use jsonb::Number as JsonbNumber; use jsonb::Value as JsonbValue; use num_traits::AsPrimitive; diff --git a/src/query/functions/src/scalars/decimal/src/comparison.rs b/src/query/functions/src/scalars/decimal/src/comparison.rs index c8e61c6014b15..6419c4b909ad6 100644 --- a/src/query/functions/src/scalars/decimal/src/comparison.rs +++ b/src/query/functions/src/scalars/decimal/src/comparison.rs @@ -17,12 +17,6 @@ use std::hint::unlikely; use std::ops::*; use std::sync::Arc; -use databend_common_expression::types::compute_view::ComputeView; -use databend_common_expression::types::decimal::*; -use databend_common_expression::types::i256; -use databend_common_expression::types::*; -use databend_common_expression::vectorize_cmp_2_arg; -use databend_common_expression::with_decimal_mapped_type; use databend_common_expression::Domain; use databend_common_expression::EvalContext; use databend_common_expression::Function; @@ -33,6 +27,12 @@ use databend_common_expression::FunctionRegistry; use databend_common_expression::FunctionSignature; use databend_common_expression::SimpleDomainCmp; use databend_common_expression::Value; +use databend_common_expression::types::compute_view::ComputeView; +use databend_common_expression::types::decimal::*; +use databend_common_expression::types::i256; +use databend_common_expression::types::*; +use databend_common_expression::vectorize_cmp_2_arg; +use databend_common_expression::with_decimal_mapped_type; #[inline] fn compare_multiplier(scale_a: u8, scale_b: u8) -> (u8, u8) { diff --git a/src/query/functions/src/scalars/decimal/src/hash.rs b/src/query/functions/src/scalars/decimal/src/hash.rs index 5d3bfa5b29d57..d8e203dbdb317 100644 --- a/src/query/functions/src/scalars/decimal/src/hash.rs +++ b/src/query/functions/src/scalars/decimal/src/hash.rs @@ -16,29 +16,29 @@ use std::hash::Hash; use std::hash::Hasher; use std::sync::Arc; -use databend_common_expression::types::decimal::*; +use databend_common_expression::EvalContext; +use databend_common_expression::Function; +use databend_common_expression::FunctionDomain; +use databend_common_expression::FunctionEval; +use databend_common_expression::FunctionFactory; +use databend_common_expression::FunctionRegistry; +use databend_common_expression::FunctionSignature; +use databend_common_expression::Value; use databend_common_expression::types::AccessType; use databend_common_expression::types::AnyType; use databend_common_expression::types::ArgType; use databend_common_expression::types::DataType; +use databend_common_expression::types::F32; +use databend_common_expression::types::F64; use databend_common_expression::types::Number; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::NumberType; use databend_common_expression::types::UInt32Type; use databend_common_expression::types::UInt64Type; -use databend_common_expression::types::F32; -use databend_common_expression::types::F64; +use databend_common_expression::types::decimal::*; use databend_common_expression::vectorize_with_builder_1_arg; use databend_common_expression::vectorize_with_builder_2_arg; use databend_common_expression::with_integer_mapped_type; -use databend_common_expression::EvalContext; -use databend_common_expression::Function; -use databend_common_expression::FunctionDomain; -use databend_common_expression::FunctionEval; -use databend_common_expression::FunctionFactory; -use databend_common_expression::FunctionRegistry; -use databend_common_expression::FunctionSignature; -use databend_common_expression::Value; pub fn register_decimal_hash(registry: &mut FunctionRegistry) { registry.register_function_factory( @@ -113,19 +113,23 @@ fn decimal_hash_factory_2_arg( ) -> Option> { let (nullable, size, seed_type) = match args_type { [DataType::Null, DataType::Number(number)] => (true, DecimalSize::default_128(), *number), - [DataType::Nullable(box DataType::Decimal(size)), DataType::Number(number)] => { - (true, *size, *number) - } + [ + DataType::Nullable(box DataType::Decimal(size)), + DataType::Number(number), + ] => (true, *size, *number), [DataType::Decimal(size), DataType::Number(number)] => (false, *size, *number), - [DataType::Null, DataType::Nullable(box DataType::Number(number))] => { - (true, DecimalSize::default_128(), *number) - } - [DataType::Nullable(box DataType::Decimal(size)), DataType::Nullable(box DataType::Number(number))] => { - (true, *size, *number) - } - [DataType::Decimal(size), DataType::Nullable(box DataType::Number(number))] => { - (true, *size, *number) - } + [ + DataType::Null, + DataType::Nullable(box DataType::Number(number)), + ] => (true, DecimalSize::default_128(), *number), + [ + DataType::Nullable(box DataType::Decimal(size)), + DataType::Nullable(box DataType::Number(number)), + ] => (true, *size, *number), + [ + DataType::Decimal(size), + DataType::Nullable(box DataType::Number(number)), + ] => (true, *size, *number), _ => return None, }; diff --git a/src/query/functions/src/scalars/decimal/src/lib.rs b/src/query/functions/src/scalars/decimal/src/lib.rs index af7e7451fb503..50e9f75b95eda 100644 --- a/src/query/functions/src/scalars/decimal/src/lib.rs +++ b/src/query/functions/src/scalars/decimal/src/lib.rs @@ -16,12 +16,12 @@ #![allow(clippy::uninlined_format_args)] #![allow(clippy::ptr_arg)] #![allow(clippy::type_complexity)] +#![allow(clippy::collapsible_if)] #![feature(box_patterns)] #![feature(type_ascription)] #![feature(try_blocks)] #![feature(downcast_unchecked)] #![feature(likely_unlikely)] -#![feature(let_chains)] mod arithmetic; mod cast; diff --git a/src/query/functions/src/scalars/decimal/src/math.rs b/src/query/functions/src/scalars/decimal/src/math.rs index a4216a14ad70d..c62bf03620152 100644 --- a/src/query/functions/src/scalars/decimal/src/math.rs +++ b/src/query/functions/src/scalars/decimal/src/math.rs @@ -15,12 +15,6 @@ use std::ops::*; use std::sync::Arc; -use databend_common_expression::types::compute_view::Compute; -use databend_common_expression::types::compute_view::ComputeView; -use databend_common_expression::types::decimal::*; -use databend_common_expression::types::*; -use databend_common_expression::vectorize_1_arg; -use databend_common_expression::with_decimal_mapped_type; use databend_common_expression::EvalContext; use databend_common_expression::Function; use databend_common_expression::FunctionDomain; @@ -30,6 +24,12 @@ use databend_common_expression::FunctionRegistry; use databend_common_expression::FunctionSignature; use databend_common_expression::Scalar; use databend_common_expression::Value; +use databend_common_expression::types::compute_view::Compute; +use databend_common_expression::types::compute_view::ComputeView; +use databend_common_expression::types::decimal::*; +use databend_common_expression::types::*; +use databend_common_expression::vectorize_1_arg; +use databend_common_expression::with_decimal_mapped_type; pub fn register_decimal_math(registry: &mut FunctionRegistry) { let factory_rounds = |params: &[Scalar], args_type: &[DataType], round_mode: RoundMode| { diff --git a/src/query/functions/src/scalars/decimal/src/uuid.rs b/src/query/functions/src/scalars/decimal/src/uuid.rs index 3045783ad7717..0c8bbd1bdd82b 100644 --- a/src/query/functions/src/scalars/decimal/src/uuid.rs +++ b/src/query/functions/src/scalars/decimal/src/uuid.rs @@ -13,14 +13,6 @@ // limitations under the License. use databend_common_base::base::uuid::Uuid; -use databend_common_expression::types::string::StringColumnBuilder; -use databend_common_expression::types::DataType; -use databend_common_expression::types::Decimal128Type; -use databend_common_expression::types::Decimal256As128Type; -use databend_common_expression::types::Decimal64As128Type; -use databend_common_expression::types::DecimalDataType; -use databend_common_expression::types::StringType; -use databend_common_expression::vectorize_with_builder_1_arg; use databend_common_expression::EvalContext; use databend_common_expression::Function; use databend_common_expression::FunctionDomain; @@ -28,6 +20,14 @@ use databend_common_expression::FunctionEval; use databend_common_expression::FunctionFactoryHelper; use databend_common_expression::FunctionRegistry; use databend_common_expression::FunctionSignature; +use databend_common_expression::types::DataType; +use databend_common_expression::types::Decimal64As128Type; +use databend_common_expression::types::Decimal128Type; +use databend_common_expression::types::Decimal256As128Type; +use databend_common_expression::types::DecimalDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::string::StringColumnBuilder; +use databend_common_expression::vectorize_with_builder_1_arg; pub fn register_decimal_to_uuid(registry: &mut FunctionRegistry) { let factory = FunctionFactoryHelper::create_1_arg_passthrough_nullable(|_, arg| { diff --git a/src/query/functions/src/scalars/geographic/Cargo.toml b/src/query/functions/src/scalars/geographic/Cargo.toml index 62a1d99bc967c..98e76016f51ae 100644 --- a/src/query/functions/src/scalars/geographic/Cargo.toml +++ b/src/query/functions/src/scalars/geographic/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "databend-functions-scalar-geo" version = "0.1.0" -edition = "2021" +edition = "2024" [dependencies] databend-common-exception = { workspace = true } diff --git a/src/query/functions/src/scalars/geographic/src/geo.rs b/src/query/functions/src/scalars/geographic/src/geo.rs index b6f9007ad5704..a3778d3d6c7a3 100644 --- a/src/query/functions/src/scalars/geographic/src/geo.rs +++ b/src/query/functions/src/scalars/geographic/src/geo.rs @@ -17,22 +17,6 @@ use std::num::Wrapping; use std::sync::Arc; use std::sync::Once; -use databend_common_expression::types::map::KvPair; -use databend_common_expression::types::number::Float64Type; -use databend_common_expression::types::number::NumberColumnBuilder; -use databend_common_expression::types::number::NumberScalar; -use databend_common_expression::types::number::F32; -use databend_common_expression::types::number::F64; -use databend_common_expression::types::AccessType; -use databend_common_expression::types::AnyType; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::NumberType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::UInt8Type; -use databend_common_expression::vectorize_with_builder_1_arg; -use databend_common_expression::vectorize_with_builder_2_arg; -use databend_common_expression::vectorize_with_builder_3_arg; use databend_common_expression::Column; use databend_common_expression::EvalContext; use databend_common_expression::Function; @@ -44,11 +28,27 @@ use databend_common_expression::FunctionSignature; use databend_common_expression::Scalar; use databend_common_expression::ScalarRef; use databend_common_expression::Value; -use geo::coord; +use databend_common_expression::types::AccessType; +use databend_common_expression::types::AnyType; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::NumberType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::UInt8Type; +use databend_common_expression::types::map::KvPair; +use databend_common_expression::types::number::F32; +use databend_common_expression::types::number::F64; +use databend_common_expression::types::number::Float64Type; +use databend_common_expression::types::number::NumberColumnBuilder; +use databend_common_expression::types::number::NumberScalar; +use databend_common_expression::vectorize_with_builder_1_arg; +use databend_common_expression::vectorize_with_builder_2_arg; +use databend_common_expression::vectorize_with_builder_3_arg; use geo::Contains; use geo::Coord; use geo::LineString; use geo::Polygon; +use geo::coord; use h3o::LatLng; use h3o::Resolution; use once_cell::sync::OnceCell; diff --git a/src/query/functions/src/scalars/geographic/src/geo_h3.rs b/src/query/functions/src/scalars/geographic/src/geo_h3.rs index 17fbf78ebd606..007251e160c50 100644 --- a/src/query/functions/src/scalars/geographic/src/geo_h3.rs +++ b/src/query/functions/src/scalars/geographic/src/geo_h3.rs @@ -12,20 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_expression::types::map::KvPair; +use databend_common_expression::FunctionDomain; +use databend_common_expression::FunctionRegistry; use databend_common_expression::types::ArrayType; use databend_common_expression::types::BooleanType; +use databend_common_expression::types::F64; use databend_common_expression::types::Float64Type; use databend_common_expression::types::Int32Type; use databend_common_expression::types::StringType; +use databend_common_expression::types::UInt8Type; use databend_common_expression::types::UInt32Type; use databend_common_expression::types::UInt64Type; -use databend_common_expression::types::UInt8Type; -use databend_common_expression::types::F64; +use databend_common_expression::types::map::KvPair; use databend_common_expression::vectorize_with_builder_1_arg; use databend_common_expression::vectorize_with_builder_2_arg; -use databend_common_expression::FunctionDomain; -use databend_common_expression::FunctionRegistry; use h3o::CellIndex; use h3o::DirectedEdgeIndex; use h3o::LatLng; diff --git a/src/query/functions/src/scalars/geographic/src/geography.rs b/src/query/functions/src/scalars/geographic/src/geography.rs index 8dd4c9954f8c9..2a5beb5ca1b48 100644 --- a/src/query/functions/src/scalars/geographic/src/geography.rs +++ b/src/query/functions/src/scalars/geographic/src/geography.rs @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_expression::FunctionDomain; +use databend_common_expression::FunctionRegistry; use databend_common_expression::types::*; use databend_common_expression::vectorize_with_builder_1_arg; use databend_common_expression::vectorize_with_builder_2_arg; -use databend_common_expression::FunctionDomain; -use databend_common_expression::FunctionRegistry; use databend_common_io::geography::geography_from_ewkt; use databend_common_io::wkb::make_point; diff --git a/src/query/functions/src/scalars/geographic/src/geometry.rs b/src/query/functions/src/scalars/geographic/src/geometry.rs index 536063dc43022..ad396c9bd5fb4 100644 --- a/src/query/functions/src/scalars/geographic/src/geometry.rs +++ b/src/query/functions/src/scalars/geographic/src/geometry.rs @@ -13,23 +13,25 @@ // limitations under the License. use databend_common_exception::ErrorCode; -use databend_common_expression::types::geometry::GeometryType; +use databend_common_expression::EvalContext; +use databend_common_expression::FunctionDomain; +use databend_common_expression::FunctionRegistry; use databend_common_expression::types::BinaryType; use databend_common_expression::types::BooleanType; +use databend_common_expression::types::F64; use databend_common_expression::types::Int32Type; use databend_common_expression::types::NullableType; use databend_common_expression::types::NumberType; use databend_common_expression::types::StringType; use databend_common_expression::types::UInt32Type; use databend_common_expression::types::VariantType; -use databend_common_expression::types::F64; +use databend_common_expression::types::geometry::GeometryType; use databend_common_expression::vectorize_with_builder_1_arg; use databend_common_expression::vectorize_with_builder_2_arg; use databend_common_expression::vectorize_with_builder_3_arg; use databend_common_expression::vectorize_with_builder_4_arg; -use databend_common_expression::EvalContext; -use databend_common_expression::FunctionDomain; -use databend_common_expression::FunctionRegistry; +use databend_common_io::Axis; +use databend_common_io::Extremum; use databend_common_io::ewkb_to_geo; use databend_common_io::geo_to_ewkb; use databend_common_io::geo_to_ewkt; @@ -41,10 +43,6 @@ use databend_common_io::geometry_format; use databend_common_io::geometry_from_ewkt; use databend_common_io::geometry_type_name; use databend_common_io::read_srid; -use databend_common_io::Axis; -use databend_common_io::Extremum; -use geo::coord; -use geo::dimensions::Dimensions; use geo::Area; use geo::BoundingRect; use geo::Contains; @@ -67,19 +65,21 @@ use geo::ToDegrees; use geo::ToRadians; use geo::Triangle; use geo::Within; +use geo::coord; +use geo::dimensions::Dimensions; use geohash::decode_bbox; use geohash::encode; -use geozero::geojson::GeoJson; -use geozero::wkb::Ewkb; use geozero::CoordDimensions; use geozero::GeozeroGeometry; use geozero::ToGeo; use geozero::ToWkb; -use jsonb::parse_owned_jsonb_with_buf; +use geozero::geojson::GeoJson; +use geozero::wkb::Ewkb; use jsonb::RawJsonb; +use jsonb::parse_owned_jsonb_with_buf; use num_traits::AsPrimitive; -use proj4rs::transform::transform; use proj4rs::Proj; +use proj4rs::transform::transform; pub fn register(registry: &mut FunctionRegistry) { // aliases diff --git a/src/query/functions/src/scalars/geographic/src/lib.rs b/src/query/functions/src/scalars/geographic/src/lib.rs index f952e74c1d8f8..9b985fd9d1698 100644 --- a/src/query/functions/src/scalars/geographic/src/lib.rs +++ b/src/query/functions/src/scalars/geographic/src/lib.rs @@ -14,8 +14,10 @@ #![allow(clippy::arc_with_non_send_sync)] #![allow(clippy::uninlined_format_args)] +#![allow(clippy::collapsible_if)] #![allow(clippy::ptr_arg)] #![allow(clippy::type_complexity)] +#![allow(clippy::redundant_iter_cloned)] #![allow(internal_features)] #![feature(core_intrinsics)] #![feature(box_patterns)] diff --git a/src/query/functions/src/scalars/hash.rs b/src/query/functions/src/scalars/hash.rs index 16cbcb6e876cc..159a93a799470 100644 --- a/src/query/functions/src/scalars/hash.rs +++ b/src/query/functions/src/scalars/hash.rs @@ -16,10 +16,11 @@ use std::collections::hash_map::DefaultHasher; use std::hash::Hash; use std::hash::Hasher; -use databend_common_expression::types::i256; -use databend_common_expression::types::number::NumberScalar; -use databend_common_expression::types::number::F32; -use databend_common_expression::types::number::F64; +use databend_common_expression::FunctionDomain; +use databend_common_expression::FunctionRegistry; +use databend_common_expression::Scalar; +use databend_common_expression::types::ALL_INTEGER_TYPES; +use databend_common_expression::types::ALL_NUMBER_CLASSES; use databend_common_expression::types::ArgType; use databend_common_expression::types::BitmapType; use databend_common_expression::types::BooleanType; @@ -31,19 +32,18 @@ use databend_common_expression::types::NumberType; use databend_common_expression::types::StringType; use databend_common_expression::types::TimestampType; use databend_common_expression::types::VariantType; -use databend_common_expression::types::ALL_INTEGER_TYPES; -use databend_common_expression::types::ALL_NUMBER_CLASSES; +use databend_common_expression::types::i256; +use databend_common_expression::types::number::F32; +use databend_common_expression::types::number::F64; +use databend_common_expression::types::number::NumberScalar; use databend_common_expression::vectorize_with_builder_1_arg; use databend_common_expression::vectorize_with_builder_2_arg; use databend_common_expression::with_integer_mapped_type; use databend_common_expression::with_number_mapped_type; -use databend_common_expression::FunctionDomain; -use databend_common_expression::FunctionRegistry; -use databend_common_expression::Scalar; -use databend_functions_scalar_decimal::register_decimal_hash; -use databend_functions_scalar_decimal::register_decimal_hash_with_seed; use databend_functions_scalar_decimal::HashFunction; use databend_functions_scalar_decimal::HashFunctionWithSeed; +use databend_functions_scalar_decimal::register_decimal_hash; +use databend_functions_scalar_decimal::register_decimal_hash_with_seed; use md5::Digest; use md5::Md5 as Md5Hasher; use naive_cityhash::cityhash64_with_seed; @@ -349,7 +349,7 @@ for_all_integer_types! { integer_impl } impl DFHash for i256 { #[inline] fn hash(&self, state: &mut H) { - Hash::hash(self.0 .0.as_slice(), state); + Hash::hash(self.0.0.as_slice(), state); } } diff --git a/src/query/functions/src/scalars/hilbert.rs b/src/query/functions/src/scalars/hilbert.rs index 060fe5ab97abe..e83c4db3eee8c 100644 --- a/src/query/functions/src/scalars/hilbert.rs +++ b/src/query/functions/src/scalars/hilbert.rs @@ -14,6 +14,16 @@ use std::sync::Arc; +use databend_common_expression::Column; +use databend_common_expression::FixedLengthEncoding; +use databend_common_expression::Function; +use databend_common_expression::FunctionDomain; +use databend_common_expression::FunctionEval; +use databend_common_expression::FunctionFactory; +use databend_common_expression::FunctionRegistry; +use databend_common_expression::FunctionSignature; +use databend_common_expression::ScalarRef; +use databend_common_expression::Value; use databend_common_expression::hilbert_index; use databend_common_expression::types::ArgType; use databend_common_expression::types::ArrayType; @@ -25,16 +35,6 @@ use databend_common_expression::types::NumberType; use databend_common_expression::types::ReturnType; use databend_common_expression::types::ValueType; use databend_common_expression::vectorize_with_builder_2_arg; -use databend_common_expression::Column; -use databend_common_expression::FixedLengthEncoding; -use databend_common_expression::Function; -use databend_common_expression::FunctionDomain; -use databend_common_expression::FunctionEval; -use databend_common_expression::FunctionFactory; -use databend_common_expression::FunctionRegistry; -use databend_common_expression::FunctionSignature; -use databend_common_expression::ScalarRef; -use databend_common_expression::Value; /// Registers Hilbert curve related functions with the function registry. pub fn register(registry: &mut FunctionRegistry) { diff --git a/src/query/functions/src/scalars/integer_arithmetic/Cargo.toml b/src/query/functions/src/scalars/integer_arithmetic/Cargo.toml index e4de947242668..443b70bb467f3 100644 --- a/src/query/functions/src/scalars/integer_arithmetic/Cargo.toml +++ b/src/query/functions/src/scalars/integer_arithmetic/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "databend-functions-scalar-integer-basic-arithmetic" version = "0.1.0" -edition = "2021" +edition = "2024" [dependencies] databend-common-expression = { workspace = true } diff --git a/src/query/functions/src/scalars/integer_arithmetic/src/integer_arithmetic.rs b/src/query/functions/src/scalars/integer_arithmetic/src/integer_arithmetic.rs index ec7e14d6e1539..03fb732f65fc9 100644 --- a/src/query/functions/src/scalars/integer_arithmetic/src/integer_arithmetic.rs +++ b/src/query/functions/src/scalars/integer_arithmetic/src/integer_arithmetic.rs @@ -12,17 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_expression::FunctionDomain; +use databend_common_expression::FunctionRegistry; use databend_common_expression::arithmetics_type::ResultTypeOfBinary; +use databend_common_expression::types::ALL_INTEGER_TYPES; +use databend_common_expression::types::F64; use databend_common_expression::types::Number; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::NumberType; use databend_common_expression::types::SimpleDomain; -use databend_common_expression::types::ALL_INTEGER_TYPES; -use databend_common_expression::types::F64; use databend_common_expression::vectorize_with_builder_2_arg; use databend_common_expression::with_integer_mapped_type; -use databend_common_expression::FunctionDomain; -use databend_common_expression::FunctionRegistry; use databend_functions_scalar_numeric_basic_arithmetic::numeric_basic_arithmetic::divide_function; use databend_functions_scalar_numeric_basic_arithmetic::register_basic_arithmetic; use databend_functions_scalar_numeric_basic_arithmetic::register_divide; diff --git a/src/query/functions/src/scalars/map.rs b/src/query/functions/src/scalars/map.rs index 2675e96727c0b..678a31ed0868f 100644 --- a/src/query/functions/src/scalars/map.rs +++ b/src/query/functions/src/scalars/map.rs @@ -16,8 +16,15 @@ use std::collections::HashSet; use std::hash::Hash; use std::sync::Arc; -use databend_common_expression::types::map::KvColumn; -use databend_common_expression::types::nullable::NullableDomain; +use databend_common_expression::ColumnBuilder; +use databend_common_expression::Function; +use databend_common_expression::FunctionDomain; +use databend_common_expression::FunctionEval; +use databend_common_expression::FunctionFactory; +use databend_common_expression::FunctionRegistry; +use databend_common_expression::FunctionSignature; +use databend_common_expression::ScalarRef; +use databend_common_expression::Value; use databend_common_expression::types::AccessType; use databend_common_expression::types::AnyType; use databend_common_expression::types::ArrayType; @@ -32,19 +39,12 @@ use databend_common_expression::types::NullableType; use databend_common_expression::types::NumberType; use databend_common_expression::types::ReturnType; use databend_common_expression::types::SimpleDomain; +use databend_common_expression::types::map::KvColumn; +use databend_common_expression::types::nullable::NullableDomain; use databend_common_expression::vectorize_1_arg; use databend_common_expression::vectorize_with_builder_2_arg; use databend_common_expression::vectorize_with_builder_3_arg; use databend_common_expression::vectorize_with_builder_4_arg; -use databend_common_expression::ColumnBuilder; -use databend_common_expression::Function; -use databend_common_expression::FunctionDomain; -use databend_common_expression::FunctionEval; -use databend_common_expression::FunctionFactory; -use databend_common_expression::FunctionRegistry; -use databend_common_expression::FunctionSignature; -use databend_common_expression::ScalarRef; -use databend_common_expression::Value; use databend_common_hashtable::StackHashSet; use siphasher::sip128::Hasher128; use siphasher::sip128::SipHasher24; diff --git a/src/query/functions/src/scalars/mathematics/Cargo.toml b/src/query/functions/src/scalars/mathematics/Cargo.toml index cf07dbdbb693e..e59a87f2c9c49 100644 --- a/src/query/functions/src/scalars/mathematics/Cargo.toml +++ b/src/query/functions/src/scalars/mathematics/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "databend-functions-scalar-math" version = "0.1.0" -edition = "2021" +edition = "2024" [dependencies] crc32fast = { workspace = true } diff --git a/src/query/functions/src/scalars/mathematics/src/math.rs b/src/query/functions/src/scalars/mathematics/src/math.rs index 5e566d0c994d9..a4299bc49518b 100644 --- a/src/query/functions/src/scalars/mathematics/src/math.rs +++ b/src/query/functions/src/scalars/mathematics/src/math.rs @@ -18,21 +18,21 @@ use std::f64::consts::PI; use std::marker::PhantomData; use databend_common_base::base::OrderedFloat; -use databend_common_expression::types::boolean::BooleanDomain; -use databend_common_expression::types::number::SimpleDomain; -use databend_common_expression::types::number::F64; +use databend_common_expression::FunctionDomain; +use databend_common_expression::FunctionRegistry; +use databend_common_expression::Value; +use databend_common_expression::types::ALL_FLOAT_TYPES; +use databend_common_expression::types::ALL_INTEGER_TYPES; +use databend_common_expression::types::ALL_NUMERICS_TYPES; use databend_common_expression::types::BooleanType; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::NumberType; use databend_common_expression::types::StringType; -use databend_common_expression::types::ALL_FLOAT_TYPES; -use databend_common_expression::types::ALL_INTEGER_TYPES; -use databend_common_expression::types::ALL_NUMERICS_TYPES; +use databend_common_expression::types::boolean::BooleanDomain; +use databend_common_expression::types::number::F64; +use databend_common_expression::types::number::SimpleDomain; use databend_common_expression::vectorize_with_builder_1_arg; use databend_common_expression::with_number_mapped_type; -use databend_common_expression::FunctionDomain; -use databend_common_expression::FunctionRegistry; -use databend_common_expression::Value; use databend_functions_scalar_decimal::register_decimal_math; use num_traits::AsPrimitive; use num_traits::Float; @@ -509,9 +509,5 @@ type Log10Function = GenericLogFunction; type Log2Function = GenericLogFunction; fn factorial(n: i64) -> i64 { - if n <= 0 { - 1 - } else { - n * factorial(n - 1) - } + if n <= 0 { 1 } else { n * factorial(n - 1) } } diff --git a/src/query/functions/src/scalars/numeric_basic_arithmetic/Cargo.toml b/src/query/functions/src/scalars/numeric_basic_arithmetic/Cargo.toml index f5efcd023d08a..ad8799cd4e83e 100644 --- a/src/query/functions/src/scalars/numeric_basic_arithmetic/Cargo.toml +++ b/src/query/functions/src/scalars/numeric_basic_arithmetic/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "databend-functions-scalar-numeric-basic-arithmetic" version = "0.1.0" -edition = "2021" +edition = "2024" [dependencies] databend-common-expression = { workspace = true } diff --git a/src/query/functions/src/scalars/numeric_basic_arithmetic/src/arithmetic_modulo.rs b/src/query/functions/src/scalars/numeric_basic_arithmetic/src/arithmetic_modulo.rs index 412f765332d2b..afc2e334c845b 100644 --- a/src/query/functions/src/scalars/numeric_basic_arithmetic/src/arithmetic_modulo.rs +++ b/src/query/functions/src/scalars/numeric_basic_arithmetic/src/arithmetic_modulo.rs @@ -14,19 +14,19 @@ use std::ops::Rem; -use databend_common_expression::types::number::*; -use databend_common_expression::types::ReturnType; -use databend_common_expression::types::ValueType; use databend_common_expression::EvalContext; use databend_common_expression::Value; +use databend_common_expression::types::ReturnType; +use databend_common_expression::types::ValueType; +use databend_common_expression::types::number::*; use num_traits::AsPrimitive; +use strength_reduce::StrengthReducedU8; use strength_reduce::StrengthReducedU16; use strength_reduce::StrengthReducedU32; use strength_reduce::StrengthReducedU64; -use strength_reduce::StrengthReducedU8; -pub fn vectorize_modulo( -) -> impl Fn(Value>, Value>, &mut EvalContext) -> Value> + Copy +pub fn vectorize_modulo() +-> impl Fn(Value>, Value>, &mut EvalContext) -> Value> + Copy where L: Number + AsPrimitive, R: Number + AsPrimitive + AsPrimitive, diff --git a/src/query/functions/src/scalars/numeric_basic_arithmetic/src/numeric_basic_arithmetic.rs b/src/query/functions/src/scalars/numeric_basic_arithmetic/src/numeric_basic_arithmetic.rs index fc9fecff48f53..567a40abff3dd 100644 --- a/src/query/functions/src/scalars/numeric_basic_arithmetic/src/numeric_basic_arithmetic.rs +++ b/src/query/functions/src/scalars/numeric_basic_arithmetic/src/numeric_basic_arithmetic.rs @@ -12,23 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_expression::EvalContext; +use databend_common_expression::FunctionDomain; +use databend_common_expression::FunctionRegistry; use databend_common_expression::arithmetics_type::ResultTypeOfBinary; use databend_common_expression::arithmetics_type::ResultTypeOfUnary; +use databend_common_expression::types::ALL_FLOAT_TYPES; +use databend_common_expression::types::ALL_INTEGER_TYPES; +use databend_common_expression::types::F64; use databend_common_expression::types::NullableType; use databend_common_expression::types::Number; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::NumberType; use databend_common_expression::types::SimpleDomain; -use databend_common_expression::types::ALL_FLOAT_TYPES; -use databend_common_expression::types::ALL_INTEGER_TYPES; -use databend_common_expression::types::F64; use databend_common_expression::vectorize_2_arg; use databend_common_expression::vectorize_with_builder_2_arg; use databend_common_expression::with_float_mapped_type; use databend_common_expression::with_integer_mapped_type; -use databend_common_expression::EvalContext; -use databend_common_expression::FunctionDomain; -use databend_common_expression::FunctionRegistry; use num_traits::AsPrimitive; use crate::arithmetic_modulo::vectorize_modulo; diff --git a/src/query/functions/src/scalars/obfuscator.rs b/src/query/functions/src/scalars/obfuscator.rs index 0dac03718b50a..5e00cb076c248 100644 --- a/src/query/functions/src/scalars/obfuscator.rs +++ b/src/query/functions/src/scalars/obfuscator.rs @@ -14,20 +14,20 @@ use std::hash::Hasher; -use databend_common_base::obfuscator::generate; use databend_common_base::obfuscator::CodePoint; use databend_common_base::obfuscator::Histogram; use databend_common_base::obfuscator::NGramHash; use databend_common_base::obfuscator::Table; +use databend_common_base::obfuscator::generate; use databend_common_column::buffer::Buffer; -use databend_common_expression::types::map::KvPair; -use databend_common_expression::types::*; -use databend_common_expression::vectorize_with_builder_2_arg; -use databend_common_expression::with_signed_integer_mapped_type; use databend_common_expression::Column; use databend_common_expression::FunctionDomain; use databend_common_expression::FunctionRegistry; use databend_common_expression::Value; +use databend_common_expression::types::map::KvPair; +use databend_common_expression::types::*; +use databend_common_expression::vectorize_with_builder_2_arg; +use databend_common_expression::with_signed_integer_mapped_type; struct ColumnHistogram { total: u32, diff --git a/src/query/functions/src/scalars/other.rs b/src/query/functions/src/scalars/other.rs index db1194eefa365..b36e30398c748 100644 --- a/src/query/functions/src/scalars/other.rs +++ b/src/query/functions/src/scalars/other.rs @@ -17,20 +17,23 @@ use std::net::Ipv4Addr; use std::sync::Arc; use std::time::Duration; +use databend_common_base::base::OrderedFloat; use databend_common_base::base::convert_byte_size; use databend_common_base::base::convert_number_size; use databend_common_base::base::uuid::Uuid; -use databend_common_base::base::OrderedFloat; +use databend_common_expression::Column; +use databend_common_expression::Domain; +use databend_common_expression::EvalContext; +use databend_common_expression::Function; +use databend_common_expression::FunctionDomain; +use databend_common_expression::FunctionEval; +use databend_common_expression::FunctionFactory; +use databend_common_expression::FunctionProperty; +use databend_common_expression::FunctionRegistry; +use databend_common_expression::FunctionSignature; +use databend_common_expression::Scalar; +use databend_common_expression::Value; use databend_common_expression::error_to_null; -use databend_common_expression::types::boolean::BooleanDomain; -use databend_common_expression::types::nullable::NullableColumn; -use databend_common_expression::types::number::Float32Type; -use databend_common_expression::types::number::Float64Type; -use databend_common_expression::types::number::Int64Type; -use databend_common_expression::types::number::UInt32Type; -use databend_common_expression::types::number::UInt8Type; -use databend_common_expression::types::number::F64; -use databend_common_expression::types::string::StringColumnBuilder; use databend_common_expression::types::AccessType; use databend_common_expression::types::DataType; use databend_common_expression::types::DateType; @@ -46,21 +49,18 @@ use databend_common_expression::types::SimpleDomain; use databend_common_expression::types::StringType; use databend_common_expression::types::TimestampType; use databend_common_expression::types::ValueType; +use databend_common_expression::types::boolean::BooleanDomain; +use databend_common_expression::types::nullable::NullableColumn; +use databend_common_expression::types::number::F64; +use databend_common_expression::types::number::Float32Type; +use databend_common_expression::types::number::Float64Type; +use databend_common_expression::types::number::Int64Type; +use databend_common_expression::types::number::UInt8Type; +use databend_common_expression::types::number::UInt32Type; +use databend_common_expression::types::string::StringColumnBuilder; use databend_common_expression::vectorize_2_arg; use databend_common_expression::vectorize_with_builder_1_arg; use databend_common_expression::vectorize_with_builder_2_arg; -use databend_common_expression::Column; -use databend_common_expression::Domain; -use databend_common_expression::EvalContext; -use databend_common_expression::Function; -use databend_common_expression::FunctionDomain; -use databend_common_expression::FunctionEval; -use databend_common_expression::FunctionFactory; -use databend_common_expression::FunctionProperty; -use databend_common_expression::FunctionRegistry; -use databend_common_expression::FunctionSignature; -use databend_common_expression::Scalar; -use databend_common_expression::Value; use databend_common_io::number::FmtCacheEntry; use rand::Rng; use rand::SeedableRng; @@ -153,7 +153,7 @@ pub fn register(registry: &mut FunctionRegistry) { rand::rngs::SmallRng::from_entropy() }; let rand_nums = (0..ctx.num_rows) - .map(|_| rng.gen::()) + .map(|_| rng.r#gen::()) .collect::>(); Value::Column(rand_nums.into()) }, @@ -169,7 +169,7 @@ pub fn register(registry: &mut FunctionRegistry) { }, |val, _| { let mut rng = rand::rngs::SmallRng::seed_from_u64(val); - rng.gen::() + rng.r#gen::() }, ); diff --git a/src/query/functions/src/scalars/string.rs b/src/query/functions/src/scalars/string.rs index 2be9f887d57ca..f75c14000051b 100644 --- a/src/query/functions/src/scalars/string.rs +++ b/src/query/functions/src/scalars/string.rs @@ -15,21 +15,21 @@ use std::cmp::Ordering; use std::io::Write; +use databend_common_expression::FunctionDomain; +use databend_common_expression::FunctionRegistry; +use databend_common_expression::types::ArrayType; +use databend_common_expression::types::NumberType; +use databend_common_expression::types::StringType; use databend_common_expression::types::number::SimpleDomain; use databend_common_expression::types::number::UInt64Type; use databend_common_expression::types::string::StringColumnBuilder; use databend_common_expression::types::string::StringDomain; -use databend_common_expression::types::ArrayType; -use databend_common_expression::types::NumberType; -use databend_common_expression::types::StringType; use databend_common_expression::unify_string; use databend_common_expression::vectorize_1_arg; use databend_common_expression::vectorize_with_builder_1_arg; use databend_common_expression::vectorize_with_builder_2_arg; use databend_common_expression::vectorize_with_builder_3_arg; use databend_common_expression::vectorize_with_builder_4_arg; -use databend_common_expression::FunctionDomain; -use databend_common_expression::FunctionRegistry; use databend_functions_scalar_decimal::register_decimal_to_uuid; use stringslice::StringSlice; @@ -922,8 +922,8 @@ pub fn register(registry: &mut FunctionRegistry) { } pub(crate) mod soundex { - use databend_common_expression::types::string::StringColumnBuilder; use databend_common_expression::EvalContext; + use databend_common_expression::types::string::StringColumnBuilder; pub fn soundex(val: &str, output: &mut StringColumnBuilder, _eval_context: &mut EvalContext) { let mut last = None; diff --git a/src/query/functions/src/scalars/string_multi_args.rs b/src/query/functions/src/scalars/string_multi_args.rs index adee7c9927dc3..ec67e4c555282 100644 --- a/src/query/functions/src/scalars/string_multi_args.rs +++ b/src/query/functions/src/scalars/string_multi_args.rs @@ -14,15 +14,6 @@ use std::sync::Arc; -use databend_common_expression::passthrough_nullable; -use databend_common_expression::types::array::ArrayColumnBuilder; -use databend_common_expression::types::nullable::NullableColumn; -use databend_common_expression::types::number::Int64Type; -use databend_common_expression::types::number::NumberScalar; -use databend_common_expression::types::string::StringDomain; -use databend_common_expression::types::MutableBitmap; -use databend_common_expression::types::NumberColumn; -use databend_common_expression::types::*; use databend_common_expression::Column; use databend_common_expression::Domain; use databend_common_expression::EvalContext; @@ -34,6 +25,15 @@ use databend_common_expression::FunctionRegistry; use databend_common_expression::FunctionSignature; use databend_common_expression::Scalar; use databend_common_expression::Value; +use databend_common_expression::passthrough_nullable; +use databend_common_expression::types::MutableBitmap; +use databend_common_expression::types::NumberColumn; +use databend_common_expression::types::array::ArrayColumnBuilder; +use databend_common_expression::types::nullable::NullableColumn; +use databend_common_expression::types::number::Int64Type; +use databend_common_expression::types::number::NumberScalar; +use databend_common_expression::types::string::StringDomain; +use databend_common_expression::types::*; use regex::Match; use string::StringColumnBuilder; diff --git a/src/query/functions/src/scalars/timestamp/Cargo.toml b/src/query/functions/src/scalars/timestamp/Cargo.toml index 83c14fe0dcdec..a8ab949918a42 100644 --- a/src/query/functions/src/scalars/timestamp/Cargo.toml +++ b/src/query/functions/src/scalars/timestamp/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "databend-functions-scalar-datetime" version = "0.1.0" -edition = "2021" +edition = "2024" [dependencies] chrono = { workspace = true } diff --git a/src/query/functions/src/scalars/timestamp/src/datetime.rs b/src/query/functions/src/scalars/timestamp/src/datetime.rs index 09968c4ffbe57..c2bd8658c2a7e 100644 --- a/src/query/functions/src/scalars/timestamp/src/datetime.rs +++ b/src/query/functions/src/scalars/timestamp/src/datetime.rs @@ -24,42 +24,48 @@ use databend_common_base::runtime::catch_unwind; use databend_common_column::types::months_days_micros; use databend_common_column::types::timestamp_tz; use databend_common_exception::ErrorCode; +use databend_common_expression::EvalContext; +use databend_common_expression::FunctionContext; +use databend_common_expression::FunctionDomain; +use databend_common_expression::FunctionProperty; +use databend_common_expression::FunctionRegistry; +use databend_common_expression::Value; use databend_common_expression::error_to_null; use databend_common_expression::serialize::EPOCH_DAYS_FROM_CE; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::DataType; +use databend_common_expression::types::DateType; +use databend_common_expression::types::F64; +use databend_common_expression::types::Float64Type; +use databend_common_expression::types::Int32Type; +use databend_common_expression::types::IntervalType; +use databend_common_expression::types::NullableType; +use databend_common_expression::types::NumberType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::TimestampType; +use databend_common_expression::types::date::DATE_MAX; +use databend_common_expression::types::date::DATE_MIN; use databend_common_expression::types::date::clamp_date; use databend_common_expression::types::date::date_to_string; use databend_common_expression::types::date::string_to_date; -use databend_common_expression::types::date::DATE_MAX; -use databend_common_expression::types::date::DATE_MIN; use databend_common_expression::types::nullable::NullableColumn; use databend_common_expression::types::nullable::NullableDomain; use databend_common_expression::types::number::Int64Type; use databend_common_expression::types::number::SimpleDomain; +use databend_common_expression::types::number::UInt8Type; use databend_common_expression::types::number::UInt16Type; use databend_common_expression::types::number::UInt32Type; use databend_common_expression::types::number::UInt64Type; -use databend_common_expression::types::number::UInt8Type; use databend_common_expression::types::string::StringDomain; -use databend_common_expression::types::timestamp::clamp_timestamp; -use databend_common_expression::types::timestamp::string_to_timestamp; -use databend_common_expression::types::timestamp::timestamp_to_string; use databend_common_expression::types::timestamp::MICROS_PER_MILLI; use databend_common_expression::types::timestamp::MICROS_PER_SEC; use databend_common_expression::types::timestamp::TIMESTAMP_MAX; use databend_common_expression::types::timestamp::TIMESTAMP_MIN; -use databend_common_expression::types::timestamp_tz::string_to_timestamp_tz; +use databend_common_expression::types::timestamp::clamp_timestamp; +use databend_common_expression::types::timestamp::string_to_timestamp; +use databend_common_expression::types::timestamp::timestamp_to_string; use databend_common_expression::types::timestamp_tz::TimestampTzType; -use databend_common_expression::types::Bitmap; -use databend_common_expression::types::DataType; -use databend_common_expression::types::DateType; -use databend_common_expression::types::Float64Type; -use databend_common_expression::types::Int32Type; -use databend_common_expression::types::IntervalType; -use databend_common_expression::types::NullableType; -use databend_common_expression::types::NumberType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::TimestampType; -use databend_common_expression::types::F64; +use databend_common_expression::types::timestamp_tz::string_to_timestamp_tz; use databend_common_expression::utils::date_helper::*; use databend_common_expression::vectorize_1_arg; use databend_common_expression::vectorize_2_arg; @@ -67,23 +73,17 @@ use databend_common_expression::vectorize_4_arg; use databend_common_expression::vectorize_with_builder_1_arg; use databend_common_expression::vectorize_with_builder_2_arg; use databend_common_expression::vectorize_with_builder_4_arg; -use databend_common_expression::EvalContext; -use databend_common_expression::FunctionContext; -use databend_common_expression::FunctionDomain; -use databend_common_expression::FunctionProperty; -use databend_common_expression::FunctionRegistry; -use databend_common_expression::Value; use databend_common_timezone::fast_components_from_timestamp; use databend_common_timezone::fast_utc_from_local; use dtparse::parse; -use jiff::civil::date; +use jiff::Timestamp; +use jiff::Unit; use jiff::civil::Date; use jiff::civil::Weekday; +use jiff::civil::date; use jiff::fmt::strtime::BrokenDownTime; use jiff::tz::Offset; use jiff::tz::TimeZone; -use jiff::Timestamp; -use jiff::Unit; use num_traits::AsPrimitive; pub fn register(registry: &mut FunctionRegistry) { @@ -210,7 +210,7 @@ fn timestamp_tz_domain_to_timestamp_domain( // jiff don't support local formats: // https://github.com/BurntSushi/jiff/issues/219 -fn replace_time_format(format: &str) -> Cow { +fn replace_time_format(format: &str) -> Cow<'_, str> { if ["%c", "x", "X"].iter().any(|f| format.contains(f)) { let format = format .replace("%c", "%x %X") diff --git a/src/query/functions/src/scalars/timestamp/src/interval.rs b/src/query/functions/src/scalars/timestamp/src/interval.rs index 38a951f971a54..dbad3ff64261d 100644 --- a/src/query/functions/src/scalars/timestamp/src/interval.rs +++ b/src/query/functions/src/scalars/timestamp/src/interval.rs @@ -17,35 +17,35 @@ use std::io::Write; use databend_common_column::types::months_days_micros; use databend_common_column::types::timestamp_tz; use databend_common_exception::Result; -use databend_common_expression::date_helper::calc_date_to_timestamp; -use databend_common_expression::date_helper::timestamp_tz_components_via_lut; -use databend_common_expression::date_helper::today_date; +use databend_common_expression::EvalContext; +use databend_common_expression::FunctionDomain; +use databend_common_expression::FunctionRegistry; +use databend_common_expression::Value; use databend_common_expression::date_helper::DateConverter; use databend_common_expression::date_helper::EvalDaysImpl; use databend_common_expression::date_helper::EvalMonthsImpl; +use databend_common_expression::date_helper::calc_date_to_timestamp; +use databend_common_expression::date_helper::timestamp_tz_components_via_lut; +use databend_common_expression::date_helper::today_date; use databend_common_expression::error_to_null; -use databend_common_expression::types::interval::interval_to_string; -use databend_common_expression::types::interval::string_to_interval; -use databend_common_expression::types::timestamp_tz::TimestampTzType; use databend_common_expression::types::DateType; use databend_common_expression::types::Float64Type; use databend_common_expression::types::Int64Type; use databend_common_expression::types::IntervalType; use databend_common_expression::types::StringType; use databend_common_expression::types::TimestampType; +use databend_common_expression::types::interval::interval_to_string; +use databend_common_expression::types::interval::string_to_interval; +use databend_common_expression::types::timestamp_tz::TimestampTzType; use databend_common_expression::vectorize_2_arg; use databend_common_expression::vectorize_with_builder_1_arg; use databend_common_expression::vectorize_with_builder_2_arg; -use databend_common_expression::EvalContext; -use databend_common_expression::FunctionDomain; -use databend_common_expression::FunctionRegistry; -use databend_common_expression::Value; -use databend_common_timezone::fast_components_from_timestamp; use databend_common_timezone::DateTimeComponents; -use jiff::tz::Offset; -use jiff::tz::TimeZone; +use databend_common_timezone::fast_components_from_timestamp; use jiff::Timestamp; use jiff::Zoned; +use jiff::tz::Offset; +use jiff::tz::TimeZone; pub fn register(registry: &mut FunctionRegistry) { // cast(xx AS interval) diff --git a/src/query/functions/src/scalars/timestamp/src/lib.rs b/src/query/functions/src/scalars/timestamp/src/lib.rs index 4aa622cbda85d..b316e75941ac9 100644 --- a/src/query/functions/src/scalars/timestamp/src/lib.rs +++ b/src/query/functions/src/scalars/timestamp/src/lib.rs @@ -17,6 +17,8 @@ #![allow(clippy::ptr_arg)] #![allow(clippy::type_complexity)] #![allow(internal_features)] +#![allow(clippy::collapsible_if)] +#![allow(clippy::unnecessary_unwrap)] #![feature(core_intrinsics)] #![feature(box_patterns)] #![feature(type_ascription)] diff --git a/src/query/functions/src/scalars/tuple.rs b/src/query/functions/src/scalars/tuple.rs index f16870d5720f4..ea0aa7a569d94 100644 --- a/src/query/functions/src/scalars/tuple.rs +++ b/src/query/functions/src/scalars/tuple.rs @@ -14,9 +14,6 @@ use std::sync::Arc; -use databend_common_expression::types::nullable::NullableColumn; -use databend_common_expression::types::nullable::NullableDomain; -use databend_common_expression::types::DataType; use databend_common_expression::Column; use databend_common_expression::ColumnBuilder; use databend_common_expression::Domain; @@ -28,6 +25,9 @@ use databend_common_expression::FunctionRegistry; use databend_common_expression::FunctionSignature; use databend_common_expression::Scalar; use databend_common_expression::Value; +use databend_common_expression::types::DataType; +use databend_common_expression::types::nullable::NullableColumn; +use databend_common_expression::types::nullable::NullableDomain; pub fn register(registry: &mut FunctionRegistry) { let tuple = FunctionFactory::Closure(Box::new(|_, args_type: &[DataType]| { diff --git a/src/query/functions/src/scalars/variant.rs b/src/query/functions/src/scalars/variant.rs index 86e7615b8fd09..189aba6f52915 100644 --- a/src/query/functions/src/scalars/variant.rs +++ b/src/query/functions/src/scalars/variant.rs @@ -20,20 +20,21 @@ use std::sync::Arc; use bstr::ByteSlice; use databend_common_column::types::months_days_micros; +use databend_common_expression::Column; +use databend_common_expression::ColumnBuilder; +use databend_common_expression::Domain; +use databend_common_expression::EvalContext; +use databend_common_expression::Function; +use databend_common_expression::FunctionDomain; +use databend_common_expression::FunctionEval; +use databend_common_expression::FunctionFactory; +use databend_common_expression::FunctionRegistry; +use databend_common_expression::FunctionSignature; +use databend_common_expression::Scalar; +use databend_common_expression::ScalarRef; +use databend_common_expression::Value; use databend_common_expression::display::scalar_ref_to_string; -use databend_common_expression::types::binary::BinaryColumnBuilder; -use databend_common_expression::types::date::clamp_date; -use databend_common_expression::types::date::string_to_date; -use databend_common_expression::types::interval::string_to_interval; -use databend_common_expression::types::nullable::NullableColumn; -use databend_common_expression::types::nullable::NullableColumnBuilder; -use databend_common_expression::types::nullable::NullableDomain; -use databend_common_expression::types::number::*; -use databend_common_expression::types::string::StringColumnBuilder; -use databend_common_expression::types::timestamp::clamp_timestamp; -use databend_common_expression::types::timestamp::string_to_timestamp; -use databend_common_expression::types::variant::cast_scalar_to_variant; -use databend_common_expression::types::variant::cast_scalars_to_variants; +use databend_common_expression::types::ALL_NUMERICS_TYPES; use databend_common_expression::types::AnyType; use databend_common_expression::types::ArrayType; use databend_common_expression::types::BinaryType; @@ -51,36 +52,35 @@ use databend_common_expression::types::NumberType; use databend_common_expression::types::StringType; use databend_common_expression::types::TimestampType; use databend_common_expression::types::VariantType; -use databend_common_expression::types::ALL_NUMERICS_TYPES; +use databend_common_expression::types::binary::BinaryColumnBuilder; +use databend_common_expression::types::date::clamp_date; +use databend_common_expression::types::date::string_to_date; +use databend_common_expression::types::interval::string_to_interval; +use databend_common_expression::types::nullable::NullableColumn; +use databend_common_expression::types::nullable::NullableColumnBuilder; +use databend_common_expression::types::nullable::NullableDomain; +use databend_common_expression::types::number::*; +use databend_common_expression::types::string::StringColumnBuilder; +use databend_common_expression::types::timestamp::clamp_timestamp; +use databend_common_expression::types::timestamp::string_to_timestamp; +use databend_common_expression::types::variant::cast_scalar_to_variant; +use databend_common_expression::types::variant::cast_scalars_to_variants; use databend_common_expression::vectorize_1_arg; use databend_common_expression::vectorize_with_builder_1_arg; use databend_common_expression::vectorize_with_builder_2_arg; use databend_common_expression::vectorize_with_builder_3_arg; use databend_common_expression::with_number_mapped_type; -use databend_common_expression::Column; -use databend_common_expression::ColumnBuilder; -use databend_common_expression::Domain; -use databend_common_expression::EvalContext; -use databend_common_expression::Function; -use databend_common_expression::FunctionDomain; -use databend_common_expression::FunctionEval; -use databend_common_expression::FunctionFactory; -use databend_common_expression::FunctionRegistry; -use databend_common_expression::FunctionSignature; -use databend_common_expression::Scalar; -use databend_common_expression::ScalarRef; -use databend_common_expression::Value; use databend_common_io::Interval; +use jiff::Unit; use jiff::civil::date; use jiff::tz::TimeZone; -use jiff::Unit; +use jsonb::OwnedJsonb; +use jsonb::RawJsonb; +use jsonb::Value as JsonbValue; use jsonb::jsonpath::parse_json_path; use jsonb::keypath::parse_key_paths; use jsonb::parse_owned_jsonb; use jsonb::parse_owned_jsonb_with_buf; -use jsonb::OwnedJsonb; -use jsonb::RawJsonb; -use jsonb::Value as JsonbValue; pub fn register(registry: &mut FunctionRegistry) { registry.register_aliases("to_string", &["json_to_string"]); @@ -1166,7 +1166,7 @@ pub fn register(registry: &mut FunctionRegistry) { Value::Column(col) => { let validity = match col { Column::Null { len } => Some(Bitmap::new_constant(false, *len)), - Column::Nullable(box ref nullable_column) => { + Column::Nullable(box nullable_column) => { Some(nullable_column.validity.clone()) } _ => None, @@ -1876,11 +1876,7 @@ pub fn register(registry: &mut FunctionRegistry) { start as usize } else { let start = vals.len() as i64 + start; - if start >= 0 { - start as usize - } else { - 0 - } + if start >= 0 { start as usize } else { 0 } }; let mut new_vals = vec![]; for (i, val) in vals.into_iter().enumerate() { diff --git a/src/query/functions/src/scalars/vector.rs b/src/query/functions/src/scalars/vector.rs index 6471f7d5e9d9a..c5e4cb6d4390f 100644 --- a/src/query/functions/src/scalars/vector.rs +++ b/src/query/functions/src/scalars/vector.rs @@ -17,10 +17,23 @@ use std::sync::Arc; use databend_common_column::bitmap::Bitmap; use databend_common_column::bitmap::MutableBitmap; use databend_common_exception::Result; +use databend_common_expression::Column; +use databend_common_expression::EvalContext; +use databend_common_expression::Function; +use databend_common_expression::FunctionDomain; +use databend_common_expression::FunctionEval; +use databend_common_expression::FunctionFactory; +use databend_common_expression::FunctionRegistry; +use databend_common_expression::FunctionSignature; +use databend_common_expression::Scalar; +use databend_common_expression::ScalarRef; +use databend_common_expression::Value; use databend_common_expression::types::AnyType; use databend_common_expression::types::ArrayType; use databend_common_expression::types::Buffer; use databend_common_expression::types::DataType; +use databend_common_expression::types::F32; +use databend_common_expression::types::F64; use databend_common_expression::types::Float32Type; use databend_common_expression::types::Float64Type; use databend_common_expression::types::NullableType; @@ -29,20 +42,7 @@ use databend_common_expression::types::NumberDataType; use databend_common_expression::types::NumberScalar; use databend_common_expression::types::VectorDataType; use databend_common_expression::types::VectorScalarRef; -use databend_common_expression::types::F32; -use databend_common_expression::types::F64; use databend_common_expression::vectorize_with_builder_2_arg; -use databend_common_expression::Column; -use databend_common_expression::EvalContext; -use databend_common_expression::Function; -use databend_common_expression::FunctionDomain; -use databend_common_expression::FunctionEval; -use databend_common_expression::FunctionFactory; -use databend_common_expression::FunctionRegistry; -use databend_common_expression::FunctionSignature; -use databend_common_expression::Scalar; -use databend_common_expression::ScalarRef; -use databend_common_expression::Value; use databend_common_vector::cosine_distance; use databend_common_vector::cosine_distance_64; use databend_common_vector::inner_product; diff --git a/src/query/functions/src/srfs/array.rs b/src/query/functions/src/srfs/array.rs index 6212e58790a03..c1d5da99b819b 100644 --- a/src/query/functions/src/srfs/array.rs +++ b/src/query/functions/src/srfs/array.rs @@ -14,8 +14,6 @@ use std::sync::Arc; -use databend_common_expression::types::nullable::NullableColumn; -use databend_common_expression::types::DataType; use databend_common_expression::Column; use databend_common_expression::Function; use databend_common_expression::FunctionEval; @@ -27,6 +25,8 @@ use databend_common_expression::FunctionSignature; use databend_common_expression::Scalar; use databend_common_expression::ScalarRef; use databend_common_expression::Value; +use databend_common_expression::types::DataType; +use databend_common_expression::types::nullable::NullableColumn; use crate::srfs::variant::unnest_variant_array; @@ -38,11 +38,13 @@ pub fn register(registry: &mut FunctionRegistry) { let unnest = FunctionFactory::Closure(Box::new(|_, arg_types: &[DataType]| { match arg_types { - [ty @ (DataType::Null - | DataType::EmptyArray - | DataType::Nullable(_) - | DataType::Array(_) - | DataType::Variant)] => Some(build_unnest(ty, Box::new(|ty| ty))), + [ + ty @ (DataType::Null + | DataType::EmptyArray + | DataType::Nullable(_) + | DataType::Array(_) + | DataType::Variant), + ] => Some(build_unnest(ty, Box::new(|ty| ty))), _ => { // Generate a fake function with signature `unset(Array(T0 NULL))` to have a better error message. Some(build_unnest( diff --git a/src/query/functions/src/srfs/string.rs b/src/query/functions/src/srfs/string.rs index bdf4231d6ea57..87b62d763662b 100644 --- a/src/query/functions/src/srfs/string.rs +++ b/src/query/functions/src/srfs/string.rs @@ -15,11 +15,6 @@ use std::borrow::Cow; use std::sync::Arc; -use databend_common_expression::types::string::StringColumnBuilder; -use databend_common_expression::types::AnyType; -use databend_common_expression::types::ArgType; -use databend_common_expression::types::DataType; -use databend_common_expression::types::StringType; use databend_common_expression::Column; use databend_common_expression::EvalContext; use databend_common_expression::Function; @@ -31,6 +26,11 @@ use databend_common_expression::FunctionRegistry; use databend_common_expression::FunctionSignature; use databend_common_expression::ScalarRef; use databend_common_expression::Value; +use databend_common_expression::types::AnyType; +use databend_common_expression::types::ArgType; +use databend_common_expression::types::DataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::string::StringColumnBuilder; use regex::RegexBuilder; pub fn register(registry: &mut FunctionRegistry) { diff --git a/src/query/functions/src/srfs/variant.rs b/src/query/functions/src/srfs/variant.rs index a43f0edc4605e..340b93b92bf9a 100644 --- a/src/query/functions/src/srfs/variant.rs +++ b/src/query/functions/src/srfs/variant.rs @@ -17,18 +17,6 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::binary::BinaryColumnBuilder; -use databend_common_expression::types::nullable::NullableColumnBuilder; -use databend_common_expression::types::string::StringColumnBuilder; -use databend_common_expression::types::AnyType; -use databend_common_expression::types::ArgType; -use databend_common_expression::types::Bitmap; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NullableType; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::UInt64Type; -use databend_common_expression::types::VariantType; use databend_common_expression::Column; use databend_common_expression::FromData; use databend_common_expression::Function; @@ -41,6 +29,18 @@ use databend_common_expression::FunctionSignature; use databend_common_expression::Scalar; use databend_common_expression::ScalarRef; use databend_common_expression::Value; +use databend_common_expression::types::AnyType; +use databend_common_expression::types::ArgType; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NullableType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::UInt64Type; +use databend_common_expression::types::VariantType; +use databend_common_expression::types::binary::BinaryColumnBuilder; +use databend_common_expression::types::nullable::NullableColumnBuilder; +use databend_common_expression::types::string::StringColumnBuilder; use jaq_core; use jaq_interpret::Ctx; use jaq_interpret::FilterT; @@ -49,10 +49,10 @@ use jaq_interpret::RcIter; use jaq_interpret::Val; use jaq_parse; use jaq_std; -use jsonb::from_raw_jsonb; -use jsonb::jsonpath::parse_json_path; use jsonb::OwnedJsonb; use jsonb::RawJsonb; +use jsonb::from_raw_jsonb; +use jsonb::jsonpath::parse_json_path; pub fn register(registry: &mut FunctionRegistry) { registry.properties.insert( diff --git a/src/query/functions/src/test_utils.rs b/src/query/functions/src/test_utils.rs index e01bbb7ef2b07..9341010bf9e19 100644 --- a/src/query/functions/src/test_utils.rs +++ b/src/query/functions/src/test_utils.rs @@ -23,23 +23,23 @@ use databend_common_ast::ast::Literal as ASTLiteral; use databend_common_ast::ast::MapAccessor; use databend_common_ast::ast::UnaryOperator; use databend_common_ast::ast::Weekday; +use databend_common_ast::parser::Dialect; use databend_common_ast::parser::parse_expr; use databend_common_ast::parser::tokenize_sql; -use databend_common_ast::parser::Dialect; use databend_common_base::base::OrderedFloat; +use databend_common_expression::ConstantFolder; +use databend_common_expression::FunctionContext; +use databend_common_expression::RawExpr; +use databend_common_expression::Scalar; use databend_common_expression::shrink_scalar; use databend_common_expression::type_check; -use databend_common_expression::types::decimal::DecimalScalar; -use databend_common_expression::types::decimal::DecimalSize; -use databend_common_expression::types::i256; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::NumberScalar; use databend_common_expression::types::VectorDataType; -use databend_common_expression::ConstantFolder; -use databend_common_expression::FunctionContext; -use databend_common_expression::RawExpr; -use databend_common_expression::Scalar; +use databend_common_expression::types::decimal::DecimalScalar; +use databend_common_expression::types::decimal::DecimalSize; +use databend_common_expression::types::i256; use log as _; use crate::BUILTIN_FUNCTIONS; diff --git a/src/query/functions/tests/it/aggregates/agg.rs b/src/query/functions/tests/it/aggregates/agg.rs index 92b131bd2e7eb..8f9f744afaaa9 100644 --- a/src/query/functions/tests/it/aggregates/agg.rs +++ b/src/query/functions/tests/it/aggregates/agg.rs @@ -15,8 +15,9 @@ use std::io::Write; use databend_common_exception::Result; -use databend_common_expression::types::number::Int64Type; -use databend_common_expression::types::number::UInt64Type; +use databend_common_expression::BlockEntry; +use databend_common_expression::Column; +use databend_common_expression::FromData; use databend_common_expression::types::BitmapType; use databend_common_expression::types::BooleanType; use databend_common_expression::types::DataType; @@ -26,18 +27,17 @@ use databend_common_expression::types::Int32Type; use databend_common_expression::types::NullableType; use databend_common_expression::types::StringType; use databend_common_expression::types::TimestampType; -use databend_common_expression::BlockEntry; -use databend_common_expression::Column; -use databend_common_expression::FromData; +use databend_common_expression::types::number::Int64Type; +use databend_common_expression::types::number::UInt64Type; use databend_common_functions::aggregates::AggregateFunctionSortDesc; use databend_common_io::HybridBitmap; use goldenfile::Mint; use itertools::Itertools; +use super::AggregationSimulator; use super::eval_aggr_for_test; use super::run_agg_ast; use super::simulate_two_groups_group_by; -use super::AggregationSimulator; fn eval_aggr( name: &str, diff --git a/src/query/functions/tests/it/aggregates/agg_hashtable.rs b/src/query/functions/tests/it/aggregates/agg_hashtable.rs index b838962d444ce..60891b2293d63 100644 --- a/src/query/functions/tests/it/aggregates/agg_hashtable.rs +++ b/src/query/functions/tests/it/aggregates/agg_hashtable.rs @@ -26,35 +26,37 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![allow(clippy::cloned_ref_to_slice_refs)] + use std::alloc::Layout; use std::sync::Arc; use bumpalo::Bump; +use databend_common_expression::AggregateHashTable; +use databend_common_expression::BlockEntry; +use databend_common_expression::DataBlock; +use databend_common_expression::FromData; +use databend_common_expression::HashTableConfig; +use databend_common_expression::PayloadFlushState; +use databend_common_expression::ProbeState; use databend_common_expression::block_debug::assert_block_value_sort_eq; use databend_common_expression::get_states_layout; -use databend_common_expression::types::i256; use databend_common_expression::types::ArgType; use databend_common_expression::types::BooleanType; use databend_common_expression::types::DataType; use databend_common_expression::types::DecimalSize; +use databend_common_expression::types::F32; +use databend_common_expression::types::F64; use databend_common_expression::types::Float32Type; use databend_common_expression::types::Float64Type; +use databend_common_expression::types::Int8Type; use databend_common_expression::types::Int16Type; use databend_common_expression::types::Int32Type; use databend_common_expression::types::Int64Type; -use databend_common_expression::types::Int8Type; use databend_common_expression::types::NullableType; use databend_common_expression::types::StringType; use databend_common_expression::types::UInt64Type; -use databend_common_expression::types::F32; -use databend_common_expression::types::F64; -use databend_common_expression::AggregateHashTable; -use databend_common_expression::BlockEntry; -use databend_common_expression::DataBlock; -use databend_common_expression::FromData; -use databend_common_expression::HashTableConfig; -use databend_common_expression::PayloadFlushState; -use databend_common_expression::ProbeState; +use databend_common_expression::types::i256; use databend_common_functions::aggregates::AggregateFunctionFactory; use databend_common_functions::aggregates::DecimalSumState; use itertools::Itertools; diff --git a/src/query/functions/tests/it/aggregates/mod.rs b/src/query/functions/tests/it/aggregates/mod.rs index adc0f885069f1..59b58d1d5c2f8 100644 --- a/src/query/functions/tests/it/aggregates/mod.rs +++ b/src/query/functions/tests/it/aggregates/mod.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![allow(clippy::cloned_ref_to_slice_refs)] + mod agg; mod agg_hashtable; @@ -21,10 +23,6 @@ use bumpalo::Bump; use comfy_table::Table; use databend_common_base::runtime::drop_guard; use databend_common_exception::Result; -use databend_common_expression::get_states_layout; -use databend_common_expression::type_check; -use databend_common_expression::types::AnyType; -use databend_common_expression::types::DataType; use databend_common_expression::AggrState; use databend_common_expression::AggregateFunctionRef; use databend_common_expression::BlockEntry; @@ -38,9 +36,13 @@ use databend_common_expression::Scalar; use databend_common_expression::StateAddr; use databend_common_expression::StatesLayout; use databend_common_expression::Value; +use databend_common_expression::get_states_layout; +use databend_common_expression::type_check; +use databend_common_expression::types::AnyType; +use databend_common_expression::types::DataType; +use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_functions::aggregates::AggregateFunctionFactory; use databend_common_functions::aggregates::AggregateFunctionSortDesc; -use databend_common_functions::BUILTIN_FUNCTIONS; use itertools::Itertools; use super::scalars::parser; diff --git a/src/query/functions/tests/it/scalars/arithmetic.rs b/src/query/functions/tests/it/scalars/arithmetic.rs index bfb04c55b8662..beb4bc0214f83 100644 --- a/src/query/functions/tests/it/scalars/arithmetic.rs +++ b/src/query/functions/tests/it/scalars/arithmetic.rs @@ -14,13 +14,13 @@ use std::io::Write; +use databend_common_expression::Column; +use databend_common_expression::FromData; +use databend_common_expression::types::Decimal64Type; use databend_common_expression::types::decimal::DecimalColumn; use databend_common_expression::types::decimal::DecimalSize; use databend_common_expression::types::i256; use databend_common_expression::types::number::*; -use databend_common_expression::types::Decimal64Type; -use databend_common_expression::Column; -use databend_common_expression::FromData; use goldenfile::Mint; use super::run_ast; diff --git a/src/query/functions/tests/it/scalars/array.rs b/src/query/functions/tests/it/scalars/array.rs index 859f215a47f89..16d50574dddd7 100644 --- a/src/query/functions/tests/it/scalars/array.rs +++ b/src/query/functions/tests/it/scalars/array.rs @@ -14,8 +14,8 @@ use std::io::Write; -use databend_common_expression::types::*; use databend_common_expression::FromData; +use databend_common_expression::types::*; use databend_common_formats::column_from_json; use goldenfile::Mint; diff --git a/src/query/functions/tests/it/scalars/binary.rs b/src/query/functions/tests/it/scalars/binary.rs index 32fc10e9ee9e9..21877961bfe3e 100644 --- a/src/query/functions/tests/it/scalars/binary.rs +++ b/src/query/functions/tests/it/scalars/binary.rs @@ -14,9 +14,9 @@ use std::io::Write; +use databend_common_expression::FromData; use databend_common_expression::types::BinaryType; use databend_common_expression::types::StringType; -use databend_common_expression::FromData; use goldenfile::Mint; use super::run_ast; @@ -122,7 +122,9 @@ fn test_to_binary(file: &mut impl Write, is_try: bool) { ); run_ast( file, - format!("{prefix}to_binary(st_geometryfromwkb(unhex('0101000020797f000066666666a9cb17411f85ebc19e325641')))"), + format!( + "{prefix}to_binary(st_geometryfromwkb(unhex('0101000020797f000066666666a9cb17411f85ebc19e325641')))" + ), &[], ); run_ast( diff --git a/src/query/functions/tests/it/scalars/bitmap.rs b/src/query/functions/tests/it/scalars/bitmap.rs index dc3fd8e2ccc79..ddfe36560d11c 100644 --- a/src/query/functions/tests/it/scalars/bitmap.rs +++ b/src/query/functions/tests/it/scalars/bitmap.rs @@ -14,8 +14,8 @@ use std::io::Write; -use databend_common_expression::types::*; use databend_common_expression::FromData; +use databend_common_expression::types::*; use goldenfile::Mint; use super::run_ast; diff --git a/src/query/functions/tests/it/scalars/boolean.rs b/src/query/functions/tests/it/scalars/boolean.rs index 04c5e8d934360..558fff5216b09 100644 --- a/src/query/functions/tests/it/scalars/boolean.rs +++ b/src/query/functions/tests/it/scalars/boolean.rs @@ -14,10 +14,10 @@ use std::io::Write; -use databend_common_expression::types::nullable::NullableColumn; -use databend_common_expression::types::*; use databend_common_expression::Column; use databend_common_expression::FromData; +use databend_common_expression::types::nullable::NullableColumn; +use databend_common_expression::types::*; use goldenfile::Mint; use super::run_ast; diff --git a/src/query/functions/tests/it/scalars/cast.rs b/src/query/functions/tests/it/scalars/cast.rs index 5a7f2f50999f7..75cd99a8c0dee 100644 --- a/src/query/functions/tests/it/scalars/cast.rs +++ b/src/query/functions/tests/it/scalars/cast.rs @@ -14,18 +14,18 @@ use std::io::Write; -use databend_common_expression::types::*; use databend_common_expression::Column; use databend_common_expression::Domain; use databend_common_expression::FromData; use databend_common_expression::FunctionContext; +use databend_common_expression::types::*; use databend_common_io::HybridBitmap; use goldenfile::Mint; use itertools::Itertools; +use super::TestContext; use super::run_ast; use super::run_ast_with_context; -use super::TestContext; #[test] fn test_cast() { diff --git a/src/query/functions/tests/it/scalars/comparison.rs b/src/query/functions/tests/it/scalars/comparison.rs index 768b0504e9b46..3274c5311f515 100644 --- a/src/query/functions/tests/it/scalars/comparison.rs +++ b/src/query/functions/tests/it/scalars/comparison.rs @@ -14,8 +14,8 @@ use std::io::Write; -use databend_common_expression::types::*; use databend_common_expression::FromData; +use databend_common_expression::types::*; use goldenfile::Mint; use super::run_ast; diff --git a/src/query/functions/tests/it/scalars/control.rs b/src/query/functions/tests/it/scalars/control.rs index dbfd93cb10753..6b626d578cb5a 100644 --- a/src/query/functions/tests/it/scalars/control.rs +++ b/src/query/functions/tests/it/scalars/control.rs @@ -14,9 +14,9 @@ use std::io::Write; -use databend_common_expression::types::*; use databend_common_expression::Column; use databend_common_expression::FromData; +use databend_common_expression::types::*; use goldenfile::Mint; use super::run_ast; diff --git a/src/query/functions/tests/it/scalars/datetime.rs b/src/query/functions/tests/it/scalars/datetime.rs index ba82c6bd44d04..0fe1d04c7fec7 100644 --- a/src/query/functions/tests/it/scalars/datetime.rs +++ b/src/query/functions/tests/it/scalars/datetime.rs @@ -15,16 +15,16 @@ use std::io::Write; use std::str::FromStr; -use databend_common_expression::types::*; use databend_common_expression::FromData; use databend_common_expression::FunctionContext; +use databend_common_expression::types::*; use goldenfile::Mint; -use jiff::tz::TimeZone; use jiff::Timestamp; +use jiff::tz::TimeZone; +use super::TestContext; use super::run_ast; use super::run_ast_with_context; -use super::TestContext; #[test] fn test_datetime() { diff --git a/src/query/functions/tests/it/scalars/geo.rs b/src/query/functions/tests/it/scalars/geo.rs index 4557602634b82..c01859f356220 100644 --- a/src/query/functions/tests/it/scalars/geo.rs +++ b/src/query/functions/tests/it/scalars/geo.rs @@ -14,8 +14,8 @@ use std::io::Write; -use databend_common_expression::types::*; use databend_common_expression::FromData; +use databend_common_expression::types::*; use goldenfile::Mint; use super::run_ast; diff --git a/src/query/functions/tests/it/scalars/geo_h3.rs b/src/query/functions/tests/it/scalars/geo_h3.rs index b691162368f91..47ccb67f0f3fc 100644 --- a/src/query/functions/tests/it/scalars/geo_h3.rs +++ b/src/query/functions/tests/it/scalars/geo_h3.rs @@ -14,8 +14,8 @@ use std::io::Write; -use databend_common_expression::types::*; use databend_common_expression::FromData; +use databend_common_expression::types::*; use goldenfile::Mint; use super::run_ast; diff --git a/src/query/functions/tests/it/scalars/geography.rs b/src/query/functions/tests/it/scalars/geography.rs index 613f93c48cf56..bd56becc3facb 100644 --- a/src/query/functions/tests/it/scalars/geography.rs +++ b/src/query/functions/tests/it/scalars/geography.rs @@ -14,8 +14,8 @@ use std::io::Write; -use databend_common_expression::types::*; use databend_common_expression::FromData; +use databend_common_expression::types::*; use goldenfile::Mint; use crate::scalars::run_ast; diff --git a/src/query/functions/tests/it/scalars/geometry.rs b/src/query/functions/tests/it/scalars/geometry.rs index cdb3f9d5d9dd9..8564c3cc8f0c2 100644 --- a/src/query/functions/tests/it/scalars/geometry.rs +++ b/src/query/functions/tests/it/scalars/geometry.rs @@ -14,10 +14,10 @@ use std::io::Write; +use databend_common_expression::FromData; use databend_common_expression::types::Float64Type; use databend_common_expression::types::Int32Type; use databend_common_expression::types::StringType; -use databend_common_expression::FromData; use goldenfile::Mint; use crate::scalars::run_ast; diff --git a/src/query/functions/tests/it/scalars/hash.rs b/src/query/functions/tests/it/scalars/hash.rs index 3480d82045e9a..0a8439e7a39f4 100644 --- a/src/query/functions/tests/it/scalars/hash.rs +++ b/src/query/functions/tests/it/scalars/hash.rs @@ -14,8 +14,8 @@ use std::io::Write; -use databend_common_expression::types::*; use databend_common_expression::FromData; +use databend_common_expression::types::*; use goldenfile::Mint; use super::run_ast; diff --git a/src/query/functions/tests/it/scalars/map.rs b/src/query/functions/tests/it/scalars/map.rs index b7714227249d1..8e51c66441de3 100644 --- a/src/query/functions/tests/it/scalars/map.rs +++ b/src/query/functions/tests/it/scalars/map.rs @@ -14,8 +14,8 @@ use std::io::Write; -use databend_common_expression::types::*; use databend_common_expression::FromData; +use databend_common_expression::types::*; use goldenfile::Mint; use super::run_ast; diff --git a/src/query/functions/tests/it/scalars/math.rs b/src/query/functions/tests/it/scalars/math.rs index 2877b0b02b924..5fe6f6b70049b 100644 --- a/src/query/functions/tests/it/scalars/math.rs +++ b/src/query/functions/tests/it/scalars/math.rs @@ -14,8 +14,8 @@ use std::io::Write; -use databend_common_expression::types::*; use databend_common_expression::FromData; +use databend_common_expression::types::*; use goldenfile::Mint; use super::run_ast; diff --git a/src/query/functions/tests/it/scalars/misc.rs b/src/query/functions/tests/it/scalars/misc.rs index a4cd4d9c514fd..84b2ec751ff75 100644 --- a/src/query/functions/tests/it/scalars/misc.rs +++ b/src/query/functions/tests/it/scalars/misc.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_expression::types::*; use databend_common_expression::FromData; +use databend_common_expression::types::*; use goldenfile::Mint; use super::run_ast; diff --git a/src/query/functions/tests/it/scalars/mod.rs b/src/query/functions/tests/it/scalars/mod.rs index ba4cb7e5282bb..05616bbedca52 100644 --- a/src/query/functions/tests/it/scalars/mod.rs +++ b/src/query/functions/tests/it/scalars/mod.rs @@ -17,8 +17,6 @@ use std::io::Write; use comfy_table::Table; use databend_common_exception::Result; -use databend_common_expression::type_check; -use databend_common_expression::types::NullableColumn; use databend_common_expression::BlockEntry; use databend_common_expression::Column; use databend_common_expression::ConstantFolder; @@ -28,6 +26,8 @@ use databend_common_expression::Evaluator; use databend_common_expression::FunctionContext; use databend_common_expression::FunctionFactory; use databend_common_expression::Value; +use databend_common_expression::type_check; +use databend_common_expression::types::NullableColumn; use databend_common_functions::BUILTIN_FUNCTIONS; use goldenfile::Mint; use itertools::Itertools; @@ -363,9 +363,9 @@ fn check_ambiguity() { #[test] fn test_if_function() -> Result<()> { - use databend_common_expression::types::*; use databend_common_expression::FromData; use databend_common_expression::Scalar; + use databend_common_expression::types::*; let raw_expr = parser::parse_raw_expr("if(eq(n,1), sum_sid + 1,100)", &[ ("n", UInt8Type::data_type()), ("sum_sid", Int32Type::data_type().wrap_nullable()), diff --git a/src/query/functions/tests/it/scalars/obfuscator.rs b/src/query/functions/tests/it/scalars/obfuscator.rs index 3becbd01c83e2..5c9323a792e5e 100644 --- a/src/query/functions/tests/it/scalars/obfuscator.rs +++ b/src/query/functions/tests/it/scalars/obfuscator.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_expression::types::*; use databend_common_expression::FromData; +use databend_common_expression::types::*; use goldenfile::Mint; use super::run_ast; diff --git a/src/query/functions/tests/it/scalars/other.rs b/src/query/functions/tests/it/scalars/other.rs index c74d27209486c..e35c107a648e1 100644 --- a/src/query/functions/tests/it/scalars/other.rs +++ b/src/query/functions/tests/it/scalars/other.rs @@ -14,10 +14,10 @@ use std::io::Write; +use databend_common_expression::FromData; use databend_common_expression::types::Float64Type; -use databend_common_expression::types::UInt16Type; use databend_common_expression::types::UInt8Type; -use databend_common_expression::FromData; +use databend_common_expression::types::UInt16Type; use goldenfile::Mint; use super::run_ast; diff --git a/src/query/functions/tests/it/scalars/regexp.rs b/src/query/functions/tests/it/scalars/regexp.rs index cf19a76670106..7d2945c6209a3 100644 --- a/src/query/functions/tests/it/scalars/regexp.rs +++ b/src/query/functions/tests/it/scalars/regexp.rs @@ -14,9 +14,9 @@ use std::io::Write; -use databend_common_expression::types::number::Int64Type; -use databend_common_expression::types::StringType; use databend_common_expression::FromData; +use databend_common_expression::types::StringType; +use databend_common_expression::types::number::Int64Type; use goldenfile::Mint; use super::run_ast; @@ -714,8 +714,16 @@ fn test_regexp_extract(file: &mut impl Write) { ), ]); - run_ast(file, "regexp_extract('name: John, age: 30', 'name: ([A-Za-z]+), age: ([0-9]+)', ['name', 'age'])", &[]); - run_ast(file, "regexp_extract('name: John, age: 30', 'name: ([A-Za-z]+), age: ([0-9]+)', ['name', 'age'])", &[]); + run_ast( + file, + "regexp_extract('name: John, age: 30', 'name: ([A-Za-z]+), age: ([0-9]+)', ['name', 'age'])", + &[], + ); + run_ast( + file, + "regexp_extract('name: John, age: 30', 'name: ([A-Za-z]+), age: ([0-9]+)', ['name', 'age'])", + &[], + ); run_ast( file, "regexp_extract('name: John, age: 30', NULL, ['name', 'age'])", @@ -726,7 +734,11 @@ fn test_regexp_extract(file: &mut impl Write) { "regexp_extract('name: John, age: 30', 'name: ([A-Za-z]+), age: ([0-9]+)', [])", &[], ); - run_ast(file, "regexp_extract('name: John, age: 30', 'name: ([A-Za-z]+), age: ([0-9]+)', ['name', 'age'])", &[]); + run_ast( + file, + "regexp_extract('name: John, age: 30', 'name: ([A-Za-z]+), age: ([0-9]+)', ['name', 'age'])", + &[], + ); run_ast( file, @@ -750,7 +762,11 @@ fn test_regexp_extract(file: &mut impl Write) { run_ast(file, "regexp_extract_all('abc def ghi', NULL)", &[]); run_ast(file, "regexp_extract_all('', '[a-z]+')", &[]); run_ast(file, "regexp_extract_all('123 456', '[a-z]+')", &[]); - run_ast(file, "regexp_extract_all('name: John, age: 30; name: Jane, age: 25', 'name: ([A-Za-z]+), age: ([0-9]+)')", &[]); + run_ast( + file, + "regexp_extract_all('name: John, age: 30; name: Jane, age: 25', 'name: ([A-Za-z]+), age: ([0-9]+)')", + &[], + ); run_ast( file, diff --git a/src/query/functions/tests/it/scalars/string.rs b/src/query/functions/tests/it/scalars/string.rs index 8e9e99a70a3d7..1fc280178817a 100644 --- a/src/query/functions/tests/it/scalars/string.rs +++ b/src/query/functions/tests/it/scalars/string.rs @@ -14,14 +14,14 @@ use std::io::Write; -use databend_common_expression::types::i256; -use databend_common_expression::types::number::*; +use databend_common_expression::FromData; use databend_common_expression::types::BooleanType; use databend_common_expression::types::Decimal128Type; use databend_common_expression::types::Decimal256Type; use databend_common_expression::types::DecimalSize; use databend_common_expression::types::StringType; -use databend_common_expression::FromData; +use databend_common_expression::types::i256; +use databend_common_expression::types::number::*; use goldenfile::Mint; use super::run_ast; diff --git a/src/query/functions/tests/it/scalars/testdata/decimal_to_decimal_cast.txt b/src/query/functions/tests/it/scalars/testdata/decimal_to_decimal_cast.txt index 9c993850183fc..c2a0c7e4fad21 100644 --- a/src/query/functions/tests/it/scalars/testdata/decimal_to_decimal_cast.txt +++ b/src/query/functions/tests/it/scalars/testdata/decimal_to_decimal_cast.txt @@ -67,7 +67,7 @@ error: --> SQL:1:1 | 1 | CAST(123.45::DECIMAL(5,2) AS DECIMAL(5,4)) - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Decimal overflow at line : 959 while evaluating function `to_decimal(5, 4)(123.45)` in expr `CAST(123.45 AS Decimal(5, 4))` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Decimal overflow at line : 955 while evaluating function `to_decimal(5, 4)(123.45)` in expr `CAST(123.45 AS Decimal(5, 4))` @@ -75,7 +75,7 @@ error: --> SQL:1:1 | 1 | CAST(123::DECIMAL(3,0) AS DECIMAL(3,2)) - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Decimal overflow at line : 959 while evaluating function `to_decimal(3, 2)(123)` in expr `CAST(CAST(123 AS Decimal(3, 0)) AS Decimal(3, 2))` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Decimal overflow at line : 955 while evaluating function `to_decimal(3, 2)(123)` in expr `CAST(CAST(123 AS Decimal(3, 0)) AS Decimal(3, 2))` @@ -449,7 +449,7 @@ error: --> SQL:1:1 | 1 | CAST(12345.67::DECIMAL(7,2) AS DECIMAL(5,2)) - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Decimal overflow at line : 959 while evaluating function `to_decimal(5, 2)(12345.67)` in expr `CAST(12345.67 AS Decimal(5, 2))` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Decimal overflow at line : 955 while evaluating function `to_decimal(5, 2)(12345.67)` in expr `CAST(12345.67 AS Decimal(5, 2))` @@ -457,7 +457,7 @@ error: --> SQL:1:1 | 1 | CAST(999.99::DECIMAL(5,2) AS DECIMAL(4,2)) - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Decimal overflow at line : 959 while evaluating function `to_decimal(4, 2)(999.99)` in expr `CAST(999.99 AS Decimal(4, 2))` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Decimal overflow at line : 955 while evaluating function `to_decimal(4, 2)(999.99)` in expr `CAST(999.99 AS Decimal(4, 2))` @@ -465,7 +465,7 @@ error: --> SQL:1:1 | 1 | CAST(99.9::DECIMAL(3,1) AS DECIMAL(2,1)) - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Decimal overflow at line : 959 while evaluating function `to_decimal(2, 1)(99.9)` in expr `CAST(99.9 AS Decimal(2, 1))` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Decimal overflow at line : 955 while evaluating function `to_decimal(2, 1)(99.9)` in expr `CAST(99.9 AS Decimal(2, 1))` @@ -482,7 +482,7 @@ error: --> SQL:1:1 | 1 | CAST(-123.45::DECIMAL(5,2) AS DECIMAL(4,2)) - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Decimal overflow at line : 959 while evaluating function `to_decimal(4, 2)(-123.45)` in expr `CAST(- 123.45 AS Decimal(4, 2))` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Decimal overflow at line : 955 while evaluating function `to_decimal(4, 2)(-123.45)` in expr `CAST(- 123.45 AS Decimal(4, 2))` @@ -490,7 +490,7 @@ error: --> SQL:1:1 | 1 | CAST(99.99::DECIMAL(4,2) AS DECIMAL(3,2)) - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Decimal overflow at line : 959 while evaluating function `to_decimal(3, 2)(99.99)` in expr `CAST(99.99 AS Decimal(3, 2))` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Decimal overflow at line : 955 while evaluating function `to_decimal(3, 2)(99.99)` in expr `CAST(99.99 AS Decimal(3, 2))` @@ -498,7 +498,7 @@ error: --> SQL:1:1 | 1 | CAST(9.99::DECIMAL(3,2) AS DECIMAL(2,2)) - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Decimal overflow at line : 959 while evaluating function `to_decimal(2, 2)(9.99)` in expr `CAST(9.99 AS Decimal(2, 2))` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Decimal overflow at line : 955 while evaluating function `to_decimal(2, 2)(9.99)` in expr `CAST(9.99 AS Decimal(2, 2))` @@ -524,7 +524,7 @@ error: --> SQL:1:1 | 1 | CAST(12345678901234567890.123456789::DECIMAL(38,9) AS DECIMAL(18,4)) - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Decimal overflow at line : 858 while evaluating function `to_decimal(18, 4)(12345678901234567890.123456789)` in expr `CAST(CAST(12345678901234567890.123456789 AS Decimal(38, 9)) AS Decimal(18, 4))` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Decimal overflow at line : 854 while evaluating function `to_decimal(18, 4)(12345678901234567890.123456789)` in expr `CAST(CAST(12345678901234567890.123456789 AS Decimal(38, 9)) AS Decimal(18, 4))` @@ -650,7 +650,7 @@ error: --> SQL:1:1 | 1 | CAST(a AS DECIMAL(5,1)) - | ^^^^^^^^^^^^^^^^^^^^^^^ Decimal overflow at line : 858 while evaluating function `to_decimal(5, 1)(1234567.89)` in expr `CAST(a AS Decimal(5, 1))` + | ^^^^^^^^^^^^^^^^^^^^^^^ Decimal overflow at line : 854 while evaluating function `to_decimal(5, 1)(1234567.89)` in expr `CAST(a AS Decimal(5, 1))` diff --git a/src/query/functions/tests/it/scalars/tuple.rs b/src/query/functions/tests/it/scalars/tuple.rs index 01ace2f3fd955..e70e3a75bd4ea 100644 --- a/src/query/functions/tests/it/scalars/tuple.rs +++ b/src/query/functions/tests/it/scalars/tuple.rs @@ -14,10 +14,10 @@ use std::io::Write; -use databend_common_expression::types::nullable::NullableColumn; -use databend_common_expression::types::StringType; use databend_common_expression::Column; use databend_common_expression::FromData; +use databend_common_expression::types::StringType; +use databend_common_expression::types::nullable::NullableColumn; use goldenfile::Mint; use super::run_ast; diff --git a/src/query/functions/tests/it/scalars/variant.rs b/src/query/functions/tests/it/scalars/variant.rs index fda6cb69ef8b5..d56658fac5180 100644 --- a/src/query/functions/tests/it/scalars/variant.rs +++ b/src/query/functions/tests/it/scalars/variant.rs @@ -14,8 +14,8 @@ use std::io::Write; -use databend_common_expression::types::*; use databend_common_expression::FromData; +use databend_common_expression::types::*; use goldenfile::Mint; use super::run_ast; @@ -594,7 +594,11 @@ fn test_is_type(file: &mut impl Write) { "is_decimal(parse_json('99999999999999999999999999999999999999'))", &[], ); - run_ast(file, "is_decimal(parse_json('99999999999999999999999999999999999999999999999999999999999999999999999999991'))", &[]); + run_ast( + file, + "is_decimal(parse_json('99999999999999999999999999999999999999999999999999999999999999999999999999991'))", + &[], + ); run_ast(file, "is_string(parse_json('\"ab\"'))", &[]); run_ast(file, "is_string(parse_json('12.34'))", &[]); run_ast(file, "is_array(parse_json('[1,2,3]'))", &[]); diff --git a/src/query/functions/tests/it/scalars/vector.rs b/src/query/functions/tests/it/scalars/vector.rs index 14a5fd02bef86..ff0a6b3f5cd0b 100644 --- a/src/query/functions/tests/it/scalars/vector.rs +++ b/src/query/functions/tests/it/scalars/vector.rs @@ -14,8 +14,8 @@ use std::io::Write; -use databend_common_expression::types::*; use databend_common_expression::FromData; +use databend_common_expression::types::*; use goldenfile::Mint; use super::run_ast; diff --git a/src/query/functions/tests/it/type_check.rs b/src/query/functions/tests/it/type_check.rs index 609097c6761ad..ccbcae1354f2d 100644 --- a/src/query/functions/tests/it/type_check.rs +++ b/src/query/functions/tests/it/type_check.rs @@ -13,15 +13,15 @@ // limitations under the License. use databend_common_column::types::timestamp_tz; +use databend_common_expression::FromData; +use databend_common_expression::FunctionContext; +use databend_common_expression::Scalar; use databend_common_expression::filter_helper::FilterHelpers; use databend_common_expression::type_check; use databend_common_expression::types::timestamp_tz::TimestampTzType; use databend_common_expression::types::*; -use databend_common_expression::FromData; -use databend_common_expression::FunctionContext; -use databend_common_expression::Scalar; -use databend_common_functions::test_utils; use databend_common_functions::BUILTIN_FUNCTIONS; +use databend_common_functions::test_utils; use goldenfile::Mint; use jsonb::OwnedJsonb; @@ -184,13 +184,7 @@ fn test_find_leveled_eq_filters() { ], vec![], ), - - ( - "database = 'a' or c like 'xxb'", - vec![], - vec![], - vec![], - ), + ("database = 'a' or c like 'xxb'", vec![], vec![], vec![]), ( "catalog = 'x' and database = 'a' and table = 'b' and c like '%xxxx%'", vec![Scalar::String("x".to_string())], @@ -206,39 +200,25 @@ fn test_find_leveled_eq_filters() { ], vec![Scalar::String("b".to_string())], ), - ( "catalog = 'x' and (database = 'a' or database = 'b' or table = 'b') and c like '%xxxx%'", vec![Scalar::String("x".to_string())], - vec![ - ], + vec![], vec![], ), - ( "catalog = 'x' and (database = 'a' or database = 'b' or table = 'b') and c like '%xxxx%'", vec![Scalar::String("x".to_string())], - vec![ - ], vec![], - ), - ( - "catalog = 'x' and (database = 'a' or database = 'b') and database = 'c' and c like '%xxxx%'", - vec![Scalar::String("x".to_string())], - vec![ - Scalar::String("c".to_string()) - ], vec![], ), - ( - "not (database = 'default')", - vec![], - vec![], + "catalog = 'x' and (database = 'a' or database = 'b') and database = 'c' and c like '%xxxx%'", + vec![Scalar::String("x".to_string())], + vec![Scalar::String("c".to_string())], vec![], ), - - + ("not (database = 'default')", vec![], vec![], vec![]), ( "not (database = 'default' or database = 'abcd')", vec![], diff --git a/src/query/management/src/lib.rs b/src/query/management/src/lib.rs index d336a44a9e10c..4b7459ddbd2b5 100644 --- a/src/query/management/src/lib.rs +++ b/src/query/management/src/lib.rs @@ -12,9 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![feature(let_chains)] #![feature(get_mut_unchecked)] -#![allow(clippy::uninlined_format_args)] +#![allow(clippy::collapsible_if, clippy::uninlined_format_args)] mod connection; mod file_format; diff --git a/src/query/management/src/procedure/procedure_mgr.rs b/src/query/management/src/procedure/procedure_mgr.rs index c6ecab0021257..2fd3486306004 100644 --- a/src/query/management/src/procedure/procedure_mgr.rs +++ b/src/query/management/src/procedure/procedure_mgr.rs @@ -18,11 +18,8 @@ use databend_common_meta_api::kv_pb_api::KVPbApi; use databend_common_meta_api::meta_txn_error::MetaTxnError; use databend_common_meta_api::name_id_value_api::NameIdValueApi; use databend_common_meta_api::serialize_struct; +use databend_common_meta_app::KeyWithTenant; use databend_common_meta_app::data_id::DataId; -use databend_common_meta_app::principal::procedure::ProcedureInfo; -use databend_common_meta_app::principal::procedure_id_ident; -use databend_common_meta_app::principal::procedure_id_ident::ProcedureIdIdent; -use databend_common_meta_app::principal::procedure_name_ident::ProcedureName; use databend_common_meta_app::principal::CreateProcedureReply; use databend_common_meta_app::principal::CreateProcedureReq; use databend_common_meta_app::principal::GetProcedureReply; @@ -35,9 +32,12 @@ use databend_common_meta_app::principal::ProcedureIdentity; use databend_common_meta_app::principal::ProcedureMeta; use databend_common_meta_app::principal::ProcedureNameIdent; use databend_common_meta_app::principal::TenantOwnershipObjectIdent; +use databend_common_meta_app::principal::procedure::ProcedureInfo; +use databend_common_meta_app::principal::procedure_id_ident; +use databend_common_meta_app::principal::procedure_id_ident::ProcedureIdIdent; +use databend_common_meta_app::principal::procedure_name_ident::ProcedureName; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_app::tenant_key::errors::ExistError; -use databend_common_meta_app::KeyWithTenant; use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::DirName; use databend_common_meta_kvapi::kvapi::Key; diff --git a/src/query/management/src/quota/quota_mgr.rs b/src/query/management/src/quota/quota_mgr.rs index 629eecad2fd1a..b0eacf3e9213b 100644 --- a/src/query/management/src/quota/quota_mgr.rs +++ b/src/query/management/src/quota/quota_mgr.rs @@ -33,8 +33,8 @@ use databend_common_meta_types::With; use fastrace::func_name; use super::quota_api::QuotaApi; -use crate::serde::check_and_upgrade_to_pb; use crate::serde::Quota; +use crate::serde::check_and_upgrade_to_pb; pub struct QuotaMgr { kv_api: Arc>, diff --git a/src/query/management/src/role/role_api.rs b/src/query/management/src/role/role_api.rs index 0b7aa34ed03a6..135b3b69e75f8 100644 --- a/src/query/management/src/role/role_api.rs +++ b/src/query/management/src/role/role_api.rs @@ -13,10 +13,10 @@ // limitations under the License. use databend_common_exception::Result; -use databend_common_meta_app::principal::role_ident; use databend_common_meta_app::principal::OwnershipInfo; use databend_common_meta_app::principal::OwnershipObject; use databend_common_meta_app::principal::RoleInfo; +use databend_common_meta_app::principal::role_ident; use databend_common_meta_app::tenant_key::errors::ExistError; use databend_common_meta_kvapi::kvapi::ListKVReply; use databend_common_meta_types::MatchSeq; diff --git a/src/query/management/src/role/role_mgr.rs b/src/query/management/src/role/role_mgr.rs index 78608125c0797..000bec899bd66 100644 --- a/src/query/management/src/role/role_mgr.rs +++ b/src/query/management/src/role/role_mgr.rs @@ -23,9 +23,10 @@ use databend_common_meta_api::txn_backoff::txn_backoff; use databend_common_meta_api::txn_cond_seq; use databend_common_meta_api::txn_op_del; use databend_common_meta_api::txn_op_put; +use databend_common_meta_app::KeyWithTenant; use databend_common_meta_app::app_error::AppError; use databend_common_meta_app::app_error::TxnRetryMaxTimes; -use databend_common_meta_app::principal::role_ident; +use databend_common_meta_app::principal::BUILTIN_ROLE_ACCOUNT_ADMIN; use databend_common_meta_app::principal::GrantObject; use databend_common_meta_app::principal::OwnershipInfo; use databend_common_meta_app::principal::OwnershipObject; @@ -33,10 +34,9 @@ use databend_common_meta_app::principal::RoleIdent; use databend_common_meta_app::principal::RoleInfo; use databend_common_meta_app::principal::TenantOwnershipObjectIdent; use databend_common_meta_app::principal::UserPrivilegeType; -use databend_common_meta_app::principal::BUILTIN_ROLE_ACCOUNT_ADMIN; +use databend_common_meta_app::principal::role_ident; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_app::tenant_key::errors::ExistError; -use databend_common_meta_app::KeyWithTenant; use databend_common_meta_cache::Cache; use databend_common_meta_client::ClientHandle; use databend_common_meta_kvapi::kvapi; @@ -63,8 +63,8 @@ use log::info; use log::warn; use crate::role::role_api::RoleApi; -use crate::serde::check_and_upgrade_to_pb; use crate::serde::Quota; +use crate::serde::check_and_upgrade_to_pb; use crate::serialize_struct; static TXN_MAX_RETRY_TIMES: u32 = 60; @@ -265,14 +265,17 @@ impl RoleApi for RoleMgr { match &kvs { Ok(kvs) => { info!( - "RoleMgr::list_ownerships() returned from cache: {} keys; first key: {:?}; last key: {:?}", - kvs.len(), - kvs.first().map(|(k, _)| k), - kvs.last().map(|(k, _)| k), - ); + "RoleMgr::list_ownerships() returned from cache: {} keys; first key: {:?}; last key: {:?}", + kvs.len(), + kvs.first().map(|(k, _)| k), + kvs.last().map(|(k, _)| k), + ); } Err(err) => { - warn!("list ownerships from cache failed. err: {}; It is not a functional issue but may be a performance issue with more than 100_000 ownership records", err); + warn!( + "list ownerships from cache failed. err: {}; It is not a functional issue but may be a performance issue with more than 100_000 ownership records", + err + ); } } diff --git a/src/query/management/src/task/task_mgr.rs b/src/query/management/src/task/task_mgr.rs index 65949a6d83935..8757238a4abdd 100644 --- a/src/query/management/src/task/task_mgr.rs +++ b/src/query/management/src/task/task_mgr.rs @@ -22,13 +22,13 @@ use databend_common_ast::ast::AlterTaskOptions; use databend_common_ast::ast::ScheduleOptions; use databend_common_meta_api::kv_pb_api::KVPbApi; use databend_common_meta_api::kv_pb_api::UpsertPB; -use databend_common_meta_app::principal::task; -use databend_common_meta_app::principal::task::TaskMessage; -use databend_common_meta_app::principal::task_message_ident::TaskMessageIdent; use databend_common_meta_app::principal::ScheduleType; use databend_common_meta_app::principal::Status; use databend_common_meta_app::principal::Task; use databend_common_meta_app::principal::TaskIdent; +use databend_common_meta_app::principal::task; +use databend_common_meta_app::principal::task::TaskMessage; +use databend_common_meta_app::principal::task_message_ident::TaskMessageIdent; use databend_common_meta_app::schema::CreateOption; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_kvapi::kvapi; diff --git a/src/query/management/src/user/user_mgr.rs b/src/query/management/src/user/user_mgr.rs index 370adaa33861a..0fe9e55f34738 100644 --- a/src/query/management/src/user/user_mgr.rs +++ b/src/query/management/src/user/user_mgr.rs @@ -16,12 +16,12 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_meta_app::KeyWithTenant; use databend_common_meta_app::principal::TenantUserIdent; use databend_common_meta_app::principal::UserIdentity; use databend_common_meta_app::principal::UserInfo; use databend_common_meta_app::schema::CreateOption; use databend_common_meta_app::tenant::Tenant; -use databend_common_meta_app::KeyWithTenant; use databend_common_meta_kvapi::kvapi; use databend_common_meta_kvapi::kvapi::Key; use databend_common_meta_kvapi::kvapi::KvApiExt; diff --git a/src/query/management/src/warehouse/warehouse_mgr.rs b/src/query/management/src/warehouse/warehouse_mgr.rs index ff44851dcbabf..c323245060848 100644 --- a/src/query/management/src/warehouse/warehouse_mgr.rs +++ b/src/query/management/src/warehouse/warehouse_mgr.rs @@ -12,24 +12,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::hash_map::Entry; use std::collections::HashMap; use std::collections::HashSet; +use std::collections::hash_map::Entry; use std::time::Duration; +use databend_common_base::base::BuildInfoRef; +use databend_common_base::base::GlobalUniqName; use databend_common_base::base::escape_for_key; use databend_common_base::base::tokio; use databend_common_base::base::unescape_for_key; -use databend_common_base::base::BuildInfoRef; -use databend_common_base::base::GlobalUniqName; use databend_common_base::vec_ext::VecExt; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_meta_kvapi::kvapi::KVApi; use databend_common_meta_kvapi::kvapi::KvApiExt; use databend_common_meta_store::MetaStore; -use databend_common_meta_types::anyerror::AnyError; -use databend_common_meta_types::txn_op_response::Response; use databend_common_meta_types::ConditionResult; use databend_common_meta_types::InvalidReply; use databend_common_meta_types::MatchSeq; @@ -44,14 +42,16 @@ use databend_common_meta_types::TxnOp; use databend_common_meta_types::TxnOpResponse; use databend_common_meta_types::TxnReply; use databend_common_meta_types::TxnRequest; +use databend_common_meta_types::anyerror::AnyError; +use databend_common_meta_types::txn_op_response::Response; use log::error; use log::info; +use crate::warehouse::WarehouseApi; use crate::warehouse::warehouse_api::SelectedNode; use crate::warehouse::warehouse_api::SystemManagedCluster; use crate::warehouse::warehouse_api::SystemManagedWarehouse; use crate::warehouse::warehouse_api::WarehouseInfo; -use crate::warehouse::WarehouseApi; static DEFAULT_CLUSTER_ID: &str = "default"; @@ -268,7 +268,7 @@ impl WarehouseMgr { WarehouseInfo::SystemManaged(_) => { return Err(ErrorCode::WarehouseAlreadyExists( "Already exists same name system-managed warehouse.", - )) + )); } WarehouseInfo::SelfManaged(_) => { // Safe to retry with the updated seq of warehouse @@ -447,7 +447,10 @@ impl WarehouseMgr { let WarehouseInfo::SystemManaged(wh) = warehouse_snapshot.warehouse_info else { // The warehouse may have been dropped and started self-managed warehouse with same name. - log::warn!("The warehouse({}) may have been dropped and started self-managed warehouse with same name.", warehouse); + log::warn!( + "The warehouse({}) may have been dropped and started self-managed warehouse with same name.", + warehouse + ); continue; }; @@ -1319,7 +1322,10 @@ impl WarehouseApi for WarehouseMgr { )), WarehouseInfo::SystemManaged(warehouse) => { if warehouse.status.to_uppercase() != "RUNNING" { - return Err(ErrorCode::InvalidWarehouse(format!("Cannot suspend warehouse {:?}, because warehouse state is not running.", warehouse.id))); + return Err(ErrorCode::InvalidWarehouse(format!( + "Cannot suspend warehouse {:?}, because warehouse state is not running.", + warehouse.id + ))); } Ok(WarehouseInfo::SystemManaged(SystemManagedWarehouse { @@ -1781,11 +1787,20 @@ impl WarehouseApi for WarehouseMgr { let mut warehouse_snapshot = self.warehouse_snapshot(&warehouse).await?; warehouse_snapshot.warehouse_info = match warehouse_snapshot.warehouse_info { - WarehouseInfo::SelfManaged(_) => Err(ErrorCode::InvalidWarehouse(format!("Cannot rename cluster for warehouse {:?}, because it's self-managed warehouse.", warehouse))), + WarehouseInfo::SelfManaged(_) => Err(ErrorCode::InvalidWarehouse(format!( + "Cannot rename cluster for warehouse {:?}, because it's self-managed warehouse.", + warehouse + ))), WarehouseInfo::SystemManaged(mut info) => match info.clusters.contains_key(&cur) { - false => Err(ErrorCode::WarehouseClusterNotExists(format!("Warehouse cluster {:?}.{:?} not exists", warehouse, cur))), + false => Err(ErrorCode::WarehouseClusterNotExists(format!( + "Warehouse cluster {:?}.{:?} not exists", + warehouse, cur + ))), true => match info.clusters.contains_key(&to) { - true => Err(ErrorCode::WarehouseClusterAlreadyExists(format!("Warehouse cluster {:?}.{:?} already exists", warehouse, to))), + true => Err(ErrorCode::WarehouseClusterAlreadyExists(format!( + "Warehouse cluster {:?}.{:?} already exists", + warehouse, to + ))), false => { let cluster_info = info.clusters.remove(&cur); info.clusters.insert(to.clone(), cluster_info.unwrap()); @@ -1796,8 +1811,8 @@ impl WarehouseApi for WarehouseMgr { clusters: info.clusters, })) } - } - } + }, + }, }?; let warehouse_info_key = self.warehouse_info_key(&warehouse)?; @@ -2039,10 +2054,13 @@ impl WarehouseApi for WarehouseMgr { ))); } - removed_nodes = info.remove_nodes(cluster_id, nodes, &warehouse_snapshot.snapshot_nodes)?; + removed_nodes = info.remove_nodes( + cluster_id, + nodes, + &warehouse_snapshot.snapshot_nodes, + )?; } - Ok(WarehouseInfo::SystemManaged(SystemManagedWarehouse { role_id: info.role_id, status: info.status, diff --git a/src/query/management/src/workload/workload_mgr.rs b/src/query/management/src/workload/workload_mgr.rs index 52eae430831e1..b12ea1a7d77bb 100644 --- a/src/query/management/src/workload/workload_mgr.rs +++ b/src/query/management/src/workload/workload_mgr.rs @@ -14,8 +14,8 @@ use std::collections::HashMap; -use databend_common_base::base::escape_for_key; use databend_common_base::base::GlobalUniqName; +use databend_common_base::base::escape_for_key; use databend_common_base::runtime::workload_group::QuotaValue; use databend_common_base::runtime::workload_group::WorkloadGroup; use databend_common_exception::ErrorCode; diff --git a/src/query/management/src/workload/workload_resource.rs b/src/query/management/src/workload/workload_resource.rs index 15f923ed7bcbd..dee9ea74f4b63 100644 --- a/src/query/management/src/workload/workload_resource.rs +++ b/src/query/management/src/workload/workload_resource.rs @@ -13,25 +13,25 @@ // limitations under the License. use std::collections::HashMap; -use std::sync::atomic::AtomicUsize; -use std::sync::atomic::Ordering; use std::sync::Arc; use std::sync::Mutex; use std::sync::PoisonError; use std::sync::Weak; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; use databend_common_base::base::tokio::sync::Mutex as TokioMutex; -use databend_common_base::runtime::workload_group::QuotaValue; -use databend_common_base::runtime::workload_group::WorkloadGroupResource; +use databend_common_base::runtime::GLOBAL_MEM_STAT; +use databend_common_base::runtime::MemStat; use databend_common_base::runtime::workload_group::MAX_CONCURRENCY_QUOTA_KEY; use databend_common_base::runtime::workload_group::MEMORY_QUOTA_KEY; -use databend_common_base::runtime::MemStat; -use databend_common_base::runtime::GLOBAL_MEM_STAT; +use databend_common_base::runtime::workload_group::QuotaValue; +use databend_common_base::runtime::workload_group::WorkloadGroupResource; use databend_common_exception::Result; use tokio::sync::Semaphore; -use crate::workload::workload_mgr::WorkloadMgr; use crate::WorkloadApi; +use crate::workload::workload_mgr::WorkloadMgr; #[derive(Debug, Default)] pub struct PercentNormalizer { @@ -326,8 +326,8 @@ impl WorkloadGroupResourceManager { #[cfg(test)] mod tests { use databend_common_base::base::tokio; - use databend_common_base::runtime::workload_group::WorkloadGroup; use databend_common_base::runtime::GLOBAL_QUERIES_MANAGER; + use databend_common_base::runtime::workload_group::WorkloadGroup; use databend_common_meta_store::MetaStore; use super::*; diff --git a/src/query/management/tests/it/warehouse.rs b/src/query/management/tests/it/warehouse.rs index 0b2af649a5e6f..68cb3f60351af 100644 --- a/src/query/management/tests/it/warehouse.rs +++ b/src/query/management/tests/it/warehouse.rs @@ -12,13 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![allow(clippy::collapsible_if)] + use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; +use databend_common_base::base::GlobalUniqName; use databend_common_base::base::tokio; use databend_common_base::base::tokio::sync::Barrier; -use databend_common_base::base::GlobalUniqName; use databend_common_base::runtime::Runtime; use databend_common_base::runtime::TrySpawn; use databend_common_exception::Result; diff --git a/src/query/pipeline/src/basic/duplicate_processor.rs b/src/query/pipeline/src/basic/duplicate_processor.rs index f72214d291bd4..76183195f43e2 100644 --- a/src/query/pipeline/src/basic/duplicate_processor.rs +++ b/src/query/pipeline/src/basic/duplicate_processor.rs @@ -100,16 +100,16 @@ impl Processor for DuplicateProcessor { #[cfg(test)] mod tests { - use databend_common_expression::types::Int32Type; use databend_common_expression::DataBlock; use databend_common_expression::FromData; + use databend_common_expression::types::Int32Type; use crate::basic::duplicate_processor::DuplicateProcessor; - use crate::core::port::connect; use crate::core::Event; use crate::core::InputPort; use crate::core::OutputPort; use crate::core::Processor; + use crate::core::port::connect; #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn test_duplicate_output_finish() -> databend_common_exception::Result<()> { diff --git a/src/query/pipeline/src/basic/resize_processor.rs b/src/query/pipeline/src/basic/resize_processor.rs index f291211684337..44b46b8a84653 100644 --- a/src/query/pipeline/src/basic/resize_processor.rs +++ b/src/query/pipeline/src/basic/resize_processor.rs @@ -18,13 +18,13 @@ use std::sync::Arc; use databend_common_exception::Result; +use crate::core::PipeItem; use crate::core::port::InputPort; use crate::core::port::OutputPort; use crate::core::processor::Event; use crate::core::processor::EventCause; use crate::core::processor::Processor; use crate::core::processor::ProcessorPtr; -use crate::core::PipeItem; #[derive(PartialEq)] enum PortStatus { @@ -231,11 +231,11 @@ mod tests { use databend_common_expression::DataBlock; use crate::basic::resize_processor::ResizeProcessor; - use crate::core::port::connect; use crate::core::EventCause; use crate::core::InputPort; use crate::core::OutputPort; use crate::core::Processor; + use crate::core::port::connect; #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn test_resize_output_finish() -> databend_common_exception::Result<()> { diff --git a/src/query/pipeline/src/basic/shuffle_processor.rs b/src/query/pipeline/src/basic/shuffle_processor.rs index 4e07d72c8f795..b8134ee4a04fa 100644 --- a/src/query/pipeline/src/basic/shuffle_processor.rs +++ b/src/query/pipeline/src/basic/shuffle_processor.rs @@ -385,17 +385,17 @@ impl Processor for MergePartitionProcessor { #[cfg(test)] mod tests { - use databend_common_expression::types::Int32Type; use databend_common_expression::DataBlock; use databend_common_expression::FromData; + use databend_common_expression::types::Int32Type; use crate::basic::shuffle_processor::ShuffleProcessor; - use crate::core::port::connect; use crate::core::Event; use crate::core::EventCause; use crate::core::InputPort; use crate::core::OutputPort; use crate::core::Processor; + use crate::core::port::connect; #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn test_shuffle_output_finish() -> databend_common_exception::Result<()> { diff --git a/src/query/pipeline/src/core/finished_chain.rs b/src/query/pipeline/src/core/finished_chain.rs index db328c3711ac5..e55a16809d408 100644 --- a/src/query/pipeline/src/core/finished_chain.rs +++ b/src/query/pipeline/src/core/finished_chain.rs @@ -271,17 +271,17 @@ pub fn basic_callback(inner: T) -> BasicCallback { mod tests { use std::collections::HashMap; use std::panic::Location; + use std::sync::Arc; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; - use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; - use crate::core::always_callback; - use crate::core::basic_callback; use crate::core::ExecutionInfo; use crate::core::FinishedCallbackChain; + use crate::core::always_callback; + use crate::core::basic_callback; #[test] fn test_callback_order() -> Result<()> { @@ -509,9 +509,11 @@ mod tests { }), ); - assert!(chain - .apply(ExecutionInfo::create(Ok(()), HashMap::new())) - .is_err()); + assert!( + chain + .apply(ExecutionInfo::create(Ok(()), HashMap::new())) + .is_err() + ); assert_eq!(seq.load(Ordering::SeqCst), 13); diff --git a/src/query/pipeline/src/core/mod.rs b/src/query/pipeline/src/core/mod.rs index bbf095f7f76d4..9f46cbcf9d840 100644 --- a/src/query/pipeline/src/core/mod.rs +++ b/src/query/pipeline/src/core/mod.rs @@ -24,11 +24,11 @@ pub mod processor; pub mod profile; mod unsafe_cell_wrap; -pub use finished_chain::always_callback; -pub use finished_chain::basic_callback; pub use finished_chain::Callback; pub use finished_chain::ExecutionInfo; pub use finished_chain::FinishedCallbackChain; +pub use finished_chain::always_callback; +pub use finished_chain::basic_callback; pub use input_error::InputError; pub use lock_guard::LockGuard; pub use lock_guard::UnlockApi; diff --git a/src/query/pipeline/src/core/pipeline.rs b/src/query/pipeline/src/core/pipeline.rs index 1d62e1953b346..60c46beebaced 100644 --- a/src/query/pipeline/src/core/pipeline.rs +++ b/src/query/pipeline/src/core/pipeline.rs @@ -24,11 +24,11 @@ use databend_common_base::runtime::drop_guard; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use log::info; +use petgraph::Direction; use petgraph::graph::EdgeIndex; use petgraph::matrix_graph::Zero; use petgraph::prelude::StableGraph; use petgraph::stable_graph::NodeIndex; -use petgraph::Direction; use crate::basic::duplicate_processor::DuplicateProcessor; use crate::basic::resize_processor::ResizeProcessor; @@ -36,8 +36,6 @@ use crate::basic::sequence_group::SequenceGroupProcessor; use crate::basic::shuffle_processor::Exchange; use crate::basic::shuffle_processor::MergePartitionProcessor; use crate::basic::shuffle_processor::PartitionProcessor; -use crate::core::processor::ProcessorPtr; -use crate::core::profile::PlanScope; use crate::core::Callback; use crate::core::ExecutionInfo; use crate::core::FinishedCallbackChain; @@ -49,6 +47,8 @@ use crate::core::PipeItem; use crate::core::SinkPipeBuilder; use crate::core::SourcePipeBuilder; use crate::core::TransformPipeBuilder; +use crate::core::processor::ProcessorPtr; +use crate::core::profile::PlanScope; #[derive(Clone)] pub struct Node { diff --git a/src/query/pipeline/src/core/port.rs b/src/query/pipeline/src/core/port.rs index 8b7b781d18873..07261a779f823 100644 --- a/src/query/pipeline/src/core/port.rs +++ b/src/query/pipeline/src/core/port.rs @@ -12,16 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::Arc; use std::sync::atomic::AtomicPtr; use std::sync::atomic::Ordering; -use std::sync::Arc; -use databend_common_base::runtime::drop_guard; -use databend_common_base::runtime::profile::Profile; -use databend_common_base::runtime::profile::ProfileStatisticsName; use databend_common_base::runtime::ExecutorStats; use databend_common_base::runtime::QueryTimeSeriesProfile; use databend_common_base::runtime::TimeSeriesProfileName; +use databend_common_base::runtime::drop_guard; +use databend_common_base::runtime::profile::Profile; +use databend_common_base::runtime::profile::ProfileStatisticsName; use databend_common_exception::Result; use databend_common_expression::DataBlock; @@ -319,10 +319,12 @@ impl OutputPort { /// /// # Safety pub unsafe fn connect(input: &InputPort, output: &OutputPort) { - let shared_status = SharedStatus::create(); + unsafe { + let shared_status = SharedStatus::create(); - input.set_shared(shared_status.clone()); - output.set_shared(shared_status); + input.set_shared(shared_status.clone()); + output.set_shared(shared_status); + } } #[cfg(test)] @@ -332,13 +334,13 @@ mod tests { use databend_common_base::runtime::Thread; use databend_common_exception::ErrorCode; - use databend_common_expression::local_block_meta_serde; use databend_common_expression::BlockMetaInfo; use databend_common_expression::DataBlock; + use databend_common_expression::local_block_meta_serde; - use crate::core::port::connect; use crate::core::InputPort; use crate::core::OutputPort; + use crate::core::port::connect; #[derive(Clone, Debug)] struct TestDataMeta { diff --git a/src/query/pipeline/src/core/port_trigger.rs b/src/query/pipeline/src/core/port_trigger.rs index 1ba1a094067ea..106c55c9200f7 100644 --- a/src/query/pipeline/src/core/port_trigger.rs +++ b/src/query/pipeline/src/core/port_trigger.rs @@ -99,14 +99,16 @@ impl UpdateList { /// /// Must be thread safe call. In other words, it needs to be called in single thread or in mutex guard. pub unsafe fn trigger(&self, queue: &mut VecDeque) { - let inner = &mut *self.inner.get(); + unsafe { + let inner = &mut *self.inner.get(); - for trigger in &inner.updated_triggers { - UpdateTrigger::trigger_version(trigger.get()); - } + for trigger in &inner.updated_triggers { + UpdateTrigger::trigger_version(trigger.get()); + } - while let Some(index) = inner.updated_edges.pop() { - queue.push_front(index); + while let Some(index) = inner.updated_edges.pop() { + queue.push_front(index); + } } } @@ -120,16 +122,18 @@ impl UpdateList { self: &Arc, edge_index: EdgeIndex, ) -> Result<*mut UpdateTrigger> { - let inner = &mut *self.inner.get(); - let update_trigger = UpdateTrigger::create(edge_index, self.inner.get()); - inner - .updated_triggers - .push(Arc::new(UnsafeCell::new(update_trigger))); - inner - .updated_triggers - .last() - .map(|trigger| trigger.get()) - .ok_or_else(|| ErrorCode::Internal("Failed to get last trigger after push")) + unsafe { + let inner = &mut *self.inner.get(); + let update_trigger = UpdateTrigger::create(edge_index, self.inner.get()); + inner + .updated_triggers + .push(Arc::new(UnsafeCell::new(update_trigger))); + inner + .updated_triggers + .last() + .map(|trigger| trigger.get()) + .ok_or_else(|| ErrorCode::Internal("Failed to get last trigger after push")) + } } } @@ -158,7 +162,9 @@ impl UpdateTrigger { /// /// *mut UpdateTrigger must be a safe pointer pub unsafe fn trigger_version(self_: *mut UpdateTrigger) { - (*self_).prev_version = (*self_).version; + unsafe { + (*self_).prev_version = (*self_).version; + } } /// Trigger node input edge. Executor will schedule this edge. @@ -168,12 +174,14 @@ impl UpdateTrigger { /// *mut UpdateTrigger must be a safe pointer #[inline(always)] pub unsafe fn update_input(self_: &*mut UpdateTrigger) { - if !self_.is_null() { - let self_ = &mut **self_; - if self_.version == self_.prev_version { - self_.version += 1; - let inner = &mut *self_.update_list; - inner.updated_edges.push(DirectedEdge::Target(self_.index)); + unsafe { + if !self_.is_null() { + let self_ = &mut **self_; + if self_.version == self_.prev_version { + self_.version += 1; + let inner = &mut *self_.update_list; + inner.updated_edges.push(DirectedEdge::Target(self_.index)); + } } } } @@ -185,12 +193,14 @@ impl UpdateTrigger { /// *mut UpdateTrigger must be a safe pointer #[inline(always)] pub unsafe fn update_output(self_: &*mut UpdateTrigger) { - if !self_.is_null() { - let self_ = &mut **self_; - if self_.version == self_.prev_version { - self_.version += 1; - let inner = &mut *self_.update_list; - inner.updated_edges.push(DirectedEdge::Source(self_.index)); + unsafe { + if !self_.is_null() { + let self_ = &mut **self_; + if self_.version == self_.prev_version { + self_.version += 1; + let inner = &mut *self_.update_list; + inner.updated_edges.push(DirectedEdge::Source(self_.index)); + } } } } diff --git a/src/query/pipeline/src/core/processor.rs b/src/query/pipeline/src/core/processor.rs index 375ac68d83e4f..1ad9ce0265a81 100644 --- a/src/query/pipeline/src/core/processor.rs +++ b/src/query/pipeline/src/core/processor.rs @@ -23,8 +23,8 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use fastrace::prelude::*; -use futures::future::BoxFuture; use futures::FutureExt; +use futures::future::BoxFuture; use petgraph::graph::node_index; use petgraph::prelude::NodeIndex; @@ -92,11 +92,11 @@ pub trait Processor: Send { // we need to wrap UnsafeCell>, and make it Sync, // so that later an Arc of it could be moved into the async closure, // which async_process returns. -struct UnsafeSyncCelledProcessor(UnsafeCell>); +struct UnsafeSyncCelledProcessor(UnsafeCell>); unsafe impl Sync for UnsafeSyncCelledProcessor {} impl Deref for UnsafeSyncCelledProcessor { - type Target = UnsafeCell>; + type Target = UnsafeCell>; fn deref(&self) -> &Self::Target { &(self.0) @@ -109,8 +109,8 @@ pub struct ProcessorPtr { inner: Arc, } -impl From>> for UnsafeSyncCelledProcessor { - fn from(value: UnsafeCell>) -> Self { +impl From>> for UnsafeSyncCelledProcessor { + fn from(value: UnsafeCell>) -> Self { Self(value) } } @@ -130,105 +130,111 @@ impl ProcessorPtr { /// # Safety pub unsafe fn as_any(&mut self) -> &mut dyn Any { - (*self.inner.get()).as_any() + unsafe { (*self.inner.get()).as_any() } } /// # Safety pub unsafe fn id(&self) -> NodeIndex { - *self.id.get() + unsafe { *self.id.get() } } /// # Safety pub unsafe fn set_id(&self, id: NodeIndex) { - *self.id.get() = id; + unsafe { + *self.id.get() = id; + } } /// # Safety pub unsafe fn name(&self) -> String { - (*self.inner.get()).name() + unsafe { (*self.inner.get()).name() } } /// # Safety pub unsafe fn event(&self, cause: EventCause) -> Result { - (*self.inner.get()).event_with_cause(cause) + unsafe { (*self.inner.get()).event_with_cause(cause) } } /// # Safety pub unsafe fn un_reacted(&self, cause: EventCause) -> Result<()> { - (*self.inner.get()).un_reacted(cause, self.id().index()) + unsafe { (*self.inner.get()).un_reacted(cause, self.id().index()) } } /// # Safety pub unsafe fn interrupt(&self) { - (*self.inner.get()).interrupt() + unsafe { (*self.inner.get()).interrupt() } } /// # Safety pub unsafe fn process(&self) -> Result<()> { - let span = LocalSpan::enter_with_local_parent(format!("{}::process", self.name())) - .with_property(|| ("graph-node-id", self.id().index().to_string())); - - match (*self.inner.get()).process() { - Ok(_) => Ok(()), - Err(err) => { - let _ = span - .with_property(|| ("error", "true")) - .with_properties(|| { - [ - ("error.type", err.code().to_string()), - ("error.message", err.display_text()), - ] - }); - log::info!(error = err.to_string(); "Error in process"); - Err(err) + unsafe { + let span = LocalSpan::enter_with_local_parent(format!("{}::process", self.name())) + .with_property(|| ("graph-node-id", self.id().index().to_string())); + + match (*self.inner.get()).process() { + Ok(_) => Ok(()), + Err(err) => { + let _ = span + .with_property(|| ("error", "true")) + .with_properties(|| { + [ + ("error.type", err.code().to_string()), + ("error.message", err.display_text()), + ] + }); + log::info!(error = err.to_string(); "Error in process"); + Err(err) + } } } } /// # Safety pub unsafe fn async_process(&self) -> BoxFuture<'static, Result<()>> { - let id = self.id(); - let mut name = self.name(); - name.push_str("::async_process"); - - let task = (*self.inner.get()).async_process(); - - // The `task` may have reference to the `Processor` that hold in `self.inner`, - // so we need to move a clone of `self.inner` into the following async closure to keep the - // `Processor` from being dropped before `task` is done. - - // e.g. - // There may be scenarios where the 'ExecutingGraph' has already been dropped, - // but the async task returned by async_process is still running; in this case, - // there could be illegal memory access. - - let inner = self.inner.clone(); - async move { - let span = Span::enter_with_local_parent(name) - .with_property(|| ("graph-node-id", id.index().to_string())); - - match task.await { - Ok(_) => { - drop(inner); - Ok(()) - } - Err(err) => { - span.with_property(|| ("error", "true")).add_properties(|| { - [ - ("error.type", err.code().to_string()), - ("error.message", err.display_text()), - ] - }); - log::info!(error = err.to_string(); "Error in process"); - Err(err) + unsafe { + let id = self.id(); + let mut name = self.name(); + name.push_str("::async_process"); + + let task = (*self.inner.get()).async_process(); + + // The `task` may have reference to the `Processor` that hold in `self.inner`, + // so we need to move a clone of `self.inner` into the following async closure to keep the + // `Processor` from being dropped before `task` is done. + + // e.g. + // There may be scenarios where the 'ExecutingGraph' has already been dropped, + // but the async task returned by async_process is still running; in this case, + // there could be illegal memory access. + + let inner = self.inner.clone(); + async move { + let span = Span::enter_with_local_parent(name) + .with_property(|| ("graph-node-id", id.index().to_string())); + + match task.await { + Ok(_) => { + drop(inner); + Ok(()) + } + Err(err) => { + span.with_property(|| ("error", "true")).add_properties(|| { + [ + ("error.type", err.code().to_string()), + ("error.message", err.display_text()), + ] + }); + log::info!(error = err.to_string(); "Error in process"); + Err(err) + } } } + .boxed() } - .boxed() } /// # Safety pub unsafe fn details_status(&self) -> Option { - (*self.inner.get()).details_status() + unsafe { (*self.inner.get()).details_status() } } } diff --git a/src/query/pipeline/src/core/profile.rs b/src/query/pipeline/src/core/profile.rs index 58b0fd98169ab..850e5ce59a706 100644 --- a/src/query/pipeline/src/core/profile.rs +++ b/src/query/pipeline/src/core/profile.rs @@ -13,10 +13,10 @@ // limitations under the License. use std::cell::RefCell; -use std::collections::btree_map::Entry; use std::collections::BTreeMap; -use std::sync::atomic::Ordering; +use std::collections::btree_map::Entry; use std::sync::Arc; +use std::sync::atomic::Ordering; use databend_common_base::runtime::error_info::NodeErrorType; use databend_common_base::runtime::metrics::MetricSample; diff --git a/src/query/pipeline/src/lib.rs b/src/query/pipeline/src/lib.rs index 00329f4b3a2ca..4fd1be5728cb3 100644 --- a/src/query/pipeline/src/lib.rs +++ b/src/query/pipeline/src/lib.rs @@ -14,6 +14,7 @@ #![feature(variant_count)] #![feature(get_mut_unchecked)] +#![allow(clippy::collapsible_if)] pub mod basic; pub mod core; diff --git a/src/query/pipeline/src/sinks/async_mpsc_sink.rs b/src/query/pipeline/src/sinks/async_mpsc_sink.rs index 7d29b612db328..058c2e2687596 100644 --- a/src/query/pipeline/src/sinks/async_mpsc_sink.rs +++ b/src/query/pipeline/src/sinks/async_mpsc_sink.rs @@ -172,17 +172,17 @@ impl Processor for AsyncMpscSinker { #[cfg(test)] mod tests { + use std::sync::Arc; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; - use std::sync::Arc; use async_trait::async_trait; use databend_common_expression::DataBlock; - use crate::core::port::connect; use crate::core::Event; use crate::core::InputPort; use crate::core::OutputPort; + use crate::core::port::connect; use crate::sinks::async_mpsc_sink::AsyncMpscSink; use crate::sinks::async_mpsc_sink::AsyncMpscSinker; diff --git a/src/query/pipeline/src/sinks/async_sink.rs b/src/query/pipeline/src/sinks/async_sink.rs index fb818bd777582..b08caff476fef 100644 --- a/src/query/pipeline/src/sinks/async_sink.rs +++ b/src/query/pipeline/src/sinks/async_sink.rs @@ -16,9 +16,9 @@ use std::any::Any; use std::sync::Arc; use async_trait::async_trait; -use databend_common_base::runtime::drop_guard; use databend_common_base::runtime::GlobalIORuntime; use databend_common_base::runtime::TrySpawn; +use databend_common_base::runtime::drop_guard; use databend_common_exception::Result; use databend_common_expression::DataBlock; diff --git a/src/query/pipeline/src/sinks/sync_mpsc_sink.rs b/src/query/pipeline/src/sinks/sync_mpsc_sink.rs index d608c3279fb59..6b52c15f66c8f 100644 --- a/src/query/pipeline/src/sinks/sync_mpsc_sink.rs +++ b/src/query/pipeline/src/sinks/sync_mpsc_sink.rs @@ -169,16 +169,16 @@ impl Processor for SyncMpscSinker { #[cfg(test)] mod tests { + use std::sync::Arc; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; - use std::sync::Arc; use databend_common_expression::DataBlock; - use crate::core::port::connect; use crate::core::Event; use crate::core::InputPort; use crate::core::OutputPort; + use crate::core::port::connect; use crate::sinks::sync_mpsc_sink::SyncMpscSink; use crate::sinks::sync_mpsc_sink::SyncMpscSinker; diff --git a/src/query/pipeline/src/sources/sync_source_receiver.rs b/src/query/pipeline/src/sources/sync_source_receiver.rs index 71342854b735d..c0511ba5fb93d 100644 --- a/src/query/pipeline/src/sources/sync_source_receiver.rs +++ b/src/query/pipeline/src/sources/sync_source_receiver.rs @@ -14,8 +14,8 @@ use std::sync::Arc; -use databend_common_base::base::tokio::sync::mpsc::Receiver; use databend_common_base::base::Progress; +use databend_common_base::base::tokio::sync::mpsc::Receiver; use databend_common_exception::Result; use databend_common_expression::DataBlock; diff --git a/src/query/pipeline/transforms/src/lib.rs b/src/query/pipeline/transforms/src/lib.rs index 33c5f2f7e5d10..1e9706150e77e 100644 --- a/src/query/pipeline/transforms/src/lib.rs +++ b/src/query/pipeline/transforms/src/lib.rs @@ -13,10 +13,13 @@ // limitations under the License. #![allow(internal_features)] -#![allow(clippy::unconditional_recursion)] +#![allow( + clippy::collapsible_if, + clippy::let_and_return, + clippy::unconditional_recursion +)] #![feature(core_intrinsics)] #![feature(int_roundings)] -#![feature(let_chains)] #![feature(iter_map_windows)] #![feature(assert_matches)] #![feature(debug_closure_helpers)] diff --git a/src/query/pipeline/transforms/src/processors/memory_settings.rs b/src/query/pipeline/transforms/src/processors/memory_settings.rs index 8df278371c97f..ae6ccd5d2f5c4 100644 --- a/src/query/pipeline/transforms/src/processors/memory_settings.rs +++ b/src/query/pipeline/transforms/src/processors/memory_settings.rs @@ -13,13 +13,13 @@ // limitations under the License. use std::fmt::Debug; -use std::sync::atomic::Ordering; use std::sync::Arc; +use std::sync::atomic::Ordering; use bytesize::ByteSize; +use databend_common_base::runtime::GLOBAL_MEM_STAT; use databend_common_base::runtime::MemStat; use databend_common_base::runtime::ThreadTracker; -use databend_common_base::runtime::GLOBAL_MEM_STAT; #[derive(Clone)] #[non_exhaustive] diff --git a/src/query/pipeline/transforms/src/processors/transforms/blocks/block_cast.rs b/src/query/pipeline/transforms/src/processors/transforms/blocks/block_cast.rs index d556f48d354ab..081a9e854d8a0 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/blocks/block_cast.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/blocks/block_cast.rs @@ -15,7 +15,6 @@ use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::type_check::check_cast; use databend_common_expression::BlockEntry; use databend_common_expression::ColumnRef; use databend_common_expression::DataBlock; @@ -23,6 +22,7 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::Evaluator; use databend_common_expression::Expr; use databend_common_expression::FunctionContext; +use databend_common_expression::type_check::check_cast; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_pipeline::core::InputPort; use databend_common_pipeline::core::OutputPort; diff --git a/src/query/pipeline/transforms/src/processors/transforms/blocks/block_operator.rs b/src/query/pipeline/transforms/src/processors/transforms/blocks/block_operator.rs index 3bcba61c71485..2b875a8df997d 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/blocks/block_operator.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/blocks/block_operator.rs @@ -20,8 +20,8 @@ use databend_common_expression::FunctionContext; use databend_common_pipeline::core::InputPort; use databend_common_pipeline::core::OutputPort; use databend_common_pipeline::core::Processor; -use databend_common_sql::evaluator::apply_cse; use databend_common_sql::evaluator::BlockOperator; +use databend_common_sql::evaluator::apply_cse; use crate::Transform; use crate::Transformer; diff --git a/src/query/pipeline/transforms/src/processors/transforms/columns/column_add_computed.rs b/src/query/pipeline/transforms/src/processors/transforms/columns/column_add_computed.rs index 3ea69b10637c1..d61d8bc704921 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/columns/column_add_computed.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/columns/column_add_computed.rs @@ -17,19 +17,19 @@ use std::sync::Arc; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::type_check::check_cast; use databend_common_expression::ColumnRef; use databend_common_expression::ComputedExpr; use databend_common_expression::DataBlock; use databend_common_expression::DataSchemaRef; +use databend_common_expression::type_check::check_cast; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_license::license::Feature::ComputedColumn; use databend_common_license::license_manager::LicenseManagerSwitch; use databend_common_sql::evaluator::BlockOperator; use databend_common_sql::parse_computed_expr; -use crate::blocks::CompoundBlockOperator; use crate::Transform; +use crate::blocks::CompoundBlockOperator; pub struct TransformAddComputedColumns { expression_transform: CompoundBlockOperator, diff --git a/src/query/pipeline/transforms/src/processors/transforms/columns/column_add_const.rs b/src/query/pipeline/transforms/src/processors/transforms/columns/column_add_const.rs index d30895e3c4b2a..886c782006c4b 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/columns/column_add_const.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/columns/column_add_const.rs @@ -21,8 +21,8 @@ use databend_common_expression::FunctionContext; use databend_common_expression::Scalar as DataScalar; use databend_common_sql::evaluator::BlockOperator; -use crate::blocks::CompoundBlockOperator; use crate::Transform; +use crate::blocks::CompoundBlockOperator; pub struct TransformAddConstColumns { expression_transform: CompoundBlockOperator, diff --git a/src/query/pipeline/transforms/src/processors/transforms/columns/column_nullif.rs b/src/query/pipeline/transforms/src/processors/transforms/columns/column_nullif.rs index faafa3e76fe78..c73569bfed58d 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/columns/column_nullif.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/columns/column_nullif.rs @@ -16,9 +16,6 @@ use std::sync::Arc; use databend_common_ast::Span; use databend_common_exception::Result; -use databend_common_expression::type_check::check_function; -use databend_common_expression::types::DataType; -use databend_common_expression::types::StringType; use databend_common_expression::BlockEntry; use databend_common_expression::ColumnIndex; use databend_common_expression::ColumnRef; @@ -33,6 +30,9 @@ use databend_common_expression::FromData; use databend_common_expression::FunctionContext; use databend_common_expression::FunctionRegistry; use databend_common_expression::Scalar; +use databend_common_expression::type_check::check_function; +use databend_common_expression::types::DataType; +use databend_common_expression::types::StringType; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_pipeline::core::InputPort; use databend_common_pipeline::core::OutputPort; diff --git a/src/query/pipeline/transforms/src/processors/transforms/filters/filter_predicate.rs b/src/query/pipeline/transforms/src/processors/transforms/filters/filter_predicate.rs index ec798b113f768..b952ef6d5826c 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/filters/filter_predicate.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/filters/filter_predicate.rs @@ -17,11 +17,11 @@ use std::sync::Arc; use databend_common_catalog::plan::AggIndexMeta; use databend_common_exception::Result; -use databend_common_expression::filter::FilterExecutor; use databend_common_expression::BlockMetaInfoDowncast; use databend_common_expression::DataBlock; use databend_common_expression::Expr; use databend_common_expression::FunctionContext; +use databend_common_expression::filter::FilterExecutor; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_pipeline::core::InputPort; use databend_common_pipeline::core::OutputPort; diff --git a/src/query/pipeline/transforms/src/processors/transforms/sorts/core/algorithm.rs b/src/query/pipeline/transforms/src/processors/transforms/sorts/core/algorithm.rs index 1ccf4ed519240..c95d4d5eb6937 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/sorts/core/algorithm.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/sorts/core/algorithm.rs @@ -15,16 +15,16 @@ use core::fmt; use std::cmp::Ordering; use std::cmp::Reverse; -use std::collections::binary_heap; use std::collections::BinaryHeap; +use std::collections::binary_heap; use std::ops::Deref; use std::ops::DerefMut; -use super::loser_tree; -use super::utils::find_bigger_child_of_root; use super::Cursor as RawCursor; use super::CursorOrder; use super::Rows; +use super::loser_tree; +use super::utils::find_bigger_child_of_root; pub type Cursor = RawCursor; diff --git a/src/query/pipeline/transforms/src/processors/transforms/sorts/core/bounds.rs b/src/query/pipeline/transforms/src/processors/transforms/sorts/core/bounds.rs index 46f3bceb152cf..308bed43f405c 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/sorts/core/bounds.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/sorts/core/bounds.rs @@ -205,8 +205,8 @@ impl SortedStream for Bounds { #[cfg(test)] mod tests { - use databend_common_expression::types::Int32Type; use databend_common_expression::FromData; + use databend_common_expression::types::Int32Type; use super::*; use crate::sorts::core::SimpleRowsAsc; diff --git a/src/query/pipeline/transforms/src/processors/transforms/sorts/core/k_way_merge_sort_partition.rs b/src/query/pipeline/transforms/src/processors/transforms/sorts/core/k_way_merge_sort_partition.rs index 124e5fe6bded8..7c9f51a042067 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/sorts/core/k_way_merge_sort_partition.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/sorts/core/k_way_merge_sort_partition.rs @@ -19,12 +19,12 @@ use databend_common_exception::Result; use databend_common_expression::DataBlock; use databend_common_expression::DataSchemaRef; +use super::Rows; +use super::SortedStream; use super::list_domain::Candidate; use super::list_domain::EndDomain; use super::list_domain::List; use super::list_domain::Partition; -use super::Rows; -use super::SortedStream; use crate::sorts::SortTaskMeta; pub struct KWaySortPartitioner diff --git a/src/query/pipeline/transforms/src/processors/transforms/sorts/core/list_domain.rs b/src/query/pipeline/transforms/src/processors/transforms/sorts/core/list_domain.rs index 60fda04bac06e..1c9ff7f11d701 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/sorts/core/list_domain.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/sorts/core/list_domain.rs @@ -503,9 +503,9 @@ impl From> for EndDomain { mod tests { use std::iter::repeat_with; - use rand::rngs::StdRng; use rand::Rng; use rand::SeedableRng; + use rand::rngs::StdRng; use super::*; diff --git a/src/query/pipeline/transforms/src/processors/transforms/sorts/core/merger.rs b/src/query/pipeline/transforms/src/processors/transforms/sorts/core/merger.rs index b3ad7f144988e..d7c42aff232ed 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/sorts/core/merger.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/sorts/core/merger.rs @@ -20,8 +20,8 @@ use databend_common_expression::Column; use databend_common_expression::DataBlock; use databend_common_expression::DataSchemaRef; -use super::algorithm::*; use super::Rows; +use super::algorithm::*; #[async_trait::async_trait] pub trait SortedStream { diff --git a/src/query/pipeline/transforms/src/processors/transforms/sorts/core/row_convert/fixed.rs b/src/query/pipeline/transforms/src/processors/transforms/sorts/core/row_convert/fixed.rs index fd1f35ca899e5..8eab0ad827876 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/sorts/core/row_convert/fixed.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/sorts/core/row_convert/fixed.rs @@ -19,7 +19,13 @@ use databend_common_column::buffer::Buffer; use databend_common_column::types::months_days_micros; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::i256; +use databend_common_expression::BlockEntry; +use databend_common_expression::Column; +use databend_common_expression::DataSchemaRef; +use databend_common_expression::FixedLengthEncoding; +use databend_common_expression::Scalar; +use databend_common_expression::SortColumnDescription; +use databend_common_expression::SortField; use databend_common_expression::types::AccessType; use databend_common_expression::types::ArgType; use databend_common_expression::types::BooleanType; @@ -36,21 +42,15 @@ use databend_common_expression::types::NumberType; use databend_common_expression::types::OpaqueType; use databend_common_expression::types::TimestampType; use databend_common_expression::types::ValueType; +use databend_common_expression::types::i256; use databend_common_expression::visitor::ValueVisitor; use databend_common_expression::with_decimal_mapped_type; use databend_common_expression::with_number_mapped_type; -use databend_common_expression::BlockEntry; -use databend_common_expression::Column; -use databend_common_expression::DataSchemaRef; -use databend_common_expression::FixedLengthEncoding; -use databend_common_expression::Scalar; -use databend_common_expression::SortColumnDescription; -use databend_common_expression::SortField; -use super::fixed_encode::fixed_encode; -use super::fixed_encode::fixed_encode_const; use super::RowConverter; use super::Rows; +use super::fixed_encode::fixed_encode; +use super::fixed_encode::fixed_encode_const; pub fn choose_encode_method(fields: &[SortField]) -> Option { let mut total_len = 0; @@ -174,10 +174,12 @@ impl FixedRowConverter { /// Convert columns into fixed-size row format. fn convert_columns(&self, entries: &[BlockEntry], num_rows: usize) -> FixedRows { debug_assert_eq!(entries.len(), self.fields.len()); - debug_assert!(entries - .iter() - .zip(self.fields.iter()) - .all(|(entry, f)| entry.len() == num_rows && entry.data_type() == f.data_type)); + debug_assert!( + entries + .iter() + .zip(self.fields.iter()) + .all(|(entry, f)| entry.len() == num_rows && entry.data_type() == f.data_type) + ); let (mut buffer, mut offsets) = self.new_empty_rows(num_rows); for (entry, field) in entries.iter().zip(self.fields.iter()) { diff --git a/src/query/pipeline/transforms/src/processors/transforms/sorts/core/row_convert/fixed_encode.rs b/src/query/pipeline/transforms/src/processors/transforms/sorts/core/row_convert/fixed_encode.rs index 186d1af89c47a..5edb25bbc11e0 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/sorts/core/row_convert/fixed_encode.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/sorts/core/row_convert/fixed_encode.rs @@ -13,9 +13,9 @@ // limitations under the License. use databend_common_column::bitmap::Bitmap; -use databend_common_expression::types::AccessType; use databend_common_expression::ColumnViewIter; use databend_common_expression::FixedLengthEncoding; +use databend_common_expression::types::AccessType; use super::null_sentinel; diff --git a/src/query/pipeline/transforms/src/processors/transforms/sorts/core/row_convert/mod.rs b/src/query/pipeline/transforms/src/processors/transforms/sorts/core/row_convert/mod.rs index 96141dab87898..9fb2d6da4d16d 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/sorts/core/row_convert/mod.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/sorts/core/row_convert/mod.rs @@ -15,8 +15,6 @@ use std::fmt::Debug; use databend_common_exception::Result; -use databend_common_expression::types::*; -use databend_common_expression::with_number_mapped_type; use databend_common_expression::BlockEntry; use databend_common_expression::Column; use databend_common_expression::DataBlock; @@ -24,6 +22,8 @@ use databend_common_expression::DataSchema; use databend_common_expression::DataSchemaRef; use databend_common_expression::SortColumnDescription; use databend_common_expression::SortField; +use databend_common_expression::types::*; +use databend_common_expression::with_number_mapped_type; use match_template::match_template; use super::Rows; @@ -252,11 +252,7 @@ pub fn order_field_type( } fn null_sentinel(nulls_first: bool) -> u8 { - if nulls_first { - 0 - } else { - 0xFF - } + if nulls_first { 0 } else { 0xFF } } mod fixed; diff --git a/src/query/pipeline/transforms/src/processors/transforms/sorts/core/row_convert/simple.rs b/src/query/pipeline/transforms/src/processors/transforms/sorts/core/row_convert/simple.rs index b76f5ad14034f..2d384ac4579f4 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/sorts/core/row_convert/simple.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/sorts/core/row_convert/simple.rs @@ -18,14 +18,14 @@ use std::ops::Range; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::ArgType; -use databend_common_expression::types::DataType; -use databend_common_expression::types::ValueType; use databend_common_expression::BlockEntry; use databend_common_expression::Column; use databend_common_expression::DataSchemaRef; use databend_common_expression::Scalar; use databend_common_expression::SortColumnDescription; +use databend_common_expression::types::ArgType; +use databend_common_expression::types::DataType; +use databend_common_expression::types::ValueType; use super::RowConverter; use super::Rows; diff --git a/src/query/pipeline/transforms/src/processors/transforms/sorts/core/row_convert/test_util.rs b/src/query/pipeline/transforms/src/processors/transforms/sorts/core/row_convert/test_util.rs index 284b9ab7698f2..fcfbd28b14d03 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/sorts/core/row_convert/test_util.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/sorts/core/row_convert/test_util.rs @@ -16,11 +16,11 @@ use arrow_array::ArrayRef; use arrow_ord::sort::LexicographicalComparator; use arrow_ord::sort::SortColumn; use arrow_schema::SortOptions; -use databend_common_expression::types::*; use databend_common_expression::BlockEntry; use databend_common_expression::FromData; use databend_common_expression::Scalar; use databend_common_expression::SortField; +use databend_common_expression::types::*; use proptest::prelude::*; pub fn print_row(entries: &[BlockEntry], row: usize) -> String { diff --git a/src/query/pipeline/transforms/src/processors/transforms/sorts/core/row_convert/variable.rs b/src/query/pipeline/transforms/src/processors/transforms/sorts/core/row_convert/variable.rs index 4e229dad9e40c..5e6f75be306ee 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/sorts/core/row_convert/variable.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/sorts/core/row_convert/variable.rs @@ -20,11 +20,13 @@ use databend_common_column::types::months_days_micros; use databend_common_column::types::timestamp_tz; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::binary::BinaryColumn; -use databend_common_expression::types::binary::BinaryColumnBuilder; -use databend_common_expression::types::i256; -use databend_common_expression::types::nullable::NullableColumn; -use databend_common_expression::types::timestamp_tz::TimestampTzType; +use databend_common_expression::BlockEntry; +use databend_common_expression::Column; +use databend_common_expression::DataSchemaRef; +use databend_common_expression::FixedLengthEncoding; +use databend_common_expression::Scalar; +use databend_common_expression::SortColumnDescription; +use databend_common_expression::SortField; use databend_common_expression::types::AccessType; use databend_common_expression::types::BinaryType; use databend_common_expression::types::BooleanType; @@ -41,25 +43,23 @@ use databend_common_expression::types::NumberType; use databend_common_expression::types::StringColumn; use databend_common_expression::types::TimestampType; use databend_common_expression::types::ValueType; +use databend_common_expression::types::binary::BinaryColumn; +use databend_common_expression::types::binary::BinaryColumnBuilder; +use databend_common_expression::types::i256; +use databend_common_expression::types::nullable::NullableColumn; +use databend_common_expression::types::timestamp_tz::TimestampTzType; use databend_common_expression::visitor::ValueVisitor; use databend_common_expression::with_decimal_mapped_type; use databend_common_expression::with_number_mapped_type; use databend_common_expression::with_number_type; -use databend_common_expression::BlockEntry; -use databend_common_expression::Column; -use databend_common_expression::DataSchemaRef; -use databend_common_expression::FixedLengthEncoding; -use databend_common_expression::Scalar; -use databend_common_expression::SortColumnDescription; -use databend_common_expression::SortField; use jsonb::RawJsonb; +use super::RowConverter; +use super::Rows; use super::fixed_encode::fixed_encode; use super::fixed_encode::fixed_encode_const; use super::variable_encode::encoded_len; use super::variable_encode::var_encode; -use super::RowConverter; -use super::Rows; pub type VariableRows = BinaryColumn; @@ -165,10 +165,12 @@ impl VariableRowConverter { /// Convert columns into [`BinaryColumn`] represented comparable row format. fn convert_columns(&self, entries: &[BlockEntry], num_rows: usize) -> BinaryColumn { debug_assert_eq!(entries.len(), self.fields.len()); - debug_assert!(entries - .iter() - .zip(self.fields.iter()) - .all(|(entry, f)| entry.len() == num_rows && entry.data_type() == f.data_type)); + debug_assert!( + entries + .iter() + .zip(self.fields.iter()) + .all(|(entry, f)| entry.len() == num_rows && entry.data_type() == f.data_type) + ); let entries: Vec = entries .iter() @@ -817,10 +819,10 @@ impl ValueVisitor for EncodeVisitor<'_> { #[cfg(test)] mod tests { use databend_common_base::base::OrderedFloat; - use databend_common_expression::types::*; use databend_common_expression::Column; use databend_common_expression::FromData; use databend_common_expression::SortField; + use databend_common_expression::types::*; use jsonb::OwnedJsonb; use proptest::prelude::*; use proptest::strategy::ValueTree; diff --git a/src/query/pipeline/transforms/src/processors/transforms/sorts/core/rows.rs b/src/query/pipeline/transforms/src/processors/transforms/sorts/core/rows.rs index 25d324a2a3cef..b447e290449a1 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/sorts/core/rows.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/sorts/core/rows.rs @@ -16,10 +16,10 @@ use std::fmt::Debug; use std::ops::Range; use databend_common_exception::Result; -use databend_common_expression::types::ArgType; -use databend_common_expression::types::DataType; use databend_common_expression::Column; use databend_common_expression::Scalar; +use databend_common_expression::types::ArgType; +use databend_common_expression::types::DataType; /// Rows can be compared. pub trait Rows diff --git a/src/query/pipeline/transforms/src/processors/transforms/sorts/mod.rs b/src/query/pipeline/transforms/src/processors/transforms/sorts/mod.rs index 0ab8cf95bb9dc..19d10f2a05aa3 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/sorts/mod.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/sorts/mod.rs @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_expression::local_block_meta_serde; use databend_common_expression::BlockMetaInfo; use databend_common_expression::BlockMetaInfoDowncast; use databend_common_expression::DataBlock; use databend_common_expression::DataSchemaRef; +use databend_common_expression::local_block_meta_serde; use enum_as_inner::EnumAsInner; pub use sort_broadcast::*; pub use sort_collect::*; @@ -32,8 +32,8 @@ pub use sort_restore::*; pub use sort_route::*; pub use sort_spill::*; -pub use self::core::utils; use self::core::Bounds; +pub use self::core::utils; pub mod core; mod sort_broadcast; diff --git a/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_broadcast.rs b/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_broadcast.rs index 2c13ed000dfdb..76dd17839b755 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_broadcast.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_broadcast.rs @@ -27,10 +27,10 @@ use databend_common_pipeline::core::InputPort; use databend_common_pipeline::core::OutputPort; use databend_common_pipeline::core::Processor; -use super::core::Bounds; -use super::core::Rows; use super::SortCollectedMeta; use super::SortExchangeMeta; +use super::core::Bounds; +use super::core::Rows; use crate::HookTransform; use crate::HookTransformer; diff --git a/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_collect.rs b/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_collect.rs index 95fe075442886..5aac50f285a0f 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_collect.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_collect.rs @@ -13,9 +13,9 @@ // limitations under the License. use std::any::Any; +use std::sync::Arc; use std::sync::atomic; use std::sync::atomic::AtomicBool; -use std::sync::Arc; use bytesize::ByteSize; use databend_common_exception::Result; @@ -26,17 +26,17 @@ use databend_common_pipeline::core::InputPort; use databend_common_pipeline::core::OutputPort; use databend_common_pipeline::core::Processor; -use super::core::algorithm::SortAlgorithm; -use super::core::RowConverter; -use super::core::Rows; use super::Base; use super::MergeSort; use super::RowsStat; use super::SortSpill; use super::SortSpillParams; use super::TransformSortMergeLimit; -use crate::traits::DataBlockSpill; +use super::core::RowConverter; +use super::core::Rows; +use super::core::algorithm::SortAlgorithm; use crate::MemorySettings; +use crate::traits::DataBlockSpill; #[allow(clippy::large_enum_variant)] enum Inner { diff --git a/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_k_way_merge.rs b/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_k_way_merge.rs index 5924d127330f4..7ddb1e9f18aaf 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_k_way_merge.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_k_way_merge.rs @@ -34,18 +34,18 @@ use databend_common_pipeline::core::Pipeline; use databend_common_pipeline::core::Processor; use databend_common_pipeline::core::ProcessorPtr; -use super::core::algorithm::HeapSort; -use super::core::algorithm::LoserTreeSort; -use super::core::algorithm::SortAlgorithm; -use super::core::select_row_type; +use super::InputBlockStream; +use super::SortTaskMeta; use super::core::KWaySortPartitioner; use super::core::Merger; use super::core::RowConverter; use super::core::Rows; use super::core::RowsTypeVisitor; use super::core::SortedStream; -use super::InputBlockStream; -use super::SortTaskMeta; +use super::core::algorithm::HeapSort; +use super::core::algorithm::LoserTreeSort; +use super::core::algorithm::SortAlgorithm; +use super::core::select_row_type; pub fn add_k_way_merge_sort( pipeline: &mut Pipeline, diff --git a/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_local_merge.rs b/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_local_merge.rs index c46bf687574a7..09a2fcb96702e 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_local_merge.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_local_merge.rs @@ -15,9 +15,9 @@ use std::any::Any; use std::collections::VecDeque; use std::intrinsics::unlikely; +use std::sync::Arc; use std::sync::atomic; use std::sync::atomic::AtomicBool; -use std::sync::Arc; use bytesize::ByteSize; use databend_common_exception::Result; @@ -29,10 +29,6 @@ use databend_common_pipeline::core::InputPort; use databend_common_pipeline::core::OutputPort; use databend_common_pipeline::core::Processor; -use super::core::algorithm::SortAlgorithm; -use super::core::RowConverter; -use super::core::Rows; -use super::create_memory_merger; use super::Base; use super::MemoryMerger; use super::MergeSort; @@ -41,8 +37,12 @@ use super::RowsStat; use super::SortSpill; use super::SortSpillParams; use super::TransformSortMergeLimit; -use crate::traits::DataBlockSpill; +use super::core::RowConverter; +use super::core::Rows; +use super::core::algorithm::SortAlgorithm; +use super::create_memory_merger; use crate::MemorySettings; +use crate::traits::DataBlockSpill; #[derive(Debug)] enum State { diff --git a/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_merge.rs b/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_merge.rs index df11705d5f540..8af0acceabc57 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_merge.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_merge.rs @@ -14,9 +14,9 @@ use std::intrinsics::unlikely; use std::marker::PhantomData; +use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; -use std::sync::Arc; use bytesize::ByteSize; use databend_common_exception::ErrorCode; @@ -26,14 +26,14 @@ use databend_common_expression::DataBlock; use databend_common_expression::DataSchemaRef; use databend_common_expression::SortColumnDescription; -use super::core::algorithm::HeapSort; -use super::core::algorithm::LoserTreeSort; -use super::core::algorithm::SortAlgorithm; use super::core::Merger; use super::core::Rows; use super::core::SortedStream; use super::core::VariableRowConverter; use super::core::VariableRows; +use super::core::algorithm::HeapSort; +use super::core::algorithm::LoserTreeSort; +use super::core::algorithm::SortAlgorithm; use super::sort_merge_base::MergeSort; use super::sort_merge_base::TransformSortMergeBase; use super::utils::has_order_field; diff --git a/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_merge_stream.rs b/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_merge_stream.rs index 953a4f063667b..7ba627a87781a 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_merge_stream.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_merge_stream.rs @@ -27,12 +27,12 @@ use databend_common_pipeline::core::InputPort; use databend_common_pipeline::core::OutputPort; use databend_common_pipeline::core::Processor; -use super::core::algorithm::SortAlgorithm; +use super::SortBound; +use super::SortBoundNext; use super::core::Merger; use super::core::Rows; use super::core::SortedStream; -use super::SortBound; -use super::SortBoundNext; +use super::core::algorithm::SortAlgorithm; type Stream = BoundedInputStream<::Rows>; @@ -293,8 +293,8 @@ impl BoundedInputStream { mod tests { use std::sync::Arc; - use databend_common_expression::types::Int32Type; use databend_common_expression::FromData; + use databend_common_expression::types::Int32Type; use databend_common_pipeline::core::port::connect; use super::*; diff --git a/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_multi_merge.rs b/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_multi_merge.rs index f6bb7fb9917b8..54fe4273cea44 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_multi_merge.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_multi_merge.rs @@ -30,15 +30,15 @@ use databend_common_pipeline::core::Pipeline; use databend_common_pipeline::core::Processor; use databend_common_pipeline::core::ProcessorPtr; -use super::core::algorithm::HeapSort; -use super::core::algorithm::LoserTreeSort; -use super::core::algorithm::SortAlgorithm; -use super::core::select_row_type; use super::core::Merger; use super::core::RowConverter; use super::core::Rows; use super::core::RowsTypeVisitor; use super::core::SortedStream; +use super::core::algorithm::HeapSort; +use super::core::algorithm::LoserTreeSort; +use super::core::algorithm::SortAlgorithm; +use super::core::select_row_type; use super::utils::ORDER_COL_NAME; pub fn try_add_multi_sort_merge( diff --git a/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_restore.rs b/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_restore.rs index 5729d3d384e57..a94cc20183968 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_restore.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_restore.rs @@ -23,17 +23,17 @@ use databend_common_pipeline::core::InputPort; use databend_common_pipeline::core::OutputPort; use databend_common_pipeline::core::Processor; -use super::core::algorithm::SortAlgorithm; -use super::sort_spill::OutputData; -use super::sort_spill::SortSpill; use super::Base; use super::SortBound; use super::SortBoundNext; use super::SortCollectedMeta; -use crate::traits::DataBlockSpill; +use super::core::algorithm::SortAlgorithm; +use super::sort_spill::OutputData; +use super::sort_spill::SortSpill; use crate::HookTransform; use crate::HookTransformer; use crate::MemorySettings; +use crate::traits::DataBlockSpill; pub struct TransformSortRestore { input: Vec, diff --git a/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_spill.rs b/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_spill.rs index 0c3b2eb9369d9..83f9228c5f0d1 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_spill.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/sorts/sort_spill.rs @@ -25,26 +25,26 @@ use std::sync::atomic::AtomicBool; use databend_common_column::bitmap::MutableBitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::sampler::FixedRateSampler; use databend_common_expression::Column; use databend_common_expression::DataBlock; use databend_common_expression::DataSchemaRef; use databend_common_expression::Scalar; -use rand::rngs::StdRng; +use databend_common_expression::sampler::FixedRateSampler; use rand::SeedableRng; +use rand::rngs::StdRng; -use super::core::algorithm::SortAlgorithm; -use super::core::Bounds; -use super::core::Merger; -use super::core::Rows; -use super::core::SortedStream; use super::Base; use super::RowsStat; use super::SortCollectedMeta; use super::SortSpillParams; +use super::core::Bounds; +use super::core::Merger; +use super::core::Rows; +use super::core::SortedStream; +use super::core::algorithm::SortAlgorithm; +use crate::MemorySettings; use crate::traits::DataBlockSpill; use crate::traits::Location; -use crate::MemorySettings; pub struct SortSpill { base: Base, @@ -975,21 +975,21 @@ mod tests { use std::sync::Mutex; use databend_common_base::base::GlobalUniqName; - use databend_common_expression::types::DataType; - use databend_common_expression::types::Int32Type; - use databend_common_expression::types::NumberDataType; - use databend_common_expression::types::NumberScalar; - use databend_common_expression::types::StringType; use databend_common_expression::Column; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRefExt; use databend_common_expression::FromData; use databend_common_expression::SortColumnDescription; + use databend_common_expression::types::DataType; + use databend_common_expression::types::Int32Type; + use databend_common_expression::types::NumberDataType; + use databend_common_expression::types::NumberScalar; + use databend_common_expression::types::StringType; use super::*; - use crate::sorts::core::convert_rows; use crate::sorts::core::SimpleRowsAsc; use crate::sorts::core::SimpleRowsDesc; + use crate::sorts::core::convert_rows; fn test_data() -> (DataSchemaRef, DataBlock) { let col1 = Int32Type::from_data(vec![7, 7, 8, 11, 3, 5, 10, 11]); diff --git a/src/query/pipeline/transforms/src/processors/transforms/transform_compact_block.rs b/src/query/pipeline/transforms/src/processors/transforms/transform_compact_block.rs index a299853c9e0d4..31db751d61432 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/transform_compact_block.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/transform_compact_block.rs @@ -15,15 +15,15 @@ use std::fmt::Debug; use std::fmt::Formatter; use std::intrinsics::unlikely; +use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; -use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::local_block_meta_serde; use databend_common_expression::BlockMetaInfo; use databend_common_expression::DataBlock; +use databend_common_expression::local_block_meta_serde; use crate::processors::BlockMetaTransform; use crate::processors::UnknownMode; diff --git a/src/query/pipeline/transforms/tests/it/merger.rs b/src/query/pipeline/transforms/tests/it/merger.rs index df740624d8d1d..47e7700c840ea 100644 --- a/src/query/pipeline/transforms/tests/it/merger.rs +++ b/src/query/pipeline/transforms/tests/it/merger.rs @@ -16,24 +16,24 @@ use std::collections::VecDeque; use databend_common_base::base::tokio; use databend_common_exception::Result; -use databend_common_expression::block_debug::pretty_format_blocks; -use databend_common_expression::types::DataType; -use databend_common_expression::types::Int32Type; -use databend_common_expression::types::NumberDataType; use databend_common_expression::Column; use databend_common_expression::DataBlock; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRefExt; use databend_common_expression::FromData; -use databend_common_pipeline_transforms::sorts::core::algorithm::HeapSort; -use databend_common_pipeline_transforms::sorts::core::algorithm::LoserTreeSort; -use databend_common_pipeline_transforms::sorts::core::algorithm::SortAlgorithm; +use databend_common_expression::block_debug::pretty_format_blocks; +use databend_common_expression::types::DataType; +use databend_common_expression::types::Int32Type; +use databend_common_expression::types::NumberDataType; use databend_common_pipeline_transforms::sorts::core::Merger; use databend_common_pipeline_transforms::sorts::core::SimpleRowsAsc; use databend_common_pipeline_transforms::sorts::core::SortedStream; +use databend_common_pipeline_transforms::sorts::core::algorithm::HeapSort; +use databend_common_pipeline_transforms::sorts::core::algorithm::LoserTreeSort; +use databend_common_pipeline_transforms::sorts::core::algorithm::SortAlgorithm; use itertools::Itertools; -use rand::rngs::ThreadRng; use rand::Rng; +use rand::rngs::ThreadRng; struct TestStream { data: VecDeque, diff --git a/src/query/script/src/compiler.rs b/src/query/script/src/compiler.rs index e05a9a44cf93c..54783364fe7ea 100644 --- a/src/query/script/src/compiler.rs +++ b/src/query/script/src/compiler.rs @@ -17,6 +17,7 @@ use std::fmt; use std::fmt::Display; use std::vec; +use databend_common_ast::Span; use databend_common_ast::ast::BinaryOperator; use databend_common_ast::ast::ColumnID; use databend_common_ast::ast::ColumnRef; @@ -36,7 +37,6 @@ use databend_common_ast::ast::SetExpr; use databend_common_ast::ast::Statement; use databend_common_ast::ast::TableReference; use databend_common_ast::ast::UnaryOperator; -use databend_common_ast::Span; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use derive_visitor::DriveMut; @@ -238,16 +238,18 @@ impl Compiler { let set = match iterable { IterableItem::Resultset(name) => { // Try to look up as a resultset first, then as a cursor - if let Ok(set) = self.lookup_set(name) { - set - } else if let Ok(cursor_set) = self.lookup_cursor(name) { - cursor_set - } else { - return Err(ErrorCode::ScriptSemanticError(format!( - "`{}` is not a resultset or cursor", - name.name - )) - .set_span(name.span)); + match self.lookup_set(name) { + Ok(set) => set, + _ => match self.lookup_cursor(name) { + Ok(cursor_set) => cursor_set, + _ => { + return Err(ErrorCode::ScriptSemanticError(format!( + "`{}` is not a resultset or cursor", + name.name + )) + .set_span(name.span)); + } + }, } } IterableItem::Cursor(cursor) => { diff --git a/src/query/script/src/executor.rs b/src/query/script/src/executor.rs index 222ff2507262e..37bb6b11e81f7 100644 --- a/src/query/script/src/executor.rs +++ b/src/query/script/src/executor.rs @@ -14,8 +14,8 @@ use std::collections::HashMap; -use databend_common_ast::ast::Expr; use databend_common_ast::Span; +use databend_common_ast::ast::Expr; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -34,7 +34,7 @@ pub trait Client { async fn query(&self, query: &str) -> Result; fn var_to_ast(&self, scalar: &Self::Var) -> Result; fn read_from_set(&self, block: &Self::Set, row: usize, col: &ColumnAccess) - -> Result; + -> Result; fn num_rows(&self, block: &Self::Set) -> usize; fn is_true(&self, scalar: &Self::Var) -> Result; fn format_error(&self, value: &Self::Var) -> Result; diff --git a/src/query/script/src/ir.rs b/src/query/script/src/ir.rs index 8bdffbabaf16c..788e14a027b60 100644 --- a/src/query/script/src/ir.rs +++ b/src/query/script/src/ir.rs @@ -16,11 +16,11 @@ use std::fmt; use std::fmt::Display; use std::hash::Hash; +use databend_common_ast::Span; use databend_common_ast::ast::Expr; use databend_common_ast::ast::Identifier; use databend_common_ast::ast::Literal; use databend_common_ast::ast::Statement; -use databend_common_ast::Span; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use derive_visitor::DriveMut; diff --git a/src/query/script/src/lib.rs b/src/query/script/src/lib.rs index f20b63c473906..281417981b1f4 100644 --- a/src/query/script/src/lib.rs +++ b/src/query/script/src/lib.rs @@ -13,7 +13,6 @@ // limitations under the License. #![feature(try_blocks)] -#![feature(let_chains)] pub mod ir; diff --git a/src/query/script/tests/it/main.rs b/src/query/script/tests/it/main.rs index 0db0e9eeb7dc7..dc05bab9d149a 100644 --- a/src/query/script/tests/it/main.rs +++ b/src/query/script/tests/it/main.rs @@ -22,20 +22,20 @@ use std::io::Write; use std::sync::Arc; use std::sync::Mutex; +use databend_common_ast::Range; use databend_common_ast::ast::Expr; use databend_common_ast::ast::Literal; +use databend_common_ast::parser::Dialect; +use databend_common_ast::parser::ParseMode; use databend_common_ast::parser::run_parser; use databend_common_ast::parser::script::script_stmts; use databend_common_ast::parser::tokenize_sql; -use databend_common_ast::parser::Dialect; -use databend_common_ast::parser::ParseMode; -use databend_common_ast::Range; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_script::compile; -use databend_common_script::ir::ColumnAccess; use databend_common_script::Client; use databend_common_script::Executor; +use databend_common_script::compile; +use databend_common_script::ir::ColumnAccess; use goldenfile::Mint; use tokio::runtime::Runtime; @@ -43,7 +43,7 @@ fn run_script(file: &mut dyn Write, src: &str) { let src = unindent::unindent(src); let src = src.trim(); - let res: Result<_> = try { + let res: Result<_, _> = try { let tokens = tokenize_sql(src).unwrap(); let ast = run_parser( &tokens, @@ -51,7 +51,9 @@ fn run_script(file: &mut dyn Write, src: &str) { ParseMode::Template, false, script_stmts, - )?; + ) + .map_err(ErrorCode::from)?; + let ir = compile(&ast)?; let client = mock_client(); let query_log = client.query_log.clone(); diff --git a/src/query/service/src/auth.rs b/src/query/service/src/auth.rs index 6138b5fd87a44..bcb97c53746ae 100644 --- a/src/query/service/src/auth.rs +++ b/src/query/service/src/auth.rs @@ -158,7 +158,9 @@ impl AuthMgr { { Ok(mut user_info) => { if user_info.auth_info != AuthInfo::JWT { - return Err(ErrorCode::AuthenticateFailure("Authentication failed: user exists but is not configured for JWT authentication")); + return Err(ErrorCode::AuthenticateFailure( + "Authentication failed: user exists but is not configured for JWT authentication", + )); } if let Some(ensure_user) = jwt.custom.ensure_user { let current_roles = user_info.grants.roles(); @@ -229,7 +231,7 @@ impl AuthMgr { return Err(ErrorCode::AuthenticateFailure(format!( "Authentication failed: {}", e.message() - ))) + ))); } } let ensure_user = jwt @@ -310,16 +312,22 @@ impl AuthMgr { hash_method: t, .. } => match p { - None => Err(ErrorCode::AuthenticateFailure("Authentication failed: password is required but was not provided")), + None => Err(ErrorCode::AuthenticateFailure( + "Authentication failed: password is required but was not provided", + )), Some(p) => { if *h == t.hash(p) { Ok(()) } else { - Err(ErrorCode::AuthenticateFailure("Authentication failed: incorrect password")) + Err(ErrorCode::AuthenticateFailure( + "Authentication failed: incorrect password", + )) } } }, - _ => Err(ErrorCode::AuthenticateFailure("Authentication failed: user exists but is not configured for password authentication")), + _ => Err(ErrorCode::AuthenticateFailure( + "Authentication failed: user exists but is not configured for password authentication", + )), }; UserApiProvider::instance() .update_user_login_result(tenant, identity, authed.is_ok(), &user) diff --git a/src/query/service/src/builtin/builtin_udfs.rs b/src/query/service/src/builtin/builtin_udfs.rs index 823ac7fbe5b12..ef28a39a594ba 100644 --- a/src/query/service/src/builtin/builtin_udfs.rs +++ b/src/query/service/src/builtin/builtin_udfs.rs @@ -18,17 +18,17 @@ use databend_common_ast::ast::Statement; use databend_common_ast::ast::TypeName; use databend_common_ast::ast::UDFArgs; use databend_common_ast::ast::UDFDefinition; +use databend_common_ast::parser::Dialect; use databend_common_ast::parser::parse_sql; use databend_common_ast::parser::tokenize_sql; -use databend_common_ast::parser::Dialect; use databend_common_config::UDFConfig; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::types::DataType; use databend_common_meta_app::principal::UserDefinedFunction; +use databend_common_sql::NameResolutionContext; use databend_common_sql::normalize_identifier; use databend_common_sql::resolve_type_name_udf; -use databend_common_sql::NameResolutionContext; use log::error; pub struct BuiltinUDFs { diff --git a/src/query/service/src/catalogs/default/database_catalog.rs b/src/query/service/src/catalogs/default/database_catalog.rs index ce53102d85d54..1ec9a8ae13b49 100644 --- a/src/query/service/src/catalogs/default/database_catalog.rs +++ b/src/query/service/src/catalogs/default/database_catalog.rs @@ -28,10 +28,8 @@ use databend_common_config::InnerConfig; use databend_common_exception::ErrorCode; use databend_common_exception::ErrorCodeResultExt; use databend_common_exception::Result; +use databend_common_meta_app::KeyWithTenant; use databend_common_meta_app::principal::UDTFServer; -use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; -use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; -use databend_common_meta_app::schema::least_visible_time_ident::LeastVisibleTimeIdent; use databend_common_meta_app::schema::CatalogInfo; use databend_common_meta_app::schema::CommitTableMetaReply; use databend_common_meta_app::schema::CommitTableMetaReq; @@ -114,8 +112,10 @@ use databend_common_meta_app::schema::UpdateMultiTableMetaReq; use databend_common_meta_app::schema::UpdateMultiTableMetaResult; use databend_common_meta_app::schema::UpsertTableOptionReply; use databend_common_meta_app::schema::UpsertTableOptionReq; +use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; +use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; +use databend_common_meta_app::schema::least_visible_time_ident::LeastVisibleTimeIdent; use databend_common_meta_app::tenant::Tenant; -use databend_common_meta_app::KeyWithTenant; use databend_common_meta_types::MetaId; use databend_common_meta_types::SeqV; use databend_common_users::GrantObjectVisibilityChecker; diff --git a/src/query/service/src/catalogs/default/immutable_catalog.rs b/src/query/service/src/catalogs/default/immutable_catalog.rs index 61d91c5165a5e..5c37d69054f5a 100644 --- a/src/query/service/src/catalogs/default/immutable_catalog.rs +++ b/src/query/service/src/catalogs/default/immutable_catalog.rs @@ -25,8 +25,6 @@ use databend_common_config::InnerConfig; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_meta_app::principal::UDTFServer; -use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; -use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; use databend_common_meta_app::schema::CatalogInfo; use databend_common_meta_app::schema::CommitTableMetaReply; use databend_common_meta_app::schema::CommitTableMetaReq; @@ -100,6 +98,8 @@ use databend_common_meta_app::schema::UpdateIndexReply; use databend_common_meta_app::schema::UpdateIndexReq; use databend_common_meta_app::schema::UpsertTableOptionReply; use databend_common_meta_app::schema::UpsertTableOptionReq; +use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; +use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_types::MetaId; use databend_common_meta_types::SeqV; diff --git a/src/query/service/src/catalogs/default/mutable_catalog.rs b/src/query/service/src/catalogs/default/mutable_catalog.rs index 79b62c0c81e34..e7c316fa62b09 100644 --- a/src/query/service/src/catalogs/default/mutable_catalog.rs +++ b/src/query/service/src/catalogs/default/mutable_catalog.rs @@ -26,8 +26,6 @@ use databend_common_catalog::table_function::TableFunction; use databend_common_config::InnerConfig; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_meta_api::kv_app_error::KVAppError; -use databend_common_meta_api::name_id_value_api::NameIdValueApiCompat; use databend_common_meta_api::AutoIncrementApi; use databend_common_meta_api::DatabaseApi; use databend_common_meta_api::DictionaryApi; @@ -37,13 +35,11 @@ use databend_common_meta_api::LockApi; use databend_common_meta_api::SecurityApi; use databend_common_meta_api::SequenceApi; use databend_common_meta_api::TableApi; +use databend_common_meta_api::kv_app_error::KVAppError; +use databend_common_meta_api::name_id_value_api::NameIdValueApiCompat; +use databend_common_meta_app::KeyWithTenant; use databend_common_meta_app::app_error::AppError; use databend_common_meta_app::principal::UDTFServer; -use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; -use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; -use databend_common_meta_app::schema::index_id_ident::IndexId; -use databend_common_meta_app::schema::index_id_ident::IndexIdIdent; -use databend_common_meta_app::schema::least_visible_time_ident::LeastVisibleTimeIdent; use databend_common_meta_app::schema::CatalogInfo; use databend_common_meta_app::schema::CommitTableMetaReply; use databend_common_meta_app::schema::CommitTableMetaReq; @@ -133,10 +129,14 @@ use databend_common_meta_app::schema::UpdateMultiTableMetaReq; use databend_common_meta_app::schema::UpdateMultiTableMetaResult; use databend_common_meta_app::schema::UpsertTableOptionReply; use databend_common_meta_app::schema::UpsertTableOptionReq; +use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; +use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; +use databend_common_meta_app::schema::index_id_ident::IndexId; +use databend_common_meta_app::schema::index_id_ident::IndexIdIdent; +use databend_common_meta_app::schema::least_visible_time_ident::LeastVisibleTimeIdent; use databend_common_meta_app::storage::S3StorageClass; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_app::tenant_key::errors::UnknownError; -use databend_common_meta_app::KeyWithTenant; use databend_common_meta_store::MetaStoreProvider; use databend_common_meta_types::MetaId; use databend_common_meta_types::SeqV; diff --git a/src/query/service/src/catalogs/default/session_catalog.rs b/src/query/service/src/catalogs/default/session_catalog.rs index 01c487b5a6efa..49a2400b17ec5 100644 --- a/src/query/service/src/catalogs/default/session_catalog.rs +++ b/src/query/service/src/catalogs/default/session_catalog.rs @@ -25,9 +25,6 @@ use databend_common_catalog::table_function::TableFunction; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_meta_app::principal::UDTFServer; -use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; -use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; -use databend_common_meta_app::schema::least_visible_time_ident::LeastVisibleTimeIdent; use databend_common_meta_app::schema::CatalogInfo; use databend_common_meta_app::schema::CommitTableMetaReply; use databend_common_meta_app::schema::CommitTableMetaReq; @@ -110,6 +107,9 @@ use databend_common_meta_app::schema::UpdateMultiTableMetaReq; use databend_common_meta_app::schema::UpdateMultiTableMetaResult; use databend_common_meta_app::schema::UpsertTableOptionReply; use databend_common_meta_app::schema::UpsertTableOptionReq; +use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; +use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; +use databend_common_meta_app::schema::least_visible_time_ident::LeastVisibleTimeIdent; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_types::MetaId; use databend_common_meta_types::SeqV; @@ -121,8 +121,8 @@ use databend_storages_common_session::TxnState; use databend_storages_common_table_meta::table::OPT_KEY_TEMP_PREFIX; use databend_storages_common_table_meta::table_id_ranges::is_temp_table_id; -use crate::catalogs::default::MutableCatalog; use crate::catalogs::Catalog; +use crate::catalogs::default::MutableCatalog; use crate::servers::http::v1::ClientSessionManager; #[derive(Clone, Debug)] diff --git a/src/query/service/src/catalogs/default/table_memory_meta.rs b/src/query/service/src/catalogs/default/table_memory_meta.rs index 0bb61a8fd985c..219f927f3afbe 100644 --- a/src/query/service/src/catalogs/default/table_memory_meta.rs +++ b/src/query/service/src/catalogs/default/table_memory_meta.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::Arc; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; -use std::sync::Arc; use dashmap::DashMap; use databend_common_exception::ErrorCode; diff --git a/src/query/service/src/catalogs/iceberg/iceberg_catalog.rs b/src/query/service/src/catalogs/iceberg/iceberg_catalog.rs index 0625b47a4f4ec..ba0d0129c8324 100644 --- a/src/query/service/src/catalogs/iceberg/iceberg_catalog.rs +++ b/src/query/service/src/catalogs/iceberg/iceberg_catalog.rs @@ -28,9 +28,8 @@ use databend_common_catalog::table_function::TableFunction; use databend_common_exception::ErrorCode; use databend_common_exception::ErrorCodeResultExt; use databend_common_exception::Result; +use databend_common_meta_app::KeyWithTenant; use databend_common_meta_app::principal::UDTFServer; -use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; -use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; use databend_common_meta_app::schema::CatalogInfo; use databend_common_meta_app::schema::CreateDatabaseReply; use databend_common_meta_app::schema::CreateDatabaseReq; @@ -98,8 +97,9 @@ use databend_common_meta_app::schema::UpdateIndexReply; use databend_common_meta_app::schema::UpdateIndexReq; use databend_common_meta_app::schema::UpsertTableOptionReply; use databend_common_meta_app::schema::UpsertTableOptionReq; +use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; +use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; use databend_common_meta_app::tenant::Tenant; -use databend_common_meta_app::KeyWithTenant; use databend_common_meta_types::MetaId; use databend_common_meta_types::SeqV; use databend_common_storages_iceberg::IcebergMutableCatalog; diff --git a/src/query/service/src/catalogs/mod.rs b/src/query/service/src/catalogs/mod.rs index 7ae8deedfec6d..344653e77f825 100644 --- a/src/query/service/src/catalogs/mod.rs +++ b/src/query/service/src/catalogs/mod.rs @@ -17,8 +17,8 @@ mod iceberg; pub use databend_common_catalog::catalog::Catalog; pub use databend_common_storages_hive as hive; -pub use default::table_memory_meta::InMemoryMetas; pub use default::DatabaseCatalog; +pub use default::table_memory_meta::InMemoryMetas; pub use iceberg::IcebergCreator; /// Merges two iterators of Option, preferring left (primary) values over right (fallback) values. diff --git a/src/query/service/src/clusters/cluster.rs b/src/query/service/src/clusters/cluster.rs index fcd6880b49e34..f517de54b0a9d 100644 --- a/src/query/service/src/clusters/cluster.rs +++ b/src/query/service/src/clusters/cluster.rs @@ -12,28 +12,28 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::hash_map::Entry; use std::collections::HashMap; +use std::collections::hash_map::Entry; use std::net::SocketAddr; use std::ops::RangeInclusive; use std::str::FromStr; +use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; -use std::sync::Arc; use std::time::Duration; use std::time::Instant; use arrow_flight::flight_service_client::FlightServiceClient; -use databend_common_base::base::tokio::sync::Mutex; -use databend_common_base::base::tokio::sync::Notify; -use databend_common_base::base::tokio::task::JoinHandle; -use databend_common_base::base::tokio::time::sleep as tokio_async_sleep; use databend_common_base::base::BuildInfoRef; use databend_common_base::base::DummySignalStream; use databend_common_base::base::GlobalInstance; use databend_common_base::base::GlobalUniqName; use databend_common_base::base::SignalStream; use databend_common_base::base::SignalType; +use databend_common_base::base::tokio::sync::Mutex; +use databend_common_base::base::tokio::sync::Notify; +use databend_common_base::base::tokio::task::JoinHandle; +use databend_common_base::base::tokio::time::sleep as tokio_async_sleep; use databend_common_cache::Cache; use databend_common_cache::LruCache; use databend_common_cache::MemSized; @@ -60,15 +60,15 @@ use databend_common_version::DATABEND_TELEMETRY_API_KEY; use databend_common_version::DATABEND_TELEMETRY_ENDPOINT; use databend_common_version::DATABEND_TELEMETRY_SOURCE; use databend_enterprise_resources_management::ResourcesManagement; -use futures::future::select; -use futures::future::Either; use futures::Future; use futures::StreamExt; +use futures::future::Either; +use futures::future::select; use log::error; use log::info; use log::warn; -use rand::thread_rng; use rand::Rng; +use rand::thread_rng; use serde::Deserialize; use serde::Serialize; use tokio::time::sleep; @@ -694,7 +694,8 @@ impl ClusterDiscovery { let total_cpu_nums = nodes.iter().map(|x| x.cpu_nums).sum::(); - info!("[Cluster] current resource status - online nodes: [{}], available CPU cores: [{}] for tenant [{}].", + info!( + "[Cluster] current resource status - online nodes: [{}], available CPU cores: [{}] for tenant [{}].", nodes.len(), total_cpu_nums, self.tenant_id diff --git a/src/query/service/src/databases/database_factory.rs b/src/query/service/src/databases/database_factory.rs index ee4e57ad1a2b0..ef8ec14a7d91a 100644 --- a/src/query/service/src/databases/database_factory.rs +++ b/src/query/service/src/databases/database_factory.rs @@ -19,9 +19,9 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_meta_app::schema::DatabaseInfo; -use crate::databases::default::DefaultDatabase; use crate::databases::Database; use crate::databases::DatabaseContext; +use crate::databases::default::DefaultDatabase; pub struct DatabaseFactory {} diff --git a/src/query/service/src/databases/information_schema/information_schema_database.rs b/src/query/service/src/databases/information_schema/information_schema_database.rs index a5cbe7dcfcdb7..7dee5a3897473 100644 --- a/src/query/service/src/databases/information_schema/information_schema_database.rs +++ b/src/query/service/src/databases/information_schema/information_schema_database.rs @@ -14,10 +14,10 @@ use std::sync::Arc; -use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_app::schema::DatabaseId; use databend_common_meta_app::schema::DatabaseInfo; use databend_common_meta_app::schema::DatabaseMeta; +use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_types::SeqV; use databend_common_storages_information_schema::CharacterSetsTable; diff --git a/src/query/service/src/databases/system/system_database.rs b/src/query/service/src/databases/system/system_database.rs index 76dc0eeb5bcee..0eeac29acf42f 100644 --- a/src/query/service/src/databases/system/system_database.rs +++ b/src/query/service/src/databases/system/system_database.rs @@ -16,10 +16,10 @@ use std::collections::HashMap; use std::sync::Arc; use databend_common_config::InnerConfig; -use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_app::schema::DatabaseId; use databend_common_meta_app::schema::DatabaseInfo; use databend_common_meta_app::schema::DatabaseMeta; +use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_types::SeqV; use databend_common_storages_system::BacktraceTable; diff --git a/src/query/service/src/global_services.rs b/src/query/service/src/global_services.rs index b5ed3b8a0fcbc..6447ff00a19ff 100644 --- a/src/query/service/src/global_services.rs +++ b/src/query/service/src/global_services.rs @@ -17,9 +17,9 @@ use std::sync::Arc; use databend_common_base::base::BuildInfoRef; use databend_common_base::base::GlobalInstance; +use databend_common_base::runtime::GLOBAL_QUERIES_MANAGER; use databend_common_base::runtime::GlobalIORuntime; use databend_common_base::runtime::GlobalQueryRuntime; -use databend_common_base::runtime::GLOBAL_QUERIES_MANAGER; use databend_common_catalog::catalog::CatalogCreator; use databend_common_catalog::catalog::CatalogManager; use databend_common_cloud_control::cloud_api::CloudControlApiProvider; @@ -36,9 +36,9 @@ use databend_common_storage::DataOperator; use databend_common_storage::ShareTableConfig; use databend_common_storages_hive::HiveCreator; use databend_common_tracing::GlobalLogger; -use databend_common_users::builtin::BuiltIn; use databend_common_users::RoleCacheManager; use databend_common_users::UserApiProvider; +use databend_common_users::builtin::BuiltIn; use databend_common_version::BUILD_INFO; use databend_enterprise_resources_management::DummyResourcesManagement; use databend_storages_common_cache::CacheManager; diff --git a/src/query/service/src/history_tables/alter_table.rs b/src/query/service/src/history_tables/alter_table.rs index 2e1193b598128..146712a9678be 100644 --- a/src/query/service/src/history_tables/alter_table.rs +++ b/src/query/service/src/history_tables/alter_table.rs @@ -23,9 +23,9 @@ use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::TableSchemaRef; +use databend_common_sql::Planner; use databend_common_sql::binder::parse_uri_location; use databend_common_sql::plans::Plan; -use databend_common_sql::Planner; use log::info; use crate::history_tables::external::ExternalStorageConnection; @@ -160,7 +160,7 @@ pub async fn should_reset( current_storage_params, new_storage_params ); return Err(ErrorCode::InvalidConfig( - "Cannot change storage parameters of external history table, please drop the tables and stage first." + "Cannot change storage parameters of external history table, please drop the tables and stage first.", )); } } diff --git a/src/query/service/src/history_tables/external.rs b/src/query/service/src/history_tables/external.rs index bd349547cea56..8e1014a98b8c6 100644 --- a/src/query/service/src/history_tables/external.rs +++ b/src/query/service/src/history_tables/external.rs @@ -15,8 +15,8 @@ use std::collections::BTreeMap; use databend_common_meta_app::storage::StorageParams; +use databend_common_storage::Scheme; use opendal::raw::normalize_root; -use opendal::Scheme; #[derive(Debug)] pub struct ExternalStorageConnection { diff --git a/src/query/service/src/history_tables/global_history_log.rs b/src/query/service/src/history_tables/global_history_log.rs index 1dcd5b4d2de76..e1761db2de5be 100644 --- a/src/query/service/src/history_tables/global_history_log.rs +++ b/src/query/service/src/history_tables/global_history_log.rs @@ -12,18 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; -use std::sync::Arc; use std::time::Duration; use databend_common_base::base::GlobalInstance; -use databend_common_base::runtime::spawn; use databend_common_base::runtime::CaptureLogSettings; use databend_common_base::runtime::MemStat; use databend_common_base::runtime::Runtime; use databend_common_base::runtime::ThreadTracker; use databend_common_base::runtime::TrySpawn; +use databend_common_base::runtime::spawn; use databend_common_catalog::table_context::TableContext; use databend_common_config::GlobalConfig; use databend_common_config::InnerConfig; @@ -35,14 +35,14 @@ use databend_common_license::license_manager::LicenseManagerSwitch; use databend_common_meta_app::storage::StorageParams; use databend_common_meta_client::MetaGrpcClient; use databend_common_sql::Planner; -use databend_common_storage::init_operator; use databend_common_storage::DataOperator; -use databend_common_tracing::get_all_history_table_names; -use databend_common_tracing::init_history_tables; +use databend_common_storage::init_operator; use databend_common_tracing::GlobalLogger; use databend_common_tracing::HistoryTable; -use futures_util::future::join_all; +use databend_common_tracing::get_all_history_table_names; +use databend_common_tracing::init_history_tables; use futures_util::TryStreamExt; +use futures_util::future::join_all; use log::debug; use log::error; use log::info; @@ -50,18 +50,18 @@ use log::warn; use opendal::raw::normalize_root; use parking_lot::Mutex; use rand::random; -use tokio::time::sleep; use tokio::time::Instant; +use tokio::time::sleep; use uuid::Uuid; use crate::clusters::ClusterDiscovery; use crate::history_tables::alter_table::get_alter_table_sql; use crate::history_tables::alter_table::get_log_table; use crate::history_tables::alter_table::should_reset; -use crate::history_tables::error_handling::is_temp_error; use crate::history_tables::error_handling::ErrorCounters; -use crate::history_tables::external::get_external_storage_connection; +use crate::history_tables::error_handling::is_temp_error; use crate::history_tables::external::ExternalStorageConnection; +use crate::history_tables::external::get_external_storage_connection; use crate::history_tables::meta::HistoryMetaHandle; use crate::history_tables::session::create_session; use crate::interpreters::InterpreterFactory; @@ -361,11 +361,12 @@ impl GlobalHistoryLog { let vacuum = format!("VACUUM TABLE system_history.{}", table.name); self.execute_sql(&vacuum).await?; } - info!("periodic retention operation on history log table '{}' completed successfully (delete {} secs, vacuum {} secs)", - table.name, - delete_elapsed, - start.elapsed().as_secs() - delete_elapsed - ); + info!( + "periodic retention operation on history log table '{}' completed successfully (delete {} secs, vacuum {} secs)", + table.name, + delete_elapsed, + start.elapsed().as_secs() - delete_elapsed + ); return Ok(true); } Ok(false) diff --git a/src/query/service/src/history_tables/meta.rs b/src/query/service/src/history_tables/meta.rs index 2f2a2ccf75d58..f110a1e33f7ea 100644 --- a/src/query/service/src/history_tables/meta.rs +++ b/src/query/service/src/history_tables/meta.rs @@ -14,20 +14,20 @@ use std::future::Future; use std::pin::pin; -use std::sync::atomic::AtomicBool; use std::sync::Arc; +use std::sync::atomic::AtomicBool; use std::time::Duration; -use databend_common_base::runtime::spawn_named; use databend_common_base::runtime::CaptureLogSettings; use databend_common_base::runtime::ThreadTracker; +use databend_common_base::runtime::spawn_named; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_meta_client::ClientHandle; use databend_common_meta_kvapi::kvapi::KVApi; use databend_common_meta_kvapi::kvapi::KvApiExt; -use databend_common_meta_semaphore::acquirer::Permit; use databend_common_meta_semaphore::Semaphore; +use databend_common_meta_semaphore::acquirer::Permit; use databend_common_meta_types::MatchSeq; use databend_common_meta_types::Operation; use databend_common_meta_types::TxnCondition; @@ -338,8 +338,8 @@ impl HistoryMetaHandle { #[cfg(test)] mod tests { use std::ops::Deref; - use std::sync::atomic::Ordering; use std::sync::Arc; + use std::sync::atomic::Ordering; use std::time::Duration; use databend_common_base::runtime::spawn; diff --git a/src/query/service/src/interpreters/access/accessor.rs b/src/query/service/src/interpreters/access/accessor.rs index 9085abae00b5c..542f19a896af7 100644 --- a/src/query/service/src/interpreters/access/accessor.rs +++ b/src/query/service/src/interpreters/access/accessor.rs @@ -17,8 +17,8 @@ use std::sync::Arc; use databend_common_exception::Result; -use crate::interpreters::access::PrivilegeAccess; use crate::interpreters::ManagementModeAccess; +use crate::interpreters::access::PrivilegeAccess; use crate::sessions::QueryContext; use crate::sql::plans::Plan; diff --git a/src/query/service/src/interpreters/access/management_mode_access.rs b/src/query/service/src/interpreters/access/management_mode_access.rs index 45f5c77275195..9e9da6e6bafed 100644 --- a/src/query/service/src/interpreters/access/management_mode_access.rs +++ b/src/query/service/src/interpreters/access/management_mode_access.rs @@ -43,7 +43,7 @@ impl AccessChecker for ManagementModeAccess { Plan::Query {rewrite_kind, .. } => { use databend_common_sql::plans::RewriteKind; match rewrite_kind { - Some(ref v) => matches!(v, + Some( v) => matches!(v, RewriteKind::ShowDatabases | RewriteKind::ShowDropDatabases | RewriteKind::ShowTables(_, _) diff --git a/src/query/service/src/interpreters/access/privilege_access.rs b/src/query/service/src/interpreters/access/privilege_access.rs index 3c354b9e4d188..4a1a6953d71a0 100644 --- a/src/query/service/src/interpreters/access/privilege_access.rs +++ b/src/query/service/src/interpreters/access/privilege_access.rs @@ -16,8 +16,8 @@ use std::collections::HashSet; use std::sync::Arc; use databend_common_base::base::GlobalInstance; -use databend_common_catalog::catalog::Catalog; use databend_common_catalog::catalog::CATALOG_DEFAULT; +use databend_common_catalog::catalog::Catalog; use databend_common_catalog::plan::DataSourceInfo; use databend_common_catalog::table_context::TableContext; use databend_common_config::GlobalConfig; @@ -33,16 +33,17 @@ use databend_common_meta_app::principal::GrantObject; use databend_common_meta_app::principal::OwnershipInfo; use databend_common_meta_app::principal::OwnershipObject; use databend_common_meta_app::principal::ProcedureNameIdent; +use databend_common_meta_app::principal::SENSITIVE_SYSTEM_RESOURCE; +use databend_common_meta_app::principal::SYSTEM_TABLES_ALLOW_LIST; use databend_common_meta_app::principal::StageInfo; use databend_common_meta_app::principal::StageType; use databend_common_meta_app::principal::UserGrantSet; use databend_common_meta_app::principal::UserPrivilegeSet; use databend_common_meta_app::principal::UserPrivilegeType; -use databend_common_meta_app::principal::SENSITIVE_SYSTEM_RESOURCE; -use databend_common_meta_app::principal::SYSTEM_TABLES_ALLOW_LIST; use databend_common_meta_app::row_access_policy::RowAccessPolicyNameIdent; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_types::SeqV; +use databend_common_sql::Planner; use databend_common_sql::binder::MutationType; use databend_common_sql::plans::InsertInputSource; use databend_common_sql::plans::ModifyColumnAction; @@ -50,10 +51,9 @@ use databend_common_sql::plans::Mutation; use databend_common_sql::plans::OptimizeCompactBlock; use databend_common_sql::plans::PresignAction; use databend_common_sql::plans::RewriteKind; -use databend_common_sql::Planner; +use databend_common_users::BUILTIN_ROLE_ACCOUNT_ADMIN; use databend_common_users::RoleCacheManager; use databend_common_users::UserApiProvider; -use databend_common_users::BUILTIN_ROLE_ACCOUNT_ADMIN; use databend_enterprise_resources_management::ResourcesManagement; use databend_storages_common_table_meta::table::OPT_KEY_TEMP_PREFIX; @@ -210,12 +210,10 @@ impl PrivilegeAccess { UserPrivilegeType::Select | UserPrivilegeType::Drop ) { - return Err(ErrorCode::PermissionDenied( - format!( - "Permission Denied: Operation '{:?}' on database 'default.system_history' is not allowed. This sensitive system resource only supports 'SELECT' and 'DROP'", - privilege - ), - )); + return Err(ErrorCode::PermissionDenied(format!( + "Permission Denied: Operation '{:?}' on database 'default.system_history' is not allowed. This sensitive system resource only supports 'SELECT' and 'DROP'", + privilege + ))); } } (None, None, Some(stage_name)) => { @@ -531,8 +529,7 @@ impl PrivilegeAccess { } else { Some(Err(ErrorCode::PermissionDenied(format!( "Permission denied: Ownership is required on WAREHOUSE '{}' for user {}", - warehouse, - current_user + warehouse, current_user )))) } } diff --git a/src/query/service/src/interpreters/access_log/access_logger.rs b/src/query/service/src/interpreters/access_log/access_logger.rs index fb50f5272765b..f72ed153fc3d1 100644 --- a/src/query/service/src/interpreters/access_log/access_logger.rs +++ b/src/query/service/src/interpreters/access_log/access_logger.rs @@ -17,13 +17,13 @@ use std::sync::Arc; use databend_common_catalog::plan::DataSourceInfo; use databend_common_catalog::table_context::TableContext; +use databend_common_sql::InsertInputSource; +use databend_common_sql::MetadataRef; use databend_common_sql::plans::CopyIntoLocationPlan; use databend_common_sql::plans::CopyIntoTablePlan; use databend_common_sql::plans::Insert; use databend_common_sql::plans::InsertMultiTable; use databend_common_sql::plans::Plan; -use databend_common_sql::InsertInputSource; -use databend_common_sql::MetadataRef; use log::info; use crate::interpreters::access_log::log_entry::AccessLogEntry; @@ -32,8 +32,8 @@ use crate::interpreters::access_log::log_entry::AccessObjectColumn; use crate::interpreters::access_log::log_entry::DDLOperationType; use crate::interpreters::access_log::log_entry::ModifyByDDLObject; use crate::interpreters::access_log::log_entry::ObjectDomain; -use crate::sessions::convert_query_log_timestamp; use crate::sessions::QueryContext; +use crate::sessions::convert_query_log_timestamp; pub struct AccessLogger { entry: AccessLogEntry, diff --git a/src/query/service/src/interpreters/common/grant.rs b/src/query/service/src/interpreters/common/grant.rs index f9747673a438e..47d47b7ccbbb7 100644 --- a/src/query/service/src/interpreters/common/grant.rs +++ b/src/query/service/src/interpreters/common/grant.rs @@ -125,7 +125,7 @@ pub async fn validate_grant_object_exists( return match ctx.get_connection(c).await { Ok(_c) => Ok(()), Err(e) => Err(e), - } + }; } GrantObject::Sequence(c) => { let catalog = ctx.get_default_catalog()?; diff --git a/src/query/service/src/interpreters/common/metrics.rs b/src/query/service/src/interpreters/common/metrics.rs index 203ececaeaa89..5e2249cfeba9f 100644 --- a/src/query/service/src/interpreters/common/metrics.rs +++ b/src/query/service/src/interpreters/common/metrics.rs @@ -14,8 +14,8 @@ use std::sync::LazyLock; -use databend_common_base::runtime::metrics::register_histogram_family_in_milliseconds; use databend_common_base::runtime::metrics::FamilyHistogram; +use databend_common_base::runtime::metrics::register_histogram_family_in_milliseconds; static COMPACT_HOOK_EXECUTION_MS: LazyLock>> = LazyLock::new(|| register_histogram_family_in_milliseconds("compact_hook_execution_ms")); diff --git a/src/query/service/src/interpreters/common/notification.rs b/src/query/service/src/interpreters/common/notification.rs index 663facb151414..fec3d663b9f95 100644 --- a/src/query/service/src/interpreters/common/notification.rs +++ b/src/query/service/src/interpreters/common/notification.rs @@ -16,8 +16,8 @@ use std::sync::Arc; use std::time::Duration; use databend_common_catalog::table_context::TableContext; -use databend_common_cloud_control::client_config::build_client_config; use databend_common_cloud_control::client_config::ClientConfig; +use databend_common_cloud_control::client_config::build_client_config; use databend_common_exception::Result; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/common/query_log.rs b/src/query/service/src/interpreters/common/query_log.rs index 696568026a10e..3ad022203d39e 100644 --- a/src/query/service/src/interpreters/common/query_log.rs +++ b/src/query/service/src/interpreters/common/query_log.rs @@ -26,9 +26,9 @@ use log::error; use log::info; use serde_json; -use crate::sessions::convert_query_log_timestamp; use crate::sessions::QueryContext; use crate::sessions::TableContext; +use crate::sessions::convert_query_log_timestamp; pub struct InterpreterQueryLog; diff --git a/src/query/service/src/interpreters/common/table.rs b/src/query/service/src/interpreters/common/table.rs index f8394342fe736..ba3ec2bed8c93 100644 --- a/src/query/service/src/interpreters/common/table.rs +++ b/src/query/service/src/interpreters/common/table.rs @@ -29,8 +29,8 @@ pub fn check_referenced_computed_columns( for f in schema.fields() { if let Some(computed_expr) = f.computed_expr() { let expr = match computed_expr { - ComputedExpr::Stored(ref expr) => expr, - ComputedExpr::Virtual(ref expr) => expr, + ComputedExpr::Stored(expr) => expr, + ComputedExpr::Virtual(expr) => expr, }; match parse_computed_expr(ctx.clone(), schema.clone(), expr) { Ok(expr) => { diff --git a/src/query/service/src/interpreters/common/task.rs b/src/query/service/src/interpreters/common/task.rs index 522c85938cf5e..cadb9b8b83d65 100644 --- a/src/query/service/src/interpreters/common/task.rs +++ b/src/query/service/src/interpreters/common/task.rs @@ -17,8 +17,8 @@ use std::time::Duration; use databend_common_ast::ast::ScheduleOptions; use databend_common_catalog::table_context::TableContext; -use databend_common_cloud_control::client_config::build_client_config; use databend_common_cloud_control::client_config::ClientConfig; +use databend_common_cloud_control::client_config::build_client_config; use databend_common_cloud_control::pb::schedule_options::ScheduleType; use databend_common_exception::Result; diff --git a/src/query/service/src/interpreters/hook/compact_hook.rs b/src/query/service/src/interpreters/hook/compact_hook.rs index 6521e16771f90..96503f1982908 100644 --- a/src/query/service/src/interpreters/hook/compact_hook.rs +++ b/src/query/service/src/interpreters/hook/compact_hook.rs @@ -23,9 +23,9 @@ use databend_common_catalog::lock::LockTableOption; use databend_common_catalog::table::CompactionLimits; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_pipeline::core::always_callback; use databend_common_pipeline::core::ExecutionInfo; use databend_common_pipeline::core::Pipeline; +use databend_common_pipeline::core::always_callback; use databend_common_sql::optimizer::ir::SExpr; use databend_common_sql::plans::OptimizeCompactBlock; use databend_common_sql::plans::ReclusterPlan; @@ -33,14 +33,14 @@ use databend_common_sql::plans::RelOperator; use databend_storages_common_table_meta::table::ClusterType; use log::info; +use crate::interpreters::Interpreter; +use crate::interpreters::OptimizeCompactBlockInterpreter; +use crate::interpreters::ReclusterTableInterpreter; use crate::interpreters::common::metrics_inc_compact_hook_compact_time_ms; use crate::interpreters::common::metrics_inc_compact_hook_main_operation_time_ms; use crate::interpreters::hook::vacuum_hook::hook_clear_m_cte_temp_table; use crate::interpreters::hook::vacuum_hook::hook_disk_temp_dir; use crate::interpreters::hook::vacuum_hook::hook_vacuum_temp_files; -use crate::interpreters::Interpreter; -use crate::interpreters::OptimizeCompactBlockInterpreter; -use crate::interpreters::ReclusterTableInterpreter; use crate::pipelines::executor::ExecutorSettings; use crate::pipelines::executor::PipelineCompleteExecutor; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/hook/hook.rs b/src/query/service/src/interpreters/hook/hook.rs index d670ef1c6c75a..080b389cd4cb6 100644 --- a/src/query/service/src/interpreters/hook/hook.rs +++ b/src/query/service/src/interpreters/hook/hook.rs @@ -25,13 +25,13 @@ use databend_common_sql::executor::physical_plans::MutationKind; use log::info; use log::warn; -use crate::interpreters::hook::analyze_hook::hook_analyze; use crate::interpreters::hook::analyze_hook::AnalyzeDesc; -use crate::interpreters::hook::compact_hook::hook_compact; +use crate::interpreters::hook::analyze_hook::hook_analyze; use crate::interpreters::hook::compact_hook::CompactHookTraceCtx; use crate::interpreters::hook::compact_hook::CompactTargetTableDescription; -use crate::interpreters::hook::refresh_hook::hook_refresh; +use crate::interpreters::hook::compact_hook::hook_compact; use crate::interpreters::hook::refresh_hook::RefreshDesc; +use crate::interpreters::hook::refresh_hook::hook_refresh; use crate::sessions::QueryContext; /// Hook operator. diff --git a/src/query/service/src/interpreters/hook/refresh_hook.rs b/src/query/service/src/interpreters/hook/refresh_hook.rs index 3fe2eecb39176..fc037ba29bc4d 100644 --- a/src/query/service/src/interpreters/hook/refresh_hook.rs +++ b/src/query/service/src/interpreters/hook/refresh_hook.rs @@ -27,26 +27,26 @@ use databend_common_meta_app::schema::IndexMeta; use databend_common_meta_app::schema::ListIndexesByIdReq; use databend_common_meta_app::schema::TableIndexType; use databend_common_meta_types::MetaId; -use databend_common_pipeline::core::always_callback; use databend_common_pipeline::core::ExecutionInfo; use databend_common_pipeline::core::Pipeline; -use databend_common_sql::plans::Plan; -use databend_common_sql::plans::RefreshIndexPlan; -use databend_common_sql::plans::RefreshTableIndexPlan; +use databend_common_pipeline::core::always_callback; use databend_common_sql::BindContext; use databend_common_sql::Binder; use databend_common_sql::Metadata; use databend_common_sql::NameResolutionContext; +use databend_common_sql::plans::Plan; +use databend_common_sql::plans::RefreshIndexPlan; +use databend_common_sql::plans::RefreshTableIndexPlan; use databend_storages_common_table_meta::meta::Location; use log::info; use parking_lot::RwLock; -use crate::interpreters::hook::vacuum_hook::hook_clear_m_cte_temp_table; -use crate::interpreters::hook::vacuum_hook::hook_disk_temp_dir; -use crate::interpreters::hook::vacuum_hook::hook_vacuum_temp_files; use crate::interpreters::Interpreter; use crate::interpreters::RefreshIndexInterpreter; use crate::interpreters::RefreshTableIndexInterpreter; +use crate::interpreters::hook::vacuum_hook::hook_clear_m_cte_temp_table; +use crate::interpreters::hook::vacuum_hook::hook_disk_temp_dir; +use crate::interpreters::hook::vacuum_hook::hook_vacuum_temp_files; use crate::pipelines::executor::ExecutorSettings; use crate::pipelines::executor::PipelineCompleteExecutor; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/interpreter.rs b/src/query/service/src/interpreters/interpreter.rs index 94c7601529839..69ca0f65dd041 100644 --- a/src/query/service/src/interpreters/interpreter.rs +++ b/src/query/service/src/interpreters/interpreter.rs @@ -27,9 +27,9 @@ use databend_common_ast::ast::OptimizeTableAction; use databend_common_ast::ast::OptimizeTableStmt; use databend_common_ast::ast::Statement; use databend_common_base::base::short_sql; -use databend_common_base::runtime::profile::get_statistics_desc; use databend_common_base::runtime::profile::ProfileDesc; use databend_common_base::runtime::profile::ProfileStatisticsName; +use databend_common_base::runtime::profile::get_statistics_desc; use databend_common_catalog::query_kind::QueryKind; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; @@ -38,13 +38,13 @@ use databend_common_exception::ResultExt; use databend_common_expression::DataSchemaRef; use databend_common_expression::SendableDataBlockStream; use databend_common_license::license_manager::LicenseManagerSwitch; -use databend_common_pipeline::core::always_callback; use databend_common_pipeline::core::ExecutionInfo; use databend_common_pipeline::core::PlanProfile; use databend_common_pipeline::core::SourcePipeBuilder; -use databend_common_sql::plans::Plan; +use databend_common_pipeline::core::always_callback; use databend_common_sql::PlanExtras; use databend_common_sql::Planner; +use databend_common_sql::plans::Plan; use databend_storages_common_cache::CacheManager; use derive_visitor::DriveMut; use derive_visitor::VisitorMut; @@ -53,16 +53,16 @@ use log::info; use md5::Digest; use md5::Md5; +use super::InterpreterMetrics; +use super::InterpreterQueryLog; use super::hook::vacuum_hook::hook_clear_m_cte_temp_table; use super::hook::vacuum_hook::hook_disk_temp_dir; use super::hook::vacuum_hook::hook_vacuum_temp_files; -use super::InterpreterMetrics; -use super::InterpreterQueryLog; use crate::interpreters::interpreter_txn_commit::execute_commit_statement; +use crate::pipelines::PipelineBuildResult; use crate::pipelines::executor::ExecutorSettings; use crate::pipelines::executor::PipelineCompleteExecutor; use crate::pipelines::executor::PipelinePullingExecutor; -use crate::pipelines::PipelineBuildResult; use crate::schedulers::ServiceQueryExecutor; use crate::sessions::AcquireQueueGuard; use crate::sessions::QueriesQueueManager; @@ -123,9 +123,10 @@ pub trait Interpreter: Sync + Send { match LicenseManagerSwitch::instance().check_license(license_key.clone()) { Ok(_) => true, Err(e) if !license_key.is_empty() => { - let msg = - format!("CRITICAL ALERT: License validation FAILED - enterprise features DISABLED, System may operate in DEGRADED MODE with LIMITED CAPABILITIES and REDUCED PERFORMANCE. Please contact us at https://www.databend.com/contact-us/ or email hi@databend.com to restore full functionality: {}", - e); + let msg = format!( + "CRITICAL ALERT: License validation FAILED - enterprise features DISABLED, System may operate in DEGRADED MODE with LIMITED CAPABILITIES and REDUCED PERFORMANCE. Please contact us at https://www.databend.com/contact-us/ or email hi@databend.com to restore full functionality: {}", + e + ); log::error!("{msg}"); // Also log at warning level to ensure the message could be propagated to client applications diff --git a/src/query/service/src/interpreters/interpreter_add_warehouse_cluster.rs b/src/query/service/src/interpreters/interpreter_add_warehouse_cluster.rs index 47586f4b96d0d..438678a517655 100644 --- a/src/query/service/src/interpreters/interpreter_add_warehouse_cluster.rs +++ b/src/query/service/src/interpreters/interpreter_add_warehouse_cluster.rs @@ -24,8 +24,8 @@ use databend_common_management::SelectedNode; use databend_common_sql::plans::AddWarehouseClusterPlan; use databend_enterprise_resources_management::ResourcesManagement; -use crate::interpreters::util::AuditElement; use crate::interpreters::Interpreter; +use crate::interpreters::util::AuditElement; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/interpreter_assign_warehouse_nodes.rs b/src/query/service/src/interpreters/interpreter_assign_warehouse_nodes.rs index 5b3a8621989f2..8b61d8a2b96c5 100644 --- a/src/query/service/src/interpreters/interpreter_assign_warehouse_nodes.rs +++ b/src/query/service/src/interpreters/interpreter_assign_warehouse_nodes.rs @@ -24,8 +24,8 @@ use databend_common_management::SelectedNode; use databend_common_sql::plans::AssignWarehouseNodesPlan; use databend_enterprise_resources_management::ResourcesManagement; -use crate::interpreters::util::AuditElement; use crate::interpreters::Interpreter; +use crate::interpreters::util::AuditElement; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/interpreter_catalog_show_create.rs b/src/query/service/src/interpreters/interpreter_catalog_show_create.rs index 6a2f369227803..3cc164c00f2c4 100644 --- a/src/query/service/src/interpreters/interpreter_catalog_show_create.rs +++ b/src/query/service/src/interpreters/interpreter_catalog_show_create.rs @@ -15,10 +15,10 @@ use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::BlockEntry; use databend_common_expression::DataBlock; use databend_common_expression::Scalar; +use databend_common_expression::types::DataType; use databend_common_meta_app::schema::CatalogOption; use databend_common_meta_app::schema::IcebergCatalogOption; use databend_common_meta_app::storage::StorageParams; diff --git a/src/query/service/src/interpreters/interpreter_connection_desc.rs b/src/query/service/src/interpreters/interpreter_connection_desc.rs index 16c21ebc19c67..b0a157f074b2c 100644 --- a/src/query/service/src/interpreters/interpreter_connection_desc.rs +++ b/src/query/service/src/interpreters/interpreter_connection_desc.rs @@ -16,9 +16,9 @@ use std::sync::Arc; use databend_common_ast::ast::Connection; use databend_common_exception::Result; -use databend_common_expression::types::StringType; use databend_common_expression::DataBlock; use databend_common_expression::FromData; +use databend_common_expression::types::StringType; use databend_common_sql::plans::DescConnectionPlan; use databend_common_users::UserApiProvider; diff --git a/src/query/service/src/interpreters/interpreter_connection_show.rs b/src/query/service/src/interpreters/interpreter_connection_show.rs index f2c6f4d3597d9..96506cf054626 100644 --- a/src/query/service/src/interpreters/interpreter_connection_show.rs +++ b/src/query/service/src/interpreters/interpreter_connection_show.rs @@ -16,9 +16,9 @@ use std::sync::Arc; use databend_common_ast::ast::Connection; use databend_common_exception::Result; -use databend_common_expression::types::StringType; use databend_common_expression::DataBlock; use databend_common_expression::FromData; +use databend_common_expression::types::StringType; use databend_common_users::Object; use databend_common_users::UserApiProvider; use log::debug; diff --git a/src/query/service/src/interpreters/interpreter_copy_into_location.rs b/src/query/service/src/interpreters/interpreter_copy_into_location.rs index 38ff44bb28d85..d8a6c24a5bb5d 100644 --- a/src/query/service/src/interpreters/interpreter_copy_into_location.rs +++ b/src/query/service/src/interpreters/interpreter_copy_into_location.rs @@ -23,10 +23,10 @@ use databend_storages_common_stage::CopyIntoLocationInfo; use log::debug; use log::info; -use crate::interpreters::common::check_deduplicate_label; -use crate::interpreters::common::dml_build_update_stream_req; use crate::interpreters::Interpreter; use crate::interpreters::SelectInterpreter; +use crate::interpreters::common::check_deduplicate_label; +use crate::interpreters::common::dml_build_update_stream_req; use crate::physical_plans::CopyIntoLocation; use crate::physical_plans::PhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; diff --git a/src/query/service/src/interpreters/interpreter_copy_into_table.rs b/src/query/service/src/interpreters/interpreter_copy_into_table.rs index 1611041abcc32..a9200713d2dbb 100644 --- a/src/query/service/src/interpreters/interpreter_copy_into_table.rs +++ b/src/query/service/src/interpreters/interpreter_copy_into_table.rs @@ -17,16 +17,13 @@ use std::collections::HashMap; use std::sync::Arc; use arrow_schema::Field; -use databend_common_ast::ast::ColumnMatchMode; use databend_common_ast::Span; +use databend_common_ast::ast::ColumnMatchMode; use databend_common_catalog::lock::LockTableOption; use databend_common_catalog::plan::StageTableInfo; use databend_common_catalog::table::TableExt; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::DataType; -use databend_common_expression::types::Int32Type; -use databend_common_expression::types::StringType; use databend_common_expression::DataBlock; use databend_common_expression::DataSchema; use databend_common_expression::FromData; @@ -37,6 +34,9 @@ use databend_common_expression::SendableDataBlockStream; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; +use databend_common_expression::types::DataType; +use databend_common_expression::types::Int32Type; +use databend_common_expression::types::StringType; use databend_common_meta_app::principal::FileFormatParams; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::UpdateStreamMetaReq; @@ -44,9 +44,9 @@ use databend_common_pipeline::core::Pipeline; use databend_common_sql::executor::physical_plans::FragmentKind; use databend_common_sql::executor::physical_plans::MutationKind; use databend_common_sql::executor::table_read_plan::ToReadDataSourcePlan; +use databend_common_storage::StageFileInfo; use databend_common_storage::init_stage_operator; use databend_common_storage::parquet::infer_schema_with_extension; -use databend_common_storage::StageFileInfo; use databend_common_storages_fuse::FuseTable; use databend_common_storages_parquet::read_metas_in_parallel_for_copy; use databend_common_storages_stage::StageTable; @@ -56,11 +56,11 @@ use itertools::Itertools; use log::debug; use log::info; -use crate::interpreters::common::check_deduplicate_label; -use crate::interpreters::common::dml_build_update_stream_req; use crate::interpreters::HookOperator; use crate::interpreters::Interpreter; use crate::interpreters::SelectInterpreter; +use crate::interpreters::common::check_deduplicate_label; +use crate::interpreters::common::dml_build_update_stream_req; use crate::physical_plans::CopyIntoTable; use crate::physical_plans::CopyIntoTableSource; use crate::physical_plans::Exchange; diff --git a/src/query/service/src/interpreters/interpreter_create_warehouses.rs b/src/query/service/src/interpreters/interpreter_create_warehouses.rs index d75578c0a0e70..cd46808464bf3 100644 --- a/src/query/service/src/interpreters/interpreter_create_warehouses.rs +++ b/src/query/service/src/interpreters/interpreter_create_warehouses.rs @@ -29,8 +29,8 @@ use databend_common_users::RoleCacheManager; use databend_common_users::UserApiProvider; use databend_enterprise_resources_management::ResourcesManagement; -use crate::interpreters::util::AuditElement; use crate::interpreters::Interpreter; +use crate::interpreters::util::AuditElement; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/interpreter_create_workload_group.rs b/src/query/service/src/interpreters/interpreter_create_workload_group.rs index e7b3dc4bc6272..34d422230e22b 100644 --- a/src/query/service/src/interpreters/interpreter_create_workload_group.rs +++ b/src/query/service/src/interpreters/interpreter_create_workload_group.rs @@ -28,8 +28,8 @@ use databend_common_management::WorkloadApi; use databend_common_management::WorkloadMgr; use databend_common_sql::plans::CreateWorkloadGroupPlan; -use crate::interpreters::util::AuditElement; use crate::interpreters::Interpreter; +use crate::interpreters::util::AuditElement; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/interpreter_data_mask_desc.rs b/src/query/service/src/interpreters/interpreter_data_mask_desc.rs index b62ee734f6762..5ae4e0c92b9fe 100644 --- a/src/query/service/src/interpreters/interpreter_data_mask_desc.rs +++ b/src/query/service/src/interpreters/interpreter_data_mask_desc.rs @@ -16,9 +16,9 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::StringType; use databend_common_expression::DataBlock; use databend_common_expression::FromData; +use databend_common_expression::types::StringType; use databend_common_license::license::Feature; use databend_common_license::license_manager::LicenseManagerSwitch; use databend_common_sql::plans::DescDatamaskPolicyPlan; diff --git a/src/query/service/src/interpreters/interpreter_database_rename.rs b/src/query/service/src/interpreters/interpreter_database_rename.rs index 6a8970a32558d..72d4ae990d949 100644 --- a/src/query/service/src/interpreters/interpreter_database_rename.rs +++ b/src/query/service/src/interpreters/interpreter_database_rename.rs @@ -15,8 +15,8 @@ use std::sync::Arc; use databend_common_exception::Result; -use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_app::schema::RenameDatabaseReq; +use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_sql::plans::RenameDatabasePlan; use crate::interpreters::Interpreter; diff --git a/src/query/service/src/interpreters/interpreter_database_show_create.rs b/src/query/service/src/interpreters/interpreter_database_show_create.rs index 262575f510bbc..1ecba621cc1c8 100644 --- a/src/query/service/src/interpreters/interpreter_database_show_create.rs +++ b/src/query/service/src/interpreters/interpreter_database_show_create.rs @@ -16,10 +16,10 @@ use std::fmt::Write; use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::BlockEntry; use databend_common_expression::DataBlock; use databend_common_expression::Scalar; +use databend_common_expression::types::DataType; use databend_common_sql::plans::ShowCreateDatabasePlan; use crate::interpreters::Interpreter; diff --git a/src/query/service/src/interpreters/interpreter_dictionary_create.rs b/src/query/service/src/interpreters/interpreter_dictionary_create.rs index df6fa0d6c4b37..b43aa79da4e09 100644 --- a/src/query/service/src/interpreters/interpreter_dictionary_create.rs +++ b/src/query/service/src/interpreters/interpreter_dictionary_create.rs @@ -17,10 +17,10 @@ use std::sync::Arc; use databend_common_ast::ast::CreateOption; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; use databend_common_meta_app::schema::CreateDictionaryReq; use databend_common_meta_app::schema::DictionaryIdentity; use databend_common_meta_app::schema::UpdateDictionaryReq; +use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; use databend_common_sql::plans::CreateDictionaryPlan; use crate::interpreters::Interpreter; diff --git a/src/query/service/src/interpreters/interpreter_dictionary_drop.rs b/src/query/service/src/interpreters/interpreter_dictionary_drop.rs index 8ba7795e75cd5..ce0b498967ac5 100644 --- a/src/query/service/src/interpreters/interpreter_dictionary_drop.rs +++ b/src/query/service/src/interpreters/interpreter_dictionary_drop.rs @@ -16,8 +16,8 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; use databend_common_meta_app::schema::DictionaryIdentity; +use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; use databend_common_sql::plans::DropDictionaryPlan; use crate::interpreters::Interpreter; diff --git a/src/query/service/src/interpreters/interpreter_dictionary_rename.rs b/src/query/service/src/interpreters/interpreter_dictionary_rename.rs index d1303faf6f13a..e9cfdcb60ef77 100644 --- a/src/query/service/src/interpreters/interpreter_dictionary_rename.rs +++ b/src/query/service/src/interpreters/interpreter_dictionary_rename.rs @@ -16,9 +16,9 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; use databend_common_meta_app::schema::DictionaryIdentity; use databend_common_meta_app::schema::RenameDictionaryReq; +use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; use databend_common_sql::plans::RenameDictionaryPlan; use crate::interpreters::Interpreter; diff --git a/src/query/service/src/interpreters/interpreter_dictionary_show_create.rs b/src/query/service/src/interpreters/interpreter_dictionary_show_create.rs index fed5e1393ab42..cf0a036dd3852 100644 --- a/src/query/service/src/interpreters/interpreter_dictionary_show_create.rs +++ b/src/query/service/src/interpreters/interpreter_dictionary_show_create.rs @@ -14,19 +14,19 @@ use std::sync::Arc; -use databend_common_ast::ast::quote::display_ident; use databend_common_ast::ast::quote::QuotedString; +use databend_common_ast::ast::quote::display_ident; use databend_common_ast::parser::Dialect; use databend_common_catalog::catalog::Catalog; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::BlockEntry; use databend_common_expression::DataBlock; use databend_common_expression::Scalar; -use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; +use databend_common_expression::types::DataType; use databend_common_meta_app::schema::DictionaryIdentity; use databend_common_meta_app::schema::DictionaryMeta; +use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; use databend_common_sql::plans::ShowCreateDictionaryPlan; use crate::interpreters::Interpreter; diff --git a/src/query/service/src/interpreters/interpreter_drop_warehouse_cluster.rs b/src/query/service/src/interpreters/interpreter_drop_warehouse_cluster.rs index 1a5424adf515b..c4daa42192aef 100644 --- a/src/query/service/src/interpreters/interpreter_drop_warehouse_cluster.rs +++ b/src/query/service/src/interpreters/interpreter_drop_warehouse_cluster.rs @@ -22,8 +22,8 @@ use databend_common_license::license_manager::LicenseManagerSwitch; use databend_common_sql::plans::DropWarehouseClusterPlan; use databend_enterprise_resources_management::ResourcesManagement; -use crate::interpreters::util::AuditElement; use crate::interpreters::Interpreter; +use crate::interpreters::util::AuditElement; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/interpreter_drop_warehouses.rs b/src/query/service/src/interpreters/interpreter_drop_warehouses.rs index 2d4d2bdce1010..8590eaa1eacc7 100644 --- a/src/query/service/src/interpreters/interpreter_drop_warehouses.rs +++ b/src/query/service/src/interpreters/interpreter_drop_warehouses.rs @@ -27,8 +27,8 @@ use databend_common_users::RoleCacheManager; use databend_common_users::UserApiProvider; use databend_enterprise_resources_management::ResourcesManagement; -use crate::interpreters::util::AuditElement; use crate::interpreters::Interpreter; +use crate::interpreters::util::AuditElement; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/interpreter_drop_workload_group.rs b/src/query/service/src/interpreters/interpreter_drop_workload_group.rs index 7b2732f78a8e1..3be7e23543522 100644 --- a/src/query/service/src/interpreters/interpreter_drop_workload_group.rs +++ b/src/query/service/src/interpreters/interpreter_drop_workload_group.rs @@ -24,8 +24,8 @@ use databend_common_management::WorkloadApi; use databend_common_management::WorkloadMgr; use databend_common_sql::plans::DropWorkloadGroupPlan; -use crate::interpreters::util::AuditElement; use crate::interpreters::Interpreter; +use crate::interpreters::util::AuditElement; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/interpreter_execute_immediate.rs b/src/query/service/src/interpreters/interpreter_execute_immediate.rs index 1d604e7408c67..cc07bd3f800a8 100644 --- a/src/query/service/src/interpreters/interpreter_execute_immediate.rs +++ b/src/query/service/src/interpreters/interpreter_execute_immediate.rs @@ -17,22 +17,22 @@ use std::sync::Arc; use databend_common_ast::ast::DeclareItem; use databend_common_ast::ast::ScriptStatement; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::BlockEntry; use databend_common_expression::DataBlock; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use databend_common_expression::Scalar; -use databend_common_script::compile; +use databend_common_expression::types::DataType; use databend_common_script::Executor; use databend_common_script::ReturnValue; +use databend_common_script::compile; use databend_common_sql::plans::ExecuteImmediatePlan; use databend_common_storages_fuse::TableContext; use tokio::sync::Mutex; -use crate::interpreters::util::ScriptClient; use crate::interpreters::Interpreter; +use crate::interpreters::util::ScriptClient; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/interpreter_explain.rs b/src/query/service/src/interpreters/interpreter_explain.rs index 9244fde6dce10..9889e40bb1281 100644 --- a/src/query/service/src/interpreters/interpreter_explain.rs +++ b/src/query/service/src/interpreters/interpreter_explain.rs @@ -18,29 +18,29 @@ use std::sync::Arc; use databend_common_ast::ast::ExplainKind; use databend_common_ast::ast::FormatTreeNode; -use databend_common_base::runtime::profile::get_statistics_desc; use databend_common_base::runtime::profile::ProfileDesc; use databend_common_base::runtime::profile::ProfileStatisticsName; +use databend_common_base::runtime::profile::get_statistics_desc; use databend_common_catalog::plan::DataSourcePlan; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::StringType; use databend_common_expression::DataBlock; use databend_common_expression::DataSchemaRef; use databend_common_expression::FromData; -use databend_common_pipeline::core::always_callback; +use databend_common_expression::types::StringType; use databend_common_pipeline::core::ExecutionInfo; use databend_common_pipeline::core::Pipeline; use databend_common_pipeline::core::PlanProfile; -use databend_common_sql::binder::ExplainConfig; -use databend_common_sql::plans::Mutation; +use databend_common_pipeline::core::always_callback; use databend_common_sql::BindContext; use databend_common_sql::ColumnSet; use databend_common_sql::FormatOptions; use databend_common_sql::MetadataRef; -use databend_common_storages_basic::gen_result_cache_key; +use databend_common_sql::binder::ExplainConfig; +use databend_common_sql::plans::Mutation; use databend_common_storages_basic::ResultCacheReader; +use databend_common_storages_basic::gen_result_cache_key; use databend_common_storages_fuse::FuseLazyPartInfo; use databend_common_storages_fuse::FuseTable; use databend_common_users::UserApiProvider; @@ -49,22 +49,22 @@ use serde_json; use super::InsertMultiTableInterpreter; use super::InterpreterFactory; +use crate::interpreters::Interpreter; use crate::interpreters::interpreter::on_execution_finished; -use crate::interpreters::interpreter_mutation::build_mutation_info; use crate::interpreters::interpreter_mutation::MutationInterpreter; -use crate::interpreters::Interpreter; +use crate::interpreters::interpreter_mutation::build_mutation_info; use crate::physical_plans::FormatContext; use crate::physical_plans::MutationBuildInfo; use crate::physical_plans::PhysicalPlan; use crate::physical_plans::PhysicalPlanBuilder; +use crate::pipelines::PipelineBuildResult; use crate::pipelines::executor::ExecutorSettings; use crate::pipelines::executor::PipelineCompleteExecutor; use crate::pipelines::executor::PipelinePullingExecutor; use crate::pipelines::executor::QueryPipelineExecutor; -use crate::pipelines::PipelineBuildResult; -use crate::schedulers::build_query_pipeline; use crate::schedulers::Fragmenter; use crate::schedulers::QueryFragmentsActions; +use crate::schedulers::build_query_pipeline; use crate::sessions::QueryContext; use crate::sql::optimizer::ir::SExpr; use crate::sql::plans::Plan; diff --git a/src/query/service/src/interpreters/interpreter_explain_perf.rs b/src/query/service/src/interpreters/interpreter_explain_perf.rs index fc4f74fc58c50..280e47780579b 100644 --- a/src/query/service/src/interpreters/interpreter_explain_perf.rs +++ b/src/query/service/src/interpreters/interpreter_explain_perf.rs @@ -21,9 +21,9 @@ use databend_common_catalog::table_context::TableContext; use databend_common_config::GlobalConfig; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::StringType; use databend_common_expression::DataBlock; use databend_common_expression::FromData; +use databend_common_expression::types::StringType; use databend_common_meta_semaphore::acquirer::Permit; use databend_common_meta_store::MetaStoreProvider; use databend_common_sql::Planner; diff --git a/src/query/service/src/interpreters/interpreter_factory.rs b/src/query/service/src/interpreters/interpreter_factory.rs index a43aca4c76fac..6502890c9dfae 100644 --- a/src/query/service/src/interpreters/interpreter_factory.rs +++ b/src/query/service/src/interpreters/interpreter_factory.rs @@ -38,6 +38,12 @@ use super::interpreter_table_modify_connection::ModifyTableConnectionInterpreter use super::interpreter_table_set_options::SetOptionsInterpreter; use super::interpreter_user_stage_drop::DropUserStageInterpreter; use super::*; +use crate::interpreters::AlterUserInterpreter; +use crate::interpreters::CreateStreamInterpreter; +use crate::interpreters::DescUserInterpreter; +use crate::interpreters::DropStreamInterpreter; +use crate::interpreters::DropUserInterpreter; +use crate::interpreters::SetRoleInterpreter; use crate::interpreters::access::Accessor; use crate::interpreters::access_log::AccessLogger; use crate::interpreters::interpreter_add_warehouse_cluster::AddWarehouseClusterInterpreter; @@ -103,12 +109,6 @@ use crate::interpreters::interpreter_unassign_warehouse_nodes::UnassignWarehouse use crate::interpreters::interpreter_unset_workload_group_quotas::UnsetWorkloadGroupQuotasInterpreter; use crate::interpreters::interpreter_use_warehouse::UseWarehouseInterpreter; use crate::interpreters::interpreter_view_describe::DescribeViewInterpreter; -use crate::interpreters::AlterUserInterpreter; -use crate::interpreters::CreateStreamInterpreter; -use crate::interpreters::DescUserInterpreter; -use crate::interpreters::DropStreamInterpreter; -use crate::interpreters::DropUserInterpreter; -use crate::interpreters::SetRoleInterpreter; use crate::sessions::QueryContext; use crate::sql::plans::Plan; diff --git a/src/query/service/src/interpreters/interpreter_file_format_show.rs b/src/query/service/src/interpreters/interpreter_file_format_show.rs index 948a49cf0107e..ef06ad66b71be 100644 --- a/src/query/service/src/interpreters/interpreter_file_format_show.rs +++ b/src/query/service/src/interpreters/interpreter_file_format_show.rs @@ -15,9 +15,9 @@ use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::types::StringType; use databend_common_expression::DataBlock; use databend_common_expression::FromData; +use databend_common_expression::types::StringType; use databend_common_users::UserApiProvider; use log::debug; diff --git a/src/query/service/src/interpreters/interpreter_index_refresh.rs b/src/query/service/src/interpreters/interpreter_index_refresh.rs index 59fca929e33eb..dfc9854517fdb 100644 --- a/src/query/service/src/interpreters/interpreter_index_refresh.rs +++ b/src/query/service/src/interpreters/interpreter_index_refresh.rs @@ -21,12 +21,12 @@ use databend_common_catalog::plan::Partitions; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::infer_schema_type; +use databend_common_expression::BLOCK_NAME_COL_NAME; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRefExt; use databend_common_expression::TableField; use databend_common_expression::TableSchema; -use databend_common_expression::BLOCK_NAME_COL_NAME; +use databend_common_expression::infer_schema_type; use databend_common_meta_app::schema::IndexMeta; use databend_common_meta_app::schema::UpdateIndexReq; use databend_common_pipeline::core::ExecutionInfo; @@ -36,12 +36,12 @@ use databend_common_sql::evaluator::BlockOperator; use databend_common_sql::plans::Plan; use databend_common_sql::plans::RefreshIndexPlan; use databend_common_sql::plans::RelOperator; -use databend_common_storages_fuse::operations::AggIndexSink; -use databend_common_storages_fuse::pruning::create_segment_location_vector; use databend_common_storages_fuse::FuseBlockPartInfo; use databend_common_storages_fuse::FuseLazyPartInfo; use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::SegmentLocation; +use databend_common_storages_fuse::operations::AggIndexSink; +use databend_common_storages_fuse::pruning::create_segment_location_vector; use databend_storages_common_table_meta::meta::Location; use crate::interpreters::Interpreter; diff --git a/src/query/service/src/interpreters/interpreter_insert.rs b/src/query/service/src/interpreters/interpreter_insert.rs index c292f983319bb..531e6773a54a9 100644 --- a/src/query/service/src/interpreters/interpreter_insert.rs +++ b/src/query/service/src/interpreters/interpreter_insert.rs @@ -20,32 +20,32 @@ use databend_common_catalog::table::Table; use databend_common_catalog::table::TableExt; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::UInt64Type; use databend_common_expression::DataBlock; use databend_common_expression::DataSchema; use databend_common_expression::FromData; use databend_common_expression::SendableDataBlockStream; +use databend_common_expression::types::UInt64Type; use databend_common_meta_app::schema::Constraint; use databend_common_pipeline::sources::AsyncSourcer; -use databend_common_pipeline_transforms::columns::TransformAddConstColumns; use databend_common_pipeline_transforms::TransformPipelineHelper; +use databend_common_pipeline_transforms::columns::TransformAddConstColumns; +use databend_common_sql::NameResolutionContext; use databend_common_sql::binder::ConstraintExprBinder; use databend_common_sql::executor::physical_plans::MutationKind; use databend_common_sql::plans::Insert; use databend_common_sql::plans::InsertInputSource; use databend_common_sql::plans::InsertValue; use databend_common_sql::plans::Plan; -use databend_common_sql::NameResolutionContext; use databend_common_storages_fuse::operations::TransformConstraintVerify; use databend_common_storages_stage::build_streaming_load_pipeline; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use log::info; -use crate::interpreters::common::check_deduplicate_label; -use crate::interpreters::common::dml_build_update_stream_req; use crate::interpreters::HookOperator; use crate::interpreters::Interpreter; use crate::interpreters::InterpreterPtr; +use crate::interpreters::common::check_deduplicate_label; +use crate::interpreters::common::dml_build_update_stream_req; use crate::physical_plans::DistributedInsertSelect; use crate::physical_plans::Exchange; use crate::physical_plans::IPhysicalPlan; diff --git a/src/query/service/src/interpreters/interpreter_insert_multi_table.rs b/src/query/service/src/interpreters/interpreter_insert_multi_table.rs index bc4aab79d2e17..62e707fc29167 100644 --- a/src/query/service/src/interpreters/interpreter_insert_multi_table.rs +++ b/src/query/service/src/interpreters/interpreter_insert_multi_table.rs @@ -18,7 +18,6 @@ use databend_common_catalog::catalog::CATALOG_DEFAULT; use databend_common_catalog::lock::LockTableOption; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::UInt64Type; use databend_common_expression::DataBlock; use databend_common_expression::DataField; use databend_common_expression::DataSchema; @@ -26,21 +25,22 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::FromData; use databend_common_expression::RemoteExpr; use databend_common_expression::SendableDataBlockStream; +use databend_common_expression::types::UInt64Type; use databend_common_functions::BUILTIN_FUNCTIONS; +use databend_common_sql::MetadataRef; +use databend_common_sql::ScalarExpr; use databend_common_sql::executor::physical_plans::MutationKind; use databend_common_sql::plans::Else; use databend_common_sql::plans::FunctionCall; use databend_common_sql::plans::InsertMultiTable; use databend_common_sql::plans::Into; use databend_common_sql::plans::Plan; -use databend_common_sql::MetadataRef; -use databend_common_sql::ScalarExpr; use databend_common_storages_fuse::FuseTable; use super::HookOperator; -use crate::interpreters::common::dml_build_update_stream_req; use crate::interpreters::Interpreter; use crate::interpreters::InterpreterPtr; +use crate::interpreters::common::dml_build_update_stream_req; use crate::physical_plans::CastSchema; use crate::physical_plans::ChunkAppendData; use crate::physical_plans::ChunkCastSchema; diff --git a/src/query/service/src/interpreters/interpreter_inspect_warehouse.rs b/src/query/service/src/interpreters/interpreter_inspect_warehouse.rs index 0ed2b5243d421..acf4396cdb44f 100644 --- a/src/query/service/src/interpreters/interpreter_inspect_warehouse.rs +++ b/src/query/service/src/interpreters/interpreter_inspect_warehouse.rs @@ -17,10 +17,10 @@ use std::sync::Arc; use databend_common_base::base::GlobalInstance; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::ColumnBuilder; use databend_common_expression::DataBlock; use databend_common_expression::Scalar; +use databend_common_expression::types::DataType; use databend_common_license::license::Feature; use databend_common_license::license_manager::LicenseManagerSwitch; use databend_common_meta_types::NodeType; diff --git a/src/query/service/src/interpreters/interpreter_mutation.rs b/src/query/service/src/interpreters/interpreter_mutation.rs index ca19dcfded932..fbef682589a06 100644 --- a/src/query/service/src/interpreters/interpreter_mutation.rs +++ b/src/query/service/src/interpreters/interpreter_mutation.rs @@ -21,11 +21,11 @@ use databend_common_catalog::plan::Partitions; use databend_common_catalog::table::TableExt; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::UInt64Type; use databend_common_expression::DataBlock; use databend_common_expression::DataSchemaRef; use databend_common_expression::FromData; use databend_common_expression::SendableDataBlockStream; +use databend_common_expression::types::UInt64Type; use databend_common_pipeline::core::ProcessorPtr; use databend_common_pipeline::sinks::EmptySink; use databend_common_sql::binder::MutationStrategy; @@ -41,14 +41,14 @@ use databend_common_storages_fuse::TableContext; use databend_storages_common_table_meta::meta::TableSnapshot; use log::info; -use crate::interpreters::common::check_deduplicate_label; -use crate::interpreters::common::dml_build_update_stream_req; use crate::interpreters::HookOperator; use crate::interpreters::Interpreter; -use crate::physical_plans::create_push_down_filters; +use crate::interpreters::common::check_deduplicate_label; +use crate::interpreters::common::dml_build_update_stream_req; use crate::physical_plans::MutationBuildInfo; use crate::physical_plans::PhysicalPlan; use crate::physical_plans::PhysicalPlanBuilder; +use crate::physical_plans::create_push_down_filters; use crate::pipelines::PipelineBuildResult; use crate::schedulers::build_query_pipeline_without_render_result_set; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/interpreter_network_policies_show.rs b/src/query/service/src/interpreters/interpreter_network_policies_show.rs index edd641f3e032c..abe7c7d940788 100644 --- a/src/query/service/src/interpreters/interpreter_network_policies_show.rs +++ b/src/query/service/src/interpreters/interpreter_network_policies_show.rs @@ -15,9 +15,9 @@ use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::types::StringType; use databend_common_expression::DataBlock; use databend_common_expression::FromData; +use databend_common_expression::types::StringType; use databend_common_users::UserApiProvider; use crate::interpreters::Interpreter; diff --git a/src/query/service/src/interpreters/interpreter_network_policy_desc.rs b/src/query/service/src/interpreters/interpreter_network_policy_desc.rs index 9a5bdc5267310..212e21e0a1731 100644 --- a/src/query/service/src/interpreters/interpreter_network_policy_desc.rs +++ b/src/query/service/src/interpreters/interpreter_network_policy_desc.rs @@ -15,9 +15,9 @@ use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::types::StringType; use databend_common_expression::DataBlock; use databend_common_expression::FromData; +use databend_common_expression::types::StringType; use databend_common_sql::plans::DescNetworkPolicyPlan; use databend_common_users::UserApiProvider; diff --git a/src/query/service/src/interpreters/interpreter_notification_alter.rs b/src/query/service/src/interpreters/interpreter_notification_alter.rs index 958c19a6399c0..618b3a16aabf6 100644 --- a/src/query/service/src/interpreters/interpreter_notification_alter.rs +++ b/src/query/service/src/interpreters/interpreter_notification_alter.rs @@ -24,8 +24,8 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_sql::plans::AlterNotificationPlan; -use crate::interpreters::common::get_notification_client_config; use crate::interpreters::Interpreter; +use crate::interpreters::common::get_notification_client_config; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/interpreter_notification_create.rs b/src/query/service/src/interpreters/interpreter_notification_create.rs index 66260377114cd..f18ce0c661cb8 100644 --- a/src/query/service/src/interpreters/interpreter_notification_create.rs +++ b/src/query/service/src/interpreters/interpreter_notification_create.rs @@ -23,8 +23,8 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_sql::plans::CreateNotificationPlan; -use crate::interpreters::common::get_notification_client_config; use crate::interpreters::Interpreter; +use crate::interpreters::common::get_notification_client_config; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/interpreter_notification_desc.rs b/src/query/service/src/interpreters/interpreter_notification_desc.rs index ff54e07361579..496ab3d29ab6d 100644 --- a/src/query/service/src/interpreters/interpreter_notification_desc.rs +++ b/src/query/service/src/interpreters/interpreter_notification_desc.rs @@ -23,8 +23,8 @@ use databend_common_exception::Result; use databend_common_sql::plans::DescNotificationPlan; use databend_common_storages_system::parse_notifications_to_datablock; -use crate::interpreters::common::get_notification_client_config; use crate::interpreters::Interpreter; +use crate::interpreters::common::get_notification_client_config; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/interpreter_notification_drop.rs b/src/query/service/src/interpreters/interpreter_notification_drop.rs index 910eb726bdf92..a5d8a5437ac19 100644 --- a/src/query/service/src/interpreters/interpreter_notification_drop.rs +++ b/src/query/service/src/interpreters/interpreter_notification_drop.rs @@ -22,8 +22,8 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_sql::plans::DropNotificationPlan; -use crate::interpreters::common::get_notification_client_config; use crate::interpreters::Interpreter; +use crate::interpreters::common::get_notification_client_config; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/interpreter_optimize_compact_block.rs b/src/query/service/src/interpreters/interpreter_optimize_compact_block.rs index 2e4543784456e..f4c501f3cd535 100644 --- a/src/query/service/src/interpreters/interpreter_optimize_compact_block.rs +++ b/src/query/service/src/interpreters/interpreter_optimize_compact_block.rs @@ -19,13 +19,13 @@ use databend_common_catalog::lock::LockTableOption; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_pipeline::core::ExecutionInfo; -use databend_common_sql::optimizer::ir::SExpr; -use databend_common_sql::plans::OptimizeCompactBlock; use databend_common_sql::ColumnSet; use databend_common_sql::MetadataRef; +use databend_common_sql::optimizer::ir::SExpr; +use databend_common_sql::plans::OptimizeCompactBlock; -use crate::interpreters::interpreter_optimize_purge::purge; use crate::interpreters::Interpreter; +use crate::interpreters::interpreter_optimize_purge::purge; use crate::physical_plans::PhysicalPlanBuilder; use crate::pipelines::PipelineBuildResult; use crate::schedulers::build_query_pipeline_without_render_result_set; diff --git a/src/query/service/src/interpreters/interpreter_password_policy_alter.rs b/src/query/service/src/interpreters/interpreter_password_policy_alter.rs index 3131eae42a87a..54c633a2ce1f9 100644 --- a/src/query/service/src/interpreters/interpreter_password_policy_alter.rs +++ b/src/query/service/src/interpreters/interpreter_password_policy_alter.rs @@ -17,7 +17,6 @@ use std::sync::Arc; use databend_common_ast::ast::AlterPasswordAction; use databend_common_exception::Result; use databend_common_sql::plans::AlterPasswordPolicyPlan; -use databend_common_users::UserApiProvider; use databend_common_users::DEFAULT_PASSWORD_HISTORY; use databend_common_users::DEFAULT_PASSWORD_LOCKOUT_TIME_MINS; use databend_common_users::DEFAULT_PASSWORD_MAX_AGE_DAYS; @@ -27,6 +26,7 @@ use databend_common_users::DEFAULT_PASSWORD_MIN_AGE_DAYS; use databend_common_users::DEFAULT_PASSWORD_MIN_CHARS; use databend_common_users::DEFAULT_PASSWORD_MIN_LENGTH; use databend_common_users::DEFAULT_PASSWORD_MIN_SPECIAL_CHARS; +use databend_common_users::UserApiProvider; use log::debug; use crate::interpreters::Interpreter; diff --git a/src/query/service/src/interpreters/interpreter_password_policy_create.rs b/src/query/service/src/interpreters/interpreter_password_policy_create.rs index 41b86858cd7d4..7cb2d5d7d3e44 100644 --- a/src/query/service/src/interpreters/interpreter_password_policy_create.rs +++ b/src/query/service/src/interpreters/interpreter_password_policy_create.rs @@ -18,7 +18,6 @@ use chrono::Utc; use databend_common_exception::Result; use databend_common_meta_app::principal::PasswordPolicy; use databend_common_sql::plans::CreatePasswordPolicyPlan; -use databend_common_users::UserApiProvider; use databend_common_users::DEFAULT_PASSWORD_HISTORY; use databend_common_users::DEFAULT_PASSWORD_LOCKOUT_TIME_MINS; use databend_common_users::DEFAULT_PASSWORD_MAX_AGE_DAYS; @@ -28,6 +27,7 @@ use databend_common_users::DEFAULT_PASSWORD_MIN_AGE_DAYS; use databend_common_users::DEFAULT_PASSWORD_MIN_CHARS; use databend_common_users::DEFAULT_PASSWORD_MIN_LENGTH; use databend_common_users::DEFAULT_PASSWORD_MIN_SPECIAL_CHARS; +use databend_common_users::UserApiProvider; use log::debug; use crate::interpreters::Interpreter; diff --git a/src/query/service/src/interpreters/interpreter_password_policy_desc.rs b/src/query/service/src/interpreters/interpreter_password_policy_desc.rs index 3c4bd4cb69a5a..2a85b6549c2f3 100644 --- a/src/query/service/src/interpreters/interpreter_password_policy_desc.rs +++ b/src/query/service/src/interpreters/interpreter_password_policy_desc.rs @@ -15,11 +15,10 @@ use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::types::StringType; use databend_common_expression::DataBlock; use databend_common_expression::FromData; +use databend_common_expression::types::StringType; use databend_common_sql::plans::DescPasswordPolicyPlan; -use databend_common_users::UserApiProvider; use databend_common_users::DEFAULT_PASSWORD_HISTORY; use databend_common_users::DEFAULT_PASSWORD_LOCKOUT_TIME_MINS; use databend_common_users::DEFAULT_PASSWORD_MAX_AGE_DAYS; @@ -29,6 +28,7 @@ use databend_common_users::DEFAULT_PASSWORD_MIN_AGE_DAYS; use databend_common_users::DEFAULT_PASSWORD_MIN_CHARS; use databend_common_users::DEFAULT_PASSWORD_MIN_LENGTH; use databend_common_users::DEFAULT_PASSWORD_MIN_SPECIAL_CHARS; +use databend_common_users::UserApiProvider; use crate::interpreters::Interpreter; use crate::pipelines::PipelineBuildResult; diff --git a/src/query/service/src/interpreters/interpreter_presign.rs b/src/query/service/src/interpreters/interpreter_presign.rs index 5b9fd9472ea56..dc80e5b905374 100644 --- a/src/query/service/src/interpreters/interpreter_presign.rs +++ b/src/query/service/src/interpreters/interpreter_presign.rs @@ -16,10 +16,10 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::StringType; -use databend_common_expression::types::VariantType; use databend_common_expression::BlockEntry; use databend_common_expression::DataBlock; +use databend_common_expression::types::StringType; +use databend_common_expression::types::VariantType; use databend_common_storages_stage::StageTable; use jsonb::Value as JsonbValue; use log::debug; diff --git a/src/query/service/src/interpreters/interpreter_privilege_grant.rs b/src/query/service/src/interpreters/interpreter_privilege_grant.rs index 89232b25a083d..3ee7ca7765712 100644 --- a/src/query/service/src/interpreters/interpreter_privilege_grant.rs +++ b/src/query/service/src/interpreters/interpreter_privilege_grant.rs @@ -33,10 +33,10 @@ use log::debug; use log::error; use log::info; +use crate::interpreters::Interpreter; use crate::interpreters::common::validate_grant_object_exists; use crate::interpreters::util::check_system_history; use crate::interpreters::util::check_system_history_stage; -use crate::interpreters::Interpreter; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; use crate::sessions::TableContext; @@ -103,7 +103,7 @@ impl GrantPrivilegeInterpreter { db_id: *db_id, table_id: *table_id, }) - }, + } GrantObject::DatabaseById(_, db_id) => { let catalog_name = catalog_name.unwrap(); let catalog = self.ctx.get_catalog(&catalog_name).await?; @@ -112,7 +112,7 @@ impl GrantPrivilegeInterpreter { catalog_name, db_id: *db_id, }) - }, + } GrantObject::Stage(name) => { let config = GlobalConfig::instance(); let sensitive_system_stage = config.log.history.stage_name.clone(); @@ -120,22 +120,18 @@ impl GrantPrivilegeInterpreter { Ok(OwnershipObject::Stage { name: name.to_string(), }) - }, + } GrantObject::UDF(name) => Ok(OwnershipObject::UDF { name: name.to_string(), }), - GrantObject::Warehouse(id) => Ok(OwnershipObject::Warehouse { - id: id.to_string(), - }), + GrantObject::Warehouse(id) => Ok(OwnershipObject::Warehouse { id: id.to_string() }), GrantObject::Connection(name) => Ok(OwnershipObject::Connection { name: name.to_string(), }), GrantObject::Sequence(name) => Ok(OwnershipObject::Sequence { name: name.to_string(), }), - GrantObject::Procedure(p) => Ok(OwnershipObject::Procedure { - procedure_id: *p, - }), + GrantObject::Procedure(p) => Ok(OwnershipObject::Procedure { procedure_id: *p }), GrantObject::MaskingPolicy(policy_id) => Ok(OwnershipObject::MaskingPolicy { policy_id: *policy_id, }), @@ -295,7 +291,7 @@ pub fn validate_grant_privileges( && privileges.iter().any(is_create_ownership_object_privilege) { return Err(ErrorCode::IllegalGrant( - "CREATE-like privileges cannot be granted directly to USER; please grant them to a role" + "CREATE-like privileges cannot be granted directly to USER; please grant them to a role", )); } Ok(()) diff --git a/src/query/service/src/interpreters/interpreter_privilege_revoke.rs b/src/query/service/src/interpreters/interpreter_privilege_revoke.rs index ae81bb7014a74..a265e7d208382 100644 --- a/src/query/service/src/interpreters/interpreter_privilege_revoke.rs +++ b/src/query/service/src/interpreters/interpreter_privilege_revoke.rs @@ -18,13 +18,13 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_meta_app::principal::PrincipalIdentity; use databend_common_sql::plans::RevokePrivilegePlan; +use databend_common_users::BUILTIN_ROLE_ACCOUNT_ADMIN; use databend_common_users::RoleCacheManager; use databend_common_users::UserApiProvider; -use databend_common_users::BUILTIN_ROLE_ACCOUNT_ADMIN; use log::debug; -use crate::interpreters::common::validate_grant_object_exists; use crate::interpreters::Interpreter; +use crate::interpreters::common::validate_grant_object_exists; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; use crate::sessions::TableContext; diff --git a/src/query/service/src/interpreters/interpreter_procedure_call.rs b/src/query/service/src/interpreters/interpreter_procedure_call.rs index 14f87759cc999..4d9c094e236e1 100644 --- a/src/query/service/src/interpreters/interpreter_procedure_call.rs +++ b/src/query/service/src/interpreters/interpreter_procedure_call.rs @@ -18,21 +18,22 @@ use databend_common_ast::ast::DeclareItem; use databend_common_ast::ast::DeclareVar; use databend_common_ast::ast::Identifier; use databend_common_ast::ast::ScriptStatement; +use databend_common_ast::parser::ParseMode; use databend_common_ast::parser::run_parser; use databend_common_ast::parser::script::script_block; use databend_common_ast::parser::tokenize_sql; -use databend_common_ast::parser::ParseMode; +use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; -use databend_common_script::compile; use databend_common_script::Executor; use databend_common_script::ReturnValue; +use databend_common_script::compile; use databend_common_sql::plans::CallProcedurePlan; use databend_common_storages_fuse::TableContext; +use crate::interpreters::Interpreter; use crate::interpreters::interpreter_execute_immediate::ProcedureState; use crate::interpreters::util::ScriptClient; -use crate::interpreters::Interpreter; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; @@ -84,14 +85,15 @@ impl Interpreter for CallProcedureInterpreter { } let settings = self.ctx.get_settings(); let sql_dialect = settings.get_sql_dialect()?; - let tokens = tokenize_sql(&self.plan.script)?; + let tokens = tokenize_sql(&self.plan.script).map_err(ErrorCode::from)?; let mut ast = run_parser( &tokens, sql_dialect, ParseMode::Template, false, script_block, - )?; + ) + .map_err(ErrorCode::from)?; for declare in ast.declares { match declare { diff --git a/src/query/service/src/interpreters/interpreter_procedure_desc.rs b/src/query/service/src/interpreters/interpreter_procedure_desc.rs index 05036135480dc..e29783a461916 100644 --- a/src/query/service/src/interpreters/interpreter_procedure_desc.rs +++ b/src/query/service/src/interpreters/interpreter_procedure_desc.rs @@ -14,9 +14,9 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::StringType; use databend_common_expression::DataBlock; use databend_common_expression::FromData; +use databend_common_expression::types::StringType; use databend_common_meta_app::principal::GetProcedureReq; use databend_common_sql::plans::DescProcedurePlan; use databend_common_users::UserApiProvider; diff --git a/src/query/service/src/interpreters/interpreter_rename_warehouse.rs b/src/query/service/src/interpreters/interpreter_rename_warehouse.rs index b044c82bb624f..243f1341a72c2 100644 --- a/src/query/service/src/interpreters/interpreter_rename_warehouse.rs +++ b/src/query/service/src/interpreters/interpreter_rename_warehouse.rs @@ -22,8 +22,8 @@ use databend_common_license::license_manager::LicenseManagerSwitch; use databend_common_sql::plans::RenameWarehousePlan; use databend_enterprise_resources_management::ResourcesManagement; -use crate::interpreters::util::AuditElement; use crate::interpreters::Interpreter; +use crate::interpreters::util::AuditElement; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/interpreter_rename_warehouse_cluster.rs b/src/query/service/src/interpreters/interpreter_rename_warehouse_cluster.rs index 5724e2826857c..8575f88e4b510 100644 --- a/src/query/service/src/interpreters/interpreter_rename_warehouse_cluster.rs +++ b/src/query/service/src/interpreters/interpreter_rename_warehouse_cluster.rs @@ -22,8 +22,8 @@ use databend_common_license::license_manager::LicenseManagerSwitch; use databend_common_sql::plans::RenameWarehouseClusterPlan; use databend_enterprise_resources_management::ResourcesManagement; -use crate::interpreters::util::AuditElement; use crate::interpreters::Interpreter; +use crate::interpreters::util::AuditElement; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/interpreter_rename_workload_group.rs b/src/query/service/src/interpreters/interpreter_rename_workload_group.rs index 9b85d11f93bff..cbfba368cffe0 100644 --- a/src/query/service/src/interpreters/interpreter_rename_workload_group.rs +++ b/src/query/service/src/interpreters/interpreter_rename_workload_group.rs @@ -23,8 +23,8 @@ use databend_common_management::WorkloadApi; use databend_common_management::WorkloadMgr; use databend_common_sql::plans::RenameWorkloadGroupPlan; -use crate::interpreters::util::AuditElement; use crate::interpreters::Interpreter; +use crate::interpreters::util::AuditElement; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/interpreter_replace.rs b/src/query/service/src/interpreters/interpreter_replace.rs index 33554d1cbfc43..70e9a81701a55 100644 --- a/src/query/service/src/interpreters/interpreter_replace.rs +++ b/src/query/service/src/interpreters/interpreter_replace.rs @@ -25,6 +25,10 @@ use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::principal::StageInfo; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::UpdateStreamMetaReq; +use databend_common_sql::BindContext; +use databend_common_sql::Metadata; +use databend_common_sql::NameResolutionContext; +use databend_common_sql::ScalarBinder; use databend_common_sql::executor::cast_expr_to_non_null_boolean; use databend_common_sql::executor::physical_plans::FragmentKind; use databend_common_sql::executor::physical_plans::MutationKind; @@ -33,10 +37,6 @@ use databend_common_sql::plans::InsertInputSource; use databend_common_sql::plans::InsertValue; use databend_common_sql::plans::Plan; use databend_common_sql::plans::Replace; -use databend_common_sql::BindContext; -use databend_common_sql::Metadata; -use databend_common_sql::NameResolutionContext; -use databend_common_sql::ScalarBinder; use databend_common_storage::StageFileInfo; use databend_common_storages_factory::Table; use databend_common_storages_fuse::FuseTable; @@ -45,13 +45,13 @@ use databend_storages_common_table_meta::readers::snapshot_reader::TableSnapshot use databend_storages_common_table_meta::table::ClusterType; use parking_lot::RwLock; -use crate::interpreters::common::check_deduplicate_label; -use crate::interpreters::common::dml_build_update_stream_req; -use crate::interpreters::interpreter_copy_into_table::CopyIntoTableInterpreter; use crate::interpreters::HookOperator; use crate::interpreters::Interpreter; use crate::interpreters::InterpreterPtr; use crate::interpreters::SelectInterpreter; +use crate::interpreters::common::check_deduplicate_label; +use crate::interpreters::common::dml_build_update_stream_req; +use crate::interpreters::interpreter_copy_into_table::CopyIntoTableInterpreter; use crate::physical_plans::CommitSink; use crate::physical_plans::CommitType; use crate::physical_plans::Exchange; diff --git a/src/query/service/src/interpreters/interpreter_report_issue.rs b/src/query/service/src/interpreters/interpreter_report_issue.rs index 0edbbb2d3a752..c9de061f911f0 100644 --- a/src/query/service/src/interpreters/interpreter_report_issue.rs +++ b/src/query/service/src/interpreters/interpreter_report_issue.rs @@ -25,16 +25,16 @@ use databend_common_ast::ast::Statement; use databend_common_ast::ast::TableReference; use databend_common_base::runtime::CaptureLogSettings; use databend_common_base::runtime::ThreadTracker; -use databend_common_catalog::table_context::TableContext; use databend_common_catalog::BasicColumnStatistics; use databend_common_catalog::TableStatistics; +use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::StringType; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::TableField; use databend_common_expression::TableSchema; +use databend_common_expression::types::StringType; use databend_common_settings::Settings; use databend_common_sql::Planner; use databend_common_storages_basic::view_table::VIEW_ENGINE; @@ -47,8 +47,8 @@ use log::LevelFilter; use super::InterpreterFactory; use super::ShowCreateQuerySettings; use super::ShowCreateTableInterpreter; -use crate::interpreters::interpreter::auto_commit_if_not_allowed_in_transaction; use crate::interpreters::Interpreter; +use crate::interpreters::interpreter::auto_commit_if_not_allowed_in_transaction; use crate::pipelines::PipelineBuildResult; use crate::schedulers::ServiceQueryExecutor; use crate::sessions::QueryContext; @@ -387,7 +387,9 @@ impl ReportContext { let mut table_info = table.table().get_table_info().clone(); if table_info.engine() == VIEW_ENGINE || table_info.engine() == STREAM_ENGINE { - return Err(ErrorCode::Unimplemented("Reports in SQL must not contain tables that use view or stream engines.")); + return Err(ErrorCode::Unimplemented( + "Reports in SQL must not contain tables that use view or stream engines.", + )); } table_info.meta.comment = String::new(); diff --git a/src/query/service/src/interpreters/interpreter_resume_warehouse.rs b/src/query/service/src/interpreters/interpreter_resume_warehouse.rs index f763908ba3f87..06d690ac426d9 100644 --- a/src/query/service/src/interpreters/interpreter_resume_warehouse.rs +++ b/src/query/service/src/interpreters/interpreter_resume_warehouse.rs @@ -22,8 +22,8 @@ use databend_common_license::license_manager::LicenseManagerSwitch; use databend_common_sql::plans::ResumeWarehousePlan; use databend_enterprise_resources_management::ResourcesManagement; -use crate::interpreters::util::AuditElement; use crate::interpreters::Interpreter; +use crate::interpreters::util::AuditElement; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/interpreter_role_create.rs b/src/query/service/src/interpreters/interpreter_role_create.rs index 2c7bd6c997f59..cbc4fe052e534 100644 --- a/src/query/service/src/interpreters/interpreter_role_create.rs +++ b/src/query/service/src/interpreters/interpreter_role_create.rs @@ -18,10 +18,10 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_meta_app::principal::RoleInfo; use databend_common_sql::plans::CreateRolePlan; -use databend_common_users::RoleCacheManager; -use databend_common_users::UserApiProvider; use databend_common_users::BUILTIN_ROLE_ACCOUNT_ADMIN; use databend_common_users::BUILTIN_ROLE_PUBLIC; +use databend_common_users::RoleCacheManager; +use databend_common_users::UserApiProvider; use log::debug; use crate::interpreters::Interpreter; diff --git a/src/query/service/src/interpreters/interpreter_role_drop.rs b/src/query/service/src/interpreters/interpreter_role_drop.rs index eb776036e1b8d..101bc09142e61 100644 --- a/src/query/service/src/interpreters/interpreter_role_drop.rs +++ b/src/query/service/src/interpreters/interpreter_role_drop.rs @@ -17,10 +17,10 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_sql::plans::DropRolePlan; -use databend_common_users::RoleCacheManager; -use databend_common_users::UserApiProvider; use databend_common_users::BUILTIN_ROLE_ACCOUNT_ADMIN; use databend_common_users::BUILTIN_ROLE_PUBLIC; +use databend_common_users::RoleCacheManager; +use databend_common_users::UserApiProvider; use log::debug; use log::warn; diff --git a/src/query/service/src/interpreters/interpreter_row_access_policy_desc.rs b/src/query/service/src/interpreters/interpreter_row_access_policy_desc.rs index eeadf3fe747f4..f425791462286 100644 --- a/src/query/service/src/interpreters/interpreter_row_access_policy_desc.rs +++ b/src/query/service/src/interpreters/interpreter_row_access_policy_desc.rs @@ -16,9 +16,9 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::StringType; use databend_common_expression::DataBlock; use databend_common_expression::FromData; +use databend_common_expression::types::StringType; use databend_common_license::license::Feature; use databend_common_license::license_manager::LicenseManagerSwitch; use databend_common_sql::plans::DescRowAccessPolicyPlan; diff --git a/src/query/service/src/interpreters/interpreter_select.rs b/src/query/service/src/interpreters/interpreter_select.rs index 3ccbc819d51cf..7f009458254aa 100644 --- a/src/query/service/src/interpreters/interpreter_select.rs +++ b/src/query/service/src/interpreters/interpreter_select.rs @@ -21,11 +21,11 @@ use databend_common_base::runtime::GlobalIORuntime; use databend_common_catalog::table::Table; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::infer_table_schema; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use databend_common_expression::TableSchemaRef; +use databend_common_expression::infer_table_schema; use databend_common_meta_app::schema::UpdateMultiTableMetaReq; use databend_common_meta_store::MetaStore; use databend_common_pipeline::core::ExecutionInfo; @@ -35,19 +35,19 @@ use databend_common_pipeline::core::Pipe; use databend_common_pipeline::core::PipeItem; use databend_common_pipeline::core::Pipeline; use databend_common_pipeline_transforms::processors::TransformDummy; -use databend_common_sql::executor::physical_plans::FragmentKind; -use databend_common_sql::parse_result_scan_args; use databend_common_sql::ColumnBinding; use databend_common_sql::MetadataRef; -use databend_common_storages_basic::gen_result_cache_key; +use databend_common_sql::executor::physical_plans::FragmentKind; +use databend_common_sql::parse_result_scan_args; use databend_common_storages_basic::ResultCacheReader; use databend_common_storages_basic::WriteResultCacheSink; +use databend_common_storages_basic::gen_result_cache_key; use databend_common_users::UserApiProvider; use log::error; use log::info; -use crate::interpreters::common::query_build_update_stream_req; use crate::interpreters::Interpreter; +use crate::interpreters::common::query_build_update_stream_req; use crate::physical_plans::Exchange; use crate::physical_plans::PhysicalPlan; use crate::physical_plans::PhysicalPlanBuilder; @@ -56,8 +56,8 @@ use crate::pipelines::PipelineBuildResult; use crate::schedulers::build_query_pipeline; use crate::sessions::QueryContext; use crate::sessions::TableContext; -use crate::sql::optimizer::ir::SExpr; use crate::sql::BindContext; +use crate::sql::optimizer::ir::SExpr; /// Interpret SQL query with new SQL planner pub struct SelectInterpreter { diff --git a/src/query/service/src/interpreters/interpreter_sequence_create.rs b/src/query/service/src/interpreters/interpreter_sequence_create.rs index 09e9374248889..f60d6c17fa012 100644 --- a/src/query/service/src/interpreters/interpreter_sequence_create.rs +++ b/src/query/service/src/interpreters/interpreter_sequence_create.rs @@ -17,9 +17,9 @@ use std::sync::Arc; use chrono::Utc; use databend_common_exception::Result; use databend_common_management::RoleApi; +use databend_common_meta_app::KeyWithTenant; use databend_common_meta_app::principal::OwnershipObject; use databend_common_meta_app::schema::CreateSequenceReq; -use databend_common_meta_app::KeyWithTenant; use databend_common_sql::plans::CreateSequencePlan; use databend_common_storages_fuse::TableContext; use databend_common_users::RoleCacheManager; diff --git a/src/query/service/src/interpreters/interpreter_sequence_desc.rs b/src/query/service/src/interpreters/interpreter_sequence_desc.rs index ef47422d1756d..5fedc4abbc29c 100644 --- a/src/query/service/src/interpreters/interpreter_sequence_desc.rs +++ b/src/query/service/src/interpreters/interpreter_sequence_desc.rs @@ -15,12 +15,12 @@ use std::sync::Arc; use databend_common_exception::Result; +use databend_common_expression::DataBlock; +use databend_common_expression::FromData; use databend_common_expression::types::Int64Type; use databend_common_expression::types::StringType; use databend_common_expression::types::TimestampType; use databend_common_expression::types::UInt64Type; -use databend_common_expression::DataBlock; -use databend_common_expression::FromData; use databend_common_meta_app::schema::GetSequenceReq; use databend_common_sql::plans::DescSequencePlan; use databend_common_storages_fuse::TableContext; diff --git a/src/query/service/src/interpreters/interpreter_sequence_drop.rs b/src/query/service/src/interpreters/interpreter_sequence_drop.rs index c931867881b28..b6a10f8bbf3a7 100644 --- a/src/query/service/src/interpreters/interpreter_sequence_drop.rs +++ b/src/query/service/src/interpreters/interpreter_sequence_drop.rs @@ -17,9 +17,9 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_management::RoleApi; +use databend_common_meta_app::KeyWithTenant; use databend_common_meta_app::principal::OwnershipObject; use databend_common_meta_app::schema::DropSequenceReq; -use databend_common_meta_app::KeyWithTenant; use databend_common_sql::plans::DropSequencePlan; use databend_common_storages_fuse::TableContext; use databend_common_users::RoleCacheManager; diff --git a/src/query/service/src/interpreters/interpreter_set.rs b/src/query/service/src/interpreters/interpreter_set.rs index acc570318d095..1beabf8689cce 100644 --- a/src/query/service/src/interpreters/interpreter_set.rs +++ b/src/query/service/src/interpreters/interpreter_set.rs @@ -19,10 +19,10 @@ use databend_common_ast::ast::SetType; use databend_common_config::GlobalConfig; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::cast_scalar; -use databend_common_expression::types::DataType; use databend_common_expression::DataBlock; use databend_common_expression::Scalar; +use databend_common_expression::cast_scalar; +use databend_common_expression::types::DataType; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::tenant::Tenant; use databend_common_sql::plans::Plan; @@ -127,8 +127,9 @@ impl SetInterpreter { // TODO: we will remove this setting when queries executor is stable. let config = GlobalConfig::instance(); if !config.query.enable_queries_executor { - return Err( - ErrorCode::InvalidArgument("This setting is not allowed when queries executor is not enabled in the configuration")); + return Err(ErrorCode::InvalidArgument( + "This setting is not allowed when queries executor is not enabled in the configuration", + )); } if scalar.as_str() == "0" { return Err(ErrorCode::InvalidArgument( diff --git a/src/query/service/src/interpreters/interpreter_set_workload_group_quotas.rs b/src/query/service/src/interpreters/interpreter_set_workload_group_quotas.rs index f4ba53787a457..2688c89d88073 100644 --- a/src/query/service/src/interpreters/interpreter_set_workload_group_quotas.rs +++ b/src/query/service/src/interpreters/interpreter_set_workload_group_quotas.rs @@ -24,9 +24,9 @@ use databend_common_management::WorkloadApi; use databend_common_management::WorkloadMgr; use databend_common_sql::plans::SetWorkloadGroupQuotasPlan; +use crate::interpreters::Interpreter; use crate::interpreters::interpreter_create_workload_group::to_quota_value; use crate::interpreters::util::AuditElement; -use crate::interpreters::Interpreter; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/interpreter_show_online_nodes.rs b/src/query/service/src/interpreters/interpreter_show_online_nodes.rs index 21f1c6ca8d950..512391fd73f67 100644 --- a/src/query/service/src/interpreters/interpreter_show_online_nodes.rs +++ b/src/query/service/src/interpreters/interpreter_show_online_nodes.rs @@ -17,10 +17,10 @@ use std::sync::Arc; use databend_common_base::base::GlobalInstance; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::ColumnBuilder; use databend_common_expression::DataBlock; use databend_common_expression::Scalar; +use databend_common_expression::types::DataType; use databend_common_license::license::Feature; use databend_common_license::license_manager::LicenseManagerSwitch; use databend_common_meta_types::NodeType; diff --git a/src/query/service/src/interpreters/interpreter_show_warehouses.rs b/src/query/service/src/interpreters/interpreter_show_warehouses.rs index 4abfbcb06084e..d9c35261c6e35 100644 --- a/src/query/service/src/interpreters/interpreter_show_warehouses.rs +++ b/src/query/service/src/interpreters/interpreter_show_warehouses.rs @@ -17,10 +17,10 @@ use std::sync::Arc; use databend_common_base::base::GlobalInstance; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::ColumnBuilder; use databend_common_expression::DataBlock; use databend_common_expression::Scalar; +use databend_common_expression::types::DataType; use databend_common_license::license::Feature; use databend_common_license::license_manager::LicenseManagerSwitch; use databend_common_management::WarehouseInfo; diff --git a/src/query/service/src/interpreters/interpreter_show_workload_groups.rs b/src/query/service/src/interpreters/interpreter_show_workload_groups.rs index 989e2e4aea7c4..07dcf31554f62 100644 --- a/src/query/service/src/interpreters/interpreter_show_workload_groups.rs +++ b/src/query/service/src/interpreters/interpreter_show_workload_groups.rs @@ -23,10 +23,10 @@ use databend_common_base::runtime::workload_group::QUERY_QUEUED_TIMEOUT_QUOTA_KE use databend_common_base::runtime::workload_group::QUERY_TIMEOUT_QUOTA_KEY; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::ColumnBuilder; use databend_common_expression::DataBlock; use databend_common_expression::Scalar; +use databend_common_expression::types::DataType; use databend_common_license::license::Feature; use databend_common_license::license_manager::LicenseManagerSwitch; use databend_common_management::WorkloadApi; diff --git a/src/query/service/src/interpreters/interpreter_suspend_warehouse.rs b/src/query/service/src/interpreters/interpreter_suspend_warehouse.rs index 31d2691cbc877..367118fc61d1d 100644 --- a/src/query/service/src/interpreters/interpreter_suspend_warehouse.rs +++ b/src/query/service/src/interpreters/interpreter_suspend_warehouse.rs @@ -22,8 +22,8 @@ use databend_common_license::license_manager::LicenseManagerSwitch; use databend_common_sql::plans::SuspendWarehousePlan; use databend_enterprise_resources_management::ResourcesManagement; -use crate::interpreters::util::AuditElement; use crate::interpreters::Interpreter; +use crate::interpreters::util::AuditElement; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/interpreter_system_action.rs b/src/query/service/src/interpreters/interpreter_system_action.rs index 125c04a14255b..d996b635a528c 100644 --- a/src/query/service/src/interpreters/interpreter_system_action.rs +++ b/src/query/service/src/interpreters/interpreter_system_action.rs @@ -16,8 +16,8 @@ use std::collections::HashMap; use std::sync::Arc; use databend_common_catalog::table_context::TableContext; -use databend_common_exception::set_backtrace; use databend_common_exception::Result; +use databend_common_exception::set_backtrace; use databend_common_sql::plans::SystemAction; use databend_common_sql::plans::SystemPlan; use databend_common_users::RoleCacheManager; diff --git a/src/query/service/src/interpreters/interpreter_table_add_column.rs b/src/query/service/src/interpreters/interpreter_table_add_column.rs index 77ca529907bc4..257a0b32eb170 100644 --- a/src/query/service/src/interpreters/interpreter_table_add_column.rs +++ b/src/query/service/src/interpreters/interpreter_table_add_column.rs @@ -28,12 +28,12 @@ use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; use databend_common_meta_app::schema::UpdateTableMetaReq; use databend_common_meta_types::MatchSeq; +use databend_common_sql::DefaultExprBinder; +use databend_common_sql::Planner; use databend_common_sql::plans::AddColumnOption; use databend_common_sql::plans::AddTableColumnPlan; use databend_common_sql::plans::Mutation; use databend_common_sql::plans::Plan; -use databend_common_sql::DefaultExprBinder; -use databend_common_sql::Planner; use databend_common_storages_basic::view_table::VIEW_ENGINE; use databend_common_storages_fuse::FuseTable; use databend_common_storages_stream::stream_table::STREAM_ENGINE; @@ -43,10 +43,10 @@ use databend_storages_common_table_meta::readers::snapshot_reader::TableSnapshot use databend_storages_common_table_meta::table::OPT_KEY_SNAPSHOT_LOCATION; use log::info; -use crate::interpreters::interpreter_table_create::is_valid_column; -use crate::interpreters::interpreter_table_modify_column::build_select_insert_plan; use crate::interpreters::Interpreter; use crate::interpreters::MutationInterpreter; +use crate::interpreters::interpreter_table_create::is_valid_column; +use crate::interpreters::interpreter_table_modify_column::build_select_insert_plan; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; use crate::sessions::TableContext; diff --git a/src/query/service/src/interpreters/interpreter_table_add_constraint.rs b/src/query/service/src/interpreters/interpreter_table_add_constraint.rs index 2cab8b5f81c33..5205af17be4b1 100644 --- a/src/query/service/src/interpreters/interpreter_table_add_constraint.rs +++ b/src/query/service/src/interpreters/interpreter_table_add_constraint.rs @@ -23,8 +23,8 @@ use databend_common_sql::plans::AddTableConstraintPlan; use databend_common_storages_basic::view_table::VIEW_ENGINE; use databend_common_storages_stream::stream_table::STREAM_ENGINE; -use crate::interpreters::interpreter_table_add_column::commit_table_meta; use crate::interpreters::Interpreter; +use crate::interpreters::interpreter_table_add_column::commit_table_meta; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/interpreter_table_analyze.rs b/src/query/service/src/interpreters/interpreter_table_analyze.rs index 93cdcb7aca825..d008b51d63161 100644 --- a/src/query/service/src/interpreters/interpreter_table_analyze.rs +++ b/src/query/service/src/interpreters/interpreter_table_analyze.rs @@ -18,14 +18,14 @@ use std::sync::Arc; use databend_common_catalog::table::TableExt; use databend_common_exception::Result; use databend_common_pipeline::core::ProcessorPtr; -use databend_common_sql::plans::AnalyzeTablePlan; -use databend_common_sql::plans::Plan; use databend_common_sql::BindContext; use databend_common_sql::Planner; +use databend_common_sql::plans::AnalyzeTablePlan; +use databend_common_sql::plans::Plan; use databend_common_storage::DEFAULT_HISTOGRAM_BUCKETS; use databend_common_storages_factory::Table; -use databend_common_storages_fuse::operations::HistogramInfoSink; use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::operations::HistogramInfoSink; use databend_storages_common_index::Index; use databend_storages_common_index::RangeIndex; use log::info; diff --git a/src/query/service/src/interpreters/interpreter_table_create.rs b/src/query/service/src/interpreters/interpreter_table_create.rs index a92e40a0b4aa4..b73bab2a61d8b 100644 --- a/src/query/service/src/interpreters/interpreter_table_create.rs +++ b/src/query/service/src/interpreters/interpreter_table_create.rs @@ -21,8 +21,8 @@ use databend_common_base::runtime::GlobalIORuntime; use databend_common_config::GlobalConfig; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::is_internal_column; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::is_internal_column; use databend_common_license::license::Feature; use databend_common_license::license::Feature::ComputedColumn; use databend_common_license::license_manager::LicenseManagerSwitch; @@ -40,15 +40,15 @@ use databend_common_meta_app::schema::TablePartition; use databend_common_meta_app::schema::TableStatistics; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_types::MatchSeq; -use databend_common_pipeline::core::always_callback; use databend_common_pipeline::core::ExecutionInfo; -use databend_common_sql::plans::CreateTablePlan; +use databend_common_pipeline::core::always_callback; use databend_common_sql::DefaultExprBinder; -use databend_common_storages_fuse::io::MetaReaders; -use databend_common_storages_fuse::FuseSegmentFormat; -use databend_common_storages_fuse::FuseStorageFormat; +use databend_common_sql::plans::CreateTablePlan; use databend_common_storages_fuse::FUSE_OPT_KEY_ENABLE_AUTO_ANALYZE; use databend_common_storages_fuse::FUSE_OPT_KEY_ENABLE_AUTO_VACUUM; +use databend_common_storages_fuse::FuseSegmentFormat; +use databend_common_storages_fuse::FuseStorageFormat; +use databend_common_storages_fuse::io::MetaReaders; use databend_common_users::RoleCacheManager; use databend_common_users::UserApiProvider; use databend_enterprise_attach_table::get_attach_table_handler; @@ -65,6 +65,8 @@ use databend_storages_common_table_meta::table::OPT_KEY_TEMP_PREFIX; use log::error; use log::info; +use crate::interpreters::InsertInterpreter; +use crate::interpreters::Interpreter; use crate::interpreters::common::table_option_validation::is_valid_approx_distinct_columns; use crate::interpreters::common::table_option_validation::is_valid_block_per_segment; use crate::interpreters::common::table_option_validation::is_valid_bloom_index_columns; @@ -78,8 +80,6 @@ use crate::interpreters::common::table_option_validation::is_valid_row_per_block use crate::interpreters::hook::vacuum_hook::hook_clear_m_cte_temp_table; use crate::interpreters::hook::vacuum_hook::hook_disk_temp_dir; use crate::interpreters::hook::vacuum_hook::hook_vacuum_temp_files; -use crate::interpreters::InsertInterpreter; -use crate::interpreters::Interpreter; use crate::pipelines::PipelineBuildResult; use crate::servers::http::v1::ClientSessionManager; use crate::sessions::QueryContext; @@ -363,9 +363,9 @@ impl CreateTableInterpreter { && (req.table_properties.is_some() || req.table_partition.is_some()) { return Err(ErrorCode::TableOptionInvalid(format!( - "Current Catalog Type is {:?}, only Iceberg Catalog supports CREATE TABLE with PARTITION BY or PROPERTIES", - catalog.info().catalog_type() - ))); + "Current Catalog Type is {:?}, only Iceberg Catalog supports CREATE TABLE with PARTITION BY or PROPERTIES", + catalog.info().catalog_type() + ))); } let reply = catalog.create_table(req.clone()).await?; diff --git a/src/query/service/src/interpreters/interpreter_table_describe.rs b/src/query/service/src/interpreters/interpreter_table_describe.rs index 94c44e6060faa..d961844815dcc 100644 --- a/src/query/service/src/interpreters/interpreter_table_describe.rs +++ b/src/query/service/src/interpreters/interpreter_table_describe.rs @@ -16,17 +16,17 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::infer_table_schema; -use databend_common_expression::types::StringType; use databend_common_expression::DataBlock; use databend_common_expression::FromData; +use databend_common_expression::infer_table_schema; +use databend_common_expression::types::StringType; use databend_common_sql::plans::DescribeTablePlan; use databend_common_storages_basic::view_table::QUERY; use databend_common_storages_basic::view_table::VIEW_ENGINE; use databend_common_storages_fuse::TableContext; -use crate::interpreters::util::generate_desc_schema; use crate::interpreters::Interpreter; +use crate::interpreters::util::generate_desc_schema; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; use crate::sql::Planner; diff --git a/src/query/service/src/interpreters/interpreter_table_drop_column.rs b/src/query/service/src/interpreters/interpreter_table_drop_column.rs index a365798edc08b..76d344c0cad9f 100644 --- a/src/query/service/src/interpreters/interpreter_table_drop_column.rs +++ b/src/query/service/src/interpreters/interpreter_table_drop_column.rs @@ -19,17 +19,17 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::DataSchema; use databend_common_meta_app::schema::DatabaseType; -use databend_common_sql::plans::DropTableColumnPlan; use databend_common_sql::ApproxDistinctColumns; use databend_common_sql::BloomIndexColumns; +use databend_common_sql::plans::DropTableColumnPlan; use databend_common_storages_basic::view_table::VIEW_ENGINE; use databend_common_storages_stream::stream_table::STREAM_ENGINE; use databend_storages_common_table_meta::table::OPT_KEY_APPROX_DISTINCT_COLUMNS; use databend_storages_common_table_meta::table::OPT_KEY_BLOOM_INDEX_COLUMNS; +use crate::interpreters::Interpreter; use crate::interpreters::common::check_referenced_computed_columns; use crate::interpreters::interpreter_table_add_column::commit_table_meta; -use crate::interpreters::Interpreter; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; use crate::sessions::TableContext; diff --git a/src/query/service/src/interpreters/interpreter_table_drop_constraint.rs b/src/query/service/src/interpreters/interpreter_table_drop_constraint.rs index bd18a5b85868f..2342488c0f479 100644 --- a/src/query/service/src/interpreters/interpreter_table_drop_constraint.rs +++ b/src/query/service/src/interpreters/interpreter_table_drop_constraint.rs @@ -23,8 +23,8 @@ use databend_common_sql::plans::DropTableConstraintPlan; use databend_common_storages_basic::view_table::VIEW_ENGINE; use databend_common_storages_stream::stream_table::STREAM_ENGINE; -use crate::interpreters::interpreter_table_add_column::commit_table_meta; use crate::interpreters::Interpreter; +use crate::interpreters::interpreter_table_add_column::commit_table_meta; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/interpreter_table_exists.rs b/src/query/service/src/interpreters/interpreter_table_exists.rs index e567b6d21ff3d..c6d6830230bf6 100644 --- a/src/query/service/src/interpreters/interpreter_table_exists.rs +++ b/src/query/service/src/interpreters/interpreter_table_exists.rs @@ -15,12 +15,12 @@ use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::types::number::NumberScalar; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; use databend_common_expression::BlockEntry; use databend_common_expression::DataBlock; use databend_common_expression::Scalar; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::number::NumberScalar; use databend_common_sql::plans::ExistsTablePlan; use crate::interpreters::Interpreter; diff --git a/src/query/service/src/interpreters/interpreter_table_index_refresh.rs b/src/query/service/src/interpreters/interpreter_table_index_refresh.rs index fe84096b2631e..9029f5c865088 100644 --- a/src/query/service/src/interpreters/interpreter_table_index_refresh.rs +++ b/src/query/service/src/interpreters/interpreter_table_index_refresh.rs @@ -20,9 +20,9 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_meta_app::schema::TableIndexType; use databend_common_sql::plans::RefreshTableIndexPlan; -use databend_common_storages_fuse::operations::do_refresh_table_index; use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::TableContext; +use databend_common_storages_fuse::operations::do_refresh_table_index; use crate::interpreters::Interpreter; use crate::pipelines::PipelineBuildResult; diff --git a/src/query/service/src/interpreters/interpreter_table_modify_column.rs b/src/query/service/src/interpreters/interpreter_table_modify_column.rs index 8fc5e87db418e..8f8b8e1b0ca13 100644 --- a/src/query/service/src/interpreters/interpreter_table_modify_column.rs +++ b/src/query/service/src/interpreters/interpreter_table_modify_column.rs @@ -20,7 +20,6 @@ use databend_common_catalog::table::Table; use databend_common_catalog::table::TableExt; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::ComputedExpr; use databend_common_expression::DataSchema; use databend_common_expression::Scalar; @@ -28,6 +27,7 @@ use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRef; +use databend_common_expression::types::DataType; use databend_common_license::license::Feature::ComputedColumn; use databend_common_license::license::Feature::DataMask; use databend_common_license::license_manager::LicenseManagerSwitch; @@ -40,14 +40,14 @@ use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; use databend_common_meta_app::schema::UpdateTableMetaReq; use databend_common_meta_types::MatchSeq; -use databend_common_sql::plans::ModifyColumnAction; -use databend_common_sql::plans::ModifyTableColumnPlan; -use databend_common_sql::plans::Plan; -use databend_common_sql::resolve_type_name_by_str; use databend_common_sql::ApproxDistinctColumns; use databend_common_sql::BloomIndexColumns; use databend_common_sql::DefaultExprBinder; use databend_common_sql::Planner; +use databend_common_sql::plans::ModifyColumnAction; +use databend_common_sql::plans::ModifyTableColumnPlan; +use databend_common_sql::plans::Plan; +use databend_common_sql::resolve_type_name_by_str; use databend_common_storages_basic::view_table::VIEW_ENGINE; use databend_common_storages_fuse::FuseTable; use databend_common_storages_stream::stream_table::STREAM_ENGINE; @@ -61,9 +61,9 @@ use databend_storages_common_table_meta::readers::snapshot_reader::TableSnapshot use databend_storages_common_table_meta::table::OPT_KEY_APPROX_DISTINCT_COLUMNS; use databend_storages_common_table_meta::table::OPT_KEY_BLOOM_INDEX_COLUMNS; +use crate::interpreters::Interpreter; use crate::interpreters::common::check_referenced_computed_columns; use crate::interpreters::interpreter_table_add_column::commit_table_meta; -use crate::interpreters::Interpreter; use crate::physical_plans::DistributedInsertSelect; use crate::physical_plans::PhysicalPlan; use crate::physical_plans::PhysicalPlanBuilder; @@ -113,7 +113,7 @@ impl ModifyTableColumnInterpreter { return Err(ErrorCode::UnknownDatamask(format!( "Data mask policy {} not found", mask_name - ))) + ))); } }; diff --git a/src/query/service/src/interpreters/interpreter_table_modify_comment.rs b/src/query/service/src/interpreters/interpreter_table_modify_comment.rs index 93197b485a824..a5144f0e4b540 100644 --- a/src/query/service/src/interpreters/interpreter_table_modify_comment.rs +++ b/src/query/service/src/interpreters/interpreter_table_modify_comment.rs @@ -22,8 +22,8 @@ use databend_common_sql::plans::ModifyTableCommentPlan; use databend_common_storages_basic::view_table::VIEW_ENGINE; use databend_common_storages_stream::stream_table::STREAM_ENGINE; -use crate::interpreters::interpreter_table_add_column::commit_table_meta; use crate::interpreters::Interpreter; +use crate::interpreters::interpreter_table_add_column::commit_table_meta; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; use crate::sessions::TableContext; diff --git a/src/query/service/src/interpreters/interpreter_table_modify_connection.rs b/src/query/service/src/interpreters/interpreter_table_modify_connection.rs index 1e16373d1f424..715470b95b323 100644 --- a/src/query/service/src/interpreters/interpreter_table_modify_connection.rs +++ b/src/query/service/src/interpreters/interpreter_table_modify_connection.rs @@ -27,8 +27,8 @@ use databend_common_storages_basic::view_table::VIEW_ENGINE; use databend_common_storages_stream::stream_table::STREAM_ENGINE; use log::debug; -use crate::interpreters::interpreter_table_add_column::commit_table_meta; use crate::interpreters::Interpreter; +use crate::interpreters::interpreter_table_add_column::commit_table_meta; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; use crate::sessions::TableContext; diff --git a/src/query/service/src/interpreters/interpreter_table_recluster.rs b/src/query/service/src/interpreters/interpreter_table_recluster.rs index d78c9622c96ae..4d11d198c70ea 100644 --- a/src/query/service/src/interpreters/interpreter_table_recluster.rs +++ b/src/query/service/src/interpreters/interpreter_table_recluster.rs @@ -27,30 +27,30 @@ use databend_common_catalog::table::Table; use databend_common_catalog::table::TableExt; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::type_check::check_function; use databend_common_expression::DataBlock; use databend_common_expression::Scalar; +use databend_common_expression::type_check::check_function; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_license::license::Feature; use databend_common_license::license_manager::LicenseManagerSwitch; use databend_common_meta_app::schema::TableInfo; -use databend_common_pipeline::core::always_callback; use databend_common_pipeline::core::ExecutionInfo; +use databend_common_pipeline::core::always_callback; +use databend_common_sql::IdentifierNormalizer; +use databend_common_sql::MetadataRef; +use databend_common_sql::NameResolutionContext; +use databend_common_sql::ScalarExpr; +use databend_common_sql::TypeChecker; use databend_common_sql::bind_table; use databend_common_sql::executor::cast_expr_to_non_null_boolean; use databend_common_sql::executor::physical_plans::FragmentKind; use databend_common_sql::executor::physical_plans::MutationKind; -use databend_common_sql::plans::plan_hilbert_sql; -use databend_common_sql::plans::replace_with_constant; -use databend_common_sql::plans::set_update_stream_columns; use databend_common_sql::plans::BoundColumnRef; use databend_common_sql::plans::Plan; use databend_common_sql::plans::ReclusterPlan; -use databend_common_sql::IdentifierNormalizer; -use databend_common_sql::MetadataRef; -use databend_common_sql::NameResolutionContext; -use databend_common_sql::ScalarExpr; -use databend_common_sql::TypeChecker; +use databend_common_sql::plans::plan_hilbert_sql; +use databend_common_sql::plans::replace_with_constant; +use databend_common_sql::plans::set_update_stream_columns; use databend_enterprise_hilbert_clustering::get_hilbert_clustering_handler; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use databend_storages_common_table_meta::meta::TableSnapshot; @@ -59,13 +59,12 @@ use derive_visitor::DriveMut; use log::error; use log::warn; +use crate::interpreters::Interpreter; +use crate::interpreters::InterpreterClusteringHistory; use crate::interpreters::hook::vacuum_hook::hook_clear_m_cte_temp_table; use crate::interpreters::hook::vacuum_hook::hook_disk_temp_dir; use crate::interpreters::hook::vacuum_hook::hook_vacuum_temp_files; use crate::interpreters::interpreter_insert_multi_table::scalar_expr_to_remote_expr; -use crate::interpreters::Interpreter; -use crate::interpreters::InterpreterClusteringHistory; -use crate::physical_plans::physical_plan_builder::PhysicalPlanBuilder; use crate::physical_plans::CommitSink; use crate::physical_plans::CommitType; use crate::physical_plans::CompactSource; @@ -75,11 +74,12 @@ use crate::physical_plans::PhysicalPlan; use crate::physical_plans::PhysicalPlanCast; use crate::physical_plans::PhysicalPlanMeta; use crate::physical_plans::Recluster; +use crate::physical_plans::physical_plan_builder::PhysicalPlanBuilder; +use crate::pipelines::PipelineBuildResult; use crate::pipelines::executor::ExecutorSettings; use crate::pipelines::executor::PipelineCompleteExecutor; -use crate::pipelines::PipelineBuildResult; -use crate::schedulers::build_query_pipeline_without_render_result_set; use crate::schedulers::ServiceQueryExecutor; +use crate::schedulers::build_query_pipeline_without_render_result_set; use crate::sessions::QueryContext; use crate::sessions::TableContext; diff --git a/src/query/service/src/interpreters/interpreter_table_rename_column.rs b/src/query/service/src/interpreters/interpreter_table_rename_column.rs index 2664d1c26a157..5f8107c8657fe 100644 --- a/src/query/service/src/interpreters/interpreter_table_rename_column.rs +++ b/src/query/service/src/interpreters/interpreter_table_rename_column.rs @@ -19,17 +19,17 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::DataSchema; use databend_common_meta_app::schema::DatabaseType; -use databend_common_sql::plans::RenameTableColumnPlan; use databend_common_sql::BloomIndexColumns; +use databend_common_sql::plans::RenameTableColumnPlan; use databend_common_storages_basic::view_table::VIEW_ENGINE; use databend_common_storages_iceberg::table::ICEBERG_ENGINE; use databend_common_storages_stream::stream_table::STREAM_ENGINE; use databend_storages_common_table_meta::table::OPT_KEY_BLOOM_INDEX_COLUMNS; +use crate::interpreters::Interpreter; use crate::interpreters::common::check_referenced_computed_columns; use crate::interpreters::interpreter_table_add_column::commit_table_meta; use crate::interpreters::interpreter_table_create::is_valid_column; -use crate::interpreters::Interpreter; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; use crate::sessions::TableContext; diff --git a/src/query/service/src/interpreters/interpreter_table_row_access_add.rs b/src/query/service/src/interpreters/interpreter_table_row_access_add.rs index b69a80c3b5cd5..3afb1a89ebb40 100644 --- a/src/query/service/src/interpreters/interpreter_table_row_access_add.rs +++ b/src/query/service/src/interpreters/interpreter_table_row_access_add.rs @@ -113,9 +113,10 @@ impl Interpreter for AddTableRowAccessPolicyInterpreter { if columns.len() != policy_data_types.len() { return Err(ErrorCode::UnmatchColumnDataType(format!( - "Number of columns ({}) does not match the number of row access policy arguments ({})", - columns.len(), policy_data_types.len() - ))); + "Number of columns ({}) does not match the number of row access policy arguments ({})", + columns.len(), + policy_data_types.len() + ))); } for (column, policy_data_type) in columns.iter().zip(policy_data_types.into_iter()) { diff --git a/src/query/service/src/interpreters/interpreter_table_set_options.rs b/src/query/service/src/interpreters/interpreter_table_set_options.rs index fd7ca4cb78caf..6514ec6d3c2af 100644 --- a/src/query/service/src/interpreters/interpreter_table_set_options.rs +++ b/src/query/service/src/interpreters/interpreter_table_set_options.rs @@ -27,20 +27,20 @@ use databend_common_meta_types::MatchSeq; use databend_common_pipeline::core::Pipeline; use databend_common_sql::plans::SetOptionsPlan; use databend_common_storages_factory::Table; -use databend_common_storages_fuse::io::read::RowOrientedSegmentReader; -use databend_common_storages_fuse::io::SegmentsIO; -use databend_common_storages_fuse::segment_format_from_location; +use databend_common_storages_fuse::FUSE_OPT_KEY_ENABLE_AUTO_ANALYZE; +use databend_common_storages_fuse::FUSE_OPT_KEY_ENABLE_AUTO_VACUUM; use databend_common_storages_fuse::FuseSegmentFormat; use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::TableContext; -use databend_common_storages_fuse::FUSE_OPT_KEY_ENABLE_AUTO_ANALYZE; -use databend_common_storages_fuse::FUSE_OPT_KEY_ENABLE_AUTO_VACUUM; -use databend_storages_common_table_meta::meta::column_oriented_segment::AbstractSegment; -use databend_storages_common_table_meta::meta::column_oriented_segment::ColumnOrientedSegmentBuilder; -use databend_storages_common_table_meta::meta::column_oriented_segment::SegmentBuilder; +use databend_common_storages_fuse::io::SegmentsIO; +use databend_common_storages_fuse::io::read::RowOrientedSegmentReader; +use databend_common_storages_fuse::segment_format_from_location; use databend_storages_common_table_meta::meta::SegmentInfo; use databend_storages_common_table_meta::meta::TableSnapshot; use databend_storages_common_table_meta::meta::Versioned; +use databend_storages_common_table_meta::meta::column_oriented_segment::AbstractSegment; +use databend_storages_common_table_meta::meta::column_oriented_segment::ColumnOrientedSegmentBuilder; +use databend_storages_common_table_meta::meta::column_oriented_segment::SegmentBuilder; use databend_storages_common_table_meta::table::OPT_KEY_CHANGE_TRACKING; use databend_storages_common_table_meta::table::OPT_KEY_CHANGE_TRACKING_BEGIN_VER; use databend_storages_common_table_meta::table::OPT_KEY_CLUSTER_TYPE; @@ -51,6 +51,7 @@ use databend_storages_common_table_meta::table::OPT_KEY_STORAGE_FORMAT; use databend_storages_common_table_meta::table::OPT_KEY_TEMP_PREFIX; use log::error; +use crate::interpreters::Interpreter; use crate::interpreters::common::table_option_validation::is_valid_approx_distinct_columns; use crate::interpreters::common::table_option_validation::is_valid_block_per_segment; use crate::interpreters::common::table_option_validation::is_valid_bloom_index_columns; @@ -59,10 +60,9 @@ use crate::interpreters::common::table_option_validation::is_valid_data_retentio use crate::interpreters::common::table_option_validation::is_valid_fuse_parquet_dictionary_opt; use crate::interpreters::common::table_option_validation::is_valid_option_of_type; use crate::interpreters::common::table_option_validation::is_valid_row_per_block; -use crate::interpreters::Interpreter; +use crate::pipelines::PipelineBuildResult; use crate::pipelines::executor::ExecutorSettings; use crate::pipelines::executor::PipelineCompleteExecutor; -use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; pub struct SetOptionsInterpreter { diff --git a/src/query/service/src/interpreters/interpreter_table_show_create.rs b/src/query/service/src/interpreters/interpreter_table_show_create.rs index ba903dd08e644..87c61511220de 100644 --- a/src/query/service/src/interpreters/interpreter_table_show_create.rs +++ b/src/query/service/src/interpreters/interpreter_table_show_create.rs @@ -14,30 +14,30 @@ use std::sync::Arc; -use databend_common_ast::ast::quote::display_ident; use databend_common_ast::ast::quote::QuotedString; +use databend_common_ast::ast::quote::display_ident; use databend_common_ast::parser::Dialect; use databend_common_catalog::catalog::Catalog; use databend_common_catalog::table::Table; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::StringType; use databend_common_expression::BlockEntry; use databend_common_expression::ComputedExpr; use databend_common_expression::DataBlock; +use databend_common_expression::types::StringType; use databend_common_meta_app::schema::TableInfo; use databend_common_sql::plans::ShowCreateTablePlan; use databend_common_storages_basic::view_table::QUERY; use databend_common_storages_basic::view_table::VIEW_ENGINE; use databend_common_storages_fuse::FUSE_OPT_KEY_ATTACH_COLUMN_IDS; -use databend_common_storages_stream::stream_table::StreamTable; use databend_common_storages_stream::stream_table::STREAM_ENGINE; -use databend_storages_common_table_meta::table::is_internal_opt_key; -use databend_storages_common_table_meta::table::StreamMode; +use databend_common_storages_stream::stream_table::StreamTable; use databend_storages_common_table_meta::table::OPT_KEY_CLUSTER_TYPE; use databend_storages_common_table_meta::table::OPT_KEY_STORAGE_PREFIX; use databend_storages_common_table_meta::table::OPT_KEY_TABLE_ATTACHED_DATA_URI; use databend_storages_common_table_meta::table::OPT_KEY_TEMP_PREFIX; +use databend_storages_common_table_meta::table::StreamMode; +use databend_storages_common_table_meta::table::is_internal_opt_key; use itertools::Itertools; use crate::interpreters::Interpreter; @@ -284,16 +284,15 @@ impl ShowCreateTableInterpreter { } if !hide_options_in_show_create_table || engine == "ICEBERG" || engine == "DELTA" { - table_create_sql.push_str({ - let mut opts = table_info.options().iter().collect::>(); - opts.sort_by_key(|(k, _)| *k); - opts.iter() - .filter(|(k, _)| !is_internal_opt_key(k)) - .map(|(k, v)| format!(" {}='{}'", k.to_uppercase(), v)) - .collect::>() - .join("") - .as_str() - }); + let mut opts = table_info.options().iter().collect::>(); + opts.sort_by_key(|(k, _)| *k); + let s = opts + .iter() + .filter(|(k, _)| !is_internal_opt_key(k)) + .map(|(k, v)| format!(" {}='{}'", k.to_uppercase(), v)) + .collect::>() + .join(""); + table_create_sql.push_str(&s); } if engine != "ICEBERG" && engine != "DELTA" { diff --git a/src/query/service/src/interpreters/interpreter_table_unset_options.rs b/src/query/service/src/interpreters/interpreter_table_unset_options.rs index 3b835d4c57f77..9ad195b7e4e58 100644 --- a/src/query/service/src/interpreters/interpreter_table_unset_options.rs +++ b/src/query/service/src/interpreters/interpreter_table_unset_options.rs @@ -23,8 +23,8 @@ use databend_common_meta_types::MatchSeq; use databend_common_sql::plans::UnsetOptionsPlan; use databend_common_storages_fuse::TableContext; -use crate::interpreters::common::table_option_validation::UNSET_TABLE_OPTIONS_WHITE_LIST; use crate::interpreters::Interpreter; +use crate::interpreters::common::table_option_validation::UNSET_TABLE_OPTIONS_WHITE_LIST; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/interpreter_table_vacuum.rs b/src/query/service/src/interpreters/interpreter_table_vacuum.rs index 78a05c04c683a..651db809eaa39 100644 --- a/src/query/service/src/interpreters/interpreter_table_vacuum.rs +++ b/src/query/service/src/interpreters/interpreter_table_vacuum.rs @@ -16,18 +16,18 @@ use std::sync::Arc; use databend_common_catalog::table::TableExt; use databend_common_exception::Result; -use databend_common_expression::types::StringType; -use databend_common_expression::types::UInt64Type; use databend_common_expression::DataBlock; use databend_common_expression::FromData; +use databend_common_expression::types::StringType; +use databend_common_expression::types::UInt64Type; use databend_common_license::license::Feature::Vacuum; use databend_common_license::license_manager::LicenseManagerSwitch; use databend_common_sql::plans::VacuumTablePlan; -use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::FUSE_TBL_BLOCK_PREFIX; use databend_common_storages_fuse::FUSE_TBL_SEGMENT_PREFIX; use databend_common_storages_fuse::FUSE_TBL_SNAPSHOT_PREFIX; use databend_common_storages_fuse::FUSE_TBL_XOR_BLOOM_INDEX_PREFIX; +use databend_common_storages_fuse::FuseTable; use databend_enterprise_vacuum_handler::get_vacuum_handler; use crate::interpreters::Interpreter; diff --git a/src/query/service/src/interpreters/interpreter_task_alter.rs b/src/query/service/src/interpreters/interpreter_task_alter.rs index 98d8911ad5bd9..89966f1a9b0b3 100644 --- a/src/query/service/src/interpreters/interpreter_task_alter.rs +++ b/src/query/service/src/interpreters/interpreter_task_alter.rs @@ -17,9 +17,9 @@ use std::sync::Arc; use databend_common_exception::Result; use databend_common_sql::plans::AlterTaskPlan; +use crate::interpreters::Interpreter; use crate::interpreters::task::TaskInterpreter; use crate::interpreters::task::TaskInterpreterManager; -use crate::interpreters::Interpreter; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/interpreter_task_create.rs b/src/query/service/src/interpreters/interpreter_task_create.rs index d6a31f17007f5..467fee2b9b081 100644 --- a/src/query/service/src/interpreters/interpreter_task_create.rs +++ b/src/query/service/src/interpreters/interpreter_task_create.rs @@ -17,9 +17,9 @@ use std::sync::Arc; use databend_common_exception::Result; use databend_common_sql::plans::CreateTaskPlan; +use crate::interpreters::Interpreter; use crate::interpreters::task::TaskInterpreter; use crate::interpreters::task::TaskInterpreterManager; -use crate::interpreters::Interpreter; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/interpreter_task_describe.rs b/src/query/service/src/interpreters/interpreter_task_describe.rs index a7796259cec2f..045a0144b76d1 100644 --- a/src/query/service/src/interpreters/interpreter_task_describe.rs +++ b/src/query/service/src/interpreters/interpreter_task_describe.rs @@ -18,9 +18,9 @@ use databend_common_exception::Result; use databend_common_sql::plans::DescribeTaskPlan; use databend_common_storages_system::parse_tasks_to_datablock; +use crate::interpreters::Interpreter; use crate::interpreters::task::TaskInterpreter; use crate::interpreters::task::TaskInterpreterManager; -use crate::interpreters::Interpreter; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/interpreter_task_drop.rs b/src/query/service/src/interpreters/interpreter_task_drop.rs index 6713c5a11725d..4d65d565f4c7e 100644 --- a/src/query/service/src/interpreters/interpreter_task_drop.rs +++ b/src/query/service/src/interpreters/interpreter_task_drop.rs @@ -17,9 +17,9 @@ use std::sync::Arc; use databend_common_exception::Result; use databend_common_sql::plans::DropTaskPlan; +use crate::interpreters::Interpreter; use crate::interpreters::task::TaskInterpreter; use crate::interpreters::task::TaskInterpreterManager; -use crate::interpreters::Interpreter; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/interpreter_task_execute.rs b/src/query/service/src/interpreters/interpreter_task_execute.rs index 70c12e67e4849..ad2addf6261e2 100644 --- a/src/query/service/src/interpreters/interpreter_task_execute.rs +++ b/src/query/service/src/interpreters/interpreter_task_execute.rs @@ -17,9 +17,9 @@ use std::sync::Arc; use databend_common_exception::Result; use databend_common_sql::plans::ExecuteTaskPlan; +use crate::interpreters::Interpreter; use crate::interpreters::task::TaskInterpreter; use crate::interpreters::task::TaskInterpreterManager; -use crate::interpreters::Interpreter; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/interpreter_tasks_show.rs b/src/query/service/src/interpreters/interpreter_tasks_show.rs index 1cc8e514ae474..24fb81ae7b4ee 100644 --- a/src/query/service/src/interpreters/interpreter_tasks_show.rs +++ b/src/query/service/src/interpreters/interpreter_tasks_show.rs @@ -18,9 +18,9 @@ use databend_common_exception::Result; use databend_common_sql::plans::ShowTasksPlan; use databend_common_storages_system::parse_tasks_to_datablock; +use crate::interpreters::Interpreter; use crate::interpreters::task::TaskInterpreter; use crate::interpreters::task::TaskInterpreterManager; -use crate::interpreters::Interpreter; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/interpreter_txn_commit.rs b/src/query/service/src/interpreters/interpreter_txn_commit.rs index 6f13fd58647f4..e47d73c6bfef7 100644 --- a/src/query/service/src/interpreters/interpreter_txn_commit.rs +++ b/src/query/service/src/interpreters/interpreter_txn_commit.rs @@ -23,9 +23,9 @@ use databend_common_meta_app::principal::StageInfo; use databend_common_metrics::storage::metrics_inc_copy_purge_files_cost_milliseconds; use databend_common_metrics::storage::metrics_inc_copy_purge_files_counter; use databend_common_storage::init_stage_operator; +use databend_common_storages_fuse::TableContext; use databend_common_storages_fuse::commit_with_backoff; use databend_common_storages_fuse::operations::vacuum_tables_from_info; -use databend_common_storages_fuse::TableContext; use databend_enterprise_vacuum_handler::VacuumHandlerWrapper; use databend_storages_common_io::Files; use databend_storages_common_session::TxnManagerRef; @@ -103,9 +103,13 @@ pub async fn execute_commit_statement(ctx: Arc) -> Result<()> if let Err(e) = vacuum_tables_from_info(tables_need_purge, ctx.clone(), handler).await { - warn!( "Failed to vacuum tables after transaction commit (best-effort operation): {e}"); + warn!( + "Failed to vacuum tables after transaction commit (best-effort operation): {e}" + ); } else { - info!( "{num_tables} tables vacuumed after transaction commit in a best-effort manner" ); + info!( + "{num_tables} tables vacuumed after transaction commit in a best-effort manner" + ); } } else { warn!("EE feature is not enabled, vacuum after transaction commit is skipped"); diff --git a/src/query/service/src/interpreters/interpreter_unassign_warehouse_nodes.rs b/src/query/service/src/interpreters/interpreter_unassign_warehouse_nodes.rs index 4f5b5d6cee0d3..c7b33f4ba867e 100644 --- a/src/query/service/src/interpreters/interpreter_unassign_warehouse_nodes.rs +++ b/src/query/service/src/interpreters/interpreter_unassign_warehouse_nodes.rs @@ -24,8 +24,8 @@ use databend_common_management::SelectedNode; use databend_common_sql::plans::UnassignWarehouseNodesPlan; use databend_enterprise_resources_management::ResourcesManagement; -use crate::interpreters::util::AuditElement; use crate::interpreters::Interpreter; +use crate::interpreters::util::AuditElement; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/interpreter_unset_workload_group_quotas.rs b/src/query/service/src/interpreters/interpreter_unset_workload_group_quotas.rs index 564260b2bff79..544e78835e985 100644 --- a/src/query/service/src/interpreters/interpreter_unset_workload_group_quotas.rs +++ b/src/query/service/src/interpreters/interpreter_unset_workload_group_quotas.rs @@ -23,8 +23,8 @@ use databend_common_management::WorkloadApi; use databend_common_management::WorkloadMgr; use databend_common_sql::plans::UnsetWorkloadGroupQuotasPlan; -use crate::interpreters::util::AuditElement; use crate::interpreters::Interpreter; +use crate::interpreters::util::AuditElement; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; diff --git a/src/query/service/src/interpreters/interpreter_user_desc.rs b/src/query/service/src/interpreters/interpreter_user_desc.rs index a1d03206343e8..858aee959f14f 100644 --- a/src/query/service/src/interpreters/interpreter_user_desc.rs +++ b/src/query/service/src/interpreters/interpreter_user_desc.rs @@ -16,10 +16,10 @@ use std::sync::Arc; use databend_common_base::base::GlobalInstance; use databend_common_exception::Result; -use databend_common_expression::types::BooleanType; -use databend_common_expression::types::StringType; use databend_common_expression::DataBlock; use databend_common_expression::FromData; +use databend_common_expression::types::BooleanType; +use databend_common_expression::types::StringType; use databend_common_management::WorkloadApi; use databend_common_management::WorkloadMgr; use databend_common_sql::plans::DescUserPlan; diff --git a/src/query/service/src/interpreters/interpreter_vacuum_drop_tables.rs b/src/query/service/src/interpreters/interpreter_vacuum_drop_tables.rs index 9a408bd41ff77..6b1808570e56f 100644 --- a/src/query/service/src/interpreters/interpreter_vacuum_drop_tables.rs +++ b/src/query/service/src/interpreters/interpreter_vacuum_drop_tables.rs @@ -22,10 +22,10 @@ use chrono::Duration; use databend_common_catalog::catalog::Catalog; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::StringType; -use databend_common_expression::types::UInt64Type; use databend_common_expression::DataBlock; use databend_common_expression::FromData; +use databend_common_expression::types::StringType; +use databend_common_expression::types::UInt64Type; use databend_common_license::license::Feature::Vacuum; use databend_common_license::license_manager::LicenseManagerSwitch; use databend_common_meta_api::GarbageCollectionApi; diff --git a/src/query/service/src/interpreters/interpreter_vacuum_temporary_files.rs b/src/query/service/src/interpreters/interpreter_vacuum_temporary_files.rs index 98ba5f4b55554..6cc04ac8f7b91 100644 --- a/src/query/service/src/interpreters/interpreter_vacuum_temporary_files.rs +++ b/src/query/service/src/interpreters/interpreter_vacuum_temporary_files.rs @@ -15,9 +15,9 @@ use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::types::UInt64Type; use databend_common_expression::DataBlock; use databend_common_expression::FromData; +use databend_common_expression::types::UInt64Type; use databend_common_license::license::Feature::Vacuum; use databend_common_license::license_manager::LicenseManagerSwitch; use databend_common_sql::plans::VacuumTemporaryFilesPlan; diff --git a/src/query/service/src/interpreters/interpreter_view_alter.rs b/src/query/service/src/interpreters/interpreter_view_alter.rs index 16b7c4ae460b9..a9e2e539a8874 100644 --- a/src/query/service/src/interpreters/interpreter_view_alter.rs +++ b/src/query/service/src/interpreters/interpreter_view_alter.rs @@ -19,8 +19,8 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_meta_app::schema::UpsertTableOptionReq; use databend_common_meta_types::MatchSeq; -use databend_common_sql::plans::AlterViewPlan; use databend_common_sql::Planner; +use databend_common_sql::plans::AlterViewPlan; use crate::interpreters::Interpreter; use crate::pipelines::PipelineBuildResult; diff --git a/src/query/service/src/interpreters/interpreter_view_create.rs b/src/query/service/src/interpreters/interpreter_view_create.rs index d9e92f6740273..3a505936f16d5 100644 --- a/src/query/service/src/interpreters/interpreter_view_create.rs +++ b/src/query/service/src/interpreters/interpreter_view_create.rs @@ -20,9 +20,9 @@ use databend_common_exception::Result; use databend_common_meta_app::schema::CreateTableReq; use databend_common_meta_app::schema::TableMeta; use databend_common_meta_app::schema::TableNameIdent; +use databend_common_sql::Planner; use databend_common_sql::plans::CreateViewPlan; use databend_common_sql::plans::Plan; -use databend_common_sql::Planner; use databend_common_storages_basic::view_table::QUERY; use databend_common_storages_basic::view_table::VIEW_ENGINE; diff --git a/src/query/service/src/interpreters/interpreter_view_describe.rs b/src/query/service/src/interpreters/interpreter_view_describe.rs index 3b2b08f4f6dfc..13d7e8a3c5662 100644 --- a/src/query/service/src/interpreters/interpreter_view_describe.rs +++ b/src/query/service/src/interpreters/interpreter_view_describe.rs @@ -16,18 +16,18 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::infer_table_schema; -use databend_common_expression::types::StringType; use databend_common_expression::DataBlock; use databend_common_expression::FromData; +use databend_common_expression::infer_table_schema; +use databend_common_expression::types::StringType; use databend_common_sql::plans::DescribeViewPlan; use databend_common_storages_basic::view_table::QUERY; use databend_common_storages_basic::view_table::VIEW_ENGINE; use databend_common_storages_fuse::TableContext; use databend_common_storages_stream::stream_table::STREAM_ENGINE; -use crate::interpreters::util::generate_desc_schema; use crate::interpreters::Interpreter; +use crate::interpreters::util::generate_desc_schema; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; use crate::sql::Planner; diff --git a/src/query/service/src/interpreters/mod.rs b/src/query/service/src/interpreters/mod.rs index 1d014c1965c99..4cf23cab10fe5 100644 --- a/src/query/service/src/interpreters/mod.rs +++ b/src/query/service/src/interpreters/mod.rs @@ -187,9 +187,9 @@ mod util; pub use access::ManagementModeAccess; pub use common::InterpreterQueryLog; pub use hook::HookOperator; -pub use interpreter::interpreter_plan_sql; pub use interpreter::Interpreter; pub use interpreter::InterpreterPtr; +pub use interpreter::interpreter_plan_sql; pub use interpreter_catalog_use::UseCatalogInterpreter; pub use interpreter_cluster_key_alter::AlterTableClusterKeyInterpreter; pub use interpreter_cluster_key_drop::DropTableClusterKeyInterpreter; diff --git a/src/query/service/src/interpreters/task/cloud.rs b/src/query/service/src/interpreters/task/cloud.rs index 0e389638ee955..5b2c54405e9d1 100644 --- a/src/query/service/src/interpreters/task/cloud.rs +++ b/src/query/service/src/interpreters/task/cloud.rs @@ -20,7 +20,6 @@ use databend_common_catalog::table_context::TableContext; use databend_common_cloud_control::client_config::make_request; use databend_common_cloud_control::cloud_api::CloudControlApiProvider; use databend_common_cloud_control::pb; -use databend_common_cloud_control::pb::alter_task_request::AlterTaskType; use databend_common_cloud_control::pb::AlterTaskRequest; use databend_common_cloud_control::pb::CreateTaskRequest; use databend_common_cloud_control::pb::DescribeTaskRequest; @@ -28,6 +27,7 @@ use databend_common_cloud_control::pb::DropTaskRequest; use databend_common_cloud_control::pb::ExecuteTaskRequest; use databend_common_cloud_control::pb::ShowTasksRequest; use databend_common_cloud_control::pb::WarehouseOptions; +use databend_common_cloud_control::pb::alter_task_request::AlterTaskType; use databend_common_cloud_control::task_utils; use databend_common_config::GlobalConfig; use databend_common_exception::ErrorCode; diff --git a/src/query/service/src/interpreters/task/private.rs b/src/query/service/src/interpreters/task/private.rs index 50ed7c43da273..2f819a3e9ca69 100644 --- a/src/query/service/src/interpreters/task/private.rs +++ b/src/query/service/src/interpreters/task/private.rs @@ -20,8 +20,8 @@ use databend_common_catalog::table_context::TableContext; use databend_common_cloud_control::task_utils; use databend_common_exception::Result; use databend_common_management::task::TaskMgr; -use databend_common_meta_app::principal::task::EMPTY_TASK_ID; use databend_common_meta_app::principal::Status; +use databend_common_meta_app::principal::task::EMPTY_TASK_ID; use databend_common_sql::plans::AlterTaskPlan; use databend_common_sql::plans::CreateTaskPlan; use databend_common_sql::plans::DescribeTaskPlan; diff --git a/src/query/service/src/interpreters/util.rs b/src/query/service/src/interpreters/util.rs index 5575c425505bc..492ac797d6107 100644 --- a/src/query/service/src/interpreters/util.rs +++ b/src/query/service/src/interpreters/util.rs @@ -15,27 +15,27 @@ use std::sync::Arc; use databend_common_ast::ast::Expr; +use databend_common_ast::parser::Dialect; use databend_common_ast::parser::parse_expr; use databend_common_ast::parser::tokenize_sql; -use databend_common_ast::parser::Dialect; use databend_common_catalog::catalog::Catalog; use databend_common_exception::ErrorCode; -use databend_common_expression::display::scalar_ref_to_string; use databend_common_expression::ComputedExpr; use databend_common_expression::DataBlock; use databend_common_expression::DataSchemaRef; use databend_common_expression::Scalar; use databend_common_expression::TableSchemaRef; -use databend_common_meta_app::principal::UserInfo; +use databend_common_expression::display::scalar_ref_to_string; use databend_common_meta_app::principal::SENSITIVE_SYSTEM_RESOURCE; -use databend_common_script::ir::ColumnAccess; +use databend_common_meta_app::principal::UserInfo; use databend_common_script::Client; +use databend_common_script::ir::ColumnAccess; use databend_common_sql::Planner; use futures_util::TryStreamExt; use itertools::Itertools; -use crate::interpreters::interpreter::auto_commit_if_not_allowed_in_transaction; use crate::interpreters::InterpreterFactory; +use crate::interpreters::interpreter::auto_commit_if_not_allowed_in_transaction; use crate::sessions::QueryContext; pub fn check_system_history( diff --git a/src/query/service/src/lib.rs b/src/query/service/src/lib.rs index d7684a3d87788..88d51d9171d46 100644 --- a/src/query/service/src/lib.rs +++ b/src/query/service/src/lib.rs @@ -14,8 +14,18 @@ #![feature(int_roundings)] #![allow(internal_features)] -#![allow(clippy::useless_asref)] -#![allow(clippy::uninlined_format_args)] +#![allow( + clippy::derivable_impls, + clippy::cloned_ref_to_slice_refs, + clippy::collapsible_if, + clippy::iter_kv_map, + clippy::let_and_return, + clippy::manual_is_multiple_of, + clippy::needless_return, + clippy::unnecessary_unwrap, + clippy::uninlined_format_args, + clippy::useless_asref +)] #![feature(iter_map_windows)] #![feature(core_intrinsics)] #![feature(arbitrary_self_types)] @@ -24,15 +34,12 @@ #![feature(trusted_len)] #![feature(box_patterns)] #![feature(sync_unsafe_cell)] -#![allow(elided_named_lifetimes)] -#![feature(result_flattening)] +#![allow(mismatched_lifetime_syntaxes)] #![feature(iterator_try_reduce)] #![feature(cursor_split)] -#![feature(vec_into_raw_parts)] #![allow(clippy::large_enum_variant)] #![feature(impl_trait_in_assoc_type)] #![feature(iterator_try_collect)] -#![feature(let_chains)] #![feature(try_blocks)] #![feature(variant_count)] #![feature(duration_constructors)] diff --git a/src/query/service/src/locks/lock_holder.rs b/src/query/service/src/locks/lock_holder.rs index 14c7a21dd96e8..39f3fe48abc10 100644 --- a/src/query/service/src/locks/lock_holder.rs +++ b/src/query/service/src/locks/lock_holder.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; -use std::sync::Arc; use std::time::Duration; use std::time::Instant; @@ -34,17 +34,17 @@ use databend_common_meta_app::schema::ExtendLockRevReq; use databend_common_meta_app::schema::ListLockRevReq; use databend_common_meta_app::schema::TableLockIdent; use databend_common_meta_kvapi::kvapi::Key; -use databend_common_meta_types::protobuf::watch_request::FilterType; use databend_common_meta_types::protobuf::WatchRequest; +use databend_common_meta_types::protobuf::watch_request::FilterType; use databend_common_metrics::lock::record_acquired_lock_nums; use databend_common_metrics::lock::record_created_lock_nums; use databend_common_storages_fuse::operations::set_backoff; use databend_common_users::UserApiProvider; -use futures::future::select; use futures::future::Either; +use futures::future::select; use futures_util::StreamExt; -use rand::thread_rng; use rand::Rng; +use rand::thread_rng; use crate::sessions::SessionManager; diff --git a/src/query/service/src/locks/lock_manager.rs b/src/query/service/src/locks/lock_manager.rs index 675bdb74dabf6..3aa144e934d92 100644 --- a/src/query/service/src/locks/lock_manager.rs +++ b/src/query/service/src/locks/lock_manager.rs @@ -16,8 +16,8 @@ use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; -use databend_common_base::base::tokio::sync::mpsc; use databend_common_base::base::GlobalInstance; +use databend_common_base::base::tokio::sync::mpsc; use databend_common_base::runtime::GlobalIORuntime; use databend_common_base::runtime::TrySpawn; use databend_common_catalog::lock::Lock; diff --git a/src/query/service/src/physical_plans/format/common.rs b/src/query/service/src/physical_plans/format/common.rs index b40ce4e69b5db..22e353080d9fb 100644 --- a/src/query/service/src/physical_plans/format/common.rs +++ b/src/query/service/src/physical_plans/format/common.rs @@ -20,14 +20,14 @@ use databend_common_base::runtime::profile::get_statistics_desc; use databend_common_catalog::plan::PartStatistics; use databend_common_catalog::runtime_filter_info::RuntimeFilterReport; use databend_common_expression::DataSchemaRef; -use databend_common_sql::executor::physical_plans::AggregateFunctionDesc; use databend_common_sql::IndexType; use databend_common_sql::Metadata; +use databend_common_sql::executor::physical_plans::AggregateFunctionDesc; -use crate::physical_plans::explain::PlanStatsInfo; -use crate::physical_plans::format::physical_format::PhysicalFormat; use crate::physical_plans::PhysicalPlanMeta; use crate::physical_plans::PhysicalRuntimeFilter; +use crate::physical_plans::explain::PlanStatsInfo; +use crate::physical_plans::format::physical_format::PhysicalFormat; pub struct FormatContext<'a> { pub metadata: &'a Metadata, diff --git a/src/query/service/src/physical_plans/format/format_add_stream_column.rs b/src/query/service/src/physical_plans/format/format_add_stream_column.rs index 2afbf288713a5..7ae54fa4a5271 100644 --- a/src/query/service/src/physical_plans/format/format_add_stream_column.rs +++ b/src/query/service/src/physical_plans/format/format_add_stream_column.rs @@ -15,11 +15,11 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::AddStreamColumn; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; pub struct AddStreamColumnFormatter<'a> { inner: &'a AddStreamColumn, diff --git a/src/query/service/src/physical_plans/format/format_aggregate_expand.rs b/src/query/service/src/physical_plans/format/format_aggregate_expand.rs index bad5c6b66a1aa..b6e9f2389129e 100644 --- a/src/query/service/src/physical_plans/format/format_aggregate_expand.rs +++ b/src/query/service/src/physical_plans/format/format_aggregate_expand.rs @@ -15,13 +15,13 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; -use crate::physical_plans::format::format_output_columns; -use crate::physical_plans::format::plan_stats_info_to_format_tree; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::AggregateExpand; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; +use crate::physical_plans::format::format_output_columns; +use crate::physical_plans::format::plan_stats_info_to_format_tree; pub struct AggregateExpandFormatter<'a> { inner: &'a AggregateExpand, diff --git a/src/query/service/src/physical_plans/format/format_aggregate_final.rs b/src/query/service/src/physical_plans/format/format_aggregate_final.rs index 042d2d99669f2..46ea747b46374 100644 --- a/src/query/service/src/physical_plans/format/format_aggregate_final.rs +++ b/src/query/service/src/physical_plans/format/format_aggregate_final.rs @@ -15,14 +15,14 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; -use crate::physical_plans::format::format_output_columns; -use crate::physical_plans::format::plan_stats_info_to_format_tree; -use crate::physical_plans::format::pretty_display_agg_desc; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::AggregateFinal; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; +use crate::physical_plans::format::format_output_columns; +use crate::physical_plans::format::plan_stats_info_to_format_tree; +use crate::physical_plans::format::pretty_display_agg_desc; pub struct AggregateFinalFormatter<'a> { inner: &'a AggregateFinal, diff --git a/src/query/service/src/physical_plans/format/format_aggregate_partial.rs b/src/query/service/src/physical_plans/format/format_aggregate_partial.rs index 0625179093a95..5a71bf5e480e2 100644 --- a/src/query/service/src/physical_plans/format/format_aggregate_partial.rs +++ b/src/query/service/src/physical_plans/format/format_aggregate_partial.rs @@ -16,13 +16,13 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; use itertools::Itertools; -use crate::physical_plans::format::plan_stats_info_to_format_tree; -use crate::physical_plans::format::pretty_display_agg_desc; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::AggregatePartial; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; +use crate::physical_plans::format::plan_stats_info_to_format_tree; +use crate::physical_plans::format::pretty_display_agg_desc; pub struct AggregatePartialFormatter<'a> { inner: &'a AggregatePartial, diff --git a/src/query/service/src/physical_plans/format/format_async_func.rs b/src/query/service/src/physical_plans/format/format_async_func.rs index c4ff5bffc5dfd..a880251f3d4ac 100644 --- a/src/query/service/src/physical_plans/format/format_async_func.rs +++ b/src/query/service/src/physical_plans/format/format_async_func.rs @@ -15,13 +15,13 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; -use crate::physical_plans::format::format_output_columns; -use crate::physical_plans::format::plan_stats_info_to_format_tree; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::AsyncFunction; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; +use crate::physical_plans::format::format_output_columns; +use crate::physical_plans::format::plan_stats_info_to_format_tree; pub struct AsyncFunctionFormatter<'a> { inner: &'a AsyncFunction, diff --git a/src/query/service/src/physical_plans/format/format_broadcast_sink.rs b/src/query/service/src/physical_plans/format/format_broadcast_sink.rs index 34027393a0a8b..f0f4e4cb2c7fb 100644 --- a/src/query/service/src/physical_plans/format/format_broadcast_sink.rs +++ b/src/query/service/src/physical_plans/format/format_broadcast_sink.rs @@ -15,11 +15,11 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::BroadcastSink; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; pub struct BroadcastSinkFormatter<'a> { inner: &'a BroadcastSink, diff --git a/src/query/service/src/physical_plans/format/format_cache_scan.rs b/src/query/service/src/physical_plans/format/format_cache_scan.rs index 99791bed4aa78..64acf5270921f 100644 --- a/src/query/service/src/physical_plans/format/format_cache_scan.rs +++ b/src/query/service/src/physical_plans/format/format_cache_scan.rs @@ -16,12 +16,12 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; use databend_common_sql::plans::CacheSource; -use crate::physical_plans::format::format_output_columns; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::CacheScan; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; +use crate::physical_plans::format::format_output_columns; pub struct CacheScanFormatter<'a> { inner: &'a CacheScan, diff --git a/src/query/service/src/physical_plans/format/format_chunk_cast_schema.rs b/src/query/service/src/physical_plans/format/format_chunk_cast_schema.rs index aeffb67137f38..5cd2565a549c5 100644 --- a/src/query/service/src/physical_plans/format/format_chunk_cast_schema.rs +++ b/src/query/service/src/physical_plans/format/format_chunk_cast_schema.rs @@ -15,11 +15,11 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::ChunkCastSchema; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; pub struct ChunkCastSchemaFormatter<'a> { inner: &'a ChunkCastSchema, diff --git a/src/query/service/src/physical_plans/format/format_chunk_eval_scalar.rs b/src/query/service/src/physical_plans/format/format_chunk_eval_scalar.rs index 4f116aa37769d..e76f7d2a79751 100644 --- a/src/query/service/src/physical_plans/format/format_chunk_eval_scalar.rs +++ b/src/query/service/src/physical_plans/format/format_chunk_eval_scalar.rs @@ -17,11 +17,11 @@ use databend_common_exception::Result; use databend_common_functions::BUILTIN_FUNCTIONS; use itertools::Itertools; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::ChunkEvalScalar; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; pub struct ChunkEvalScalarFormatter<'a> { inner: &'a ChunkEvalScalar, diff --git a/src/query/service/src/physical_plans/format/format_chunk_fill_and_reorder.rs b/src/query/service/src/physical_plans/format/format_chunk_fill_and_reorder.rs index afec3c770835e..26877cb40d01e 100644 --- a/src/query/service/src/physical_plans/format/format_chunk_fill_and_reorder.rs +++ b/src/query/service/src/physical_plans/format/format_chunk_fill_and_reorder.rs @@ -15,11 +15,11 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::ChunkFillAndReorder; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; pub struct ChunkFillAndReorderFormatter<'a> { inner: &'a ChunkFillAndReorder, diff --git a/src/query/service/src/physical_plans/format/format_chunk_filter.rs b/src/query/service/src/physical_plans/format/format_chunk_filter.rs index 76824360d4bd8..98745711d3726 100644 --- a/src/query/service/src/physical_plans/format/format_chunk_filter.rs +++ b/src/query/service/src/physical_plans/format/format_chunk_filter.rs @@ -16,11 +16,11 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; use databend_common_functions::BUILTIN_FUNCTIONS; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::ChunkFilter; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; pub struct ChunkFilterFormatter<'a> { inner: &'a ChunkFilter, diff --git a/src/query/service/src/physical_plans/format/format_chunk_merge.rs b/src/query/service/src/physical_plans/format/format_chunk_merge.rs index 3970ed2290a56..979b3bfba587a 100644 --- a/src/query/service/src/physical_plans/format/format_chunk_merge.rs +++ b/src/query/service/src/physical_plans/format/format_chunk_merge.rs @@ -15,11 +15,11 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::ChunkMerge; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; pub struct ChunkMergeFormatter<'a> { inner: &'a ChunkMerge, diff --git a/src/query/service/src/physical_plans/format/format_column_mutation.rs b/src/query/service/src/physical_plans/format/format_column_mutation.rs index fdc9ec4706ac9..2aac117fae45e 100644 --- a/src/query/service/src/physical_plans/format/format_column_mutation.rs +++ b/src/query/service/src/physical_plans/format/format_column_mutation.rs @@ -15,11 +15,11 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::ColumnMutation; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; pub struct ColumnMutationFormatter<'a> { inner: &'a ColumnMutation, diff --git a/src/query/service/src/physical_plans/format/format_constant_table_scan.rs b/src/query/service/src/physical_plans/format/format_constant_table_scan.rs index b6106050f6b10..f50f53ec25ff4 100644 --- a/src/query/service/src/physical_plans/format/format_constant_table_scan.rs +++ b/src/query/service/src/physical_plans/format/format_constant_table_scan.rs @@ -16,12 +16,12 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; use itertools::Itertools; -use crate::physical_plans::format::format_output_columns; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::ConstantTableScan; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; +use crate::physical_plans::format::format_output_columns; pub struct ConstantTableScanFormatter<'a> { inner: &'a ConstantTableScan, diff --git a/src/query/service/src/physical_plans/format/format_copy_into_table.rs b/src/query/service/src/physical_plans/format/format_copy_into_table.rs index 825c3a524e1d3..0cd9f314c9657 100644 --- a/src/query/service/src/physical_plans/format/format_copy_into_table.rs +++ b/src/query/service/src/physical_plans/format/format_copy_into_table.rs @@ -15,12 +15,12 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::CopyIntoTable; use crate::physical_plans::CopyIntoTableSource; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; pub struct CopyIntoTableFormatter<'a> { inner: &'a CopyIntoTable, diff --git a/src/query/service/src/physical_plans/format/format_cte_consumer.rs b/src/query/service/src/physical_plans/format/format_cte_consumer.rs index 40dc5f028419e..27b8f9bfa2c28 100644 --- a/src/query/service/src/physical_plans/format/format_cte_consumer.rs +++ b/src/query/service/src/physical_plans/format/format_cte_consumer.rs @@ -15,13 +15,13 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; -use crate::physical_plans::format::format_output_columns; -use crate::physical_plans::format::plan_stats_info_to_format_tree; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::MaterializeCTERef; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; +use crate::physical_plans::format::format_output_columns; +use crate::physical_plans::format::plan_stats_info_to_format_tree; pub struct MaterializeCTERefFormatter<'a> { inner: &'a MaterializeCTERef, diff --git a/src/query/service/src/physical_plans/format/format_duplicate.rs b/src/query/service/src/physical_plans/format/format_duplicate.rs index 93ac59dae42ac..dd2ea4a828fef 100644 --- a/src/query/service/src/physical_plans/format/format_duplicate.rs +++ b/src/query/service/src/physical_plans/format/format_duplicate.rs @@ -15,11 +15,11 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::Duplicate; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; pub struct DuplicateFormatter<'a> { inner: &'a Duplicate, diff --git a/src/query/service/src/physical_plans/format/format_eval_scalar.rs b/src/query/service/src/physical_plans/format/format_eval_scalar.rs index fac804227c98a..6cb7c0af0c976 100644 --- a/src/query/service/src/physical_plans/format/format_eval_scalar.rs +++ b/src/query/service/src/physical_plans/format/format_eval_scalar.rs @@ -16,13 +16,13 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; use databend_common_functions::BUILTIN_FUNCTIONS; -use crate::physical_plans::format::format_output_columns; -use crate::physical_plans::format::plan_stats_info_to_format_tree; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::EvalScalar; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; +use crate::physical_plans::format::format_output_columns; +use crate::physical_plans::format::plan_stats_info_to_format_tree; pub struct EvalScalarFormatter<'a> { inner: &'a EvalScalar, diff --git a/src/query/service/src/physical_plans/format/format_exchange.rs b/src/query/service/src/physical_plans/format/format_exchange.rs index c9f3e7648ec9d..32a9adde00108 100644 --- a/src/query/service/src/physical_plans/format/format_exchange.rs +++ b/src/query/service/src/physical_plans/format/format_exchange.rs @@ -17,12 +17,12 @@ use databend_common_exception::Result; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_sql::executor::physical_plans::FragmentKind; -use crate::physical_plans::format::format_output_columns; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::Exchange; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; +use crate::physical_plans::format::format_output_columns; pub struct ExchangeFormatter<'a> { inner: &'a Exchange, diff --git a/src/query/service/src/physical_plans/format/format_exchange_sink.rs b/src/query/service/src/physical_plans/format/format_exchange_sink.rs index 7a6877443e297..c9f718b5e9338 100644 --- a/src/query/service/src/physical_plans/format/format_exchange_sink.rs +++ b/src/query/service/src/physical_plans/format/format_exchange_sink.rs @@ -15,12 +15,12 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; -use crate::physical_plans::format::format_output_columns; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::ExchangeSink; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; +use crate::physical_plans::format::format_output_columns; pub struct ExchangeSinkFormatter<'a> { inner: &'a ExchangeSink, diff --git a/src/query/service/src/physical_plans/format/format_exchange_source.rs b/src/query/service/src/physical_plans/format/format_exchange_source.rs index c7fb4519cd95f..46190e0b2eecf 100644 --- a/src/query/service/src/physical_plans/format/format_exchange_source.rs +++ b/src/query/service/src/physical_plans/format/format_exchange_source.rs @@ -15,12 +15,12 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; -use crate::physical_plans::format::format_output_columns; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::ExchangeSource; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; +use crate::physical_plans::format::format_output_columns; pub struct ExchangeSourceFormatter<'a> { inner: &'a ExchangeSource, diff --git a/src/query/service/src/physical_plans/format/format_expression_scan.rs b/src/query/service/src/physical_plans/format/format_expression_scan.rs index cdaa09e9e4b8e..46594a119faef 100644 --- a/src/query/service/src/physical_plans/format/format_expression_scan.rs +++ b/src/query/service/src/physical_plans/format/format_expression_scan.rs @@ -17,12 +17,12 @@ use databend_common_exception::Result; use databend_common_functions::BUILTIN_FUNCTIONS; use itertools::Itertools; -use crate::physical_plans::format::format_output_columns; +use crate::physical_plans::IPhysicalPlan; +use crate::physical_plans::PhysicalPlanMeta; use crate::physical_plans::format::FormatContext; use crate::physical_plans::format::PhysicalFormat; +use crate::physical_plans::format::format_output_columns; use crate::physical_plans::physical_expression_scan::ExpressionScan; -use crate::physical_plans::IPhysicalPlan; -use crate::physical_plans::PhysicalPlanMeta; pub struct ExpressionScanFormatter<'a> { inner: &'a ExpressionScan, diff --git a/src/query/service/src/physical_plans/format/format_filter.rs b/src/query/service/src/physical_plans/format/format_filter.rs index 0641d4f920621..0a079c3d10716 100644 --- a/src/query/service/src/physical_plans/format/format_filter.rs +++ b/src/query/service/src/physical_plans/format/format_filter.rs @@ -17,14 +17,14 @@ use databend_common_exception::Result; use databend_common_functions::BUILTIN_FUNCTIONS; use itertools::Itertools; -use crate::physical_plans::format::append_output_rows_info; -use crate::physical_plans::format::format_output_columns; -use crate::physical_plans::format::plan_stats_info_to_format_tree; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::Filter; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; +use crate::physical_plans::format::append_output_rows_info; +use crate::physical_plans::format::format_output_columns; +use crate::physical_plans::format::plan_stats_info_to_format_tree; pub struct FilterFormatter<'a> { inner: &'a Filter, diff --git a/src/query/service/src/physical_plans/format/format_hash_join.rs b/src/query/service/src/physical_plans/format/format_hash_join.rs index c242702853cff..c902d3ae73988 100644 --- a/src/query/service/src/physical_plans/format/format_hash_join.rs +++ b/src/query/service/src/physical_plans/format/format_hash_join.rs @@ -17,14 +17,14 @@ use databend_common_exception::Result; use databend_common_functions::BUILTIN_FUNCTIONS; use itertools::Itertools; -use crate::physical_plans::format::append_output_rows_info; -use crate::physical_plans::format::format_output_columns; -use crate::physical_plans::format::plan_stats_info_to_format_tree; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::HashJoin; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; +use crate::physical_plans::format::append_output_rows_info; +use crate::physical_plans::format::format_output_columns; +use crate::physical_plans::format::plan_stats_info_to_format_tree; pub struct HashJoinFormatter<'a> { inner: &'a HashJoin, diff --git a/src/query/service/src/physical_plans/format/format_limit.rs b/src/query/service/src/physical_plans/format/format_limit.rs index 96b3a8d675ef9..32d7bcda75df6 100644 --- a/src/query/service/src/physical_plans/format/format_limit.rs +++ b/src/query/service/src/physical_plans/format/format_limit.rs @@ -15,13 +15,13 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; -use crate::physical_plans::format::format_output_columns; -use crate::physical_plans::format::plan_stats_info_to_format_tree; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::Limit; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; +use crate::physical_plans::format::format_output_columns; +use crate::physical_plans::format::plan_stats_info_to_format_tree; pub struct LimitFormatter<'a> { inner: &'a Limit, diff --git a/src/query/service/src/physical_plans/format/format_materialized_cte.rs b/src/query/service/src/physical_plans/format/format_materialized_cte.rs index 01eae37d9e4f3..1e83915492632 100644 --- a/src/query/service/src/physical_plans/format/format_materialized_cte.rs +++ b/src/query/service/src/physical_plans/format/format_materialized_cte.rs @@ -15,11 +15,11 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::MaterializedCTE; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; pub struct MaterializedCTEFormatter<'a> { inner: &'a MaterializedCTE, diff --git a/src/query/service/src/physical_plans/format/format_multi_table_insert.rs b/src/query/service/src/physical_plans/format/format_multi_table_insert.rs index b6d68d86ff702..4f52555fe8d1f 100644 --- a/src/query/service/src/physical_plans/format/format_multi_table_insert.rs +++ b/src/query/service/src/physical_plans/format/format_multi_table_insert.rs @@ -15,11 +15,11 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::ChunkAppendData; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; pub struct ChunkAppendDataFormatter<'a> { inner: &'a ChunkAppendData, diff --git a/src/query/service/src/physical_plans/format/format_mutation.rs b/src/query/service/src/physical_plans/format/format_mutation.rs index 607bc2c414455..5a10389b318a2 100644 --- a/src/query/service/src/physical_plans/format/format_mutation.rs +++ b/src/query/service/src/physical_plans/format/format_mutation.rs @@ -15,11 +15,11 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::Mutation; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; pub struct MutationFormatter<'a> { inner: &'a Mutation, diff --git a/src/query/service/src/physical_plans/format/format_mutation_into_organize.rs b/src/query/service/src/physical_plans/format/format_mutation_into_organize.rs index 24ad6ba14534e..c0243e91a1f10 100644 --- a/src/query/service/src/physical_plans/format/format_mutation_into_organize.rs +++ b/src/query/service/src/physical_plans/format/format_mutation_into_organize.rs @@ -15,11 +15,11 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::MutationOrganize; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; pub struct MutationOrganizeFormatter<'a> { inner: &'a MutationOrganize, diff --git a/src/query/service/src/physical_plans/format/format_mutation_into_split.rs b/src/query/service/src/physical_plans/format/format_mutation_into_split.rs index b7125bb5ac2a4..b55b8989107e4 100644 --- a/src/query/service/src/physical_plans/format/format_mutation_into_split.rs +++ b/src/query/service/src/physical_plans/format/format_mutation_into_split.rs @@ -15,11 +15,11 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::MutationSplit; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; pub struct MutationSplitFormatter<'a> { inner: &'a MutationSplit, diff --git a/src/query/service/src/physical_plans/format/format_mutation_manipulate.rs b/src/query/service/src/physical_plans/format/format_mutation_manipulate.rs index b039700a7c8c8..1aaadf283d942 100644 --- a/src/query/service/src/physical_plans/format/format_mutation_manipulate.rs +++ b/src/query/service/src/physical_plans/format/format_mutation_manipulate.rs @@ -17,11 +17,11 @@ use databend_common_exception::Result; use databend_common_functions::BUILTIN_FUNCTIONS; use itertools::Itertools; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::MutationManipulate; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; pub struct MutationManipulateFormatter<'a> { inner: &'a MutationManipulate, diff --git a/src/query/service/src/physical_plans/format/format_mutation_source.rs b/src/query/service/src/physical_plans/format/format_mutation_source.rs index 7749f1607f659..7b8ce92e9f9fe 100644 --- a/src/query/service/src/physical_plans/format/format_mutation_source.rs +++ b/src/query/service/src/physical_plans/format/format_mutation_source.rs @@ -17,14 +17,14 @@ use databend_common_exception::Result; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_sql::binder::MutationType; -use crate::physical_plans::format::append_output_rows_info; -use crate::physical_plans::format::format_output_columns; -use crate::physical_plans::format::part_stats_info_to_format_tree; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::MutationSource; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; +use crate::physical_plans::format::append_output_rows_info; +use crate::physical_plans::format::format_output_columns; +use crate::physical_plans::format::part_stats_info_to_format_tree; pub struct MutationSourceFormatter<'a> { inner: &'a MutationSource, diff --git a/src/query/service/src/physical_plans/format/format_project_set.rs b/src/query/service/src/physical_plans/format/format_project_set.rs index 5ff4ae89adb51..d0940fa1e9b03 100644 --- a/src/query/service/src/physical_plans/format/format_project_set.rs +++ b/src/query/service/src/physical_plans/format/format_project_set.rs @@ -16,13 +16,13 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; use databend_common_functions::BUILTIN_FUNCTIONS; -use crate::physical_plans::format::format_output_columns; -use crate::physical_plans::format::plan_stats_info_to_format_tree; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; use crate::physical_plans::ProjectSet; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; +use crate::physical_plans::format::format_output_columns; +use crate::physical_plans::format::plan_stats_info_to_format_tree; pub struct ProjectSetFormatter<'a> { inner: &'a ProjectSet, diff --git a/src/query/service/src/physical_plans/format/format_range_join.rs b/src/query/service/src/physical_plans/format/format_range_join.rs index 369805e7a579a..8ff2fafef0dd1 100644 --- a/src/query/service/src/physical_plans/format/format_range_join.rs +++ b/src/query/service/src/physical_plans/format/format_range_join.rs @@ -16,15 +16,15 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; use databend_common_functions::BUILTIN_FUNCTIONS; -use crate::physical_plans::format::append_output_rows_info; -use crate::physical_plans::format::format_output_columns; -use crate::physical_plans::format::plan_stats_info_to_format_tree; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; use crate::physical_plans::RangeJoin; use crate::physical_plans::RangeJoinType; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; +use crate::physical_plans::format::append_output_rows_info; +use crate::physical_plans::format::format_output_columns; +use crate::physical_plans::format::plan_stats_info_to_format_tree; pub struct RangeJoinFormatter<'a> { inner: &'a RangeJoin, diff --git a/src/query/service/src/physical_plans/format/format_replace_into.rs b/src/query/service/src/physical_plans/format/format_replace_into.rs index 645ddbb6c3491..23afd4cb7f0e6 100644 --- a/src/query/service/src/physical_plans/format/format_replace_into.rs +++ b/src/query/service/src/physical_plans/format/format_replace_into.rs @@ -15,11 +15,11 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; use crate::physical_plans::ReplaceInto; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; pub struct ReplaceIntoFormatter<'a> { inner: &'a ReplaceInto, diff --git a/src/query/service/src/physical_plans/format/format_row_fetch.rs b/src/query/service/src/physical_plans/format/format_row_fetch.rs index f188eab1d3ab5..d4db21fbf419a 100644 --- a/src/query/service/src/physical_plans/format/format_row_fetch.rs +++ b/src/query/service/src/physical_plans/format/format_row_fetch.rs @@ -16,13 +16,13 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; use itertools::Itertools; -use crate::physical_plans::format::format_output_columns; -use crate::physical_plans::format::plan_stats_info_to_format_tree; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; use crate::physical_plans::RowFetch; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; +use crate::physical_plans::format::format_output_columns; +use crate::physical_plans::format::plan_stats_info_to_format_tree; pub struct RowFetchFormatter<'a> { inner: &'a RowFetch, diff --git a/src/query/service/src/physical_plans/format/format_secure_filter.rs b/src/query/service/src/physical_plans/format/format_secure_filter.rs index f32dce22261ed..c19f3eeb2fe7a 100644 --- a/src/query/service/src/physical_plans/format/format_secure_filter.rs +++ b/src/query/service/src/physical_plans/format/format_secure_filter.rs @@ -17,14 +17,14 @@ use databend_common_exception::Result; use databend_common_functions::BUILTIN_FUNCTIONS; use itertools::Itertools; -use crate::physical_plans::format::append_output_rows_info; -use crate::physical_plans::format::format_output_columns; -use crate::physical_plans::format::plan_stats_info_to_format_tree; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; use crate::physical_plans::SecureFilter; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; +use crate::physical_plans::format::append_output_rows_info; +use crate::physical_plans::format::format_output_columns; +use crate::physical_plans::format::plan_stats_info_to_format_tree; pub struct SecureFilterFormatter<'a> { inner: &'a SecureFilter, diff --git a/src/query/service/src/physical_plans/format/format_shuffle.rs b/src/query/service/src/physical_plans/format/format_shuffle.rs index 0eb6fe6845e52..f0bc3e10595e8 100644 --- a/src/query/service/src/physical_plans/format/format_shuffle.rs +++ b/src/query/service/src/physical_plans/format/format_shuffle.rs @@ -15,11 +15,11 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; use crate::physical_plans::Shuffle; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; pub struct ShuffleFormatter<'a> { inner: &'a Shuffle, diff --git a/src/query/service/src/physical_plans/format/format_sort.rs b/src/query/service/src/physical_plans/format/format_sort.rs index b1d8be13086ff..633b68eeafa5d 100644 --- a/src/query/service/src/physical_plans/format/format_sort.rs +++ b/src/query/service/src/physical_plans/format/format_sort.rs @@ -15,13 +15,13 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; -use crate::physical_plans::format::format_output_columns; -use crate::physical_plans::format::plan_stats_info_to_format_tree; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; use crate::physical_plans::Sort; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; +use crate::physical_plans::format::format_output_columns; +use crate::physical_plans::format::plan_stats_info_to_format_tree; pub struct SortFormatter<'a> { inner: &'a Sort, diff --git a/src/query/service/src/physical_plans/format/format_table_scan.rs b/src/query/service/src/physical_plans/format/format_table_scan.rs index 83515e96e89b8..06db799713c28 100644 --- a/src/query/service/src/physical_plans/format/format_table_scan.rs +++ b/src/query/service/src/physical_plans/format/format_table_scan.rs @@ -18,15 +18,15 @@ use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_sql::DUMMY_TABLE_INDEX; use itertools::Itertools; +use crate::physical_plans::IPhysicalPlan; +use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::TableScan; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::format::append_output_rows_info; use crate::physical_plans::format::format_output_columns; use crate::physical_plans::format::part_stats_info_to_format_tree; use crate::physical_plans::format::plan_stats_info_to_format_tree; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; -use crate::physical_plans::IPhysicalPlan; -use crate::physical_plans::PhysicalPlanMeta; -use crate::physical_plans::TableScan; pub struct TableScanFormatter<'a> { inner: &'a TableScan, diff --git a/src/query/service/src/physical_plans/format/format_udf.rs b/src/query/service/src/physical_plans/format/format_udf.rs index fd97d7ff33946..38775f727e64d 100644 --- a/src/query/service/src/physical_plans/format/format_udf.rs +++ b/src/query/service/src/physical_plans/format/format_udf.rs @@ -15,13 +15,13 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; -use crate::physical_plans::format::format_output_columns; -use crate::physical_plans::format::plan_stats_info_to_format_tree; +use crate::physical_plans::IPhysicalPlan; +use crate::physical_plans::PhysicalPlanMeta; use crate::physical_plans::format::FormatContext; use crate::physical_plans::format::PhysicalFormat; +use crate::physical_plans::format::format_output_columns; +use crate::physical_plans::format::plan_stats_info_to_format_tree; use crate::physical_plans::physical_udf::Udf; -use crate::physical_plans::IPhysicalPlan; -use crate::physical_plans::PhysicalPlanMeta; pub struct UdfFormatter<'a> { inner: &'a Udf, diff --git a/src/query/service/src/physical_plans/format/format_union_all.rs b/src/query/service/src/physical_plans/format/format_union_all.rs index c5d827eac5a1b..9808c9c2c90b5 100644 --- a/src/query/service/src/physical_plans/format/format_union_all.rs +++ b/src/query/service/src/physical_plans/format/format_union_all.rs @@ -15,14 +15,14 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; -use crate::physical_plans::format::append_output_rows_info; -use crate::physical_plans::format::format_output_columns; -use crate::physical_plans::format::plan_stats_info_to_format_tree; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; use crate::physical_plans::UnionAll; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; +use crate::physical_plans::format::append_output_rows_info; +use crate::physical_plans::format::format_output_columns; +use crate::physical_plans::format::plan_stats_info_to_format_tree; pub struct UnionAllFormatter<'a> { inner: &'a UnionAll, diff --git a/src/query/service/src/physical_plans/format/format_window.rs b/src/query/service/src/physical_plans/format/format_window.rs index 312f7e2cafa77..274d8cd08d088 100644 --- a/src/query/service/src/physical_plans/format/format_window.rs +++ b/src/query/service/src/physical_plans/format/format_window.rs @@ -15,14 +15,14 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; -use crate::physical_plans::format::format_output_columns; -use crate::physical_plans::format::pretty_display_agg_desc; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; use crate::physical_plans::Window; use crate::physical_plans::WindowFunction; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; +use crate::physical_plans::format::format_output_columns; +use crate::physical_plans::format::pretty_display_agg_desc; pub struct WindowFormatter<'a> { inner: &'a Window, diff --git a/src/query/service/src/physical_plans/format/format_window_partition.rs b/src/query/service/src/physical_plans/format/format_window_partition.rs index 6d6cc8cafcdfa..2e71a13fe97ec 100644 --- a/src/query/service/src/physical_plans/format/format_window_partition.rs +++ b/src/query/service/src/physical_plans/format/format_window_partition.rs @@ -15,13 +15,13 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; -use crate::physical_plans::format::format_output_columns; -use crate::physical_plans::format::plan_stats_info_to_format_tree; -use crate::physical_plans::format::FormatContext; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlanMeta; use crate::physical_plans::WindowPartition; +use crate::physical_plans::format::FormatContext; +use crate::physical_plans::format::PhysicalFormat; +use crate::physical_plans::format::format_output_columns; +use crate::physical_plans::format::plan_stats_info_to_format_tree; pub struct WindowPartitionFormatter<'a> { inner: &'a WindowPartition, diff --git a/src/query/service/src/physical_plans/format/physical_format.rs b/src/query/service/src/physical_plans/format/physical_format.rs index 4952783c16985..0d96916898113 100644 --- a/src/query/service/src/physical_plans/format/physical_format.rs +++ b/src/query/service/src/physical_plans/format/physical_format.rs @@ -16,8 +16,8 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_base::runtime::profile::get_statistics_desc; use databend_common_exception::Result; -use crate::physical_plans::format::FormatContext; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::format::FormatContext; pub trait PhysicalFormat { #[recursive::recursive] diff --git a/src/query/service/src/physical_plans/mod.rs b/src/query/service/src/physical_plans/mod.rs index a6008acbace85..d5780e1ec116f 100644 --- a/src/query/service/src/physical_plans/mod.rs +++ b/src/query/service/src/physical_plans/mod.rs @@ -46,7 +46,6 @@ mod physical_project_set; mod physical_r_cte_scan; mod physical_range_join; mod physical_recluster; -mod physical_refresh_index; mod physical_replace_async_source; mod physical_replace_deduplicate; mod physical_replace_into; diff --git a/src/query/service/src/physical_plans/physical_add_stream_column.rs b/src/query/service/src/physical_plans/physical_add_stream_column.rs index 68c6d31adf2aa..6b91b0703470b 100644 --- a/src/query/service/src/physical_plans/physical_add_stream_column.rs +++ b/src/query/service/src/physical_plans/physical_add_stream_column.rs @@ -19,25 +19,25 @@ use databend_common_catalog::plan::StreamColumn; use databend_common_catalog::plan::StreamColumnType; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::RemoteExpr; use databend_common_expression::ORIGIN_BLOCK_ID_COL_NAME; use databend_common_expression::ORIGIN_BLOCK_ROW_NUM_COL_NAME; use databend_common_expression::ORIGIN_VERSION_COL_NAME; +use databend_common_expression::RemoteExpr; use databend_common_functions::BUILTIN_FUNCTIONS; -use databend_common_pipeline_transforms::columns::TransformAddStreamColumns; use databend_common_pipeline_transforms::TransformPipelineHelper; -use databend_common_sql::evaluator::BlockOperator; -use databend_common_sql::plans::BoundColumnRef; -use databend_common_sql::plans::ConstantExpr; -use databend_common_sql::plans::FunctionCall; +use databend_common_pipeline_transforms::columns::TransformAddStreamColumns; use databend_common_sql::Binder; +use databend_common_sql::CURRENT_BLOCK_ID_COL_NAME; +use databend_common_sql::CURRENT_BLOCK_ROW_NUM_COL_NAME; use databend_common_sql::ColumnBindingBuilder; use databend_common_sql::MetadataRef; use databend_common_sql::ScalarExpr; use databend_common_sql::StreamContext; use databend_common_sql::Visibility; -use databend_common_sql::CURRENT_BLOCK_ID_COL_NAME; -use databend_common_sql::CURRENT_BLOCK_ROW_NUM_COL_NAME; +use databend_common_sql::evaluator::BlockOperator; +use databend_common_sql::plans::BoundColumnRef; +use databend_common_sql::plans::ConstantExpr; +use databend_common_sql::plans::FunctionCall; use crate::physical_plans::format::AddStreamColumnFormatter; use crate::physical_plans::format::PhysicalFormat; diff --git a/src/query/service/src/physical_plans/physical_aggregate_expand.rs b/src/query/service/src/physical_plans/physical_aggregate_expand.rs index 54750cc1231b4..9f5a6ce5069d2 100644 --- a/src/query/service/src/physical_plans/physical_aggregate_expand.rs +++ b/src/query/service/src/physical_plans/physical_aggregate_expand.rs @@ -15,14 +15,14 @@ use std::any::Any; use databend_common_exception::Result; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; use databend_common_pipeline_transforms::TransformPipelineHelper; -use databend_common_sql::plans::GroupingSets; use databend_common_sql::IndexType; +use databend_common_sql::plans::GroupingSets; use crate::physical_plans::explain::PlanStatsInfo; use crate::physical_plans::format::AggregateExpandFormatter; @@ -30,8 +30,8 @@ use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::physical_plan::IPhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlanMeta; -use crate::pipelines::processors::transforms::aggregator::TransformExpandGroupingSets; use crate::pipelines::PipelineBuilder; +use crate::pipelines::processors::transforms::aggregator::TransformExpandGroupingSets; /// Add dummy data before `GROUPING SETS`. #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] diff --git a/src/query/service/src/physical_plans/physical_aggregate_final.rs b/src/query/service/src/physical_plans/physical_aggregate_final.rs index 0f8bb7fe8d3fe..a87b6b6b8cc6f 100644 --- a/src/query/service/src/physical_plans/physical_aggregate_final.rs +++ b/src/query/service/src/physical_plans/physical_aggregate_final.rs @@ -22,6 +22,9 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use databend_common_expression::RemoteExpr; use databend_common_pipeline::core::ProcessorPtr; +use databend_common_sql::ColumnSet; +use databend_common_sql::IndexType; +use databend_common_sql::ScalarExpr; use databend_common_sql::executor::physical_plans::AggregateFunctionDesc; use databend_common_sql::executor::physical_plans::AggregateFunctionSignature; use databend_common_sql::executor::physical_plans::SortDesc; @@ -29,9 +32,6 @@ use databend_common_sql::optimizer::ir::SExpr; use databend_common_sql::plans::Aggregate; use databend_common_sql::plans::AggregateMode; use databend_common_sql::plans::ConstantTableScan; -use databend_common_sql::ColumnSet; -use databend_common_sql::IndexType; -use databend_common_sql::ScalarExpr; use itertools::Itertools; use super::AggregateExpand; @@ -46,10 +46,10 @@ use crate::physical_plans::physical_plan::IPhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlanMeta; use crate::physical_plans::physical_plan_builder::PhysicalPlanBuilder; -use crate::pipelines::processors::transforms::aggregator::build_partition_bucket; +use crate::pipelines::PipelineBuilder; use crate::pipelines::processors::transforms::aggregator::AggregateInjector; use crate::pipelines::processors::transforms::aggregator::FinalSingleStateAggregator; -use crate::pipelines::PipelineBuilder; +use crate::pipelines::processors::transforms::aggregator::build_partition_bucket; #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] pub struct AggregateFinal { diff --git a/src/query/service/src/physical_plans/physical_aggregate_partial.rs b/src/query/service/src/physical_plans/physical_aggregate_partial.rs index 447fd8a04bf4a..fbaeb0157d5a1 100644 --- a/src/query/service/src/physical_plans/physical_aggregate_partial.rs +++ b/src/query/service/src/physical_plans/physical_aggregate_partial.rs @@ -18,7 +18,6 @@ use std::sync::Arc; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::DataType; #[allow(unused_imports)] use databend_common_expression::DataBlock; use databend_common_expression::DataField; @@ -26,15 +25,16 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use databend_common_expression::HashTableConfig; use databend_common_expression::LimitType; -use databend_common_expression::SortColumnDescription; use databend_common_expression::MAX_AGGREGATE_HASHTABLE_BUCKETS_NUM; +use databend_common_expression::SortColumnDescription; +use databend_common_expression::types::DataType; use databend_common_functions::aggregates::AggregateFunctionFactory; use databend_common_pipeline::core::ProcessorPtr; -use databend_common_pipeline_transforms::sorts::TransformSortPartial; use databend_common_pipeline_transforms::TransformPipelineHelper; +use databend_common_pipeline_transforms::sorts::TransformSortPartial; +use databend_common_sql::IndexType; use databend_common_sql::executor::physical_plans::AggregateFunctionDesc; use databend_common_sql::executor::physical_plans::SortDesc; -use databend_common_sql::IndexType; use databend_common_storage::DataOperator; use itertools::Itertools; @@ -45,13 +45,13 @@ use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::physical_plan::IPhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlanMeta; +use crate::pipelines::PipelineBuilder; use crate::pipelines::processors::transforms::aggregator::AggregateInjector; use crate::pipelines::processors::transforms::aggregator::NewTransformPartialAggregate; use crate::pipelines::processors::transforms::aggregator::PartialSingleStateAggregator; use crate::pipelines::processors::transforms::aggregator::SharedPartitionStream; use crate::pipelines::processors::transforms::aggregator::TransformAggregateSpillWriter; use crate::pipelines::processors::transforms::aggregator::TransformPartialAggregate; -use crate::pipelines::PipelineBuilder; #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] pub struct AggregatePartial { diff --git a/src/query/service/src/physical_plans/physical_async_func.rs b/src/query/service/src/physical_plans/physical_async_func.rs index c7f7b8ed355d1..d3595c2436225 100644 --- a/src/query/service/src/physical_plans/physical_async_func.rs +++ b/src/query/service/src/physical_plans/physical_async_func.rs @@ -21,10 +21,10 @@ use databend_common_expression::DataField; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use databend_common_pipeline_transforms::TransformPipelineHelper; -use databend_common_sql::binder::AsyncFunctionDesc; -use databend_common_sql::optimizer::ir::SExpr; use databend_common_sql::ColumnSet; use databend_common_sql::ScalarExpr; +use databend_common_sql::binder::AsyncFunctionDesc; +use databend_common_sql::optimizer::ir::SExpr; use itertools::Itertools; use crate::physical_plans::explain::PlanStatsInfo; @@ -34,8 +34,8 @@ use crate::physical_plans::physical_plan::IPhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlanMeta; use crate::physical_plans::physical_plan_builder::PhysicalPlanBuilder; -use crate::pipelines::processors::transforms::TransformAsyncFunction; use crate::pipelines::PipelineBuilder; +use crate::pipelines::processors::transforms::TransformAsyncFunction; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct AsyncFunction { diff --git a/src/query/service/src/physical_plans/physical_broadcast.rs b/src/query/service/src/physical_plans/physical_broadcast.rs index 615dfd7f30934..88f961c5103ed 100644 --- a/src/query/service/src/physical_plans/physical_broadcast.rs +++ b/src/query/service/src/physical_plans/physical_broadcast.rs @@ -25,9 +25,9 @@ use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::physical_plan::IPhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlanMeta; +use crate::pipelines::PipelineBuilder; use crate::pipelines::processors::transforms::BroadcastSinkProcessor; use crate::pipelines::processors::transforms::BroadcastSourceProcessor; -use crate::pipelines::PipelineBuilder; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct BroadcastSource { diff --git a/src/query/service/src/physical_plans/physical_cache_scan.rs b/src/query/service/src/physical_plans/physical_cache_scan.rs index d713d27a4e8ca..6988ab483fbdf 100644 --- a/src/query/service/src/physical_plans/physical_cache_scan.rs +++ b/src/query/service/src/physical_plans/physical_cache_scan.rs @@ -18,8 +18,8 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; -use databend_common_sql::plans::CacheSource; use databend_common_sql::ColumnSet; +use databend_common_sql::plans::CacheSource; use crate::physical_plans::format::CacheScanFormatter; use crate::physical_plans::format::PhysicalFormat; @@ -27,12 +27,12 @@ use crate::physical_plans::physical_plan::IPhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlanMeta; use crate::physical_plans::physical_plan_builder::PhysicalPlanBuilder; +use crate::pipelines::HashJoinStateRef; +use crate::pipelines::PipelineBuilder; use crate::pipelines::processors::transforms::CacheSourceState; use crate::pipelines::processors::transforms::HashJoinCacheState; use crate::pipelines::processors::transforms::NewHashJoinCacheState; use crate::pipelines::processors::transforms::TransformCacheScan; -use crate::pipelines::HashJoinStateRef; -use crate::pipelines::PipelineBuilder; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct CacheScan { diff --git a/src/query/service/src/physical_plans/physical_column_mutation.rs b/src/query/service/src/physical_plans/physical_column_mutation.rs index 34861184c22db..c1bb9b709ec8b 100644 --- a/src/query/service/src/physical_plans/physical_column_mutation.rs +++ b/src/query/service/src/physical_plans/physical_column_mutation.rs @@ -22,12 +22,12 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::RemoteExpr; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::schema::TableInfo; -use databend_common_pipeline_transforms::blocks::CompoundBlockOperator; use databend_common_pipeline_transforms::TransformPipelineHelper; +use databend_common_pipeline_transforms::blocks::CompoundBlockOperator; use databend_common_sql::evaluator::BlockOperator; use databend_common_sql::executor::physical_plans::MutationKind; -use databend_common_storages_fuse::operations::TransformSerializeBlock; use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::operations::TransformSerializeBlock; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use crate::physical_plans::format::ColumnMutationFormatter; diff --git a/src/query/service/src/physical_plans/physical_commit_sink.rs b/src/query/service/src/physical_plans/physical_commit_sink.rs index d4914f3b77e99..c8f584484b0f0 100644 --- a/src/query/service/src/physical_plans/physical_commit_sink.rs +++ b/src/query/service/src/physical_plans/physical_commit_sink.rs @@ -25,11 +25,11 @@ use databend_common_pipeline::core::ExecutionInfo; use databend_common_pipeline_transforms::TransformPipelineHelper; use databend_common_sql::executor::physical_plans::MutationKind; use databend_common_sql::plans::TruncateMode; +use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::operations::MutationGenerator; use databend_common_storages_fuse::operations::TableMutationAggregator; use databend_common_storages_fuse::operations::TransformMergeCommitMeta; use databend_common_storages_fuse::operations::TruncateGenerator; -use databend_common_storages_fuse::FuseTable; use databend_storages_common_table_meta::meta::BlockHLLState; use databend_storages_common_table_meta::meta::ExtendedBlockMeta; use databend_storages_common_table_meta::meta::TableMetaTimestamps; diff --git a/src/query/service/src/physical_plans/physical_compact_source.rs b/src/query/service/src/physical_plans/physical_compact_source.rs index 51ffd91adef43..52fa41a4b8533 100644 --- a/src/query/service/src/physical_plans/physical_compact_source.rs +++ b/src/query/service/src/physical_plans/physical_compact_source.rs @@ -30,24 +30,24 @@ use databend_common_meta_app::schema::TableInfo; use databend_common_pipeline::sources::EmptySource; use databend_common_pipeline::sources::PrefetchAsyncSourcer; use databend_common_pipeline_transforms::TransformPipelineHelper; +use databend_common_sql::StreamContext; use databend_common_sql::executor::physical_plans::FragmentKind; use databend_common_sql::executor::physical_plans::MutationKind; -use databend_common_sql::StreamContext; +use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::operations::BlockCompactMutator; use databend_common_storages_fuse::operations::CompactLazyPartInfo; use databend_common_storages_fuse::operations::CompactTransform; use databend_common_storages_fuse::operations::TableMutationAggregator; use databend_common_storages_fuse::operations::TransformSerializeBlock; -use databend_common_storages_fuse::FuseTable; use databend_storages_common_table_meta::meta::TableMetaTimestamps; -use crate::physical_plans::physical_plan::IPhysicalPlan; -use crate::physical_plans::physical_plan::PhysicalPlan; -use crate::physical_plans::physical_plan::PhysicalPlanMeta; use crate::physical_plans::CommitSink; use crate::physical_plans::CommitType; use crate::physical_plans::Exchange; use crate::physical_plans::PhysicalPlanBuilder; +use crate::physical_plans::physical_plan::IPhysicalPlan; +use crate::physical_plans::physical_plan::PhysicalPlan; +use crate::physical_plans::physical_plan::PhysicalPlanMeta; use crate::pipelines::PipelineBuilder; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] diff --git a/src/query/service/src/physical_plans/physical_constant_table_scan.rs b/src/query/service/src/physical_plans/physical_constant_table_scan.rs index e371fa23e549d..fd9865d9e3c8b 100644 --- a/src/query/service/src/physical_plans/physical_constant_table_scan.rs +++ b/src/query/service/src/physical_plans/physical_constant_table_scan.rs @@ -22,12 +22,12 @@ use databend_common_pipeline::sources::OneBlockSource; use databend_common_sql::ColumnSet; use databend_common_sql::IndexType; +use crate::physical_plans::PhysicalPlanBuilder; use crate::physical_plans::format::ConstantTableScanFormatter; use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::physical_plan::IPhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlanMeta; -use crate::physical_plans::PhysicalPlanBuilder; use crate::pipelines::PipelineBuilder; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -101,13 +101,14 @@ impl PhysicalPlanBuilder { scan: &databend_common_sql::plans::ConstantTableScan, required: ColumnSet, ) -> Result { - debug_assert!(scan - .schema - .fields - .iter() - .map(|field| field.name().parse::().unwrap()) - .collect::() - .is_superset(&scan.columns)); + debug_assert!( + scan.schema + .fields + .iter() + .map(|field| field.name().parse::().unwrap()) + .collect::() + .is_superset(&scan.columns) + ); let used: ColumnSet = required.intersection(&scan.columns).copied().collect(); if used.len() < scan.columns.len() { diff --git a/src/query/service/src/physical_plans/physical_copy_into_location.rs b/src/query/service/src/physical_plans/physical_copy_into_location.rs index 3348f08dd17f3..982beaa49d70c 100644 --- a/src/query/service/src/physical_plans/physical_copy_into_location.rs +++ b/src/query/service/src/physical_plans/physical_copy_into_location.rs @@ -19,12 +19,12 @@ use databend_common_base::base::Version; use databend_common_catalog::plan::DataSourcePlan; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use databend_common_expression::TableSchemaRef; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; use databend_common_sql::ColumnBinding; use databend_common_storages_stage::StageSinkTable; use databend_storages_common_stage::CopyIntoLocationInfo; diff --git a/src/query/service/src/physical_plans/physical_copy_into_table.rs b/src/query/service/src/physical_plans/physical_copy_into_table.rs index f3bd4c78c7dbb..a5b3bdaa34e3a 100644 --- a/src/query/service/src/physical_plans/physical_copy_into_table.rs +++ b/src/query/service/src/physical_plans/physical_copy_into_table.rs @@ -22,9 +22,9 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use databend_common_expression::Scalar; use databend_common_meta_app::schema::TableInfo; +use databend_common_sql::ColumnBinding; use databend_common_sql::plans::CopyIntoTableMode; use databend_common_sql::plans::ValidationMode; -use databend_common_sql::ColumnBinding; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use crate::physical_plans::format::CopyIntoTableFormatter; diff --git a/src/query/service/src/physical_plans/physical_cte_consumer.rs b/src/query/service/src/physical_plans/physical_cte_consumer.rs index 602eb9e3d943e..9d221cf5fee05 100644 --- a/src/query/service/src/physical_plans/physical_cte_consumer.rs +++ b/src/query/service/src/physical_plans/physical_cte_consumer.rs @@ -19,15 +19,15 @@ use databend_common_expression::DataField; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; -use crate::physical_plans::explain::PlanStatsInfo; -use crate::physical_plans::format::MaterializeCTERefFormatter; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlan; use crate::physical_plans::PhysicalPlanBuilder; use crate::physical_plans::PhysicalPlanMeta; -use crate::pipelines::processors::transforms::CTESource; +use crate::physical_plans::explain::PlanStatsInfo; +use crate::physical_plans::format::MaterializeCTERefFormatter; +use crate::physical_plans::format::PhysicalFormat; use crate::pipelines::PipelineBuilder; +use crate::pipelines::processors::transforms::CTESource; /// This is a leaf operator that consumes the result of a materialized CTE. #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] diff --git a/src/query/service/src/physical_plans/physical_distributed_insert_select.rs b/src/query/service/src/physical_plans/physical_distributed_insert_select.rs index 07db2b1af9b62..3c7280778f4a6 100644 --- a/src/query/service/src/physical_plans/physical_distributed_insert_select.rs +++ b/src/query/service/src/physical_plans/physical_distributed_insert_select.rs @@ -18,8 +18,8 @@ use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use databend_common_meta_app::schema::TableInfo; -use databend_common_pipeline_transforms::blocks::TransformCastSchema; use databend_common_pipeline_transforms::TransformPipelineHelper; +use databend_common_pipeline_transforms::blocks::TransformCastSchema; use databend_common_sql::ColumnBinding; use databend_storages_common_table_meta::meta::TableMetaTimestamps; diff --git a/src/query/service/src/physical_plans/physical_eval_scalar.rs b/src/query/service/src/physical_plans/physical_eval_scalar.rs index e3714bf6641c4..dfa77296fbf31 100644 --- a/src/query/service/src/physical_plans/physical_eval_scalar.rs +++ b/src/query/service/src/physical_plans/physical_eval_scalar.rs @@ -27,8 +27,11 @@ use databend_common_expression::DataSchemaRefExt; use databend_common_expression::RemoteExpr; use databend_common_expression::Scalar; use databend_common_functions::BUILTIN_FUNCTIONS; -use databend_common_pipeline_transforms::blocks::CompoundBlockOperator; use databend_common_pipeline_transforms::TransformPipelineHelper; +use databend_common_pipeline_transforms::blocks::CompoundBlockOperator; +use databend_common_sql::ColumnSet; +use databend_common_sql::IndexType; +use databend_common_sql::TypeCheck; use databend_common_sql::evaluator::BlockOperator; use databend_common_sql::optimizer::ir::Matcher; use databend_common_sql::optimizer::ir::SExpr; @@ -40,9 +43,6 @@ use databend_common_sql::plans::RelOperator; use databend_common_sql::plans::ScalarExpr; use databend_common_sql::plans::ScalarItem; use databend_common_sql::plans::Visitor; -use databend_common_sql::ColumnSet; -use databend_common_sql::IndexType; -use databend_common_sql::TypeCheck; use itertools::Itertools; use crate::physical_plans::explain::PlanStatsInfo; @@ -384,11 +384,7 @@ impl PhysicalPlanBuilder { } } } - if has_flatten { - Some(project_set) - } else { - None - } + if has_flatten { Some(project_set) } else { None } } } diff --git a/src/query/service/src/physical_plans/physical_exchange.rs b/src/query/service/src/physical_plans/physical_exchange.rs index 0aca8deef5ab0..ba19faecbb359 100644 --- a/src/query/service/src/physical_plans/physical_exchange.rs +++ b/src/query/service/src/physical_plans/physical_exchange.rs @@ -19,17 +19,17 @@ use databend_common_exception::Result; use databend_common_expression::ConstantFolder; use databend_common_expression::RemoteExpr; use databend_common_functions::BUILTIN_FUNCTIONS; -use databend_common_sql::executor::physical_plans::FragmentKind; -use databend_common_sql::optimizer::ir::SExpr; use databend_common_sql::ColumnSet; use databend_common_sql::TypeCheck; +use databend_common_sql::executor::physical_plans::FragmentKind; +use databend_common_sql::optimizer::ir::SExpr; +use crate::physical_plans::PhysicalPlanBuilder; use crate::physical_plans::format::ExchangeFormatter; use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::physical_plan::IPhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlanMeta; -use crate::physical_plans::PhysicalPlanBuilder; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct Exchange { diff --git a/src/query/service/src/physical_plans/physical_expression_scan.rs b/src/query/service/src/physical_plans/physical_expression_scan.rs index 8b29f5e16c228..8d13ffbe9ed7f 100644 --- a/src/query/service/src/physical_plans/physical_expression_scan.rs +++ b/src/query/service/src/physical_plans/physical_expression_scan.rs @@ -20,18 +20,18 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::RemoteExpr; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_pipeline::core::ProcessorPtr; -use databend_common_sql::optimizer::ir::SExpr; use databend_common_sql::ColumnSet; use databend_common_sql::TypeCheck; +use databend_common_sql::optimizer::ir::SExpr; +use crate::physical_plans::PhysicalPlanBuilder; use crate::physical_plans::format::ExpressionScanFormatter; use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::physical_plan::IPhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlanMeta; -use crate::physical_plans::PhysicalPlanBuilder; -use crate::pipelines::processors::transforms::TransformExpressionScan; use crate::pipelines::PipelineBuilder; +use crate::pipelines::processors::transforms::TransformExpressionScan; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ExpressionScan { diff --git a/src/query/service/src/physical_plans/physical_filter.rs b/src/query/service/src/physical_plans/physical_filter.rs index 8efa752e7d23b..d9d0f15d9edff 100644 --- a/src/query/service/src/physical_plans/physical_filter.rs +++ b/src/query/service/src/physical_plans/physical_filter.rs @@ -22,18 +22,18 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use databend_common_expression::RemoteExpr; use databend_common_functions::BUILTIN_FUNCTIONS; -use databend_common_sql::executor::cast_expr_to_non_null_boolean; -use databend_common_sql::optimizer::ir::SExpr; use databend_common_sql::ColumnSet; use databend_common_sql::TypeCheck; +use databend_common_sql::executor::cast_expr_to_non_null_boolean; +use databend_common_sql::optimizer::ir::SExpr; +use crate::physical_plans::PhysicalPlanBuilder; use crate::physical_plans::explain::PlanStatsInfo; use crate::physical_plans::format::FilterFormatter; use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::physical_plan::IPhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlanMeta; -use crate::physical_plans::PhysicalPlanBuilder; use crate::pipelines::PipelineBuilder; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] diff --git a/src/query/service/src/physical_plans/physical_hash_join.rs b/src/query/service/src/physical_plans/physical_hash_join.rs index f5c1bbef6464b..9a2e5802f105f 100644 --- a/src/query/service/src/physical_plans/physical_hash_join.rs +++ b/src/query/service/src/physical_plans/physical_hash_join.rs @@ -19,33 +19,35 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::type_check::check_cast; -use databend_common_expression::type_check::common_super_type; -use databend_common_expression::types::DataType; use databend_common_expression::ConstantFolder; use databend_common_expression::DataBlock; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use databend_common_expression::RemoteExpr; +use databend_common_expression::type_check::check_cast; +use databend_common_expression::type_check::common_super_type; +use databend_common_expression::types::DataType; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_pipeline::core::InputPort; use databend_common_pipeline::core::OutputPort; use databend_common_pipeline::core::Pipe; use databend_common_pipeline::core::PipeItem; use databend_common_pipeline::core::ProcessorPtr; -use databend_common_sql::optimizer::ir::SExpr; -use databend_common_sql::plans::Join; -use databend_common_sql::plans::JoinType; use databend_common_sql::ColumnEntry; use databend_common_sql::ColumnSet; use databend_common_sql::IndexType; use databend_common_sql::ScalarExpr; use databend_common_sql::TypeCheck; +use databend_common_sql::optimizer::ir::SExpr; +use databend_common_sql::plans::Join; +use databend_common_sql::plans::JoinType; use tokio::sync::Barrier; -use super::runtime_filter::PhysicalRuntimeFilters; use super::PhysicalPlanCast; +use super::runtime_filter::PhysicalRuntimeFilters; +use crate::physical_plans::Exchange; +use crate::physical_plans::PhysicalPlanBuilder; use crate::physical_plans::explain::PlanStatsInfo; use crate::physical_plans::format::HashJoinFormatter; use crate::physical_plans::format::PhysicalFormat; @@ -53,19 +55,17 @@ use crate::physical_plans::physical_plan::IPhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlanMeta; use crate::physical_plans::runtime_filter::build_runtime_filter; -use crate::physical_plans::Exchange; -use crate::physical_plans::PhysicalPlanBuilder; +use crate::pipelines::HashJoinStateRef; +use crate::pipelines::PipelineBuilder; +use crate::pipelines::processors::HashJoinBuildState; +use crate::pipelines::processors::HashJoinDesc; +use crate::pipelines::processors::HashJoinState; use crate::pipelines::processors::transforms::HashJoinFactory; use crate::pipelines::processors::transforms::HashJoinProbeState; use crate::pipelines::processors::transforms::RuntimeFiltersDesc; use crate::pipelines::processors::transforms::TransformHashJoin; use crate::pipelines::processors::transforms::TransformHashJoinBuild; use crate::pipelines::processors::transforms::TransformHashJoinProbe; -use crate::pipelines::processors::HashJoinBuildState; -use crate::pipelines::processors::HashJoinDesc; -use crate::pipelines::processors::HashJoinState; -use crate::pipelines::HashJoinStateRef; -use crate::pipelines::PipelineBuilder; // Type aliases to simplify complex return types type JoinConditionsResult = ( diff --git a/src/query/service/src/physical_plans/physical_join.rs b/src/query/service/src/physical_plans/physical_join.rs index bc48246312e38..1d21ace7e8b53 100644 --- a/src/query/service/src/physical_plans/physical_join.rs +++ b/src/query/service/src/physical_plans/physical_join.rs @@ -14,18 +14,18 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_sql::ColumnSet; +use databend_common_sql::ScalarExpr; use databend_common_sql::binder::is_range_join_condition; use databend_common_sql::optimizer::ir::RelExpr; use databend_common_sql::optimizer::ir::SExpr; use databend_common_sql::plans::FunctionCall; use databend_common_sql::plans::Join; use databend_common_sql::plans::JoinType; -use databend_common_sql::ColumnSet; -use databend_common_sql::ScalarExpr; +use crate::physical_plans::PhysicalPlanBuilder; use crate::physical_plans::explain::PlanStatsInfo; use crate::physical_plans::physical_plan::PhysicalPlan; -use crate::physical_plans::PhysicalPlanBuilder; enum PhysicalJoinType { Hash, diff --git a/src/query/service/src/physical_plans/physical_limit.rs b/src/query/service/src/physical_plans/physical_limit.rs index 0eaca599acbd3..33dd52a60d495 100644 --- a/src/query/service/src/physical_plans/physical_limit.rs +++ b/src/query/service/src/physical_plans/physical_limit.rs @@ -22,10 +22,11 @@ use databend_common_expression::DataField; use databend_common_expression::ROW_ID_COL_NAME; use databend_common_pipeline::core::ProcessorPtr; use databend_common_pipeline_transforms::filters::TransformLimit; -use databend_common_sql::optimizer::ir::SExpr; use databend_common_sql::ColumnEntry; use databend_common_sql::ColumnSet; +use databend_common_sql::optimizer::ir::SExpr; +use crate::physical_plans::PhysicalPlanBuilder; use crate::physical_plans::explain::PlanStatsInfo; use crate::physical_plans::format::LimitFormatter; use crate::physical_plans::format::PhysicalFormat; @@ -33,7 +34,6 @@ use crate::physical_plans::physical_plan::IPhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlanMeta; use crate::physical_plans::physical_row_fetch::RowFetch; -use crate::physical_plans::PhysicalPlanBuilder; use crate::pipelines::PipelineBuilder; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] diff --git a/src/query/service/src/physical_plans/physical_materialized_cte.rs b/src/query/service/src/physical_plans/physical_materialized_cte.rs index 22f9c2ae7a8f4..fc6f6b476db38 100644 --- a/src/query/service/src/physical_plans/physical_materialized_cte.rs +++ b/src/query/service/src/physical_plans/physical_materialized_cte.rs @@ -16,19 +16,19 @@ use std::any::Any; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; +use databend_common_sql::ColumnBinding; use databend_common_sql::optimizer::ir::RelExpr; use databend_common_sql::optimizer::ir::SExpr; -use databend_common_sql::ColumnBinding; -use crate::physical_plans::explain::PlanStatsInfo; -use crate::physical_plans::format::MaterializedCTEFormatter; -use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlan; use crate::physical_plans::PhysicalPlanBuilder; use crate::physical_plans::PhysicalPlanMeta; -use crate::pipelines::processors::transforms::MaterializedCteSink; +use crate::physical_plans::explain::PlanStatsInfo; +use crate::physical_plans::format::MaterializedCTEFormatter; +use crate::physical_plans::format::PhysicalFormat; use crate::pipelines::PipelineBuilder; +use crate::pipelines::processors::transforms::MaterializedCteSink; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct MaterializedCTE { diff --git a/src/query/service/src/physical_plans/physical_multi_table_insert.rs b/src/query/service/src/physical_plans/physical_multi_table_insert.rs index 9bd08c7ca6763..d9959e5b95a11 100644 --- a/src/query/service/src/physical_plans/physical_multi_table_insert.rs +++ b/src/query/service/src/physical_plans/physical_multi_table_insert.rs @@ -34,8 +34,8 @@ use databend_common_pipeline::sinks::AsyncSinker; use databend_common_pipeline_transforms::blocks::CompoundBlockOperator; use databend_common_pipeline_transforms::sorts::TransformSortPartial; use databend_common_sql::ColumnSet; -use databend_common_storages_fuse::operations::CommitMultiTableInsert; use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::operations::CommitMultiTableInsert; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use crate::physical_plans::format::ChunkAppendDataFormatter; diff --git a/src/query/service/src/physical_plans/physical_mutation.rs b/src/query/service/src/physical_plans/physical_mutation.rs index f6f49d93f53a7..8ce678a6d1c5a 100644 --- a/src/query/service/src/physical_plans/physical_mutation.rs +++ b/src/query/service/src/physical_plans/physical_mutation.rs @@ -22,30 +22,40 @@ use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::expr::*; -use databend_common_expression::type_check::check_cast; -use databend_common_expression::type_check::check_function; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; use databend_common_expression::ComputedExpr; use databend_common_expression::ConstantFolder; use databend_common_expression::DataField; use databend_common_expression::DataSchema; use databend_common_expression::DataSchemaRef; use databend_common_expression::FieldIndex; +use databend_common_expression::PREDICATE_COLUMN_NAME; +use databend_common_expression::ROW_ID_COL_NAME; use databend_common_expression::RemoteExpr; use databend_common_expression::Scalar; use databend_common_expression::TableSchema; -use databend_common_expression::PREDICATE_COLUMN_NAME; -use databend_common_expression::ROW_ID_COL_NAME; +use databend_common_expression::expr::*; +use databend_common_expression::type_check::check_cast; +use databend_common_expression::type_check::check_function; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::schema::TableInfo; use databend_common_pipeline::core::InputPort; use databend_common_pipeline::core::OutputPort; use databend_common_pipeline::core::Pipe; -use databend_common_sql::binder::wrap_cast; +use databend_common_sql::BindContext; +use databend_common_sql::ColumnBindingBuilder; +use databend_common_sql::ColumnEntry; +use databend_common_sql::ColumnSet; +use databend_common_sql::DUMMY_COLUMN_INDEX; +use databend_common_sql::IndexType; +use databend_common_sql::MetadataRef; +use databend_common_sql::ScalarExpr; +use databend_common_sql::TypeCheck; +use databend_common_sql::Visibility; use databend_common_sql::binder::MutationStrategy; use databend_common_sql::binder::MutationType; +use databend_common_sql::binder::wrap_cast; use databend_common_sql::executor::physical_plans::FragmentKind; use databend_common_sql::executor::physical_plans::MutationKind; use databend_common_sql::optimizer::ir::SExpr; @@ -54,32 +64,17 @@ use databend_common_sql::plans::BoundColumnRef; use databend_common_sql::plans::ConstantExpr; use databend_common_sql::plans::FunctionCall; use databend_common_sql::plans::TruncateMode; -use databend_common_sql::BindContext; -use databend_common_sql::ColumnBindingBuilder; -use databend_common_sql::ColumnEntry; -use databend_common_sql::ColumnSet; -use databend_common_sql::IndexType; -use databend_common_sql::MetadataRef; -use databend_common_sql::ScalarExpr; -use databend_common_sql::TypeCheck; -use databend_common_sql::Visibility; -use databend_common_sql::DUMMY_COLUMN_INDEX; -use databend_common_storages_fuse::operations::TransformSerializeBlock; use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::operations::TransformSerializeBlock; use databend_storages_common_table_meta::meta::Location; -use databend_storages_common_table_meta::meta::TableMetaTimestamps; use databend_storages_common_table_meta::meta::NUM_BLOCK_ID_BITS; +use databend_storages_common_table_meta::meta::TableMetaTimestamps; use databend_storages_common_table_meta::readers::snapshot_reader::TableSnapshotAccessor; use itertools::Itertools; use tokio::sync::Semaphore; use super::ColumnMutation; use super::CommitType; -use crate::physical_plans::format::MutationFormatter; -use crate::physical_plans::format::PhysicalFormat; -use crate::physical_plans::physical_plan::IPhysicalPlan; -use crate::physical_plans::physical_plan::PhysicalPlan; -use crate::physical_plans::physical_plan::PhysicalPlanMeta; use crate::physical_plans::CommitSink; use crate::physical_plans::Exchange; use crate::physical_plans::MutationManipulate; @@ -87,6 +82,11 @@ use crate::physical_plans::MutationOrganize; use crate::physical_plans::MutationSplit; use crate::physical_plans::PhysicalPlanBuilder; use crate::physical_plans::RowFetch; +use crate::physical_plans::format::MutationFormatter; +use crate::physical_plans::format::PhysicalFormat; +use crate::physical_plans::physical_plan::IPhysicalPlan; +use crate::physical_plans::physical_plan::PhysicalPlan; +use crate::physical_plans::physical_plan::PhysicalPlanMeta; use crate::pipelines::PipelineBuilder; // The predicate_column_index should not be conflict with update expr's column_binding's index. diff --git a/src/query/service/src/physical_plans/physical_mutation_source.rs b/src/query/service/src/physical_plans/physical_mutation_source.rs index 5924b417a5119..68ab0c632f610 100644 --- a/src/query/service/src/physical_plans/physical_mutation_source.rs +++ b/src/query/service/src/physical_plans/physical_mutation_source.rs @@ -24,39 +24,39 @@ use databend_common_catalog::plan::Projection; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::type_check::check_function; -use databend_common_expression::types::DataType; use databend_common_expression::ConstantFolder; use databend_common_expression::DataBlock; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use databend_common_expression::FunctionContext; +use databend_common_expression::type_check::check_function; +use databend_common_expression::types::DataType; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::schema::TableInfo; use databend_common_pipeline::sources::OneBlockSource; -use databend_common_pipeline_transforms::columns::TransformAddStreamColumns; use databend_common_pipeline_transforms::TransformPipelineHelper; -use databend_common_sql::binder::MutationType; -use databend_common_sql::executor::cast_expr_to_non_null_boolean; +use databend_common_pipeline_transforms::columns::TransformAddStreamColumns; use databend_common_sql::ColumnSet; use databend_common_sql::IndexType; use databend_common_sql::ScalarExpr; use databend_common_sql::StreamContext; +use databend_common_sql::binder::MutationType; +use databend_common_sql::executor::cast_expr_to_non_null_boolean; +use databend_common_storages_fuse::FuseLazyPartInfo; +use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::SegmentLocation; use databend_common_storages_fuse::operations::CommitMeta; use databend_common_storages_fuse::operations::ConflictResolveContext; use databend_common_storages_fuse::operations::MutationAction; use databend_common_storages_fuse::operations::MutationBlockPruningContext; -use databend_common_storages_fuse::FuseLazyPartInfo; -use databend_common_storages_fuse::FuseTable; -use databend_common_storages_fuse::SegmentLocation; +use crate::physical_plans::PhysicalPlanBuilder; use crate::physical_plans::format::MutationSourceFormatter; use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::physical_plan::IPhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlanMeta; -use crate::physical_plans::PhysicalPlanBuilder; use crate::pipelines::PipelineBuilder; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] diff --git a/src/query/service/src/physical_plans/physical_plan.rs b/src/query/service/src/physical_plans/physical_plan.rs index 19a06f1b4fc29..27cc04c134d07 100644 --- a/src/query/service/src/physical_plans/physical_plan.rs +++ b/src/query/service/src/physical_plans/physical_plan.rs @@ -34,11 +34,11 @@ use dyn_clone::DynClone; use serde::Deserializer; use serde::Serializer; +use crate::physical_plans::ExchangeSink; +use crate::physical_plans::MutationSource; use crate::physical_plans::format::FormatContext; use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::format::SimplePhysicalFormat; -use crate::physical_plans::ExchangeSink; -use crate::physical_plans::MutationSource; use crate::pipelines::PipelineBuilder; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] diff --git a/src/query/service/src/physical_plans/physical_plan_builder.rs b/src/query/service/src/physical_plans/physical_plan_builder.rs index 5190a683affe3..11e336c122cbe 100644 --- a/src/query/service/src/physical_plans/physical_plan_builder.rs +++ b/src/query/service/src/physical_plans/physical_plan_builder.rs @@ -21,11 +21,11 @@ use databend_common_exception::Result; use databend_common_expression::FunctionContext; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::UpdateStreamMetaReq; +use databend_common_sql::ColumnSet; +use databend_common_sql::MetadataRef; use databend_common_sql::optimizer::ir::RelExpr; use databend_common_sql::optimizer::ir::SExpr; use databend_common_sql::plans::RelOperator; -use databend_common_sql::ColumnSet; -use databend_common_sql::MetadataRef; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use databend_storages_common_table_meta::meta::TableSnapshot; diff --git a/src/query/service/src/physical_plans/physical_project_set.rs b/src/query/service/src/physical_plans/physical_project_set.rs index 8043ca08dc980..7b3caf514ea24 100644 --- a/src/query/service/src/physical_plans/physical_project_set.rs +++ b/src/query/service/src/physical_plans/physical_project_set.rs @@ -23,21 +23,21 @@ use databend_common_expression::DataSchemaRefExt; use databend_common_expression::RemoteExpr; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_pipeline::core::ProcessorPtr; -use databend_common_sql::optimizer::ir::SExpr; use databend_common_sql::ColumnSet; use databend_common_sql::IndexType; use databend_common_sql::TypeCheck; +use databend_common_sql::optimizer::ir::SExpr; use itertools::Itertools; +use crate::physical_plans::PhysicalPlanBuilder; use crate::physical_plans::explain::PlanStatsInfo; use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::format::ProjectSetFormatter; use crate::physical_plans::physical_plan::IPhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlanMeta; -use crate::physical_plans::PhysicalPlanBuilder; -use crate::pipelines::processors::transforms::TransformSRF; use crate::pipelines::PipelineBuilder; +use crate::pipelines::processors::transforms::TransformSRF; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ProjectSet { diff --git a/src/query/service/src/physical_plans/physical_r_cte_scan.rs b/src/query/service/src/physical_plans/physical_r_cte_scan.rs index 325818154776c..94d478dadb0c0 100644 --- a/src/query/service/src/physical_plans/physical_r_cte_scan.rs +++ b/src/query/service/src/physical_plans/physical_r_cte_scan.rs @@ -19,13 +19,13 @@ use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; +use crate::physical_plans::PhysicalPlanBuilder; use crate::physical_plans::explain::PlanStatsInfo; use crate::physical_plans::physical_plan::IPhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlanMeta; -use crate::physical_plans::PhysicalPlanBuilder; -use crate::pipelines::processors::transforms::TransformRecursiveCteScan; use crate::pipelines::PipelineBuilder; +use crate::pipelines::processors::transforms::TransformRecursiveCteScan; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct RecursiveCteScan { diff --git a/src/query/service/src/physical_plans/physical_range_join.rs b/src/query/service/src/physical_plans/physical_range_join.rs index 9c9723edf6e73..b96091b477d84 100644 --- a/src/query/service/src/physical_plans/physical_range_join.rs +++ b/src/query/service/src/physical_plans/physical_range_join.rs @@ -17,35 +17,35 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::type_check::common_super_type; use databend_common_expression::DataSchema; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use databend_common_expression::RemoteExpr; +use databend_common_expression::type_check::common_super_type; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_pipeline::core::ProcessorPtr; use databend_common_pipeline::sinks::Sinker; -use databend_common_sql::binder::wrap_cast; +use databend_common_sql::ColumnSet; +use databend_common_sql::ScalarExpr; +use databend_common_sql::TypeCheck; use databend_common_sql::binder::JoinPredicate; +use databend_common_sql::binder::wrap_cast; use databend_common_sql::optimizer::ir::RelExpr; use databend_common_sql::optimizer::ir::RelationalProperty; use databend_common_sql::optimizer::ir::SExpr; use databend_common_sql::plans::JoinType; -use databend_common_sql::ColumnSet; -use databend_common_sql::ScalarExpr; -use databend_common_sql::TypeCheck; +use crate::physical_plans::PhysicalPlanBuilder; use crate::physical_plans::explain::PlanStatsInfo; use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::format::RangeJoinFormatter; use crate::physical_plans::physical_plan::IPhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlanMeta; -use crate::physical_plans::PhysicalPlanBuilder; +use crate::pipelines::PipelineBuilder; use crate::pipelines::processors::transforms::range_join::RangeJoinState; use crate::pipelines::processors::transforms::range_join::TransformRangeJoinLeft; use crate::pipelines::processors::transforms::range_join::TransformRangeJoinRight; -use crate::pipelines::PipelineBuilder; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct RangeJoin { diff --git a/src/query/service/src/physical_plans/physical_recluster.rs b/src/query/service/src/physical_plans/physical_recluster.rs index 50111a2533d51..f86247763f24c 100644 --- a/src/query/service/src/physical_plans/physical_recluster.rs +++ b/src/query/service/src/physical_plans/physical_recluster.rs @@ -34,29 +34,29 @@ use databend_common_metrics::storage::metrics_inc_recluster_block_nums_to_read; use databend_common_metrics::storage::metrics_inc_recluster_row_nums_to_read; use databend_common_pipeline::core::ProcessorPtr; use databend_common_pipeline::sources::EmptySource; +use databend_common_pipeline_transforms::MemorySettings; +use databend_common_pipeline_transforms::TransformPipelineHelper; use databend_common_pipeline_transforms::blocks::CompoundBlockOperator; use databend_common_pipeline_transforms::build_compact_block_no_split_pipeline; use databend_common_pipeline_transforms::columns::TransformAddStreamColumns; -use databend_common_pipeline_transforms::MemorySettings; -use databend_common_pipeline_transforms::TransformPipelineHelper; -use databend_common_sql::executor::physical_plans::MutationKind; use databend_common_sql::StreamContext; +use databend_common_sql::executor::physical_plans::MutationKind; +use databend_common_storages_fuse::FUSE_OPT_KEY_BLOCK_IN_MEM_SIZE_THRESHOLD; +use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::operations::TransformSerializeBlock; use databend_common_storages_fuse::statistics::ClusterStatsGenerator; -use databend_common_storages_fuse::FuseTable; -use databend_common_storages_fuse::FUSE_OPT_KEY_BLOCK_IN_MEM_SIZE_THRESHOLD; use databend_storages_common_cache::TempDirManager; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use crate::physical_plans::physical_plan::IPhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlanMeta; +use crate::pipelines::PipelineBuilder; use crate::pipelines::builders::SortPipelineBuilder; use crate::pipelines::memory_settings::MemorySettingsExt; use crate::pipelines::processors::transforms::CompactStrategy; use crate::pipelines::processors::transforms::HilbertPartitionExchange; use crate::pipelines::processors::transforms::TransformWindowPartitionCollect; -use crate::pipelines::PipelineBuilder; use crate::spillers::SpillerDiskConfig; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] diff --git a/src/query/service/src/physical_plans/physical_refresh_index.rs b/src/query/service/src/physical_plans/physical_refresh_index.rs deleted file mode 100644 index d855c92c76038..0000000000000 --- a/src/query/service/src/physical_plans/physical_refresh_index.rs +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::fmt::Display; -use std::fmt::Formatter; - -use databend_common_exception::Result; -use databend_common_expression::types::DataType; -use databend_common_expression::DataField; -use databend_common_expression::DataSchemaRef; -use databend_common_expression::DataSchemaRefExt; -use databend_common_meta_app::schema::TableInfo; -use databend_common_sql::ColumnBinding; - -use crate::physical_plans::PhysicalPlan; - -#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] -pub struct RefreshIndex { - pub input: PhysicalPlan, - pub index_id: u64, - pub table_info: TableInfo, - pub select_schema: DataSchemaRef, - pub select_column_bindings: Vec, -} - -impl RefreshIndex { - pub fn _output_schema(&self) -> Result { - Ok(DataSchemaRefExt::create(vec![DataField::new( - "index_loc", - DataType::String, - )])) - } -} - -impl Display for RefreshIndex { - fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - write!(f, "RefreshIndex") - } -} diff --git a/src/query/service/src/physical_plans/physical_replace_async_source.rs b/src/query/service/src/physical_plans/physical_replace_async_source.rs index 28df3a2e62ebd..e57f78f04fb6f 100644 --- a/src/query/service/src/physical_plans/physical_replace_async_source.rs +++ b/src/query/service/src/physical_plans/physical_replace_async_source.rs @@ -18,8 +18,8 @@ use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use databend_common_pipeline::sources::AsyncSourcer; -use databend_common_sql::plans::InsertValue; use databend_common_sql::NameResolutionContext; +use databend_common_sql::plans::InsertValue; use crate::physical_plans::physical_plan::IPhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlan; diff --git a/src/query/service/src/physical_plans/physical_replace_deduplicate.rs b/src/query/service/src/physical_plans/physical_replace_deduplicate.rs index 74159f8295f39..29f1ce6d2ec89 100644 --- a/src/query/service/src/physical_plans/physical_replace_deduplicate.rs +++ b/src/query/service/src/physical_plans/physical_replace_deduplicate.rs @@ -27,14 +27,14 @@ use databend_common_expression::RemoteExpr; use databend_common_expression::TableSchemaRef; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::schema::TableInfo; +use databend_common_pipeline_transforms::TransformPipelineHelper; use databend_common_pipeline_transforms::blocks::TransformCastSchema; use databend_common_pipeline_transforms::build_compact_block_pipeline; -use databend_common_pipeline_transforms::TransformPipelineHelper; -use databend_common_sql::executor::physical_plans::OnConflictField; use databend_common_sql::ColumnBinding; +use databend_common_sql::executor::physical_plans::OnConflictField; +use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::operations::ReplaceIntoProcessor; use databend_common_storages_fuse::operations::UnbranchedReplaceIntoProcessor; -use databend_common_storages_fuse::FuseTable; use databend_storages_common_table_meta::meta::ColumnStatistics; use crate::physical_plans::physical_plan::IPhysicalPlan; diff --git a/src/query/service/src/physical_plans/physical_replace_into.rs b/src/query/service/src/physical_plans/physical_replace_into.rs index a88d4c8975122..9e502fb467a81 100644 --- a/src/query/service/src/physical_plans/physical_replace_into.rs +++ b/src/query/service/src/physical_plans/physical_replace_into.rs @@ -28,9 +28,9 @@ use databend_common_pipeline::core::Pipe; use databend_common_pipeline_transforms::create_dummy_item; use databend_common_sql::executor::physical_plans::MutationKind; use databend_common_sql::executor::physical_plans::OnConflictField; +use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::operations::BroadcastProcessor; use databend_common_storages_fuse::operations::TransformSerializeBlock; -use databend_common_storages_fuse::FuseTable; use databend_storages_common_table_meta::meta::BlockSlotDescription; use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::TableMetaTimestamps; diff --git a/src/query/service/src/physical_plans/physical_row_fetch.rs b/src/query/service/src/physical_plans/physical_row_fetch.rs index fc2a879329fd8..11fef083c340a 100644 --- a/src/query/service/src/physical_plans/physical_row_fetch.rs +++ b/src/query/service/src/physical_plans/physical_row_fetch.rs @@ -28,14 +28,14 @@ use databend_common_pipeline_transforms::create_dummy_item; use databend_common_storages_fuse::operations::row_fetch_processor; use itertools::Itertools; +use crate::physical_plans::MutationSplit; +use crate::physical_plans::PhysicalPlanCast; use crate::physical_plans::explain::PlanStatsInfo; use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::format::RowFetchFormatter; use crate::physical_plans::physical_plan::IPhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlanMeta; -use crate::physical_plans::MutationSplit; -use crate::physical_plans::PhysicalPlanCast; use crate::pipelines::PipelineBuilder; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] diff --git a/src/query/service/src/physical_plans/physical_secure_filter.rs b/src/query/service/src/physical_plans/physical_secure_filter.rs index 9e8e19230c430..851952279b83f 100644 --- a/src/query/service/src/physical_plans/physical_secure_filter.rs +++ b/src/query/service/src/physical_plans/physical_secure_filter.rs @@ -22,18 +22,18 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use databend_common_expression::RemoteExpr; use databend_common_functions::BUILTIN_FUNCTIONS; -use databend_common_sql::executor::cast_expr_to_non_null_boolean; -use databend_common_sql::optimizer::ir::SExpr; use databend_common_sql::ColumnSet; use databend_common_sql::TypeCheck; +use databend_common_sql::executor::cast_expr_to_non_null_boolean; +use databend_common_sql::optimizer::ir::SExpr; +use crate::physical_plans::PhysicalPlanBuilder; use crate::physical_plans::explain::PlanStatsInfo; use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::format::SecureFilterFormatter; use crate::physical_plans::physical_plan::IPhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlanMeta; -use crate::physical_plans::PhysicalPlanBuilder; use crate::pipelines::PipelineBuilder; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] diff --git a/src/query/service/src/physical_plans/physical_sequence.rs b/src/query/service/src/physical_plans/physical_sequence.rs index 4f14eda09a25a..4642019c961f5 100644 --- a/src/query/service/src/physical_plans/physical_sequence.rs +++ b/src/query/service/src/physical_plans/physical_sequence.rs @@ -16,14 +16,14 @@ use std::any::Any; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; -use databend_common_sql::optimizer::ir::SExpr; use databend_common_sql::ColumnSet; +use databend_common_sql::optimizer::ir::SExpr; -use crate::physical_plans::explain::PlanStatsInfo; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlan; use crate::physical_plans::PhysicalPlanBuilder; use crate::physical_plans::PhysicalPlanMeta; +use crate::physical_plans::explain::PlanStatsInfo; use crate::pipelines::PipelineBuilder; use crate::sessions::QueryContext; diff --git a/src/query/service/src/physical_plans/physical_sort.rs b/src/query/service/src/physical_plans/physical_sort.rs index 5b545709aa7a6..531e93b955b55 100644 --- a/src/query/service/src/physical_plans/physical_sort.rs +++ b/src/query/service/src/physical_plans/physical_sort.rs @@ -23,33 +23,33 @@ use databend_common_expression::DataSchema; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use databend_common_expression::SortColumnDescription; +use databend_common_pipeline_transforms::TransformPipelineHelper; use databend_common_pipeline_transforms::blocks::CompoundBlockOperator; use databend_common_pipeline_transforms::sorts::core::order_field_type; use databend_common_pipeline_transforms::sorts::utils::ORDER_COL_NAME; -use databend_common_pipeline_transforms::TransformPipelineHelper; +use databend_common_sql::ColumnSet; +use databend_common_sql::IndexType; use databend_common_sql::evaluator::BlockOperator; use databend_common_sql::executor::physical_plans::FragmentKind; use databend_common_sql::executor::physical_plans::SortDesc; use databend_common_sql::optimizer::ir::SExpr; use databend_common_sql::plans::WindowFuncType; -use databend_common_sql::ColumnSet; -use databend_common_sql::IndexType; use itertools::Itertools; -use crate::physical_plans::explain::PlanStatsInfo; -use crate::physical_plans::format::PhysicalFormat; -use crate::physical_plans::format::SortFormatter; -use crate::physical_plans::physical_plan::IPhysicalPlan; -use crate::physical_plans::physical_plan::PhysicalPlan; -use crate::physical_plans::physical_plan::PhysicalPlanMeta; use crate::physical_plans::Exchange; use crate::physical_plans::PhysicalPlanBuilder; use crate::physical_plans::PhysicalPlanCast; use crate::physical_plans::WindowPartition; use crate::physical_plans::WindowPartitionTopN; use crate::physical_plans::WindowPartitionTopNFunc; -use crate::pipelines::builders::SortPipelineBuilder; +use crate::physical_plans::explain::PlanStatsInfo; +use crate::physical_plans::format::PhysicalFormat; +use crate::physical_plans::format::SortFormatter; +use crate::physical_plans::physical_plan::IPhysicalPlan; +use crate::physical_plans::physical_plan::PhysicalPlan; +use crate::physical_plans::physical_plan::PhysicalPlanMeta; use crate::pipelines::PipelineBuilder; +use crate::pipelines::builders::SortPipelineBuilder; use crate::spillers::SortSpiller; type TransformSortBuilder = diff --git a/src/query/service/src/physical_plans/physical_table_scan.rs b/src/query/service/src/physical_plans/physical_table_scan.rs index 8162ba6685568..ecca1c5f005cf 100644 --- a/src/query/service/src/physical_plans/physical_table_scan.rs +++ b/src/query/service/src/physical_plans/physical_table_scan.rs @@ -35,31 +35,28 @@ use databend_common_catalog::plan::VirtualColumnInfo; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::type_check::check_function; -use databend_common_expression::types::DataType; use databend_common_expression::ConstantFolder; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use databend_common_expression::FieldIndex; +use databend_common_expression::ROW_ID_COL_NAME; use databend_common_expression::RemoteExpr; use databend_common_expression::Scalar; use databend_common_expression::TableDataType; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRef; -use databend_common_expression::ROW_ID_COL_NAME; +use databend_common_expression::type_check::check_function; +use databend_common_expression::types::DataType; use databend_common_functions::BUILTIN_FUNCTIONS; +use databend_common_pipeline_transforms::TransformPipelineHelper; use databend_common_pipeline_transforms::blocks::CompoundBlockOperator; use databend_common_pipeline_transforms::columns::TransformAddInternalColumns; -use databend_common_pipeline_transforms::TransformPipelineHelper; -use databend_common_sql::binder::INTERNAL_COLUMN_FACTORY; -use databend_common_sql::evaluator::BlockOperator; -use databend_common_sql::executor::cast_expr_to_non_null_boolean; -use databend_common_sql::executor::table_read_plan::ToReadDataSourcePlan; -use databend_common_sql::plans::FunctionCall; use databend_common_sql::BaseTableColumn; use databend_common_sql::ColumnEntry; use databend_common_sql::ColumnSet; +use databend_common_sql::DUMMY_COLUMN_INDEX; +use databend_common_sql::DUMMY_TABLE_INDEX; use databend_common_sql::DerivedColumn; use databend_common_sql::IndexType; use databend_common_sql::Metadata; @@ -67,22 +64,25 @@ use databend_common_sql::ScalarExpr; use databend_common_sql::TableInternalColumn; use databend_common_sql::TypeCheck; use databend_common_sql::VirtualColumn; -use databend_common_sql::DUMMY_COLUMN_INDEX; -use databend_common_sql::DUMMY_TABLE_INDEX; +use databend_common_sql::binder::INTERNAL_COLUMN_FACTORY; +use databend_common_sql::evaluator::BlockOperator; +use databend_common_sql::executor::cast_expr_to_non_null_boolean; +use databend_common_sql::executor::table_read_plan::ToReadDataSourcePlan; +use databend_common_sql::plans::FunctionCall; use jsonb::keypath::KeyPath; use jsonb::keypath::KeyPaths; use rand::distributions::Bernoulli; use rand::distributions::Distribution; use rand::thread_rng; +use crate::physical_plans::AddStreamColumn; +use crate::physical_plans::PhysicalPlanBuilder; use crate::physical_plans::explain::PlanStatsInfo; use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::format::TableScanFormatter; use crate::physical_plans::physical_plan::IPhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlanMeta; -use crate::physical_plans::AddStreamColumn; -use crate::physical_plans::PhysicalPlanBuilder; use crate::pipelines::PipelineBuilder; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -182,11 +182,9 @@ impl IPhysicalPlan for TableScan { ), self.name_mapping.keys().cloned().collect(), ), - (String::from("Total partitions"), vec![self - .source - .statistics - .partitions_total - .to_string()]), + (String::from("Total partitions"), vec![ + self.source.statistics.partitions_total.to_string(), + ]), ])) } diff --git a/src/query/service/src/physical_plans/physical_udf.rs b/src/query/service/src/physical_plans/physical_udf.rs index 1dbe22efb8682..89f75767e8acb 100644 --- a/src/query/service/src/physical_plans/physical_udf.rs +++ b/src/query/service/src/physical_plans/physical_udf.rs @@ -18,29 +18,29 @@ use std::collections::BTreeMap; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; +use databend_common_expression::types::DataType; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_pipeline_transforms::TransformPipelineHelper; -use databend_common_sql::optimizer::ir::SExpr; -use databend_common_sql::plans::UDFType; use databend_common_sql::ColumnSet; use databend_common_sql::IndexType; use databend_common_sql::ScalarExpr; +use databend_common_sql::optimizer::ir::SExpr; +use databend_common_sql::plans::UDFType; use itertools::Itertools; +use crate::physical_plans::PhysicalPlanBuilder; use crate::physical_plans::explain::PlanStatsInfo; use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::format::UdfFormatter; use crate::physical_plans::physical_plan::IPhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlanMeta; -use crate::physical_plans::PhysicalPlanBuilder; +use crate::pipelines::PipelineBuilder; use crate::pipelines::processors::transforms::TransformUdfScript; use crate::pipelines::processors::transforms::TransformUdfServer; -use crate::pipelines::PipelineBuilder; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct Udf { diff --git a/src/query/service/src/physical_plans/physical_union_all.rs b/src/query/service/src/physical_plans/physical_union_all.rs index 7c4280ae57127..c2f311951a588 100644 --- a/src/query/service/src/physical_plans/physical_union_all.rs +++ b/src/query/service/src/physical_plans/physical_union_all.rs @@ -23,23 +23,23 @@ use databend_common_expression::RemoteExpr; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_pipeline::core::ProcessorPtr; use databend_common_pipeline_transforms::blocks::CompoundBlockOperator; -use databend_common_sql::evaluator::BlockOperator; -use databend_common_sql::optimizer::ir::SExpr; use databend_common_sql::ColumnSet; use databend_common_sql::IndexType; use databend_common_sql::ScalarExpr; use databend_common_sql::TypeCheck; +use databend_common_sql::evaluator::BlockOperator; +use databend_common_sql::optimizer::ir::SExpr; use itertools::Itertools; +use crate::physical_plans::PhysicalPlanBuilder; use crate::physical_plans::explain::PlanStatsInfo; use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::format::UnionAllFormatter; use crate::physical_plans::physical_plan::IPhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlanMeta; -use crate::physical_plans::PhysicalPlanBuilder; -use crate::pipelines::processors::transforms::TransformRecursiveCteSource; use crate::pipelines::PipelineBuilder; +use crate::pipelines::processors::transforms::TransformRecursiveCteSource; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct UnionAll { diff --git a/src/query/service/src/physical_plans/physical_window.rs b/src/query/service/src/physical_plans/physical_window.rs index 883bbd9ea41f1..e60934ab3acd2 100644 --- a/src/query/service/src/physical_plans/physical_window.rs +++ b/src/query/service/src/physical_plans/physical_window.rs @@ -18,10 +18,6 @@ use std::fmt::Display; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::type_check; -use databend_common_expression::type_check::common_super_type; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; use databend_common_expression::Constant; use databend_common_expression::ConstantFolder; use databend_common_expression::DataField; @@ -30,9 +26,17 @@ use databend_common_expression::DataSchemaRefExt; use databend_common_expression::Expr; use databend_common_expression::FunctionContext; use databend_common_expression::RawExpr; +use databend_common_expression::type_check; +use databend_common_expression::type_check::common_super_type; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_pipeline::core::Processor; use databend_common_pipeline::core::ProcessorPtr; +use databend_common_sql::ColumnSet; +use databend_common_sql::IndexType; +use databend_common_sql::ScalarExpr; +use databend_common_sql::TypeCheck; use databend_common_sql::binder::wrap_cast; use databend_common_sql::executor::physical_plans::AggregateFunctionDesc; use databend_common_sql::executor::physical_plans::AggregateFunctionSignature; @@ -41,23 +45,19 @@ use databend_common_sql::optimizer::ir::SExpr; use databend_common_sql::plans::WindowFuncFrame; use databend_common_sql::plans::WindowFuncFrameBound; use databend_common_sql::plans::WindowFuncType; -use databend_common_sql::ColumnSet; -use databend_common_sql::IndexType; -use databend_common_sql::ScalarExpr; -use databend_common_sql::TypeCheck; +use crate::physical_plans::PhysicalPlanBuilder; use crate::physical_plans::explain::PlanStatsInfo; use crate::physical_plans::format::PhysicalFormat; use crate::physical_plans::format::WindowFormatter; use crate::physical_plans::physical_plan::IPhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlanMeta; -use crate::physical_plans::PhysicalPlanBuilder; +use crate::pipelines::PipelineBuilder; use crate::pipelines::processors::transforms::FrameBound; use crate::pipelines::processors::transforms::TransformWindow; use crate::pipelines::processors::transforms::WindowFunctionInfo; use crate::pipelines::processors::transforms::WindowSortDesc; -use crate::pipelines::PipelineBuilder; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct Window { @@ -372,7 +372,10 @@ impl PhysicalPlanBuilder { for scalar in start.iter_mut().chain(end.iter_mut()) { let scalar_ty = scalar.as_ref().infer_data_type(); if !scalar_ty.is_interval() { - return Err(ErrorCode::IllegalDataType(format!("when the type of the order by in window func is Timestamp, Preceding and Following can only be INTERVAL types, but get {}", scalar_ty))); + return Err(ErrorCode::IllegalDataType(format!( + "when the type of the order by in window func is Timestamp, Preceding and Following can only be INTERVAL types, but get {}", + scalar_ty + ))); } } } else if common_ty.remove_nullable().is_date() { @@ -380,13 +383,20 @@ impl PhysicalPlanBuilder { for scalar in start.iter_mut().chain(end.iter_mut()) { let scalar_ty = scalar.as_ref().infer_data_type(); if !scalar_ty.is_interval() && !scalar_ty.is_unsigned_numeric() { - return Err(ErrorCode::IllegalDataType(format!("when the type of the order by in window func is Date, Preceding and Following can only be INTERVAL or Unsigned Integer types, but get {}", scalar_ty))); + return Err(ErrorCode::IllegalDataType(format!( + "when the type of the order by in window func is Date, Preceding and Following can only be INTERVAL or Unsigned Integer types, but get {}", + scalar_ty + ))); } if last_ty.as_ref().is_none_or(|ty| ty == &scalar_ty) { last_ty = Some(scalar_ty); continue; } - return Err(ErrorCode::IllegalDataType(format!("when the type of the order by in window func is Date, Preceding and Following can only be of the same type, but get {} and {}", last_ty.unwrap(), scalar_ty))); + return Err(ErrorCode::IllegalDataType(format!( + "when the type of the order by in window func is Date, Preceding and Following can only be of the same type, but get {} and {}", + last_ty.unwrap(), + scalar_ty + ))); } } else { for scalar in start.iter_mut().chain(end.iter_mut()) { @@ -439,7 +449,8 @@ impl PhysicalPlanBuilder { } } - let default_nulls_first = self.ctx.get_settings().get_nulls_first(); + let settings = self.ctx.get_settings(); + let default_nulls_first = settings.get_nulls_first(); let order_by_items = w .order_by diff --git a/src/query/service/src/physical_plans/physical_window_partition.rs b/src/query/service/src/physical_plans/physical_window_partition.rs index 7158a06df5c53..e3c9f1f0228d4 100644 --- a/src/query/service/src/physical_plans/physical_window_partition.rs +++ b/src/query/service/src/physical_plans/physical_window_partition.rs @@ -23,8 +23,8 @@ use databend_common_exception::Result; use databend_common_expression::SortColumnDescription; use databend_common_pipeline::core::ProcessorPtr; use databend_common_pipeline_transforms::MemorySettings; -use databend_common_sql::executor::physical_plans::SortDesc; use databend_common_sql::IndexType; +use databend_common_sql::executor::physical_plans::SortDesc; use databend_storages_common_cache::TempDirManager; use crate::physical_plans::explain::PlanStatsInfo; @@ -34,12 +34,12 @@ use crate::physical_plans::physical_plan::IPhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlanMeta; use crate::physical_plans::physical_sort::SortStep; +use crate::pipelines::PipelineBuilder; use crate::pipelines::memory_settings::MemorySettingsExt; use crate::pipelines::processors::transforms::SortStrategy; use crate::pipelines::processors::transforms::TransformWindowPartitionCollect; use crate::pipelines::processors::transforms::WindowPartitionExchange; use crate::pipelines::processors::transforms::WindowPartitionTopNExchange; -use crate::pipelines::PipelineBuilder; use crate::spillers::SpillerDiskConfig; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] diff --git a/src/query/service/src/physical_plans/runtime_filter/builder.rs b/src/query/service/src/physical_plans/runtime_filter/builder.rs index 60cf42e3685b5..2833f20caa018 100644 --- a/src/query/service/src/physical_plans/runtime_filter/builder.rs +++ b/src/query/service/src/physical_plans/runtime_filter/builder.rs @@ -17,9 +17,13 @@ use std::sync::Arc; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::RemoteExpr; +use databend_common_expression::types::DataType; use databend_common_functions::BUILTIN_FUNCTIONS; +use databend_common_sql::ColumnEntry; +use databend_common_sql::IndexType; +use databend_common_sql::MetadataRef; +use databend_common_sql::TypeCheck; use databend_common_sql::optimizer::ir::SExpr; use databend_common_sql::plans::Exchange; use databend_common_sql::plans::Join; @@ -27,10 +31,6 @@ use databend_common_sql::plans::JoinEquiCondition; use databend_common_sql::plans::JoinType; use databend_common_sql::plans::RelOperator; use databend_common_sql::plans::ScalarExpr; -use databend_common_sql::ColumnEntry; -use databend_common_sql::IndexType; -use databend_common_sql::MetadataRef; -use databend_common_sql::TypeCheck; use super::types::PhysicalRuntimeFilter; use super::types::PhysicalRuntimeFilters; diff --git a/src/query/service/src/pipelines/builders/builder_aggregate.rs b/src/query/service/src/pipelines/builders/builder_aggregate.rs index 9ac541f863464..1a55e06b6aa24 100644 --- a/src/query/service/src/pipelines/builders/builder_aggregate.rs +++ b/src/query/service/src/pipelines/builders/builder_aggregate.rs @@ -20,14 +20,14 @@ use databend_common_expression::DataField; use databend_common_expression::DataSchemaRef; use databend_common_functions::aggregates::AggregateFunctionFactory; use databend_common_functions::aggregates::AggregateFunctionSortDesc; +use databend_common_sql::IndexType; use databend_common_sql::executor::physical_plans::AggregateFunctionDesc; use databend_common_sql::plans::UDFType; -use databend_common_sql::IndexType; use itertools::Itertools; -use crate::pipelines::processors::transforms::aggregator::create_udaf_script_function; -use crate::pipelines::processors::transforms::aggregator::AggregatorParams; use crate::pipelines::PipelineBuilder; +use crate::pipelines::processors::transforms::aggregator::AggregatorParams; +use crate::pipelines::processors::transforms::aggregator::create_udaf_script_function; impl PipelineBuilder { pub fn build_aggregator_params( diff --git a/src/query/service/src/pipelines/builders/builder_copy_into_table.rs b/src/query/service/src/pipelines/builders/builder_copy_into_table.rs index cf9f056ef7e13..6fbadb4561289 100644 --- a/src/query/service/src/pipelines/builders/builder_copy_into_table.rs +++ b/src/query/service/src/pipelines/builders/builder_copy_into_table.rs @@ -28,10 +28,10 @@ use databend_common_meta_app::principal::ParquetFileFormatParams; use databend_common_meta_app::schema::TableCopiedFileInfo; use databend_common_meta_app::schema::UpsertTableCopiedFileReq; use databend_common_pipeline::core::Pipeline; +use databend_common_pipeline_transforms::TransformPipelineHelper; use databend_common_pipeline_transforms::blocks::TransformCastSchema; use databend_common_pipeline_transforms::columns::TransformAddConstColumns; use databend_common_pipeline_transforms::columns::TransformNullIf; -use databend_common_pipeline_transforms::TransformPipelineHelper; use databend_common_sql::plans::CopyIntoTableMode; use databend_common_storage::StageFileInfo; use log::debug; diff --git a/src/query/service/src/pipelines/builders/builder_fill_missing_columns.rs b/src/query/service/src/pipelines/builders/builder_fill_missing_columns.rs index 99b1854d0128e..29ed83cdd116a 100644 --- a/src/query/service/src/pipelines/builders/builder_fill_missing_columns.rs +++ b/src/query/service/src/pipelines/builders/builder_fill_missing_columns.rs @@ -25,9 +25,9 @@ use databend_common_pipeline_transforms::columns::TransformAddComputedColumns; use databend_common_pipeline_transforms::processors::TransformPipelineHelper; use databend_common_sql::DefaultExprBinder; +use crate::pipelines::PipelineBuilder; use crate::pipelines::processors::transforms::TransformAsyncFunction; use crate::pipelines::processors::transforms::TransformResortAddOn; -use crate::pipelines::PipelineBuilder; use crate::sessions::QueryContext; /// This file implements append to table pipeline builder. diff --git a/src/query/service/src/pipelines/builders/builder_mutation.rs b/src/query/service/src/pipelines/builders/builder_mutation.rs index 43a90be385710..89b5ee7accc73 100644 --- a/src/query/service/src/pipelines/builders/builder_mutation.rs +++ b/src/query/service/src/pipelines/builders/builder_mutation.rs @@ -24,27 +24,27 @@ use databend_common_expression::DataSchemaRef; use databend_common_pipeline::basic::create_resize_item; use databend_common_pipeline::core::Pipe; use databend_common_pipeline::core::ProcessorPtr; -use databend_common_pipeline_transforms::blocks::build_cast_exprs; use databend_common_pipeline_transforms::blocks::CastSchemaBranch; use databend_common_pipeline_transforms::blocks::TransformBranchedCastSchema; +use databend_common_pipeline_transforms::blocks::build_cast_exprs; use databend_common_pipeline_transforms::columns::TransformAddComputedColumns; -use databend_common_pipeline_transforms::processors::create_dummy_item; use databend_common_pipeline_transforms::processors::AccumulatingTransformer; use databend_common_pipeline_transforms::processors::BlockCompactBuilder; use databend_common_pipeline_transforms::processors::BlockMetaTransformer; use databend_common_pipeline_transforms::processors::TransformCompactBlock; use databend_common_pipeline_transforms::processors::TransformPipelineHelper; +use databend_common_pipeline_transforms::processors::create_dummy_item; use databend_common_sql::DefaultExprBinder; -use databend_common_storages_fuse::operations::UnMatchedExprs; use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::TableContext; +use databend_common_storages_fuse::operations::UnMatchedExprs; -use crate::pipelines::processors::transforms::build_expression_transform; +use crate::pipelines::PipelineBuilder; use crate::pipelines::processors::transforms::AsyncFunctionBranch; use crate::pipelines::processors::transforms::TransformAsyncFunction; use crate::pipelines::processors::transforms::TransformBranchedAsyncFunction; use crate::pipelines::processors::transforms::TransformResortAddOnWithoutSourceSchema; -use crate::pipelines::PipelineBuilder; +use crate::pipelines::processors::transforms::build_expression_transform; impl PipelineBuilder { pub fn build_fill_columns_in_merge_into( diff --git a/src/query/service/src/pipelines/builders/builder_project.rs b/src/query/service/src/pipelines/builders/builder_project.rs index 44fca6911d819..2b380e896dc2a 100644 --- a/src/query/service/src/pipelines/builders/builder_project.rs +++ b/src/query/service/src/pipelines/builders/builder_project.rs @@ -20,8 +20,8 @@ use databend_common_pipeline::core::ProcessorPtr; use databend_common_pipeline::sinks::EmptySink; use databend_common_pipeline_transforms::blocks::CompoundBlockOperator; use databend_common_pipeline_transforms::processors::TransformPipelineHelper; -use databend_common_sql::evaluator::BlockOperator; use databend_common_sql::ColumnBinding; +use databend_common_sql::evaluator::BlockOperator; use crate::pipelines::PipelineBuilder; diff --git a/src/query/service/src/pipelines/builders/builder_replace_into.rs b/src/query/service/src/pipelines/builders/builder_replace_into.rs index 450e7bceb8c0a..2c633f6394c13 100644 --- a/src/query/service/src/pipelines/builders/builder_replace_into.rs +++ b/src/query/service/src/pipelines/builders/builder_replace_into.rs @@ -181,11 +181,12 @@ impl FastValuesDecodeFallback for RawValueSource { let res: Result> = try { let settings = self.ctx.get_settings(); let sql_dialect = settings.get_sql_dialect()?; - let tokens = tokenize_sql(sql)?; + let tokens = tokenize_sql(sql).map_err(ErrorCode::from)?; let mut bind_context = self.bind_context.clone(); let metadata = self.metadata.clone(); - let exprs = parse_values(&tokens, sql_dialect)? + let exprs = parse_values(&tokens, sql_dialect) + .map_err(ErrorCode::from)? .into_iter() .map(|expr| match expr { Expr::Placeholder { .. } => { @@ -205,7 +206,7 @@ impl FastValuesDecodeFallback for RawValueSource { ) .await? }; - res.map_err(|mut err| { + res.map_err(|mut err: ErrorCode| { // The input for ValueSource is a sub-section of the original SQL. This causes // the error span to have an offset, so we adjust the span accordingly. if let Some(span) = err.span() { diff --git a/src/query/service/src/pipelines/builders/builder_sort.rs b/src/query/service/src/pipelines/builders/builder_sort.rs index d397e71f41463..97ae8c8c62f93 100644 --- a/src/query/service/src/pipelines/builders/builder_sort.rs +++ b/src/query/service/src/pipelines/builders/builder_sort.rs @@ -25,12 +25,12 @@ use databend_common_pipeline::core::Pipe; use databend_common_pipeline::core::PipeItem; use databend_common_pipeline::core::Pipeline; use databend_common_pipeline::core::ProcessorPtr; +use databend_common_pipeline_transforms::MemorySettings; +use databend_common_pipeline_transforms::TransformPipelineHelper; +use databend_common_pipeline_transforms::sorts::TransformSortPartial; use databend_common_pipeline_transforms::sorts::add_k_way_merge_sort; use databend_common_pipeline_transforms::sorts::try_add_multi_sort_merge; use databend_common_pipeline_transforms::sorts::utils::add_order_field; -use databend_common_pipeline_transforms::sorts::TransformSortPartial; -use databend_common_pipeline_transforms::MemorySettings; -use databend_common_pipeline_transforms::TransformPipelineHelper; use databend_common_storage::DataOperator; use databend_common_storages_fuse::TableContext; use databend_storages_common_cache::TempDirManager; diff --git a/src/query/service/src/pipelines/builders/builder_udtf.rs b/src/query/service/src/pipelines/builders/builder_udtf.rs index b00d5eb1593d7..51db844bd4059 100644 --- a/src/query/service/src/pipelines/builders/builder_udtf.rs +++ b/src/query/service/src/pipelines/builders/builder_udtf.rs @@ -23,12 +23,12 @@ use databend_common_base::runtime::profile::ProfileStatisticsName; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::DataType; -use databend_common_expression::udf_client::error_kind; -use databend_common_expression::udf_client::UDFFlightClient; use databend_common_expression::BlockEntry; use databend_common_expression::DataBlock; use databend_common_expression::Scalar; +use databend_common_expression::types::DataType; +use databend_common_expression::udf_client::UDFFlightClient; +use databend_common_expression::udf_client::error_kind; use databend_common_metrics::external_server::record_retry_external; use databend_common_pipeline::sources::AsyncSource; use tokio::sync::Semaphore; diff --git a/src/query/service/src/pipelines/builders/transform_builder.rs b/src/query/service/src/pipelines/builders/transform_builder.rs index 00cd2ad12e5a4..ae1138eb05959 100644 --- a/src/query/service/src/pipelines/builders/transform_builder.rs +++ b/src/query/service/src/pipelines/builders/transform_builder.rs @@ -16,11 +16,11 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::type_check::check_function; -use databend_common_expression::types::DataType; use databend_common_expression::BlockThresholds; use databend_common_expression::DataSchemaRef; use databend_common_expression::RemoteExpr; +use databend_common_expression::type_check::check_function; +use databend_common_expression::types::DataType; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_pipeline::core::InputPort; use databend_common_pipeline::core::OutputPort; @@ -34,28 +34,28 @@ use databend_common_pipeline_transforms::processors::BlockCompactBuilder; use databend_common_pipeline_transforms::processors::BlockMetaTransformer; use databend_common_pipeline_transforms::processors::TransformCompactBlock; use databend_common_pipeline_transforms::processors::TransformDummy; +use databend_common_sql::ColumnSet; use databend_common_sql::evaluator::BlockOperator; use databend_common_sql::executor::physical_plans::MutationKind; -use databend_common_sql::ColumnSet; use databend_common_storages_factory::Table; -use databend_common_storages_fuse::operations::new_serialize_segment_processor; +use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::operations::TableMutationAggregator; use databend_common_storages_fuse::operations::TransformSerializeBlock; +use databend_common_storages_fuse::operations::new_serialize_segment_processor; use databend_common_storages_fuse::statistics::ClusterStatsGenerator; -use databend_common_storages_fuse::FuseTable; use databend_storages_common_table_meta::meta; use databend_storages_common_table_meta::meta::Statistics; use meta::TableMetaTimestamps; -use crate::pipelines::processors::transforms::TransformResortAddOn; use crate::pipelines::PipelineBuilder; +use crate::pipelines::processors::transforms::TransformResortAddOn; impl PipelineBuilder { pub(crate) fn filter_transform_builder( &self, predicates: &[RemoteExpr], projections: ColumnSet, - ) -> Result, Arc) -> Result> { + ) -> Result, Arc) -> Result + use<>> { let predicate = predicates .iter() .map(|expr| expr.as_expr(&BUILTIN_FUNCTIONS)) @@ -86,14 +86,14 @@ impl PipelineBuilder { pub(crate) fn dummy_transform_builder( &self, - ) -> Result, Arc) -> Result> { + ) -> Result, Arc) -> Result + use<>> { Ok(|input, output| Ok(TransformDummy::create(input, output))) } pub(crate) fn block_compact_task_builder( &self, block_thresholds: BlockThresholds, - ) -> Result, Arc) -> Result> { + ) -> Result, Arc) -> Result + use<>> { Ok(move |transform_input_port, transform_output_port| { Ok(ProcessorPtr::create(AccumulatingTransformer::create( transform_input_port, @@ -105,7 +105,7 @@ impl PipelineBuilder { pub(crate) fn block_compact_transform_builder( &self, - ) -> Result, Arc) -> Result> { + ) -> Result, Arc) -> Result + use<>> { Ok(move |transform_input_port, transform_output_port| { Ok(ProcessorPtr::create(BlockMetaTransformer::create( transform_input_port, @@ -120,7 +120,7 @@ impl PipelineBuilder { table: Arc, cluster_stats_gen: ClusterStatsGenerator, table_meta_timestamps: TableMetaTimestamps, - ) -> Result, Arc) -> Result> { + ) -> Result, Arc) -> Result + use<>> { let ctx = self.ctx.clone(); Ok(move |input, output| { let fuse_table = FuseTable::try_from_table(table.as_ref())?; @@ -142,7 +142,7 @@ impl PipelineBuilder { table: Arc, block_thresholds: BlockThresholds, table_meta_timestamps: TableMetaTimestamps, - ) -> Result, Arc) -> Result> { + ) -> Result, Arc) -> Result + use<>> { let ctx = self.ctx.clone(); Ok(move |input, output| { let fuse_table = FuseTable::try_from_table(table.as_ref())?; @@ -161,7 +161,7 @@ impl PipelineBuilder { &self, table: Arc, table_meta_timestamps: TableMetaTimestamps, - ) -> Result, Arc) -> Result> { + ) -> Result, Arc) -> Result + use<>> { let ctx = self.ctx.clone(); Ok(move |input, output| { let fuse_table = FuseTable::try_from_table(table.as_ref())?; @@ -186,7 +186,7 @@ impl PipelineBuilder { num_input_columns: usize, remote_exprs: Vec, projections: Option, - ) -> Result, Arc) -> Result> { + ) -> Result, Arc) -> Result + use<>> { let func_ctx = self.func_ctx.clone(); let exprs = remote_exprs .iter() @@ -210,7 +210,7 @@ impl PipelineBuilder { &self, source_schema: DataSchemaRef, target_schema: DataSchemaRef, - ) -> Result, Arc) -> Result> { + ) -> Result, Arc) -> Result + use<>> { let func_ctx = self.func_ctx.clone(); Ok(move |transform_input_port, transform_output_port| { TransformCastSchema::try_create( @@ -227,7 +227,7 @@ impl PipelineBuilder { &self, table: Arc, source_schema: DataSchemaRef, - ) -> Result, Arc) -> Result> { + ) -> Result, Arc) -> Result + use<>> { let ctx = self.ctx.clone(); Ok(move |transform_input_port, transform_output_port| { TransformResortAddOn::try_create( diff --git a/src/query/service/src/pipelines/executor/executor_condvar.rs b/src/query/service/src/pipelines/executor/executor_condvar.rs index caf93035cbf88..cce68aaa5650d 100644 --- a/src/query/service/src/pipelines/executor/executor_condvar.rs +++ b/src/query/service/src/pipelines/executor/executor_condvar.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; -use std::sync::Arc; use parking_lot::Condvar; use parking_lot::Mutex; diff --git a/src/query/service/src/pipelines/executor/executor_graph.rs b/src/query/service/src/pipelines/executor/executor_graph.rs index 71f53b3aa247b..43d4659187955 100644 --- a/src/query/service/src/pipelines/executor/executor_graph.rs +++ b/src/query/service/src/pipelines/executor/executor_graph.rs @@ -12,23 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::hash_map::Entry; use std::collections::HashMap; use std::collections::VecDeque; +use std::collections::hash_map::Entry; use std::fmt::Debug; use std::fmt::Formatter; +use std::sync::Arc; +use std::sync::PoisonError; use std::sync::atomic::AtomicBool; use std::sync::atomic::AtomicU64; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; -use std::sync::Arc; -use std::sync::PoisonError; use std::time::SystemTime; use databend_common_base::base::WatchNotify; -use databend_common_base::runtime::error_info::NodeErrorType; -use databend_common_base::runtime::profile::Profile; -use databend_common_base::runtime::profile::ProfileStatisticsName; use databend_common_base::runtime::ExecutorStats; use databend_common_base::runtime::ExecutorStatsSnapshot; use databend_common_base::runtime::QueryTimeSeriesProfileBuilder; @@ -36,14 +33,12 @@ use databend_common_base::runtime::ThreadTracker; use databend_common_base::runtime::TimeSeriesProfiles; use databend_common_base::runtime::TrackingPayload; use databend_common_base::runtime::TrySpawn; +use databend_common_base::runtime::error_info::NodeErrorType; +use databend_common_base::runtime::profile::Profile; +use databend_common_base::runtime::profile::ProfileStatisticsName; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_exception::ResultExt; -use databend_common_pipeline::core::port::connect; -use databend_common_pipeline::core::port_trigger::DirectedEdge; -use databend_common_pipeline::core::port_trigger::UpdateList; -use databend_common_pipeline::core::port_trigger::UpdateTrigger; -use databend_common_pipeline::core::profile::PlanScope; use databend_common_pipeline::core::Event; use databend_common_pipeline::core::EventCause; use databend_common_pipeline::core::InputPort; @@ -51,6 +46,11 @@ use databend_common_pipeline::core::OutputPort; use databend_common_pipeline::core::Pipeline; use databend_common_pipeline::core::PlanProfile; use databend_common_pipeline::core::ProcessorPtr; +use databend_common_pipeline::core::port::connect; +use databend_common_pipeline::core::port_trigger::DirectedEdge; +use databend_common_pipeline::core::port_trigger::UpdateList; +use databend_common_pipeline::core::port_trigger::UpdateTrigger; +use databend_common_pipeline::core::profile::PlanScope; use databend_common_storages_system::QueryExecutionStatsQueue; use fastrace::prelude::*; use log::debug; @@ -58,14 +58,13 @@ use log::trace; use log::warn; use parking_lot::Condvar; use parking_lot::Mutex; +use petgraph::Direction; use petgraph::dot::Config; use petgraph::dot::Dot; use petgraph::prelude::EdgeIndex; use petgraph::prelude::NodeIndex; use petgraph::prelude::StableGraph; -use petgraph::Direction; -use crate::pipelines::executor::processor_async_task::ExecutorTasksQueue; use crate::pipelines::executor::ExecutorTask; use crate::pipelines::executor::ExecutorWorkerContext; use crate::pipelines::executor::ProcessorAsyncTask; @@ -74,6 +73,7 @@ use crate::pipelines::executor::QueriesPipelineExecutor; use crate::pipelines::executor::QueryExecutorTasksQueue; use crate::pipelines::executor::QueryPipelineExecutor; use crate::pipelines::executor::WorkersCondvar; +use crate::pipelines::executor::processor_async_task::ExecutorTasksQueue; enum State { Idle, @@ -154,13 +154,15 @@ impl Node { } pub unsafe fn trigger(&self, queue: &mut VecDeque) { - self.updated_list.trigger(queue) + unsafe { self.updated_list.trigger(queue) } } pub unsafe fn create_trigger(&self, index: EdgeIndex) -> *mut UpdateTrigger { - self.updated_list - .create_trigger(index) - .expect("Failed to create trigger") + unsafe { + self.updated_list + .create_trigger(index) + .expect("Failed to create trigger") + } } } @@ -340,12 +342,14 @@ impl ExecutingGraph { capacity: usize, graph: &Arc, ) -> Result { - let mut schedule_queue = ScheduleQueue::with_capacity(capacity); - for sink_index in locker.graph.externals(Direction::Outgoing) { - ExecutingGraph::schedule_queue(locker, sink_index, &mut schedule_queue, graph)?; - } + unsafe { + let mut schedule_queue = ScheduleQueue::with_capacity(capacity); + for sink_index in locker.graph.externals(Direction::Outgoing) { + ExecutingGraph::schedule_queue(locker, sink_index, &mut schedule_queue, graph)?; + } - Ok(schedule_queue) + Ok(schedule_queue) + } } /// # Safety @@ -357,100 +361,102 @@ impl ExecutingGraph { schedule_queue: &mut ScheduleQueue, graph: &Arc, ) -> Result<()> { - let mut need_schedule_nodes = VecDeque::new(); - let mut need_schedule_edges = VecDeque::new(); + unsafe { + let mut need_schedule_nodes = VecDeque::new(); + let mut need_schedule_edges = VecDeque::new(); - need_schedule_nodes.push_back(index); + need_schedule_nodes.push_back(index); - while !need_schedule_nodes.is_empty() || !need_schedule_edges.is_empty() { - // To avoid lock too many times, we will try to cache lock. - let mut state_guard_cache = None; - let mut event_cause = EventCause::Other; + while !need_schedule_nodes.is_empty() || !need_schedule_edges.is_empty() { + // To avoid lock too many times, we will try to cache lock. + let mut state_guard_cache = None; + let mut event_cause = EventCause::Other; - if need_schedule_nodes.is_empty() { - let edge = need_schedule_edges.pop_front().unwrap(); - let target_index = DirectedEdge::get_target(&edge, &locker.graph)?; + if need_schedule_nodes.is_empty() { + let edge = need_schedule_edges.pop_front().unwrap(); + let target_index = DirectedEdge::get_target(&edge, &locker.graph)?; - event_cause = match edge { - DirectedEdge::Source(index) => { - EventCause::Input(locker.graph.edge_weight(index).unwrap().input_index) - } - DirectedEdge::Target(index) => { - EventCause::Output(locker.graph.edge_weight(index).unwrap().output_index) - } - }; + event_cause = match edge { + DirectedEdge::Source(index) => { + EventCause::Input(locker.graph.edge_weight(index).unwrap().input_index) + } + DirectedEdge::Target(index) => EventCause::Output( + locker.graph.edge_weight(index).unwrap().output_index, + ), + }; - let node = &locker.graph[target_index]; - let node_state = node.state.lock().unwrap_or_else(PoisonError::into_inner); + let node = &locker.graph[target_index]; + let node_state = node.state.lock().unwrap_or_else(PoisonError::into_inner); - if matches!(*node_state, State::Idle) { - state_guard_cache = Some(node_state); - need_schedule_nodes.push_back(target_index); - } else { - node.processor.un_reacted(event_cause.clone())?; + if matches!(*node_state, State::Idle) { + state_guard_cache = Some(node_state); + need_schedule_nodes.push_back(target_index); + } else { + node.processor.un_reacted(event_cause.clone())?; + } } - } - if let Some(schedule_index) = need_schedule_nodes.pop_front() { - let node = &locker.graph[schedule_index]; - let (event, process_rows) = { - let mut payload = node.tracking_payload.clone(); - payload.process_rows = AtomicUsize::new(0); - let guard = ThreadTracker::tracking(payload); - - if state_guard_cache.is_none() { - state_guard_cache = Some(node.state.lock().unwrap()); - } + if let Some(schedule_index) = need_schedule_nodes.pop_front() { + let node = &locker.graph[schedule_index]; + let (event, process_rows) = { + let mut payload = node.tracking_payload.clone(); + payload.process_rows = AtomicUsize::new(0); + let guard = ThreadTracker::tracking(payload); - let event = node.processor.event(event_cause)?; - let process_rows = ThreadTracker::process_rows(); - match guard.flush() { - Ok(_) => Ok((event, process_rows)), - Err(out_of_limit) => { - Err(ErrorCode::PanicError(format!("{:?}", out_of_limit))) + if state_guard_cache.is_none() { + state_guard_cache = Some(node.state.lock().unwrap()); } - } - }?; - - trace!( - "node id: {:?}, name: {:?}, event: {:?}", - node.processor.id(), - node.processor.name(), - event - ); - let processor_state = match event { - Event::Finished => { - if !matches!(state_guard_cache.as_deref(), Some(State::Finished)) { - locker.finished_nodes.fetch_add(1, Ordering::SeqCst); + + let event = node.processor.event(event_cause)?; + let process_rows = ThreadTracker::process_rows(); + match guard.flush() { + Ok(_) => Ok((event, process_rows)), + Err(out_of_limit) => { + Err(ErrorCode::PanicError(format!("{:?}", out_of_limit))) + } } + }?; - State::Finished - } - Event::NeedData | Event::NeedConsume => State::Idle, - Event::Sync => { - schedule_queue.push_sync(ProcessorWrapper { - processor: node.processor.clone(), - graph: graph.clone(), - process_rows, - }); - State::Processing - } - Event::Async => { - schedule_queue.push_async(ProcessorWrapper { - processor: node.processor.clone(), - graph: graph.clone(), - process_rows, - }); - State::Processing - } - }; + trace!( + "node id: {:?}, name: {:?}, event: {:?}", + node.processor.id(), + node.processor.name(), + event + ); + let processor_state = match event { + Event::Finished => { + if !matches!(state_guard_cache.as_deref(), Some(State::Finished)) { + locker.finished_nodes.fetch_add(1, Ordering::SeqCst); + } + + State::Finished + } + Event::NeedData | Event::NeedConsume => State::Idle, + Event::Sync => { + schedule_queue.push_sync(ProcessorWrapper { + processor: node.processor.clone(), + graph: graph.clone(), + process_rows, + }); + State::Processing + } + Event::Async => { + schedule_queue.push_async(ProcessorWrapper { + processor: node.processor.clone(), + graph: graph.clone(), + process_rows, + }); + State::Processing + } + }; - node.trigger(&mut need_schedule_edges); - *state_guard_cache.unwrap() = processor_state; + node.trigger(&mut need_schedule_edges); + *state_guard_cache.unwrap() = processor_state; + } } - } - Ok(()) + Ok(()) + } } /// Checks if a task can be performed in the current epoch, consuming a point if possible. @@ -716,16 +722,18 @@ impl RunningGraph { /// /// Method is thread unsafe and require thread safe call pub unsafe fn init_schedule_queue(self: Arc, capacity: usize) -> Result { - ExecutingGraph::init_schedule_queue(&self.0, capacity, &self) + unsafe { ExecutingGraph::init_schedule_queue(&self.0, capacity, &self) } } /// # Safety /// /// Method is thread unsafe and require thread safe call pub unsafe fn schedule_queue(self: Arc, node_index: NodeIndex) -> Result { - let mut schedule_queue = ScheduleQueue::with_capacity(0); - ExecutingGraph::schedule_queue(&self.0, node_index, &mut schedule_queue, &self)?; - Ok(schedule_queue) + unsafe { + let mut schedule_queue = ScheduleQueue::with_capacity(0); + ExecutingGraph::schedule_queue(&self.0, node_index, &mut schedule_queue, &self)?; + Ok(schedule_queue) + } } pub(crate) fn get_node_tracking_payload(&self, pid: NodeIndex) -> &TrackingPayload { diff --git a/src/query/service/src/pipelines/executor/executor_worker_context.rs b/src/query/service/src/pipelines/executor/executor_worker_context.rs index 2559533d7b2c6..17b6dd343d6e9 100644 --- a/src/query/service/src/pipelines/executor/executor_worker_context.rs +++ b/src/query/service/src/pipelines/executor/executor_worker_context.rs @@ -19,24 +19,24 @@ use std::sync::Arc; use std::time::Instant; use std::time::SystemTime; +use databend_common_base::runtime::ThreadTracker; +use databend_common_base::runtime::TrySpawn; use databend_common_base::runtime::error_info::NodeErrorType; use databend_common_base::runtime::profile::Profile; use databend_common_base::runtime::profile::ProfileStatisticsName; -use databend_common_base::runtime::ThreadTracker; -use databend_common_base::runtime::TrySpawn; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use fastrace::future::FutureExt; use fastrace::Span; +use fastrace::future::FutureExt; use petgraph::prelude::NodeIndex; -use crate::pipelines::executor::executor_graph::ProcessorWrapper; -use crate::pipelines::executor::processor_async_task::ExecutorTasksQueue; use crate::pipelines::executor::ProcessorAsyncTask; use crate::pipelines::executor::QueriesExecutorTasksQueue; use crate::pipelines::executor::QueriesPipelineExecutor; use crate::pipelines::executor::RunningGraph; use crate::pipelines::executor::WorkersCondvar; +use crate::pipelines::executor::executor_graph::ProcessorWrapper; +use crate::pipelines::executor::processor_async_task::ExecutorTasksQueue; pub enum ExecutorTask { None, @@ -126,34 +126,36 @@ impl ExecutorWorkerContext { &mut self, executor: Option<&Arc>, ) -> std::result::Result)>, Box> { - match std::mem::replace(&mut self.task, ExecutorTask::None) { - ExecutorTask::None => Err(Box::new(NodeErrorType::LocalError(ErrorCode::Internal( - "Execute none task.", - )))), - ExecutorTask::Sync(processor) => match self.execute_sync_task(processor) { - Ok(res) => Ok(res), - Err(cause) => Err(Box::new(NodeErrorType::SyncProcessError(cause))), - }, - ExecutorTask::Async(processor) => { - if let Some(executor) = executor { - match self.execute_async_task( - processor, - executor, - executor.global_tasks_queue.clone(), - ) { - Ok(res) => Ok(res), - Err(cause) => Err(Box::new(NodeErrorType::AsyncProcessError(cause))), + unsafe { + match std::mem::replace(&mut self.task, ExecutorTask::None) { + ExecutorTask::None => Err(Box::new(NodeErrorType::LocalError( + ErrorCode::Internal("Execute none task."), + ))), + ExecutorTask::Sync(processor) => match self.execute_sync_task(processor) { + Ok(res) => Ok(res), + Err(cause) => Err(Box::new(NodeErrorType::SyncProcessError(cause))), + }, + ExecutorTask::Async(processor) => { + if let Some(executor) = executor { + match self.execute_async_task( + processor, + executor, + executor.global_tasks_queue.clone(), + ) { + Ok(res) => Ok(res), + Err(cause) => Err(Box::new(NodeErrorType::AsyncProcessError(cause))), + } + } else { + Err(Box::new(NodeErrorType::LocalError(ErrorCode::Internal( + "Async task should only be executed on queries executor", + )))) } - } else { - Err(Box::new(NodeErrorType::LocalError(ErrorCode::Internal( - "Async task should only be executed on queries executor", - )))) } + ExecutorTask::AsyncCompleted(task) => match task.res { + Ok(_) => Ok(Some((task.id, task.graph))), + Err(cause) => Err(Box::new(NodeErrorType::AsyncProcessError(cause))), + }, } - ExecutorTask::AsyncCompleted(task) => match task.res { - Ok(_) => Ok(Some((task.id, task.graph))), - Err(cause) => Err(Box::new(NodeErrorType::AsyncProcessError(cause))), - }, } } @@ -162,24 +164,26 @@ impl ExecutorWorkerContext { &mut self, proc: ProcessorWrapper, ) -> Result)>> { - let payload = proc.graph.get_node_tracking_payload(proc.processor.id()); - let guard = ThreadTracker::tracking(payload.clone()); - let begin = SystemTime::now(); - let instant = Instant::now(); - - proc.processor.process()?; - let nanos = instant.elapsed().as_nanos(); - assume(nanos < 18446744073709551615_u128); - Profile::record_usize_profile(ProfileStatisticsName::CpuTime, nanos as usize); - let process_rows = proc.process_rows; - proc.graph - .record_process(begin, nanos as usize / 1_000, process_rows); - - if let Err(out_of_limit) = guard.flush() { - return Err(ErrorCode::PanicError(format!("{:?}", out_of_limit))); - } + unsafe { + let payload = proc.graph.get_node_tracking_payload(proc.processor.id()); + let guard = ThreadTracker::tracking(payload.clone()); + let begin = SystemTime::now(); + let instant = Instant::now(); + + proc.processor.process()?; + let nanos = instant.elapsed().as_nanos(); + assume(nanos < 18446744073709551615_u128); + Profile::record_usize_profile(ProfileStatisticsName::CpuTime, nanos as usize); + let process_rows = proc.process_rows; + proc.graph + .record_process(begin, nanos as usize / 1_000, process_rows); + + if let Err(out_of_limit) = guard.flush() { + return Err(ErrorCode::PanicError(format!("{:?}", out_of_limit))); + } - Ok(Some((proc.processor.id(), proc.graph))) + Ok(Some((proc.processor.id(), proc.graph))) + } } pub fn execute_async_task( diff --git a/src/query/service/src/pipelines/executor/pipeline_complete_executor.rs b/src/query/service/src/pipelines/executor/pipeline_complete_executor.rs index 3b34abdf607fc..7b6469d195772 100644 --- a/src/query/service/src/pipelines/executor/pipeline_complete_executor.rs +++ b/src/query/service/src/pipelines/executor/pipeline_complete_executor.rs @@ -14,10 +14,10 @@ use std::sync::Arc; -use databend_common_base::runtime::drop_guard; use databend_common_base::runtime::Thread; use databend_common_base::runtime::ThreadTracker; use databend_common_base::runtime::TrackingPayload; +use databend_common_base::runtime::drop_guard; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_pipeline::core::Pipeline; @@ -101,7 +101,7 @@ impl PipelineCompleteExecutor { .flatten() } - fn thread_function(&self) -> impl Fn() -> Result<()> { + fn thread_function(&self) -> impl Fn() -> Result<()> + use<> { let span = Span::enter_with_local_parent(func_path!()); let executor = self.executor.clone(); diff --git a/src/query/service/src/pipelines/executor/pipeline_executor.rs b/src/query/service/src/pipelines/executor/pipeline_executor.rs index 46a8d45bc9fcd..5e10d851c4396 100644 --- a/src/query/service/src/pipelines/executor/pipeline_executor.rs +++ b/src/query/service/src/pipelines/executor/pipeline_executor.rs @@ -18,11 +18,11 @@ use std::time::Duration; use std::time::Instant; use databend_common_base::base::WatchNotify; -use databend_common_base::runtime::catch_unwind; -use databend_common_base::runtime::defer; use databend_common_base::runtime::ExecutorStatsSnapshot; use databend_common_base::runtime::GlobalIORuntime; use databend_common_base::runtime::TrySpawn; +use databend_common_base::runtime::catch_unwind; +use databend_common_base::runtime::defer; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_pipeline::core::ExecutionInfo; @@ -30,8 +30,8 @@ use databend_common_pipeline::core::FinishedCallbackChain; use databend_common_pipeline::core::LockGuard; use databend_common_pipeline::core::Pipeline; use databend_common_pipeline::core::PlanProfile; -use futures_util::future::select; use futures_util::future::Either; +use futures_util::future::select; use log::info; use parking_lot::Condvar; use parking_lot::Mutex; diff --git a/src/query/service/src/pipelines/executor/pipeline_pulling_executor.rs b/src/query/service/src/pipelines/executor/pipeline_pulling_executor.rs index 413edd1d9c313..b975aea68895d 100644 --- a/src/query/service/src/pipelines/executor/pipeline_pulling_executor.rs +++ b/src/query/service/src/pipelines/executor/pipeline_pulling_executor.rs @@ -12,18 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; use std::sync::mpsc::Receiver; use std::sync::mpsc::RecvTimeoutError; use std::sync::mpsc::SyncSender; -use std::sync::Arc; use std::time::Duration; -use databend_common_base::runtime::drop_guard; use databend_common_base::runtime::Thread; use databend_common_base::runtime::ThreadTracker; use databend_common_base::runtime::TrackingPayload; +use databend_common_base::runtime::drop_guard; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::DataBlock; @@ -39,9 +39,9 @@ use fastrace::prelude::*; use parking_lot::Condvar; use parking_lot::Mutex; +use crate::pipelines::PipelineBuildResult; use crate::pipelines::executor::ExecutorSettings; use crate::pipelines::executor::PipelineExecutor; -use crate::pipelines::PipelineBuildResult; struct State { is_finished: AtomicBool, diff --git a/src/query/service/src/pipelines/executor/processor_async_task.rs b/src/query/service/src/pipelines/executor/processor_async_task.rs index 1ee3b47fdac31..d040a62e58044 100644 --- a/src/query/service/src/pipelines/executor/processor_async_task.rs +++ b/src/query/service/src/pipelines/executor/processor_async_task.rs @@ -21,17 +21,17 @@ use std::task::Poll; use std::time::Duration; use std::time::Instant; -use databend_common_base::base::tokio::time::sleep; use databend_common_base::base::WatchNotify; +use databend_common_base::base::tokio::time::sleep; use databend_common_base::runtime::catch_unwind; use databend_common_base::runtime::profile::Profile; use databend_common_base::runtime::profile::ProfileStatisticsName; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_pipeline::core::ProcessorPtr; +use futures_util::FutureExt; use futures_util::future::BoxFuture; use futures_util::future::Either; -use futures_util::FutureExt; use log::warn; use petgraph::prelude::NodeIndex; diff --git a/src/query/service/src/pipelines/executor/queries_executor_tasks.rs b/src/query/service/src/pipelines/executor/queries_executor_tasks.rs index d491331a05696..5054ebc953a60 100644 --- a/src/query/service/src/pipelines/executor/queries_executor_tasks.rs +++ b/src/query/service/src/pipelines/executor/queries_executor_tasks.rs @@ -14,20 +14,20 @@ use std::collections::VecDeque; use std::mem; +use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; -use std::sync::Arc; use parking_lot::Mutex; -use crate::pipelines::executor::executor_graph::ProcessorWrapper; -use crate::pipelines::executor::executor_worker_context::CompletedAsyncTask; use crate::pipelines::executor::ExecutorTask; use crate::pipelines::executor::ExecutorWorkerContext; use crate::pipelines::executor::QueriesPipelineExecutor; use crate::pipelines::executor::WatchNotify; use crate::pipelines::executor::WorkersCondvar; use crate::pipelines::executor::WorkersWaitingStatus; +use crate::pipelines::executor::executor_graph::ProcessorWrapper; +use crate::pipelines::executor::executor_worker_context::CompletedAsyncTask; struct WorkersTasks { workers_waiting_status: WorkersWaitingStatus, diff --git a/src/query/service/src/pipelines/executor/queries_pipeline_executor.rs b/src/query/service/src/pipelines/executor/queries_pipeline_executor.rs index 013d48ddc0adc..ebeb60064c563 100644 --- a/src/query/service/src/pipelines/executor/queries_pipeline_executor.rs +++ b/src/query/service/src/pipelines/executor/queries_pipeline_executor.rs @@ -12,17 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::Arc; use std::sync::atomic::AtomicU32; use std::sync::atomic::Ordering; -use std::sync::Arc; -use databend_common_base::runtime::catch_unwind; -use databend_common_base::runtime::drop_guard; -use databend_common_base::runtime::error_info::NodeErrorType; use databend_common_base::runtime::GlobalIORuntime; use databend_common_base::runtime::Runtime; use databend_common_base::runtime::Thread; use databend_common_base::runtime::ThreadJoinHandle; +use databend_common_base::runtime::catch_unwind; +use databend_common_base::runtime::drop_guard; +use databend_common_base::runtime::error_info::NodeErrorType; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use fastrace::func_path; @@ -145,82 +145,86 @@ impl QueriesPipelineExecutor { /// /// Method is thread unsafe and require thread safe call pub unsafe fn execute_single_thread(self: &Arc, thread_num: usize) -> Result<()> { - let workers_condvar = self.workers_condvar.clone(); - let mut context = ExecutorWorkerContext::create(thread_num, workers_condvar); + unsafe { + let workers_condvar = self.workers_condvar.clone(); + let mut context = ExecutorWorkerContext::create(thread_num, workers_condvar); - while !self.global_tasks_queue.is_finished() { - // Load tasks from global queue into worker context when context is empty. - // When context already contains tasks (new scheduled task triggered - // by previously executed processors in this context), - // those local tasks take priority and are executed first. - while !self.global_tasks_queue.is_finished() && !context.has_task() { - // If no tasks are available in the global tasks queue, - // this steal_tasks_to_context will block here - self.global_tasks_queue - .steal_task_to_context(&mut context, self); - } + while !self.global_tasks_queue.is_finished() { + // Load tasks from global queue into worker context when context is empty. + // When context already contains tasks (new scheduled task triggered + // by previously executed processors in this context), + // those local tasks take priority and are executed first. + while !self.global_tasks_queue.is_finished() && !context.has_task() { + // If no tasks are available in the global tasks queue, + // this steal_tasks_to_context will block here + self.global_tasks_queue + .steal_task_to_context(&mut context, self); + } - while !self.global_tasks_queue.is_finished() && context.has_task() { - let task_info = context.get_task_info(); - let execute_res = catch_unwind(|| context.execute_task(Some(self))); - match execute_res { - Ok(res) => { - match res { - Ok(Some((executed_pid, graph))) => { - // Not scheduled graph if pipeline is finished. - if !self.global_tasks_queue.is_finished() - && !graph.is_should_finish() - { - // We immediately schedule the processor again. - let schedule_queue_res = - graph.clone().schedule_queue(executed_pid); - match schedule_queue_res { - Ok(schedule_queue) => { - schedule_queue.schedule_with_condition( - &self.global_tasks_queue, - &mut context, - self, - ); - } - Err(cause) => { - graph.record_node_error( - executed_pid, - NodeErrorType::ScheduleEventError(cause.clone()), - ); - graph.should_finish(Err(cause))?; + while !self.global_tasks_queue.is_finished() && context.has_task() { + let task_info = context.get_task_info(); + let execute_res = catch_unwind(|| context.execute_task(Some(self))); + match execute_res { + Ok(res) => { + match res { + Ok(Some((executed_pid, graph))) => { + // Not scheduled graph if pipeline is finished. + if !self.global_tasks_queue.is_finished() + && !graph.is_should_finish() + { + // We immediately schedule the processor again. + let schedule_queue_res = + graph.clone().schedule_queue(executed_pid); + match schedule_queue_res { + Ok(schedule_queue) => { + schedule_queue.schedule_with_condition( + &self.global_tasks_queue, + &mut context, + self, + ); + } + Err(cause) => { + graph.record_node_error( + executed_pid, + NodeErrorType::ScheduleEventError( + cause.clone(), + ), + ); + graph.should_finish(Err(cause))?; + } } } + if graph.is_all_nodes_finished() { + graph.should_finish::<()>(Ok(()))?; + } } - if graph.is_all_nodes_finished() { - graph.should_finish::<()>(Ok(()))?; - } - } - Err(node_error) => { - let cause = node_error.get_error_code(); - warn!("Execute task error: {:?}", cause); - if let Some((graph, node_index)) = task_info { - graph.record_node_error(node_index, *node_error); - graph.should_finish(Err(cause.clone()))?; + Err(node_error) => { + let cause = node_error.get_error_code(); + warn!("Execute task error: {:?}", cause); + if let Some((graph, node_index)) = task_info { + graph.record_node_error(node_index, *node_error); + graph.should_finish(Err(cause.clone()))?; + } } + _ => {} } - _ => {} } - } - Err(panic_error) => { - warn!("Execute task error: {:?}", panic_error); - if let Some((graph, node_index)) = task_info { - graph.record_node_error( - node_index, - NodeErrorType::SyncProcessError(panic_error.clone()), - ); - graph.should_finish(Err(panic_error.clone()))?; + Err(panic_error) => { + warn!("Execute task error: {:?}", panic_error); + if let Some((graph, node_index)) = task_info { + graph.record_node_error( + node_index, + NodeErrorType::SyncProcessError(panic_error.clone()), + ); + graph.should_finish(Err(panic_error.clone()))?; + } } } } } - } - Ok(()) + Ok(()) + } } pub fn finish(&self, cause: Option) { diff --git a/src/query/service/src/pipelines/executor/query_executor_tasks.rs b/src/query/service/src/pipelines/executor/query_executor_tasks.rs index 335b16bfa323c..36f98ab67856c 100644 --- a/src/query/service/src/pipelines/executor/query_executor_tasks.rs +++ b/src/query/service/src/pipelines/executor/query_executor_tasks.rs @@ -13,19 +13,19 @@ // limitations under the License. use std::collections::VecDeque; +use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; -use std::sync::Arc; use parking_lot::Mutex; -use crate::pipelines::executor::executor_graph::ProcessorWrapper; -use crate::pipelines::executor::executor_worker_context::CompletedAsyncTask; use crate::pipelines::executor::ExecutorTask; use crate::pipelines::executor::ExecutorWorkerContext; use crate::pipelines::executor::WatchNotify; use crate::pipelines::executor::WorkersCondvar; use crate::pipelines::executor::WorkersWaitingStatus; +use crate::pipelines::executor::executor_graph::ProcessorWrapper; +use crate::pipelines::executor::executor_worker_context::CompletedAsyncTask; pub struct QueryExecutorTasksQueue { finished: Arc, @@ -130,7 +130,7 @@ impl QueryExecutorTasksQueue { } } - #[allow(unused_assignments)] + #[allow(unused)] pub fn push_tasks(&self, ctx: &mut ExecutorWorkerContext, mut tasks: VecDeque) { let mut workers_tasks = self.workers_tasks.lock(); diff --git a/src/query/service/src/pipelines/executor/query_pipeline_executor.rs b/src/query/service/src/pipelines/executor/query_pipeline_executor.rs index 940bd20e9ee93..3a7305ae35575 100644 --- a/src/query/service/src/pipelines/executor/query_pipeline_executor.rs +++ b/src/query/service/src/pipelines/executor/query_pipeline_executor.rs @@ -20,8 +20,6 @@ use std::sync::Arc; use std::time::Instant; use databend_common_base::base::tokio; -use databend_common_base::runtime::catch_unwind; -use databend_common_base::runtime::error_info::NodeErrorType; use databend_common_base::runtime::ExecutorStatsSnapshot; use databend_common_base::runtime::GlobalIORuntime; use databend_common_base::runtime::LimitMemGuard; @@ -30,6 +28,8 @@ use databend_common_base::runtime::Thread; use databend_common_base::runtime::ThreadJoinHandle; use databend_common_base::runtime::ThreadTracker; use databend_common_base::runtime::TrySpawn; +use databend_common_base::runtime::catch_unwind; +use databend_common_base::runtime::error_info::NodeErrorType; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_pipeline::core::ExecutionInfo; @@ -45,13 +45,13 @@ use log::warn; use parking_lot::Mutex; use petgraph::matrix_graph::Zero; -use crate::pipelines::executor::executor_graph::ScheduleQueue; use crate::pipelines::executor::ExecutorSettings; use crate::pipelines::executor::ExecutorWorkerContext; use crate::pipelines::executor::QueryExecutorTasksQueue; use crate::pipelines::executor::RunningGraph; use crate::pipelines::executor::WatchNotify; use crate::pipelines::executor::WorkersCondvar; +use crate::pipelines::executor::executor_graph::ScheduleQueue; pub type InitCallback = Box Result<()> + Send + Sync + 'static>; @@ -384,57 +384,59 @@ impl QueryPipelineExecutor { /// /// Method is thread unsafe and require thread safe call pub unsafe fn execute_single_thread(self: &Arc, thread_num: usize) -> Result<()> { - let workers_condvar = self.workers_condvar.clone(); - let mut context = ExecutorWorkerContext::create(thread_num, workers_condvar); + unsafe { + let workers_condvar = self.workers_condvar.clone(); + let mut context = ExecutorWorkerContext::create(thread_num, workers_condvar); - while !self.global_tasks_queue.is_finished() { - // When there are not enough tasks, the thread will be blocked, so we need loop check. - while !self.global_tasks_queue.is_finished() && !context.has_task() { - self.global_tasks_queue.steal_task_to_context(&mut context); - } + while !self.global_tasks_queue.is_finished() { + // When there are not enough tasks, the thread will be blocked, so we need loop check. + while !self.global_tasks_queue.is_finished() && !context.has_task() { + self.global_tasks_queue.steal_task_to_context(&mut context); + } - while !self.global_tasks_queue.is_finished() && context.has_task() { - let task_info = context.get_task_info(); - let execute_res = context.execute_task(None); - match execute_res { - Ok(Some((executed_pid, graph))) => { - // Not scheduled graph if pipeline is finished. - if !self.global_tasks_queue.is_finished() { - // We immediately schedule the processor again. - let schedule_queue_res = graph.clone().schedule_queue(executed_pid); - match schedule_queue_res { - Ok(schedule_queue) => { - schedule_queue.schedule( - &self.global_tasks_queue, - &mut context, - self, - ); - } - Err(cause) => { - graph.record_node_error( - executed_pid, - NodeErrorType::ScheduleEventError(cause.clone()), - ); - graph.should_finish(Err(cause.clone()))?; - return Err(cause); + while !self.global_tasks_queue.is_finished() && context.has_task() { + let task_info = context.get_task_info(); + let execute_res = context.execute_task(None); + match execute_res { + Ok(Some((executed_pid, graph))) => { + // Not scheduled graph if pipeline is finished. + if !self.global_tasks_queue.is_finished() { + // We immediately schedule the processor again. + let schedule_queue_res = graph.clone().schedule_queue(executed_pid); + match schedule_queue_res { + Ok(schedule_queue) => { + schedule_queue.schedule( + &self.global_tasks_queue, + &mut context, + self, + ); + } + Err(cause) => { + graph.record_node_error( + executed_pid, + NodeErrorType::ScheduleEventError(cause.clone()), + ); + graph.should_finish(Err(cause.clone()))?; + return Err(cause); + } } } } - } - Err(error_type) => { - let cause = error_type.get_error_code(); - if let Some((graph, node_index)) = task_info { - graph.record_node_error(node_index, *error_type); - graph.should_finish(Err(cause.clone()))?; + Err(error_type) => { + let cause = error_type.get_error_code(); + if let Some((graph, node_index)) = task_info { + graph.record_node_error(node_index, *error_type); + graph.should_finish(Err(cause.clone()))?; + } + return Err(cause); } - return Err(cause); - } - _ => {} - }; + _ => {} + }; + } } - } - Ok(()) + Ok(()) + } } pub fn format_graph_nodes(&self) -> String { diff --git a/src/query/service/src/pipelines/mod.rs b/src/query/service/src/pipelines/mod.rs index 0c56b4247d0dd..9ad2a096cff78 100644 --- a/src/query/service/src/pipelines/mod.rs +++ b/src/query/service/src/pipelines/mod.rs @@ -24,6 +24,6 @@ pub use builders::RawValueSource; pub use builders::ValueSource; pub use pipeline_build_res::PipelineBuildResult; pub use pipeline_build_res::PipelineBuilderData; -pub use pipeline_builder::attach_runtime_filter_logger; pub use pipeline_builder::HashJoinStateRef; pub use pipeline_builder::PipelineBuilder; +pub use pipeline_builder::attach_runtime_filter_logger; diff --git a/src/query/service/src/pipelines/pipeline_builder.rs b/src/query/service/src/pipelines/pipeline_builder.rs index bc923ea19aedd..d21e5498aadc8 100644 --- a/src/query/service/src/pipelines/pipeline_builder.rs +++ b/src/query/service/src/pipelines/pipeline_builder.rs @@ -19,18 +19,18 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::DataField; use databend_common_expression::FunctionContext; -use databend_common_pipeline::core::always_callback; use databend_common_pipeline::core::ExecutionInfo; use databend_common_pipeline::core::Pipeline; +use databend_common_pipeline::core::always_callback; use databend_common_settings::Settings; use super::PipelineBuilderData; use crate::interpreters::CreateTableInterpreter; use crate::physical_plans::PhysicalPlan; -use crate::pipelines::processors::transforms::BasicHashJoinState; +use crate::pipelines::PipelineBuildResult; use crate::pipelines::processors::HashJoinBuildState; use crate::pipelines::processors::HashJoinState; -use crate::pipelines::PipelineBuildResult; +use crate::pipelines::processors::transforms::BasicHashJoinState; use crate::servers::flight::v1::exchange::DefaultExchangeInjector; use crate::servers::flight::v1::exchange::ExchangeInjector; use crate::sessions::QueryContext; diff --git a/src/query/service/src/pipelines/processors/transforms/aggregator/aggregate_exchange_injector.rs b/src/query/service/src/pipelines/processors/transforms/aggregator/aggregate_exchange_injector.rs index bc244b9b93683..96e884f311b9c 100644 --- a/src/query/service/src/pipelines/processors/transforms/aggregator/aggregate_exchange_injector.rs +++ b/src/query/service/src/pipelines/processors/transforms/aggregator/aggregate_exchange_injector.rs @@ -27,13 +27,13 @@ use databend_common_pipeline::core::ProcessorPtr; use databend_common_settings::FlightCompression; use databend_common_storage::DataOperator; -use crate::pipelines::processors::transforms::aggregator::aggregate_meta::AggregateMeta; -use crate::pipelines::processors::transforms::aggregator::serde::TransformExchangeAggregateSerializer; -use crate::pipelines::processors::transforms::aggregator::serde::TransformExchangeAsyncBarrier; use crate::pipelines::processors::transforms::aggregator::AggregatorParams; use crate::pipelines::processors::transforms::aggregator::TransformAggregateDeserializer; use crate::pipelines::processors::transforms::aggregator::TransformAggregateSerializer; use crate::pipelines::processors::transforms::aggregator::TransformAggregateSpillWriter; +use crate::pipelines::processors::transforms::aggregator::aggregate_meta::AggregateMeta; +use crate::pipelines::processors::transforms::aggregator::serde::TransformExchangeAggregateSerializer; +use crate::pipelines::processors::transforms::aggregator::serde::TransformExchangeAsyncBarrier; use crate::servers::flight::v1::exchange::DataExchange; use crate::servers::flight::v1::exchange::ExchangeInjector; use crate::servers::flight::v1::exchange::ExchangeSorting; diff --git a/src/query/service/src/pipelines/processors/transforms/aggregator/aggregate_meta.rs b/src/query/service/src/pipelines/processors/transforms/aggregator/aggregate_meta.rs index bd6f265c0a5da..98f8263248462 100644 --- a/src/query/service/src/pipelines/processors/transforms/aggregator/aggregate_meta.rs +++ b/src/query/service/src/pipelines/processors/transforms/aggregator/aggregate_meta.rs @@ -19,7 +19,6 @@ use std::sync::Arc; use bumpalo::Bump; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::AggregateFunction; use databend_common_expression::AggregateHashTable; use databend_common_expression::BlockMetaInfo; @@ -31,6 +30,7 @@ use databend_common_expression::PartitionedPayload; use databend_common_expression::Payload; use databend_common_expression::ProbeState; use databend_common_expression::ProjectedBlock; +use databend_common_expression::types::DataType; use parquet::file::metadata::RowGroupMetaData; pub struct SerializedPayload { diff --git a/src/query/service/src/pipelines/processors/transforms/aggregator/aggregator_params.rs b/src/query/service/src/pipelines/processors/transforms/aggregator/aggregator_params.rs index 4bc3c5303219e..d615f3c05afc5 100644 --- a/src/query/service/src/pipelines/processors/transforms/aggregator/aggregator_params.rs +++ b/src/query/service/src/pipelines/processors/transforms/aggregator/aggregator_params.rs @@ -15,13 +15,13 @@ use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::ColumnBuilder; use databend_common_expression::DataBlock; use databend_common_expression::DataSchemaRef; -use databend_common_functions::aggregates::get_states_layout; +use databend_common_expression::types::DataType; use databend_common_functions::aggregates::AggregateFunctionRef; use databend_common_functions::aggregates::StatesLayout; +use databend_common_functions::aggregates::get_states_layout; use databend_common_sql::IndexType; use itertools::Itertools; diff --git a/src/query/service/src/pipelines/processors/transforms/aggregator/build_partition_bucket.rs b/src/query/service/src/pipelines/processors/transforms/aggregator/build_partition_bucket.rs index 0e649e8d37f3e..f226ecb33d91c 100644 --- a/src/query/service/src/pipelines/processors/transforms/aggregator/build_partition_bucket.rs +++ b/src/query/service/src/pipelines/processors/transforms/aggregator/build_partition_bucket.rs @@ -28,15 +28,15 @@ use parking_lot::Mutex; use tokio::sync::Barrier; use tokio::sync::Semaphore; +use crate::pipelines::processors::transforms::aggregator::AggregatorParams; +use crate::pipelines::processors::transforms::aggregator::TransformAggregateSpillReader; +use crate::pipelines::processors::transforms::aggregator::TransformFinalAggregate; use crate::pipelines::processors::transforms::aggregator::new_aggregate::FinalAggregateSharedState; use crate::pipelines::processors::transforms::aggregator::new_aggregate::NewAggregateSpiller; use crate::pipelines::processors::transforms::aggregator::new_aggregate::NewFinalAggregateTransform; use crate::pipelines::processors::transforms::aggregator::new_aggregate::SharedPartitionStream; use crate::pipelines::processors::transforms::aggregator::new_aggregate::TransformPartitionBucketScatter; use crate::pipelines::processors::transforms::aggregator::transform_partition_bucket::TransformPartitionBucket; -use crate::pipelines::processors::transforms::aggregator::AggregatorParams; -use crate::pipelines::processors::transforms::aggregator::TransformAggregateSpillReader; -use crate::pipelines::processors::transforms::aggregator::TransformFinalAggregate; use crate::sessions::QueryContext; fn build_partition_bucket_experimental( diff --git a/src/query/service/src/pipelines/processors/transforms/aggregator/new_aggregate/new_aggregate_spiller.rs b/src/query/service/src/pipelines/processors/transforms/aggregator/new_aggregate/new_aggregate_spiller.rs index 8bd75194199b1..45b02e1e1c69e 100644 --- a/src/query/service/src/pipelines/processors/transforms/aggregator/new_aggregate/new_aggregate_spiller.rs +++ b/src/query/service/src/pipelines/processors/transforms/aggregator/new_aggregate/new_aggregate_spiller.rs @@ -24,8 +24,8 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::BlockPartitionStream; use databend_common_expression::DataBlock; -use databend_common_pipeline_transforms::traits::Location; use databend_common_pipeline_transforms::MemorySettings; +use databend_common_pipeline_transforms::traits::Location; use databend_common_storage::DataOperator; use databend_common_storages_parquet::ReadSettings; use log::debug; @@ -377,7 +377,11 @@ impl NewAggregateSpiller { info!( "Read aggregate spill finished: (bucket: {}, location: {}, bytes: {}, rows: {}, elapsed: {:?})", - bucket, location, read_size, row_group.num_rows(), elapsed + bucket, + location, + read_size, + row_group.num_rows(), + elapsed ); if let Some(block) = data_block { @@ -426,12 +430,12 @@ mod tests { use databend_common_base::base::tokio; use databend_common_exception::Result; - use databend_common_expression::types::Int32Type; use databend_common_expression::DataBlock; use databend_common_expression::FromData; + use databend_common_expression::types::Int32Type; - use crate::pipelines::processors::transforms::aggregator::new_aggregate::SharedPartitionStream; use crate::pipelines::processors::transforms::aggregator::NewAggregateSpiller; + use crate::pipelines::processors::transforms::aggregator::new_aggregate::SharedPartitionStream; use crate::test_kits::TestFixture; #[tokio::test(flavor = "multi_thread", worker_threads = 1)] diff --git a/src/query/service/src/pipelines/processors/transforms/aggregator/new_aggregate/new_transform_aggregate_partial.rs b/src/query/service/src/pipelines/processors/transforms/aggregator/new_aggregate/new_transform_aggregate_partial.rs index 2154fd9c2c658..2cd1eb63fb28a 100644 --- a/src/query/service/src/pipelines/processors/transforms/aggregator/new_aggregate/new_transform_aggregate_partial.rs +++ b/src/query/service/src/pipelines/processors/transforms/aggregator/new_aggregate/new_transform_aggregate_partial.rs @@ -21,37 +21,37 @@ use databend_common_base::base::convert_byte_size; use databend_common_base::base::convert_number_size; use databend_common_catalog::plan::AggIndexMeta; use databend_common_exception::Result; -use databend_common_expression::types::BinaryType; -use databend_common_expression::types::Int64Type; -use databend_common_expression::types::StringType; use databend_common_expression::AggregateHashTable; use databend_common_expression::BlockMetaInfoDowncast; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::HashTableConfig; +use databend_common_expression::MAX_AGGREGATE_HASHTABLE_BUCKETS_NUM; use databend_common_expression::PartitionedPayload; use databend_common_expression::PayloadFlushState; use databend_common_expression::ProbeState; use databend_common_expression::ProjectedBlock; -use databend_common_expression::MAX_AGGREGATE_HASHTABLE_BUCKETS_NUM; +use databend_common_expression::types::BinaryType; +use databend_common_expression::types::Int64Type; +use databend_common_expression::types::StringType; use databend_common_pipeline::core::InputPort; use databend_common_pipeline::core::OutputPort; use databend_common_pipeline::core::Processor; +use databend_common_pipeline_transforms::MemorySettings; use databend_common_pipeline_transforms::processors::AccumulatingTransform; use databend_common_pipeline_transforms::processors::AccumulatingTransformer; -use databend_common_pipeline_transforms::MemorySettings; use databend_common_storages_parquet::serialize_row_group_meta_to_bytes; use crate::pipelines::memory_settings::MemorySettingsExt; -use crate::pipelines::processors::transforms::aggregator::aggregate_exchange_injector::scatter_partitioned_payload; -use crate::pipelines::processors::transforms::aggregator::aggregate_meta::AggregateMeta; -use crate::pipelines::processors::transforms::aggregator::exchange_defines; use crate::pipelines::processors::transforms::aggregator::AggregateSerdeMeta; use crate::pipelines::processors::transforms::aggregator::AggregatorParams; use crate::pipelines::processors::transforms::aggregator::FlightSerialized; use crate::pipelines::processors::transforms::aggregator::FlightSerializedMeta; use crate::pipelines::processors::transforms::aggregator::NewAggregateSpiller; use crate::pipelines::processors::transforms::aggregator::SharedPartitionStream; +use crate::pipelines::processors::transforms::aggregator::aggregate_exchange_injector::scatter_partitioned_payload; +use crate::pipelines::processors::transforms::aggregator::aggregate_meta::AggregateMeta; +use crate::pipelines::processors::transforms::aggregator::exchange_defines; use crate::servers::flight::v1::exchange::serde::serialize_block; use crate::sessions::QueryContext; diff --git a/src/query/service/src/pipelines/processors/transforms/aggregator/new_aggregate/transform_partition_bucket_scatter.rs b/src/query/service/src/pipelines/processors/transforms/aggregator/new_aggregate/transform_partition_bucket_scatter.rs index 113ccf57f4f34..40ed11c3cb101 100644 --- a/src/query/service/src/pipelines/processors/transforms/aggregator/new_aggregate/transform_partition_bucket_scatter.rs +++ b/src/query/service/src/pipelines/processors/transforms/aggregator/new_aggregate/transform_partition_bucket_scatter.rs @@ -13,8 +13,8 @@ // limitations under the License. use std::any::Any; -use std::collections::btree_map::Entry; use std::collections::BTreeMap; +use std::collections::btree_map::Entry; use std::sync::Arc; use bumpalo::Bump; @@ -30,10 +30,10 @@ use databend_common_pipeline::core::OutputPort; use databend_common_pipeline::core::Processor; use super::split_partitioned_meta_into_datablocks; -use crate::pipelines::processors::transforms::aggregator::aggregate_meta::AggregateMeta; -use crate::pipelines::processors::transforms::aggregator::aggregate_meta::SerializedPayload; use crate::pipelines::processors::transforms::aggregator::AggregatePayload; use crate::pipelines::processors::transforms::aggregator::AggregatorParams; +use crate::pipelines::processors::transforms::aggregator::aggregate_meta::AggregateMeta; +use crate::pipelines::processors::transforms::aggregator::aggregate_meta::SerializedPayload; static SINGLE_LEVEL_BUCKET_NUM: isize = -1; static MAX_PARTITION_COUNT: usize = 128; @@ -211,7 +211,7 @@ impl TransformPartitionBucketScatter { Ok(self.initialized_all_inputs) } - #[allow(unused_assignments)] + #[allow(unused)] fn add_bucket(&mut self, mut data_block: DataBlock) -> Result<(isize, usize)> { let (mut bucket, mut partition_count) = (0, 0); let mut is_empty_block = false; diff --git a/src/query/service/src/pipelines/processors/transforms/aggregator/serde/mod.rs b/src/query/service/src/pipelines/processors/transforms/aggregator/serde/mod.rs index 2608e69214d53..890e67aa0ff93 100644 --- a/src/query/service/src/pipelines/processors/transforms/aggregator/serde/mod.rs +++ b/src/query/service/src/pipelines/processors/transforms/aggregator/serde/mod.rs @@ -31,10 +31,10 @@ pub use transform_spill_reader::*; pub mod exchange_defines { use arrow_ipc::writer::IpcWriteOptions; use arrow_schema::Schema; - use databend_common_expression::types::DataType; - use databend_common_expression::types::NumberDataType; use databend_common_expression::DataField; use databend_common_expression::DataSchema; + use databend_common_expression::types::DataType; + use databend_common_expression::types::NumberDataType; pub fn spilled_schema() -> DataSchema { DataSchema::new(vec![ diff --git a/src/query/service/src/pipelines/processors/transforms/aggregator/serde/transform_aggregate_serializer.rs b/src/query/service/src/pipelines/processors/transforms/aggregator/serde/transform_aggregate_serializer.rs index f6bc3063ace86..d24b491713165 100644 --- a/src/query/service/src/pipelines/processors/transforms/aggregator/serde/transform_aggregate_serializer.rs +++ b/src/query/service/src/pipelines/processors/transforms/aggregator/serde/transform_aggregate_serializer.rs @@ -18,12 +18,12 @@ use std::pin::Pin; use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::local_block_meta_serde; use databend_common_expression::BlockMetaInfo; use databend_common_expression::BlockMetaInfoDowncast; use databend_common_expression::BlockMetaInfoPtr; use databend_common_expression::DataBlock; use databend_common_expression::PayloadFlushState; +use databend_common_expression::local_block_meta_serde; use databend_common_pipeline::core::Event; use databend_common_pipeline::core::InputPort; use databend_common_pipeline::core::OutputPort; diff --git a/src/query/service/src/pipelines/processors/transforms/aggregator/serde/transform_aggregate_spill_writer.rs b/src/query/service/src/pipelines/processors/transforms/aggregator/serde/transform_aggregate_spill_writer.rs index 090ffbf54de97..8e85a355a2f70 100644 --- a/src/query/service/src/pipelines/processors/transforms/aggregator/serde/transform_aggregate_spill_writer.rs +++ b/src/query/service/src/pipelines/processors/transforms/aggregator/serde/transform_aggregate_spill_writer.rs @@ -22,10 +22,10 @@ use databend_common_base::runtime::profile::ProfileStatisticsName; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::arrow::serialize_column; use databend_common_expression::BlockMetaInfoDowncast; use databend_common_expression::DataBlock; use databend_common_expression::PartitionedPayload; +use databend_common_expression::arrow::serialize_column; use databend_common_pipeline::core::Event; use databend_common_pipeline::core::InputPort; use databend_common_pipeline::core::OutputPort; diff --git a/src/query/service/src/pipelines/processors/transforms/aggregator/serde/transform_deserializer.rs b/src/query/service/src/pipelines/processors/transforms/aggregator/serde/transform_deserializer.rs index 41beab5b485c1..f0cce90c2687f 100644 --- a/src/query/service/src/pipelines/processors/transforms/aggregator/serde/transform_deserializer.rs +++ b/src/query/service/src/pipelines/processors/transforms/aggregator/serde/transform_deserializer.rs @@ -17,17 +17,17 @@ use std::sync::Arc; use arrow_schema::Schema as ArrowSchema; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::BlockMetaInfoDowncast; +use databend_common_expression::DataBlock; +use databend_common_expression::DataSchemaRef; use databend_common_expression::types::AccessType; use databend_common_expression::types::ArrayType; use databend_common_expression::types::BinaryType; use databend_common_expression::types::NumberType; use databend_common_expression::types::StringType; use databend_common_expression::types::UInt64Type; -use databend_common_expression::BlockMetaInfoDowncast; -use databend_common_expression::DataBlock; -use databend_common_expression::DataSchemaRef; -use databend_common_io::prelude::bincode_deserialize_from_slice; use databend_common_io::prelude::BinaryRead; +use databend_common_io::prelude::bincode_deserialize_from_slice; use databend_common_pipeline::core::InputPort; use databend_common_pipeline::core::OutputPort; use databend_common_pipeline::core::ProcessorPtr; @@ -36,16 +36,16 @@ use databend_common_pipeline_transforms::processors::BlockMetaTransformer; use databend_common_pipeline_transforms::processors::UnknownMode; use databend_common_storages_parquet::deserialize_row_group_meta_from_bytes; -use crate::pipelines::processors::transforms::aggregator::exchange_defines; use crate::pipelines::processors::transforms::aggregator::AggregateMeta; use crate::pipelines::processors::transforms::aggregator::AggregateSerdeMeta; -use crate::pipelines::processors::transforms::aggregator::BucketSpilledPayload; -use crate::pipelines::processors::transforms::aggregator::NewSpilledPayload; use crate::pipelines::processors::transforms::aggregator::BUCKET_TYPE; +use crate::pipelines::processors::transforms::aggregator::BucketSpilledPayload; use crate::pipelines::processors::transforms::aggregator::NEW_SPILLED_TYPE; +use crate::pipelines::processors::transforms::aggregator::NewSpilledPayload; use crate::pipelines::processors::transforms::aggregator::SPILLED_TYPE; -use crate::servers::flight::v1::exchange::serde::deserialize_block; +use crate::pipelines::processors::transforms::aggregator::exchange_defines; use crate::servers::flight::v1::exchange::serde::ExchangeDeserializeMeta; +use crate::servers::flight::v1::exchange::serde::deserialize_block; use crate::servers::flight::v1::packets::DataPacket; use crate::servers::flight::v1::packets::FragmentData; diff --git a/src/query/service/src/pipelines/processors/transforms/aggregator/serde/transform_exchange_aggregate_serializer.rs b/src/query/service/src/pipelines/processors/transforms/aggregator/serde/transform_exchange_aggregate_serializer.rs index 480d075562008..86727c1f27ca7 100644 --- a/src/query/service/src/pipelines/processors/transforms/aggregator/serde/transform_exchange_aggregate_serializer.rs +++ b/src/query/service/src/pipelines/processors/transforms/aggregator/serde/transform_exchange_aggregate_serializer.rs @@ -15,46 +15,46 @@ use std::sync::Arc; use std::time::Instant; -use arrow_ipc::writer::IpcWriteOptions; use arrow_ipc::CompressionType; +use arrow_ipc::writer::IpcWriteOptions; use databend_common_base::base::ProgressValues; use databend_common_base::runtime::profile::Profile; use databend_common_base::runtime::profile::ProfileStatisticsName; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; +use databend_common_expression::BlockMetaInfoDowncast; +use databend_common_expression::DataBlock; +use databend_common_expression::FromData; +use databend_common_expression::PartitionedPayload; use databend_common_expression::arrow::serialize_column; use databend_common_expression::types::ArgType; use databend_common_expression::types::ArrayType; use databend_common_expression::types::Int64Type; use databend_common_expression::types::ReturnType; use databend_common_expression::types::UInt64Type; -use databend_common_expression::BlockMetaInfoDowncast; -use databend_common_expression::DataBlock; -use databend_common_expression::FromData; -use databend_common_expression::PartitionedPayload; use databend_common_pipeline::core::InputPort; use databend_common_pipeline::core::OutputPort; use databend_common_pipeline::core::Processor; +use databend_common_pipeline_transforms::UnknownMode; use databend_common_pipeline_transforms::processors::BlockMetaTransform; use databend_common_pipeline_transforms::processors::BlockMetaTransformer; -use databend_common_pipeline_transforms::UnknownMode; use databend_common_settings::FlightCompression; use futures_util::future::BoxFuture; use log::info; use opendal::Operator; use super::SerializePayload; -use crate::pipelines::processors::transforms::aggregator::agg_spilling_aggregate_payload as local_agg_spilling_aggregate_payload; -use crate::pipelines::processors::transforms::aggregator::aggregate_exchange_injector::compute_block_number; -use crate::pipelines::processors::transforms::aggregator::aggregate_meta::AggregateMeta; -use crate::pipelines::processors::transforms::aggregator::exchange_defines; use crate::pipelines::processors::transforms::aggregator::AggregateSerdeMeta; use crate::pipelines::processors::transforms::aggregator::AggregatorParams; use crate::pipelines::processors::transforms::aggregator::FlightSerialized; use crate::pipelines::processors::transforms::aggregator::FlightSerializedMeta; use crate::pipelines::processors::transforms::aggregator::SerializeAggregateStream; -use crate::servers::flight::v1::exchange::serde::serialize_block; +use crate::pipelines::processors::transforms::aggregator::agg_spilling_aggregate_payload as local_agg_spilling_aggregate_payload; +use crate::pipelines::processors::transforms::aggregator::aggregate_exchange_injector::compute_block_number; +use crate::pipelines::processors::transforms::aggregator::aggregate_meta::AggregateMeta; +use crate::pipelines::processors::transforms::aggregator::exchange_defines; use crate::servers::flight::v1::exchange::ExchangeShuffleMeta; +use crate::servers::flight::v1::exchange::serde::serialize_block; use crate::sessions::QueryContext; use crate::spillers::Spiller; use crate::spillers::SpillerConfig; diff --git a/src/query/service/src/pipelines/processors/transforms/aggregator/serde/transform_spill_reader.rs b/src/query/service/src/pipelines/processors/transforms/aggregator/serde/transform_spill_reader.rs index 55b97ce052cea..abd72df217f1c 100644 --- a/src/query/service/src/pipelines/processors/transforms/aggregator/serde/transform_spill_reader.rs +++ b/src/query/service/src/pipelines/processors/transforms/aggregator/serde/transform_spill_reader.rs @@ -14,9 +14,9 @@ use std::any::Any; use std::collections::VecDeque; +use std::sync::Arc; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; -use std::sync::Arc; use std::time::Duration; use std::time::Instant; @@ -24,10 +24,10 @@ use databend_common_base::runtime::profile::Profile; use databend_common_base::runtime::profile::ProfileStatisticsName; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::arrow::deserialize_column; use databend_common_expression::BlockMetaInfoDowncast; use databend_common_expression::BlockMetaInfoPtr; use databend_common_expression::DataBlock; +use databend_common_expression::arrow::deserialize_column; use databend_common_pipeline::core::Event; use databend_common_pipeline::core::InputPort; use databend_common_pipeline::core::OutputPort; diff --git a/src/query/service/src/pipelines/processors/transforms/aggregator/transform_aggregate_expand.rs b/src/query/service/src/pipelines/processors/transforms/aggregator/transform_aggregate_expand.rs index 963c0a0276042..1634759d11f23 100644 --- a/src/query/service/src/pipelines/processors/transforms/aggregator/transform_aggregate_expand.rs +++ b/src/query/service/src/pipelines/processors/transforms/aggregator/transform_aggregate_expand.rs @@ -13,12 +13,12 @@ // limitations under the License. use databend_common_exception::Result; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::NumberScalar; use databend_common_expression::BlockEntry; use databend_common_expression::DataBlock; use databend_common_expression::Scalar; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::NumberScalar; use databend_common_pipeline_transforms::AccumulatingTransform; pub struct TransformExpandGroupingSets { diff --git a/src/query/service/src/pipelines/processors/transforms/aggregator/transform_aggregate_final.rs b/src/query/service/src/pipelines/processors/transforms/aggregator/transform_aggregate_final.rs index 0f259f9de6953..6707641f8267b 100644 --- a/src/query/service/src/pipelines/processors/transforms/aggregator/transform_aggregate_final.rs +++ b/src/query/service/src/pipelines/processors/transforms/aggregator/transform_aggregate_final.rs @@ -26,8 +26,8 @@ use databend_common_pipeline::core::Processor; use databend_common_pipeline_transforms::processors::BlockMetaTransform; use databend_common_pipeline_transforms::processors::BlockMetaTransformer; -use crate::pipelines::processors::transforms::aggregator::aggregate_meta::AggregateMeta; use crate::pipelines::processors::transforms::aggregator::AggregatorParams; +use crate::pipelines::processors::transforms::aggregator::aggregate_meta::AggregateMeta; pub struct TransformFinalAggregate { params: Arc, diff --git a/src/query/service/src/pipelines/processors/transforms/aggregator/transform_aggregate_partial.rs b/src/query/service/src/pipelines/processors/transforms/aggregator/transform_aggregate_partial.rs index 8e76a2ddd1c40..8c076d04e744a 100644 --- a/src/query/service/src/pipelines/processors/transforms/aggregator/transform_aggregate_partial.rs +++ b/src/query/service/src/pipelines/processors/transforms/aggregator/transform_aggregate_partial.rs @@ -31,13 +31,13 @@ use databend_common_expression::ProjectedBlock; use databend_common_pipeline::core::InputPort; use databend_common_pipeline::core::OutputPort; use databend_common_pipeline::core::Processor; +use databend_common_pipeline_transforms::MemorySettings; use databend_common_pipeline_transforms::processors::AccumulatingTransform; use databend_common_pipeline_transforms::processors::AccumulatingTransformer; -use databend_common_pipeline_transforms::MemorySettings; use crate::pipelines::memory_settings::MemorySettingsExt; -use crate::pipelines::processors::transforms::aggregator::aggregate_meta::AggregateMeta; use crate::pipelines::processors::transforms::aggregator::AggregatorParams; +use crate::pipelines::processors::transforms::aggregator::aggregate_meta::AggregateMeta; use crate::sessions::QueryContext; #[allow(clippy::enum_variant_names)] enum HashTable { diff --git a/src/query/service/src/pipelines/processors/transforms/aggregator/transform_partition_bucket.rs b/src/query/service/src/pipelines/processors/transforms/aggregator/transform_partition_bucket.rs index 86bc657657a48..6c909a7b993a9 100644 --- a/src/query/service/src/pipelines/processors/transforms/aggregator/transform_partition_bucket.rs +++ b/src/query/service/src/pipelines/processors/transforms/aggregator/transform_partition_bucket.rs @@ -13,8 +13,8 @@ // limitations under the License. use std::any::Any; -use std::collections::btree_map::Entry; use std::collections::BTreeMap; +use std::collections::btree_map::Entry; use std::sync::Arc; use bumpalo::Bump; @@ -30,9 +30,9 @@ use databend_common_pipeline::core::OutputPort; use databend_common_pipeline::core::Processor; use super::AggregatePayload; +use crate::pipelines::processors::transforms::aggregator::AggregatorParams; use crate::pipelines::processors::transforms::aggregator::aggregate_meta::AggregateMeta; use crate::pipelines::processors::transforms::aggregator::aggregate_meta::SerializedPayload; -use crate::pipelines::processors::transforms::aggregator::AggregatorParams; static SINGLE_LEVEL_BUCKET_NUM: isize = -1; static MAX_PARTITION_COUNT: usize = 128; @@ -201,7 +201,7 @@ impl TransformPartitionBucket { Ok(self.initialized_all_inputs) } - #[allow(unused_assignments)] + #[allow(unused)] fn add_bucket(&mut self, mut data_block: DataBlock) -> Result<(isize, usize)> { let (mut bucket, mut partition_count) = (0, 0); let mut is_empty_block = false; diff --git a/src/query/service/src/pipelines/processors/transforms/aggregator/udaf_script.rs b/src/query/service/src/pipelines/processors/transforms/aggregator/udaf_script.rs index c6fbf7898f4fc..6e498cc91953d 100644 --- a/src/query/service/src/pipelines/processors/transforms/aggregator/udaf_script.rs +++ b/src/query/service/src/pipelines/processors/transforms/aggregator/udaf_script.rs @@ -26,12 +26,6 @@ use arrow_udf_runtime::javascript::AggregateOptions; use databend_common_base::runtime::GlobalIORuntime; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::converts::arrow::ARROW_EXT_TYPE_VARIANT; -use databend_common_expression::converts::arrow::EXTENSION_KEY; -use databend_common_expression::types::BinaryType; -use databend_common_expression::types::Bitmap; -use databend_common_expression::types::DataType; -use databend_common_expression::types::UnaryType; use databend_common_expression::AggrState; use databend_common_expression::AggrStateRegistry; use databend_common_expression::AggrStateType; @@ -43,6 +37,12 @@ use databend_common_expression::DataField; use databend_common_expression::DataSchema; use databend_common_expression::ProjectedBlock; use databend_common_expression::StateSerdeItem; +use databend_common_expression::converts::arrow::ARROW_EXT_TYPE_VARIANT; +use databend_common_expression::converts::arrow::EXTENSION_KEY; +use databend_common_expression::types::BinaryType; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::DataType; +use databend_common_expression::types::UnaryType; use databend_common_functions::aggregates::AggrStateLoc; use databend_common_functions::aggregates::AggregateFunction; use databend_common_functions::aggregates::StateAddr; @@ -197,8 +197,10 @@ impl AggregateFunction for AggregateUdfScript { } unsafe fn drop_state(&self, place: AggrState) { - let state = place.get::(); - std::ptr::drop_in_place(state); + unsafe { + let state = place.get::(); + std::ptr::drop_in_place(state); + } } } diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/build_state.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/build_state.rs index 59dbf495053f9..4226c4d4273aa 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/build_state.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/build_state.rs @@ -15,9 +15,9 @@ use std::sync::Arc; use databend_common_catalog::runtime_filter_info::RuntimeFilterReady; -use databend_common_expression::types::DataType; use databend_common_expression::ColumnVec; use databend_common_expression::DataBlock; +use databend_common_expression::types::DataType; pub struct BuildState { // The `generation_state` is used to generate build side `DataBlock`. diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/common.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/common.rs index bb005f9d62a0e..1b4e3239edb1d 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/common.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/common.rs @@ -15,11 +15,6 @@ use databend_common_column::bitmap::Bitmap; use databend_common_column::bitmap::MutableBitmap; use databend_common_exception::Result; -use databend_common_expression::arrow::or_validities; -use databend_common_expression::types::nullable::NullableColumn; -use databend_common_expression::types::AccessType; -use databend_common_expression::types::BooleanType; -use databend_common_expression::types::NullableType; use databend_common_expression::BlockEntry; use databend_common_expression::Column; use databend_common_expression::DataBlock; @@ -28,12 +23,17 @@ use databend_common_expression::Expr; use databend_common_expression::FilterExecutor; use databend_common_expression::FunctionContext; use databend_common_expression::ProjectedBlock; +use databend_common_expression::arrow::or_validities; +use databend_common_expression::types::AccessType; +use databend_common_expression::types::BooleanType; +use databend_common_expression::types::NullableType; +use databend_common_expression::types::nullable::NullableColumn; use databend_common_functions::BUILTIN_FUNCTIONS; +use super::HashJoinState; use super::desc::MARKER_KIND_FALSE; use super::desc::MARKER_KIND_NULL; use super::desc::MARKER_KIND_TRUE; -use super::HashJoinState; use crate::pipelines::processors::transforms::hash_join::HashJoinProbeState; /// Some common methods for hash join. diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/desc.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/desc.rs index ae2a25d06733b..1bd24c4750501 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/desc.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/desc.rs @@ -14,8 +14,6 @@ use databend_common_column::bitmap::Bitmap; use databend_common_exception::Result; -use databend_common_expression::arrow::and_validities; -use databend_common_expression::type_check::check_function; use databend_common_expression::BlockEntry; use databend_common_expression::Constant; use databend_common_expression::DataBlock; @@ -24,9 +22,11 @@ use databend_common_expression::Evaluator; use databend_common_expression::Expr; use databend_common_expression::FunctionContext; use databend_common_expression::RemoteExpr; +use databend_common_expression::arrow::and_validities; +use databend_common_expression::type_check::check_function; use databend_common_functions::BUILTIN_FUNCTIONS; -use databend_common_sql::executor::cast_expr_to_non_null_boolean; use databend_common_sql::ColumnSet; +use databend_common_sql::executor::cast_expr_to_non_null_boolean; use parking_lot::RwLock; use crate::physical_plans::HashJoin; diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_build_state.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_build_state.rs index c41364a50856e..04ddaf900190b 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_build_state.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_build_state.rs @@ -15,11 +15,11 @@ use std::collections::HashSet; use std::collections::VecDeque; use std::ops::ControlFlow; +use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::AtomicU32; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; -use std::sync::Arc; use databend_common_base::base::tokio::sync::Barrier; use databend_common_catalog::runtime_filter_info::RuntimeFilterReady; @@ -27,8 +27,6 @@ use databend_common_catalog::table_context::TableContext; use databend_common_column::bitmap::Bitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::arrow::and_validities; -use databend_common_expression::types::DataType; use databend_common_expression::BlockEntry; use databend_common_expression::Column; use databend_common_expression::ColumnVec; @@ -41,16 +39,18 @@ use databend_common_expression::HashMethodSerializer; use databend_common_expression::HashMethodSingleBinary; use databend_common_expression::KeysState; use databend_common_expression::RemoteExpr; +use databend_common_expression::arrow::and_validities; +use databend_common_expression::types::DataType; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_hashtable::BinaryHashJoinHashMap; use databend_common_hashtable::HashJoinHashMap; use databend_common_hashtable::RawEntry; use databend_common_hashtable::RowPtr; -use databend_common_hashtable::StringRawEntry; use databend_common_hashtable::STRING_EARLY_SIZE; +use databend_common_hashtable::StringRawEntry; use databend_common_pipeline_transforms::MemorySettings; -use databend_common_sql::plans::JoinType; use databend_common_sql::ColumnSet; +use databend_common_sql::plans::JoinType; use ethnum::U256; use itertools::Itertools; use log::info; @@ -61,17 +61,17 @@ use super::concat_buffer::ConcatBuffer; use super::desc::RuntimeFilterDesc; use super::runtime_filter::JoinRuntimeFilterPacket; use crate::pipelines::memory_settings::MemorySettingsExt; -use crate::pipelines::processors::transforms::hash_join::common::wrap_true_validity; -use crate::pipelines::processors::transforms::hash_join::desc::MARKER_KIND_FALSE; -use crate::pipelines::processors::transforms::hash_join::transform_hash_join_build::HashTableType; +use crate::pipelines::processors::HashJoinState; +use crate::pipelines::processors::transforms::SkipDuplicatesFixedKeyHashJoinHashTable; +use crate::pipelines::processors::transforms::SkipDuplicatesSerializerHashJoinHashTable; +use crate::pipelines::processors::transforms::SkipDuplicatesSingleBinaryHashJoinHashTable; use crate::pipelines::processors::transforms::hash_join::FixedKeyHashJoinHashTable; use crate::pipelines::processors::transforms::hash_join::HashJoinHashTable; use crate::pipelines::processors::transforms::hash_join::SerializerHashJoinHashTable; use crate::pipelines::processors::transforms::hash_join::SingleBinaryHashJoinHashTable; -use crate::pipelines::processors::transforms::SkipDuplicatesFixedKeyHashJoinHashTable; -use crate::pipelines::processors::transforms::SkipDuplicatesSerializerHashJoinHashTable; -use crate::pipelines::processors::transforms::SkipDuplicatesSingleBinaryHashJoinHashTable; -use crate::pipelines::processors::HashJoinState; +use crate::pipelines::processors::transforms::hash_join::common::wrap_true_validity; +use crate::pipelines::processors::transforms::hash_join::desc::MARKER_KIND_FALSE; +use crate::pipelines::processors::transforms::hash_join::transform_hash_join_build::HashTableType; use crate::sessions::QueryContext; /// Define some shared states for all hash join build threads. diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_probe_state.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_probe_state.rs index 1632c7dab8371..f5eebbf9b4430 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_probe_state.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_probe_state.rs @@ -14,18 +14,15 @@ use std::collections::VecDeque; use std::ops::ControlFlow; +use std::sync::Arc; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; -use std::sync::Arc; use databend_common_base::base::tokio::sync::Barrier; use databend_common_column::bitmap::Bitmap; use databend_common_column::bitmap::MutableBitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::arrow::and_validities; -use databend_common_expression::types::nullable::NullableColumn; -use databend_common_expression::with_join_hash_method; use databend_common_expression::BlockEntry; use databend_common_expression::Column; use databend_common_expression::DataBlock; @@ -36,6 +33,9 @@ use databend_common_expression::HashMethod; use databend_common_expression::HashMethodKind; use databend_common_expression::RemoteExpr; use databend_common_expression::Scalar; +use databend_common_expression::arrow::and_validities; +use databend_common_expression::types::nullable::NullableColumn; +use databend_common_expression::with_join_hash_method; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_hashtable::HashJoinHashtableLike; use databend_common_hashtable::Interval; @@ -46,13 +46,13 @@ use parking_lot::RwLock; use super::ProbeState; use super::ProcessState; +use crate::pipelines::processors::HashJoinState; use crate::pipelines::processors::transforms::hash_join::common::wrap_true_validity; use crate::pipelines::processors::transforms::hash_join::desc::MARKER_KIND_FALSE; use crate::pipelines::processors::transforms::hash_join::desc::MARKER_KIND_NULL; use crate::pipelines::processors::transforms::hash_join::desc::MARKER_KIND_TRUE; use crate::pipelines::processors::transforms::hash_join::hash_join_state::HashJoinHashTable; use crate::pipelines::processors::transforms::hash_join::util::probe_schema_wrap_nullable; -use crate::pipelines::processors::HashJoinState; use crate::sessions::QueryContext; use crate::sql::planner::plans::JoinType; diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_spiller.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_spiller.rs index 957b2a780d72c..b37ab57a9c389 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_spiller.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_spiller.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::hash_map::Entry; use std::collections::HashMap; use std::collections::HashSet; +use std::collections::hash_map::Entry; use std::sync::Arc; use databend_common_base::base::ProgressValues; @@ -28,8 +28,8 @@ use databend_common_sql::plans::JoinType; use databend_common_storage::DataOperator; use databend_common_storages_fuse::TableContext; -use crate::pipelines::processors::transforms::hash_join::spill_common::get_hashes; use crate::pipelines::processors::HashJoinState; +use crate::pipelines::processors::transforms::hash_join::spill_common::get_hashes; use crate::sessions::QueryContext; use crate::spillers::BlocksWriter; use crate::spillers::PartitionBuffer; diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_state.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_state.rs index d696ca2304e7b..7c29d0c2e5c0b 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_state.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/hash_join_state.rs @@ -15,10 +15,10 @@ use std::cell::SyncUnsafeCell; use std::collections::HashMap; use std::collections::HashSet; +use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; -use std::sync::Arc; use databend_common_base::base::tokio::sync::watch; use databend_common_base::base::tokio::sync::watch::Receiver; @@ -26,7 +26,6 @@ use databend_common_base::base::tokio::sync::watch::Sender; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::BlockEntry; use databend_common_expression::ColumnVec; use databend_common_expression::DataBlock; @@ -36,21 +35,22 @@ use databend_common_expression::FixedKey; use databend_common_expression::HashMethodFixedKeys; use databend_common_expression::HashMethodSerializer; use databend_common_expression::HashMethodSingleBinary; +use databend_common_expression::types::DataType; use databend_common_hashtable::BinaryHashJoinHashMap; use databend_common_hashtable::HashJoinHashMap; use databend_common_hashtable::HashJoinHashtableLike; use databend_common_hashtable::HashtableKeyable; use databend_common_hashtable::RowPtr; -use databend_common_sql::plans::JoinType; use databend_common_sql::ColumnSet; +use databend_common_sql::plans::JoinType; use ethnum::U256; use parking_lot::RwLock; use super::merge_into_hash_join_optimization::MergeIntoState; +use crate::pipelines::processors::HashJoinDesc; use crate::pipelines::processors::transforms::hash_join::build_state::BuildState; use crate::pipelines::processors::transforms::hash_join::transform_hash_join_build::HashTableType; use crate::pipelines::processors::transforms::hash_join::util::build_schema_wrap_nullable; -use crate::pipelines::processors::HashJoinDesc; use crate::sessions::QueryContext; use crate::sql::IndexType; diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/merge_into_hash_join_optimization.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/merge_into_hash_join_optimization.rs index 5d35f2eb7d47f..dfe9bcf042442 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/merge_into_hash_join_optimization.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/merge_into_hash_join_optimization.rs @@ -28,12 +28,12 @@ use databend_common_sql::plans::JoinType; use databend_common_storages_fuse::operations::BlockMetaIndex; use log::info; -use super::build_state::BuildState; -use super::hash_join_probe_state::MergeIntoChunkPartialUnmodified; use super::HashJoinBuildState; use super::HashJoinProbeState; use super::HashJoinState; use super::TransformHashJoinProbe; +use super::build_state::BuildState; +use super::hash_join_probe_state::MergeIntoChunkPartialUnmodified; pub struct MatchedPtr(pub *mut AtomicU8); unsafe impl Send for MatchedPtr {} diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/inner_join.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/inner_join.rs index 9e579bd4a7e8d..d4bafef1127cf 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/inner_join.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/inner_join.rs @@ -22,12 +22,12 @@ use databend_common_expression::KeyAccessor; use databend_common_hashtable::HashJoinHashtableLike; use databend_common_hashtable::RowPtr; +use crate::pipelines::processors::transforms::ProcessState; +use crate::pipelines::processors::transforms::hash_join::HashJoinProbeState; +use crate::pipelines::processors::transforms::hash_join::ProbeState; use crate::pipelines::processors::transforms::hash_join::build_state::BuildBlockGenerationState; use crate::pipelines::processors::transforms::hash_join::common::wrap_true_validity; use crate::pipelines::processors::transforms::hash_join::probe_state::ProbeBlockGenerationState; -use crate::pipelines::processors::transforms::hash_join::HashJoinProbeState; -use crate::pipelines::processors::transforms::hash_join::ProbeState; -use crate::pipelines::processors::transforms::ProcessState; impl HashJoinProbeState { pub(crate) fn inner_join< @@ -38,7 +38,7 @@ impl HashJoinProbeState { >( &self, probe_state: &mut ProbeState, - keys: Box<(dyn KeyAccessor)>, + keys: Box>, hash_table: &H, ) -> Result> where diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_anti_join.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_anti_join.rs index dc93f4709506f..42716ee796d70 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_anti_join.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_anti_join.rs @@ -23,16 +23,16 @@ use databend_common_expression::KeyAccessor; use databend_common_hashtable::HashJoinHashtableLike; use databend_common_hashtable::RowPtr; -use crate::pipelines::processors::transforms::hash_join::build_state::BuildBlockGenerationState; -use crate::pipelines::processors::transforms::hash_join::probe_state::ProbeBlockGenerationState; use crate::pipelines::processors::transforms::hash_join::HashJoinProbeState; use crate::pipelines::processors::transforms::hash_join::ProbeState; +use crate::pipelines::processors::transforms::hash_join::build_state::BuildBlockGenerationState; +use crate::pipelines::processors::transforms::hash_join::probe_state::ProbeBlockGenerationState; impl HashJoinProbeState { pub(crate) fn left_anti_join<'a, H: HashJoinHashtableLike>( &self, probe_state: &mut ProbeState, - keys: Box<(dyn KeyAccessor)>, + keys: Box>, hash_table: &H, ) -> Result> where @@ -96,7 +96,7 @@ impl HashJoinProbeState { pub(crate) fn left_anti_join_with_conjunct<'a, H: HashJoinHashtableLike>( &self, probe_state: &mut ProbeState, - keys: Box<(dyn KeyAccessor)>, + keys: Box>, hash_table: &H, ) -> Result> where diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_join.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_join.rs index d8478cb0fcb19..b6261d6cbd218 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_join.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_join.rs @@ -25,18 +25,18 @@ use databend_common_expression::Scalar; use databend_common_hashtable::HashJoinHashtableLike; use databend_common_hashtable::RowPtr; +use crate::pipelines::processors::transforms::hash_join::HashJoinProbeState; +use crate::pipelines::processors::transforms::hash_join::ProbeState; use crate::pipelines::processors::transforms::hash_join::build_state::BuildBlockGenerationState; use crate::pipelines::processors::transforms::hash_join::common::wrap_true_validity; use crate::pipelines::processors::transforms::hash_join::probe_state::ProbeBlockGenerationState; -use crate::pipelines::processors::transforms::hash_join::HashJoinProbeState; -use crate::pipelines::processors::transforms::hash_join::ProbeState; use crate::sql::plans::JoinType; impl HashJoinProbeState { pub(crate) fn left_join<'a, H: HashJoinHashtableLike, const LEFT_SINGLE: bool>( &self, probe_state: &mut ProbeState, - keys: Box<(dyn KeyAccessor)>, + keys: Box>, hash_table: &H, ) -> Result> where @@ -188,7 +188,7 @@ impl HashJoinProbeState { pub(crate) fn left_join_with_conjunct<'a, H: HashJoinHashtableLike, const LEFT_SINGLE: bool>( &self, probe_state: &mut ProbeState, - keys: Box<(dyn KeyAccessor)>, + keys: Box>, hash_table: &H, ) -> Result> where diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_mark_join.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_mark_join.rs index 2b209e4f9578f..71c3be1dd7748 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_mark_join.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_mark_join.rs @@ -22,19 +22,19 @@ use databend_common_expression::KeyAccessor; use databend_common_hashtable::HashJoinHashtableLike; use databend_common_hashtable::RowPtr; +use crate::pipelines::processors::transforms::hash_join::HashJoinProbeState; +use crate::pipelines::processors::transforms::hash_join::ProbeState; use crate::pipelines::processors::transforms::hash_join::build_state::BuildBlockGenerationState; use crate::pipelines::processors::transforms::hash_join::desc::MARKER_KIND_FALSE; use crate::pipelines::processors::transforms::hash_join::desc::MARKER_KIND_NULL; use crate::pipelines::processors::transforms::hash_join::desc::MARKER_KIND_TRUE; use crate::pipelines::processors::transforms::hash_join::probe_state::ProbeBlockGenerationState; -use crate::pipelines::processors::transforms::hash_join::HashJoinProbeState; -use crate::pipelines::processors::transforms::hash_join::ProbeState; impl HashJoinProbeState { pub(crate) fn left_mark_join<'a, H: HashJoinHashtableLike>( &self, probe_state: &mut ProbeState, - keys: Box<(dyn KeyAccessor)>, + keys: Box>, hash_table: &H, ) -> Result> where @@ -128,7 +128,7 @@ impl HashJoinProbeState { pub(crate) fn left_mark_join_with_conjunct<'a, H: HashJoinHashtableLike>( &self, probe_state: &mut ProbeState, - keys: Box<(dyn KeyAccessor)>, + keys: Box>, hash_table: &H, ) -> Result> where diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_semi_join.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_semi_join.rs index c41c9b4d7a260..6cf239d1d05b7 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_semi_join.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/left_semi_join.rs @@ -22,16 +22,16 @@ use databend_common_expression::KeyAccessor; use databend_common_hashtable::HashJoinHashtableLike; use databend_common_hashtable::RowPtr; -use crate::pipelines::processors::transforms::hash_join::build_state::BuildBlockGenerationState; -use crate::pipelines::processors::transforms::hash_join::probe_state::ProbeBlockGenerationState; use crate::pipelines::processors::transforms::hash_join::HashJoinProbeState; use crate::pipelines::processors::transforms::hash_join::ProbeState; +use crate::pipelines::processors::transforms::hash_join::build_state::BuildBlockGenerationState; +use crate::pipelines::processors::transforms::hash_join::probe_state::ProbeBlockGenerationState; impl HashJoinProbeState { pub(crate) fn left_semi_join<'a, H: HashJoinHashtableLike>( &self, probe_state: &mut ProbeState, - keys: Box<(dyn KeyAccessor)>, + keys: Box>, hash_table: &H, ) -> Result> where @@ -91,7 +91,7 @@ impl HashJoinProbeState { pub(crate) fn left_semi_join_with_conjunct<'a, H: HashJoinHashtableLike>( &self, probe_state: &mut ProbeState, - keys: Box<(dyn KeyAccessor)>, + keys: Box>, hash_table: &H, ) -> Result> where diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/right_join.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/right_join.rs index dfe122404ba3b..be96f0f343725 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/right_join.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/right_join.rs @@ -23,18 +23,18 @@ use databend_common_expression::KeyAccessor; use databend_common_hashtable::HashJoinHashtableLike; use databend_common_hashtable::RowPtr; +use crate::pipelines::processors::transforms::hash_join::HashJoinProbeState; +use crate::pipelines::processors::transforms::hash_join::ProbeState; use crate::pipelines::processors::transforms::hash_join::build_state::BuildBlockGenerationState; use crate::pipelines::processors::transforms::hash_join::common::wrap_true_validity; use crate::pipelines::processors::transforms::hash_join::probe_state::ProbeBlockGenerationState; -use crate::pipelines::processors::transforms::hash_join::HashJoinProbeState; -use crate::pipelines::processors::transforms::hash_join::ProbeState; use crate::sql::plans::JoinType; impl HashJoinProbeState { pub(crate) fn probe_right_join<'a, H: HashJoinHashtableLike>( &self, probe_state: &mut ProbeState, - keys: Box<(dyn KeyAccessor)>, + keys: Box>, hash_table: &H, ) -> Result> where diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/right_mark_join.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/right_mark_join.rs index 5bfa8f5f582a5..430bfb098fb64 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/right_mark_join.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/right_mark_join.rs @@ -22,19 +22,19 @@ use databend_common_expression::KeyAccessor; use databend_common_hashtable::HashJoinHashtableLike; use databend_common_hashtable::RowPtr; +use crate::pipelines::processors::transforms::hash_join::HashJoinProbeState; +use crate::pipelines::processors::transforms::hash_join::ProbeState; use crate::pipelines::processors::transforms::hash_join::build_state::BuildBlockGenerationState; use crate::pipelines::processors::transforms::hash_join::desc::MARKER_KIND_FALSE; use crate::pipelines::processors::transforms::hash_join::desc::MARKER_KIND_NULL; use crate::pipelines::processors::transforms::hash_join::desc::MARKER_KIND_TRUE; use crate::pipelines::processors::transforms::hash_join::probe_state::ProbeBlockGenerationState; -use crate::pipelines::processors::transforms::hash_join::HashJoinProbeState; -use crate::pipelines::processors::transforms::hash_join::ProbeState; impl HashJoinProbeState { pub(crate) fn right_mark_join<'a, H: HashJoinHashtableLike>( &self, probe_state: &mut ProbeState, - keys: Box<(dyn KeyAccessor)>, + keys: Box>, hash_table: &H, ) -> Result> where @@ -93,7 +93,7 @@ impl HashJoinProbeState { pub(crate) fn right_mark_join_with_conjunct<'a, H: HashJoinHashtableLike>( &self, probe_state: &mut ProbeState, - keys: Box<(dyn KeyAccessor)>, + keys: Box>, hash_table: &H, ) -> Result> where diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/right_semi_anti_join.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/right_semi_anti_join.rs index f806203188971..41f1510487f17 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/right_semi_anti_join.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_join/right_semi_anti_join.rs @@ -22,16 +22,16 @@ use databend_common_expression::KeyAccessor; use databend_common_hashtable::HashJoinHashtableLike; use databend_common_hashtable::RowPtr; -use crate::pipelines::processors::transforms::hash_join::build_state::BuildBlockGenerationState; -use crate::pipelines::processors::transforms::hash_join::probe_state::ProbeBlockGenerationState; use crate::pipelines::processors::transforms::hash_join::HashJoinProbeState; use crate::pipelines::processors::transforms::hash_join::ProbeState; +use crate::pipelines::processors::transforms::hash_join::build_state::BuildBlockGenerationState; +use crate::pipelines::processors::transforms::hash_join::probe_state::ProbeBlockGenerationState; impl HashJoinProbeState { pub(crate) fn right_semi_anti_join<'a, H: HashJoinHashtableLike>( &self, probe_state: &mut ProbeState, - keys: Box<(dyn KeyAccessor)>, + keys: Box>, hash_table: &H, ) -> Result> where @@ -113,7 +113,7 @@ impl HashJoinProbeState { pub(crate) fn right_semi_anti_join_with_conjunct<'a, H: HashJoinHashtableLike>( &self, probe_state: &mut ProbeState, - keys: Box<(dyn KeyAccessor)>, + keys: Box>, hash_table: &H, ) -> Result> where diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_state.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_state.rs index 91beeb5beaddf..f089a8795855c 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/probe_state.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/probe_state.rs @@ -14,11 +14,11 @@ use databend_common_column::bitmap::Bitmap; use databend_common_column::bitmap::MutableBitmap; -use databend_common_expression::filter::FilterExecutor; use databend_common_expression::DataBlock; use databend_common_expression::Expr; use databend_common_expression::FunctionContext; use databend_common_expression::KeysState; +use databend_common_expression::filter::FilterExecutor; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_hashtable::RowPtr; diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/result_blocks.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/result_blocks.rs index 114f019669287..68bc514708571 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/result_blocks.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/result_blocks.rs @@ -23,8 +23,8 @@ use databend_common_expression::Scalar; use databend_common_hashtable::HashJoinHashtableLike; use super::ProbeState; -use crate::pipelines::processors::transforms::hash_join::common::wrap_true_validity; use crate::pipelines::processors::transforms::hash_join::HashJoinProbeState; +use crate::pipelines::processors::transforms::hash_join::common::wrap_true_validity; use crate::sql::planner::plans::JoinType; impl HashJoinProbeState { @@ -43,7 +43,7 @@ impl HashJoinProbeState { pub(crate) fn result_blocks<'a, H: HashJoinHashtableLike>( &self, probe_state: &mut ProbeState, - keys: Box<(dyn KeyAccessor)>, + keys: Box>, hash_table: &H, ) -> Result> where diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/runtime_filter/builder.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/runtime_filter/builder.rs index 9af1bec4b542b..90e7011adbda0 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/runtime_filter/builder.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/runtime_filter/builder.rs @@ -18,8 +18,6 @@ use std::time::Instant; use databend_common_base::runtime::profile::Profile; use databend_common_base::runtime::profile::ProfileStatisticsName; use databend_common_exception::Result; -use databend_common_expression::type_check; -use databend_common_expression::types::DataType; use databend_common_expression::Column; use databend_common_expression::DataBlock; use databend_common_expression::Evaluator; @@ -27,6 +25,8 @@ use databend_common_expression::Expr; use databend_common_expression::FunctionContext; use databend_common_expression::RawExpr; use databend_common_expression::Scalar; +use databend_common_expression::type_check; +use databend_common_expression::types::DataType; use databend_common_functions::BUILTIN_FUNCTIONS; use super::packet::JoinRuntimeFilterPacket; @@ -109,7 +109,12 @@ impl<'a> JoinRuntimeFilterPacketBuilder<'a> { log::info!( "RUNTIME-FILTER: Built filter {} - total: {:?}, min_max: {:?}, inlist: {:?}, bloom: {:?}, rows: {}", - desc.id, total_time, min_max_time, inlist_time, bloom_time, self.build_key_column.len() + desc.id, + total_time, + min_max_time, + inlist_time, + bloom_time, + self.build_key_column.len() ); Ok(RuntimeFilterPacket { diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/runtime_filter/convert.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/runtime_filter/convert.rs index 90f2a8a7737fa..e2dd6d45afea9 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/runtime_filter/convert.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/runtime_filter/convert.rs @@ -23,16 +23,16 @@ use databend_common_catalog::sbbf::Sbbf; use databend_common_catalog::sbbf::SbbfAtomic; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::type_check; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDomain; -use databend_common_expression::types::NumberScalar; use databend_common_expression::Column; use databend_common_expression::Constant; use databend_common_expression::Domain; use databend_common_expression::Expr; use databend_common_expression::RawExpr; use databend_common_expression::Scalar; +use databend_common_expression::type_check; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDomain; +use databend_common_expression::types::NumberScalar; use databend_common_functions::BUILTIN_FUNCTIONS; use super::builder::should_enable_runtime_filter; @@ -250,7 +250,7 @@ fn build_min_max_filter( return Err(ErrorCode::UnsupportedDataType(format!( "Unsupported domain {:?} for runtime filter", min_max, - ))) + ))); } }; Ok(min_max_filter) @@ -305,8 +305,6 @@ async fn build_bloom_filter( mod tests { use std::collections::HashMap; - use databend_common_expression::types::DataType; - use databend_common_expression::types::NumberDataType; use databend_common_expression::ColumnBuilder; use databend_common_expression::ColumnRef; use databend_common_expression::Constant; @@ -315,6 +313,8 @@ mod tests { use databend_common_expression::Expr; use databend_common_expression::FunctionContext; use databend_common_expression::Scalar; + use databend_common_expression::types::DataType; + use databend_common_expression::types::NumberDataType; use databend_common_functions::BUILTIN_FUNCTIONS; use super::build_inlist_filter; @@ -466,7 +466,9 @@ mod tests { scalar: Scalar::Boolean(false), .. }) => { - println!("✓ Test passed: column_b in [2000,3000] correctly evaluated to false for 1024 elements"); + println!( + "✓ Test passed: column_b in [2000,3000] correctly evaluated to false for 1024 elements" + ); } _ => { panic!( @@ -476,6 +478,8 @@ mod tests { } } - println!("✓ Large inlist filter test (1024 elements) passed - balanced binary tree working correctly"); + println!( + "✓ Large inlist filter test (1024 elements) passed - balanced binary tree working correctly" + ); } } diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/runtime_filter/interface.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/runtime_filter/interface.rs index d0f1fb003fb1e..19b29bd197de6 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/runtime_filter/interface.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/runtime_filter/interface.rs @@ -19,8 +19,8 @@ use databend_common_storages_fuse::TableContext; use super::convert::build_runtime_filter_infos; use super::global::get_global_runtime_filter_packet; -use crate::pipelines::processors::transforms::JoinRuntimeFilterPacket; use crate::pipelines::processors::HashJoinBuildState; +use crate::pipelines::processors::transforms::JoinRuntimeFilterPacket; pub async fn build_and_push_down_runtime_filter( mut packet: JoinRuntimeFilterPacket, diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/runtime_filter/local_builder.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/runtime_filter/local_builder.rs index e91b51ce1ffcf..ab91a06cc8b6b 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/runtime_filter/local_builder.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/runtime_filter/local_builder.rs @@ -13,7 +13,6 @@ // limitations under the License. use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::BlockEntry; use databend_common_expression::Column; use databend_common_expression::ColumnBuilder; @@ -24,6 +23,7 @@ use databend_common_expression::FunctionContext; use databend_common_expression::HashMethodKind; use databend_common_expression::RawExpr; use databend_common_expression::Scalar; +use databend_common_expression::types::DataType; use databend_common_functions::BUILTIN_FUNCTIONS; use crate::pipelines::processors::transforms::hash_join::desc::RuntimeFilterDesc; diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/transform_hash_join_build.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/transform_hash_join_build.rs index 2b32d4b47e179..d4012965d2dfa 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/transform_hash_join_build.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/transform_hash_join_build.rs @@ -13,8 +13,8 @@ // limitations under the License. use std::any::Any; -use std::sync::atomic::Ordering; use std::sync::Arc; +use std::sync::atomic::Ordering; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -26,9 +26,9 @@ use databend_common_pipeline_transforms::MemorySettings; use databend_common_sql::plans::JoinType; use databend_common_storages_fuse::TableContext; +use super::runtime_filter::RuntimeFilterLocalBuilder; use super::runtime_filter::build_and_push_down_runtime_filter; use super::runtime_filter::merge_join_runtime_filter_packets; -use super::runtime_filter::RuntimeFilterLocalBuilder; use crate::pipelines::processors::transforms::hash_join::HashJoinBuildState; use crate::pipelines::processors::transforms::hash_join::HashJoinSpiller; diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/transform_hash_join_probe.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/transform_hash_join_probe.rs index 8be41e570acb0..a3f9b6e41bc92 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/transform_hash_join_probe.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/transform_hash_join_probe.rs @@ -14,8 +14,8 @@ use std::any::Any; use std::collections::VecDeque; -use std::sync::atomic::Ordering; use std::sync::Arc; +use std::sync::atomic::Ordering; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -25,14 +25,14 @@ use databend_common_pipeline::core::Event; use databend_common_pipeline::core::InputPort; use databend_common_pipeline::core::OutputPort; use databend_common_pipeline::core::Processor; -use databend_common_sql::plans::JoinType; use databend_common_sql::ColumnSet; +use databend_common_sql::plans::JoinType; -use crate::pipelines::processors::transforms::hash_join::transform_hash_join_build::HashTableType; +use crate::pipelines::processors::transforms::HashJoinHashTable; use crate::pipelines::processors::transforms::hash_join::HashJoinProbeState; use crate::pipelines::processors::transforms::hash_join::HashJoinSpiller; use crate::pipelines::processors::transforms::hash_join::ProbeState; -use crate::pipelines::processors::transforms::HashJoinHashTable; +use crate::pipelines::processors::transforms::hash_join::transform_hash_join_build::HashTableType; enum FinalScanType { HashJoin, diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/util.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/util.rs index d3ce8644fbe77..5b5abd25d50b0 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/util.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/util.rs @@ -13,15 +13,15 @@ // limitations under the License. use databend_common_exception::Result; -pub use databend_common_expression::hash_util::hash_by_method; -pub use databend_common_expression::hash_util::hash_by_method_for_bloom; -use databend_common_expression::type_check; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use databend_common_expression::Expr; use databend_common_expression::RawExpr; use databend_common_expression::Scalar; +pub use databend_common_expression::hash_util::hash_by_method; +pub use databend_common_expression::hash_util::hash_by_method_for_bloom; +use databend_common_expression::type_check; use databend_common_functions::BUILTIN_FUNCTIONS; pub(crate) fn build_schema_wrap_nullable(build_schema: &DataSchemaRef) -> DataSchemaRef { diff --git a/src/query/service/src/pipelines/processors/transforms/mod.rs b/src/query/service/src/pipelines/processors/transforms/mod.rs index 50f5e2fc32d4e..55d56bb0299eb 100644 --- a/src/query/service/src/pipelines/processors/transforms/mod.rs +++ b/src/query/service/src/pipelines/processors/transforms/mod.rs @@ -59,8 +59,8 @@ pub use transform_expression_scan::TransformExpressionScan; pub use transform_recursive_cte_scan::TransformRecursiveCteScan; pub use transform_recursive_cte_source::TransformRecursiveCteSource; pub use transform_resort_addon::TransformResortAddOn; -pub use transform_resort_addon_without_source_schema::build_expression_transform; pub use transform_resort_addon_without_source_schema::TransformResortAddOnWithoutSourceSchema; +pub use transform_resort_addon_without_source_schema::build_expression_transform; pub use transform_srf::TransformSRF; pub use transform_udf_script::TransformUdfScript; pub use transform_udf_server::TransformUdfServer; diff --git a/src/query/service/src/pipelines/processors/transforms/new_hash_join/grace/grace_join.rs b/src/query/service/src/pipelines/processors/transforms/new_hash_join/grace/grace_join.rs index 2c9611c31d5a9..e81d0a6052d07 100644 --- a/src/query/service/src/pipelines/processors/transforms/new_hash_join/grace/grace_join.rs +++ b/src/query/service/src/pipelines/processors/transforms/new_hash_join/grace/grace_join.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::btree_map::Entry; use std::collections::VecDeque; +use std::collections::btree_map::Entry; use std::sync::Arc; use std::sync::PoisonError; @@ -29,14 +29,14 @@ use databend_common_pipeline_transforms::traits::Location; use databend_common_storage::DataOperator; use databend_common_storages_parquet::ReadSettings; +use crate::pipelines::processors::HashJoinDesc; +use crate::pipelines::processors::transforms::Join; use crate::pipelines::processors::transforms::get_hashes; use crate::pipelines::processors::transforms::new_hash_join::grace::grace_memory::GraceMemoryJoin; use crate::pipelines::processors::transforms::new_hash_join::grace::grace_state::GraceHashJoinState; use crate::pipelines::processors::transforms::new_hash_join::grace::grace_state::SpillMetadata; use crate::pipelines::processors::transforms::new_hash_join::join::EmptyJoinStream; use crate::pipelines::processors::transforms::new_hash_join::join::JoinStream; -use crate::pipelines::processors::transforms::Join; -use crate::pipelines::processors::HashJoinDesc; use crate::sessions::QueryContext; use crate::spillers::Layout; use crate::spillers::SpillAdapter; diff --git a/src/query/service/src/pipelines/processors/transforms/new_hash_join/grace/grace_memory.rs b/src/query/service/src/pipelines/processors/transforms/new_hash_join/grace/grace_memory.rs index a0ad98b17f19e..45b7d3409f9f3 100644 --- a/src/query/service/src/pipelines/processors/transforms/new_hash_join/grace/grace_memory.rs +++ b/src/query/service/src/pipelines/processors/transforms/new_hash_join/grace/grace_memory.rs @@ -14,11 +14,11 @@ use std::sync::PoisonError; -use crate::pipelines::processors::transforms::memory::outer_left_join::OuterLeftHashJoin; use crate::pipelines::processors::transforms::BasicHashJoinState; use crate::pipelines::processors::transforms::HashJoinHashTable; use crate::pipelines::processors::transforms::InnerHashJoin; use crate::pipelines::processors::transforms::Join; +use crate::pipelines::processors::transforms::memory::outer_left_join::OuterLeftHashJoin; pub trait GraceMemoryJoin: Join { fn reset_memory(&mut self); diff --git a/src/query/service/src/pipelines/processors/transforms/new_hash_join/grace/grace_state.rs b/src/query/service/src/pipelines/processors/transforms/new_hash_join/grace/grace_state.rs index f6a41102ec54e..1d9e431b6f75b 100644 --- a/src/query/service/src/pipelines/processors/transforms/new_hash_join/grace/grace_state.rs +++ b/src/query/service/src/pipelines/processors/transforms/new_hash_join/grace/grace_state.rs @@ -19,8 +19,8 @@ use std::sync::Mutex; use parquet::file::metadata::RowGroupMetaData; -use crate::pipelines::processors::transforms::new_hash_join::common::CStyleCell; use crate::pipelines::processors::transforms::HashJoinFactory; +use crate::pipelines::processors::transforms::new_hash_join::common::CStyleCell; use crate::sessions::QueryContext; #[derive(Debug)] diff --git a/src/query/service/src/pipelines/processors/transforms/new_hash_join/hash_join_factory.rs b/src/query/service/src/pipelines/processors/transforms/new_hash_join/hash_join_factory.rs index bc7aad3e4a36f..873a4139b1e8d 100644 --- a/src/query/service/src/pipelines/processors/transforms/new_hash_join/hash_join_factory.rs +++ b/src/query/service/src/pipelines/processors/transforms/new_hash_join/hash_join_factory.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::hash_map::Entry; use std::collections::HashMap; +use std::collections::hash_map::Entry; use std::sync::Arc; use std::sync::Mutex; use std::sync::PoisonError; @@ -26,14 +26,14 @@ use databend_common_expression::FunctionContext; use databend_common_expression::HashMethodKind; use databend_common_sql::plans::JoinType; -use crate::pipelines::processors::transforms::memory::outer_left_join::OuterLeftHashJoin; -use crate::pipelines::processors::transforms::new_hash_join::common::CStyleCell; -use crate::pipelines::processors::transforms::new_hash_join::grace::GraceHashJoinState; +use crate::pipelines::processors::HashJoinDesc; use crate::pipelines::processors::transforms::BasicHashJoinState; use crate::pipelines::processors::transforms::GraceHashJoin; use crate::pipelines::processors::transforms::InnerHashJoin; use crate::pipelines::processors::transforms::Join; -use crate::pipelines::processors::HashJoinDesc; +use crate::pipelines::processors::transforms::memory::outer_left_join::OuterLeftHashJoin; +use crate::pipelines::processors::transforms::new_hash_join::common::CStyleCell; +use crate::pipelines::processors::transforms::new_hash_join::grace::GraceHashJoinState; use crate::sessions::QueryContext; pub struct HashJoinFactory { diff --git a/src/query/service/src/pipelines/processors/transforms/new_hash_join/hashtable/fixed_keys.rs b/src/query/service/src/pipelines/processors/transforms/new_hash_join/hashtable/fixed_keys.rs index faff2f14906f1..633338f5700a6 100644 --- a/src/query/service/src/pipelines/processors/transforms/new_hash_join/hashtable/fixed_keys.rs +++ b/src/query/service/src/pipelines/processors/transforms/new_hash_join/hashtable/fixed_keys.rs @@ -26,12 +26,12 @@ use databend_common_hashtable::HashtableKeyable; use databend_common_hashtable::RawEntry; use databend_common_hashtable::RowPtr; +use crate::pipelines::processors::transforms::FixedKeyHashJoinHashTable; +use crate::pipelines::processors::transforms::new_hash_join::hashtable::ProbeData; use crate::pipelines::processors::transforms::new_hash_join::hashtable::basic::AllUnmatchedProbeStream; use crate::pipelines::processors::transforms::new_hash_join::hashtable::basic::EmptyProbeStream; use crate::pipelines::processors::transforms::new_hash_join::hashtable::basic::ProbeStream; use crate::pipelines::processors::transforms::new_hash_join::hashtable::basic::ProbedRows; -use crate::pipelines::processors::transforms::new_hash_join::hashtable::ProbeData; -use crate::pipelines::processors::transforms::FixedKeyHashJoinHashTable; impl FixedKeyHashJoinHashTable @@ -178,7 +178,7 @@ struct FixedKeyProbeStream, probe_entry_ptr: u64, - keys: Box<(dyn KeyAccessor)>, + keys: Box>, matched_num_rows: usize, } @@ -267,7 +267,7 @@ struct EarlyFilteringProbeStream<'a, Key: FixedKey + HashtableKeyable, const MAT idx: usize, pointers: Vec, probe_entry_ptr: u64, - keys: Box<(dyn KeyAccessor)>, + keys: Box>, selections: &'a [u32], unmatched_selection: &'a [u32], matched_num_rows: usize, diff --git a/src/query/service/src/pipelines/processors/transforms/new_hash_join/hashtable/serialize_keys.rs b/src/query/service/src/pipelines/processors/transforms/new_hash_join/hashtable/serialize_keys.rs index d2e25dd955fb9..3dbac52c5a062 100644 --- a/src/query/service/src/pipelines/processors/transforms/new_hash_join/hashtable/serialize_keys.rs +++ b/src/query/service/src/pipelines/processors/transforms/new_hash_join/hashtable/serialize_keys.rs @@ -24,15 +24,15 @@ use databend_common_expression::ProjectedBlock; use databend_common_hashtable::BinaryHashJoinHashMap; use databend_common_hashtable::HashJoinHashtableLike; use databend_common_hashtable::RowPtr; -use databend_common_hashtable::StringRawEntry; use databend_common_hashtable::STRING_EARLY_SIZE; +use databend_common_hashtable::StringRawEntry; +use crate::pipelines::processors::transforms::SerializerHashJoinHashTable; +use crate::pipelines::processors::transforms::new_hash_join::hashtable::ProbeData; use crate::pipelines::processors::transforms::new_hash_join::hashtable::basic::AllUnmatchedProbeStream; use crate::pipelines::processors::transforms::new_hash_join::hashtable::basic::EmptyProbeStream; use crate::pipelines::processors::transforms::new_hash_join::hashtable::basic::ProbeStream; use crate::pipelines::processors::transforms::new_hash_join::hashtable::basic::ProbedRows; -use crate::pipelines::processors::transforms::new_hash_join::hashtable::ProbeData; -use crate::pipelines::processors::transforms::SerializerHashJoinHashTable; impl SerializerHashJoinHashTable { pub fn new( @@ -197,7 +197,7 @@ impl SerializerHashJoinHashTable { pub struct BinaryKeyProbeStream { key_idx: usize, pointers: Vec, - keys: Box<(dyn KeyAccessor)>, + keys: Box>, probe_entry_ptr: u64, matched_num_rows: usize, } @@ -297,7 +297,7 @@ impl ProbeStream for BinaryKeyProbeStream { pub struct EarlyFilteringProbeStream<'a, const MATCHED: bool> { idx: usize, pointers: Vec, - keys: Box<(dyn KeyAccessor)>, + keys: Box>, probe_entry_ptr: u64, selections: &'a [u32], unmatched_selection: &'a [u32], diff --git a/src/query/service/src/pipelines/processors/transforms/new_hash_join/hashtable/single_binary_key.rs b/src/query/service/src/pipelines/processors/transforms/new_hash_join/hashtable/single_binary_key.rs index 31a9d23adb630..e3a2b6d016714 100644 --- a/src/query/service/src/pipelines/processors/transforms/new_hash_join/hashtable/single_binary_key.rs +++ b/src/query/service/src/pipelines/processors/transforms/new_hash_join/hashtable/single_binary_key.rs @@ -22,16 +22,16 @@ use databend_common_expression::ProjectedBlock; use databend_common_hashtable::BinaryHashJoinHashMap; use databend_common_hashtable::HashJoinHashtableLike; use databend_common_hashtable::RowPtr; -use databend_common_hashtable::StringRawEntry; use databend_common_hashtable::STRING_EARLY_SIZE; +use databend_common_hashtable::StringRawEntry; +use crate::pipelines::processors::transforms::SingleBinaryHashJoinHashTable; +use crate::pipelines::processors::transforms::new_hash_join::hashtable::ProbeData; use crate::pipelines::processors::transforms::new_hash_join::hashtable::basic::AllUnmatchedProbeStream; use crate::pipelines::processors::transforms::new_hash_join::hashtable::basic::EmptyProbeStream; use crate::pipelines::processors::transforms::new_hash_join::hashtable::basic::ProbeStream; use crate::pipelines::processors::transforms::new_hash_join::hashtable::serialize_keys::BinaryKeyProbeStream; use crate::pipelines::processors::transforms::new_hash_join::hashtable::serialize_keys::EarlyFilteringProbeStream; -use crate::pipelines::processors::transforms::new_hash_join::hashtable::ProbeData; -use crate::pipelines::processors::transforms::SingleBinaryHashJoinHashTable; impl SingleBinaryHashJoinHashTable { pub fn new( diff --git a/src/query/service/src/pipelines/processors/transforms/new_hash_join/memory/basic.rs b/src/query/service/src/pipelines/processors/transforms/new_hash_join/memory/basic.rs index 8168fbc081173..2e7b244f50576 100644 --- a/src/query/service/src/pipelines/processors/transforms/new_hash_join/memory/basic.rs +++ b/src/query/service/src/pipelines/processors/transforms/new_hash_join/memory/basic.rs @@ -30,7 +30,7 @@ use databend_common_hashtable::HashJoinHashMap; use databend_common_sql::plans::JoinType; use ethnum::U256; -use crate::pipelines::processors::transforms::new_hash_join::common::SquashBlocks; +use crate::pipelines::processors::HashJoinDesc; use crate::pipelines::processors::transforms::BasicHashJoinState; use crate::pipelines::processors::transforms::FixedKeyHashJoinHashTable; use crate::pipelines::processors::transforms::HashJoinHashTable; @@ -39,7 +39,7 @@ use crate::pipelines::processors::transforms::SingleBinaryHashJoinHashTable; use crate::pipelines::processors::transforms::SkipDuplicatesFixedKeyHashJoinHashTable; use crate::pipelines::processors::transforms::SkipDuplicatesSerializerHashJoinHashTable; use crate::pipelines::processors::transforms::SkipDuplicatesSingleBinaryHashJoinHashTable; -use crate::pipelines::processors::HashJoinDesc; +use crate::pipelines::processors::transforms::new_hash_join::common::SquashBlocks; use crate::sessions::QueryContext; pub struct BasicHashJoin { diff --git a/src/query/service/src/pipelines/processors/transforms/new_hash_join/memory/basic_state.rs b/src/query/service/src/pipelines/processors/transforms/new_hash_join/memory/basic_state.rs index 99db7c70456bb..5f6684ba7188c 100644 --- a/src/query/service/src/pipelines/processors/transforms/new_hash_join/memory/basic_state.rs +++ b/src/query/service/src/pipelines/processors/transforms/new_hash_join/memory/basic_state.rs @@ -16,14 +16,14 @@ use std::collections::VecDeque; use std::sync::Arc; use std::sync::Mutex; -use databend_common_expression::types::DataType; use databend_common_expression::ColumnVec; use databend_common_expression::DataBlock; +use databend_common_expression::types::DataType; -use crate::pipelines::processors::transforms::new_hash_join::common::CStyleCell; use crate::pipelines::processors::transforms::HashJoinFactory; use crate::pipelines::processors::transforms::HashJoinHashTable; use crate::pipelines::processors::transforms::JoinRuntimeFilterPacket; +use crate::pipelines::processors::transforms::new_hash_join::common::CStyleCell; pub struct BasicHashJoinState { pub mutex: Mutex<()>, diff --git a/src/query/service/src/pipelines/processors/transforms/new_hash_join/memory/inner_join.rs b/src/query/service/src/pipelines/processors/transforms/new_hash_join/memory/inner_join.rs index 9c5cb96f312e7..abc81674601f8 100644 --- a/src/query/service/src/pipelines/processors/transforms/new_hash_join/memory/inner_join.rs +++ b/src/query/service/src/pipelines/processors/transforms/new_hash_join/memory/inner_join.rs @@ -21,27 +21,27 @@ use databend_common_catalog::table_context::TableContext; use databend_common_column::bitmap::Bitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::NullableColumn; -use databend_common_expression::with_join_hash_method; use databend_common_expression::BlockEntry; use databend_common_expression::DataBlock; use databend_common_expression::FilterExecutor; use databend_common_expression::FunctionContext; use databend_common_expression::HashMethodKind; +use databend_common_expression::types::NullableColumn; +use databend_common_expression::with_join_hash_method; +use crate::pipelines::processors::HashJoinDesc; +use crate::pipelines::processors::transforms::HashJoinHashTable; +use crate::pipelines::processors::transforms::JoinRuntimeFilterPacket; use crate::pipelines::processors::transforms::merge_join_runtime_filter_packets; +use crate::pipelines::processors::transforms::new_hash_join::hashtable::ProbeData; use crate::pipelines::processors::transforms::new_hash_join::hashtable::basic::ProbeStream; use crate::pipelines::processors::transforms::new_hash_join::hashtable::basic::ProbedRows; -use crate::pipelines::processors::transforms::new_hash_join::hashtable::ProbeData; use crate::pipelines::processors::transforms::new_hash_join::join::EmptyJoinStream; use crate::pipelines::processors::transforms::new_hash_join::join::Join; use crate::pipelines::processors::transforms::new_hash_join::join::JoinStream; use crate::pipelines::processors::transforms::new_hash_join::memory::basic::BasicHashJoin; use crate::pipelines::processors::transforms::new_hash_join::memory::basic_state::BasicHashJoinState; use crate::pipelines::processors::transforms::new_hash_join::performance::PerformanceContext; -use crate::pipelines::processors::transforms::HashJoinHashTable; -use crate::pipelines::processors::transforms::JoinRuntimeFilterPacket; -use crate::pipelines::processors::HashJoinDesc; use crate::sessions::QueryContext; pub struct InnerHashJoin { diff --git a/src/query/service/src/pipelines/processors/transforms/new_hash_join/memory/outer_left_join.rs b/src/query/service/src/pipelines/processors/transforms/new_hash_join/memory/outer_left_join.rs index 8210b495bfca3..9c0dc8aa08aef 100644 --- a/src/query/service/src/pipelines/processors/transforms/new_hash_join/memory/outer_left_join.rs +++ b/src/query/service/src/pipelines/processors/transforms/new_hash_join/memory/outer_left_join.rs @@ -20,29 +20,29 @@ use databend_common_catalog::table_context::TableContext; use databend_common_column::bitmap::Bitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NullableColumn; -use databend_common_expression::with_join_hash_method; use databend_common_expression::BlockEntry; use databend_common_expression::DataBlock; use databend_common_expression::FilterExecutor; use databend_common_expression::FunctionContext; use databend_common_expression::HashMethodKind; use databend_common_expression::Scalar; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NullableColumn; +use databend_common_expression::with_join_hash_method; +use crate::pipelines::processors::HashJoinDesc; +use crate::pipelines::processors::transforms::BasicHashJoinState; +use crate::pipelines::processors::transforms::HashJoinHashTable; +use crate::pipelines::processors::transforms::Join; +use crate::pipelines::processors::transforms::new_hash_join::hashtable::ProbeData; use crate::pipelines::processors::transforms::new_hash_join::hashtable::basic::ProbeStream; use crate::pipelines::processors::transforms::new_hash_join::hashtable::basic::ProbedRows; -use crate::pipelines::processors::transforms::new_hash_join::hashtable::ProbeData; use crate::pipelines::processors::transforms::new_hash_join::join::EmptyJoinStream; use crate::pipelines::processors::transforms::new_hash_join::join::JoinStream; use crate::pipelines::processors::transforms::new_hash_join::join::OneBlockJoinStream; use crate::pipelines::processors::transforms::new_hash_join::memory::basic::BasicHashJoin; use crate::pipelines::processors::transforms::new_hash_join::performance::PerformanceContext; use crate::pipelines::processors::transforms::wrap_true_validity; -use crate::pipelines::processors::transforms::BasicHashJoinState; -use crate::pipelines::processors::transforms::HashJoinHashTable; -use crate::pipelines::processors::transforms::Join; -use crate::pipelines::processors::HashJoinDesc; use crate::sessions::QueryContext; pub struct OuterLeftHashJoin { diff --git a/src/query/service/src/pipelines/processors/transforms/new_hash_join/performance.rs b/src/query/service/src/pipelines/processors/transforms/new_hash_join/performance.rs index 5f3b71794e286..0f3f7cb8f560b 100644 --- a/src/query/service/src/pipelines/processors/transforms/new_hash_join/performance.rs +++ b/src/query/service/src/pipelines/processors/transforms/new_hash_join/performance.rs @@ -18,9 +18,9 @@ use databend_common_expression::FilterExecutor; use databend_common_expression::FunctionContext; use databend_common_functions::BUILTIN_FUNCTIONS; -use crate::pipelines::processors::transforms::new_hash_join::hashtable::basic::ProbedRows; -use crate::pipelines::processors::transforms::new_hash_join::hashtable::ProbeHashStatistics; use crate::pipelines::processors::HashJoinDesc; +use crate::pipelines::processors::transforms::new_hash_join::hashtable::ProbeHashStatistics; +use crate::pipelines::processors::transforms::new_hash_join::hashtable::basic::ProbedRows; pub struct PerformanceContext { pub probe_result: ProbedRows, diff --git a/src/query/service/src/pipelines/processors/transforms/new_hash_join/runtime_filter.rs b/src/query/service/src/pipelines/processors/transforms/new_hash_join/runtime_filter.rs index 4757d0510a7d9..37b522b543431 100644 --- a/src/query/service/src/pipelines/processors/transforms/new_hash_join/runtime_filter.rs +++ b/src/query/service/src/pipelines/processors/transforms/new_hash_join/runtime_filter.rs @@ -21,10 +21,10 @@ use databend_common_exception::Result; use databend_common_expression::FunctionContext; use crate::physical_plans::HashJoin; -use crate::pipelines::processors::transforms::build_runtime_filter_infos; -use crate::pipelines::processors::transforms::get_global_runtime_filter_packet; use crate::pipelines::processors::transforms::JoinRuntimeFilterPacket; use crate::pipelines::processors::transforms::RuntimeFilterDesc; +use crate::pipelines::processors::transforms::build_runtime_filter_infos; +use crate::pipelines::processors::transforms::get_global_runtime_filter_packet; use crate::sessions::QueryContext; pub struct RuntimeFiltersDesc { diff --git a/src/query/service/src/pipelines/processors/transforms/new_hash_join/transform_hash_join.rs b/src/query/service/src/pipelines/processors/transforms/new_hash_join/transform_hash_join.rs index 68fbaf0fd8ade..02684f029594c 100644 --- a/src/query/service/src/pipelines/processors/transforms/new_hash_join/transform_hash_join.rs +++ b/src/query/service/src/pipelines/processors/transforms/new_hash_join/transform_hash_join.rs @@ -27,10 +27,10 @@ use databend_common_pipeline::core::ProcessorPtr; use databend_common_sql::ColumnSet; use tokio::sync::Barrier; +use crate::pipelines::processors::transforms::RuntimeFilterLocalBuilder; use crate::pipelines::processors::transforms::new_hash_join::join::Join; use crate::pipelines::processors::transforms::new_hash_join::join::JoinStream; use crate::pipelines::processors::transforms::new_hash_join::runtime_filter::RuntimeFiltersDesc; -use crate::pipelines::processors::transforms::RuntimeFilterLocalBuilder; pub struct TransformHashJoin { build_port: Arc, diff --git a/src/query/service/src/pipelines/processors/transforms/range_join/ie_join_state.rs b/src/query/service/src/pipelines/processors/transforms/range_join/ie_join_state.rs index 3b5793613e72a..60354a776e135 100644 --- a/src/query/service/src/pipelines/processors/transforms/range_join/ie_join_state.rs +++ b/src/query/service/src/pipelines/processors/transforms/range_join/ie_join_state.rs @@ -20,13 +20,6 @@ use databend_common_column::bitmap::Bitmap; use databend_common_column::bitmap::MutableBitmap; use databend_common_column::buffer::Buffer; use databend_common_exception::Result; -use databend_common_expression::types::AccessType; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberColumn; -use databend_common_expression::types::NumberColumnBuilder; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::NumberScalar; -use databend_common_expression::types::UInt64Type; use databend_common_expression::BlockEntry; use databend_common_expression::Column; use databend_common_expression::DataBlock; @@ -37,14 +30,21 @@ use databend_common_expression::Scalar; use databend_common_expression::ScalarRef; use databend_common_expression::SortColumnDescription; use databend_common_expression::Value; +use databend_common_expression::types::AccessType; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberColumn; +use databend_common_expression::types::NumberColumnBuilder; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::NumberScalar; +use databend_common_expression::types::UInt64Type; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_pipeline_transforms::sorts::sort_merge; use crate::physical_plans::RangeJoin; +use crate::pipelines::processors::transforms::range_join::RangeJoinState; use crate::pipelines::processors::transforms::range_join::filter_block; use crate::pipelines::processors::transforms::range_join::order_match; use crate::pipelines::processors::transforms::range_join::probe_l1; -use crate::pipelines::processors::transforms::range_join::RangeJoinState; pub struct IEJoinState { _l1_data_type: DataType, diff --git a/src/query/service/src/pipelines/processors/transforms/range_join/ie_join_util.rs b/src/query/service/src/pipelines/processors/transforms/range_join/ie_join_util.rs index 864636c4fd7d0..66ddcad389f7b 100644 --- a/src/query/service/src/pipelines/processors/transforms/range_join/ie_join_util.rs +++ b/src/query/service/src/pipelines/processors/transforms/range_join/ie_join_util.rs @@ -12,18 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::cmp::min; use std::cmp::Ordering; +use std::cmp::min; use databend_common_exception::Result; -use databend_common_expression::types::BooleanType; -use databend_common_expression::types::DataType; use databend_common_expression::Column; use databend_common_expression::DataBlock; use databend_common_expression::Evaluator; use databend_common_expression::FunctionContext; use databend_common_expression::RemoteExpr; use databend_common_expression::ScalarRef; +use databend_common_expression::types::BooleanType; +use databend_common_expression::types::DataType; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_sql::executor::cast_expr_to_non_null_boolean; diff --git a/src/query/service/src/pipelines/processors/transforms/range_join/merge_join_state.rs b/src/query/service/src/pipelines/processors/transforms/range_join/merge_join_state.rs index 8e2d33b16e588..4405f7fc88e8c 100644 --- a/src/query/service/src/pipelines/processors/transforms/range_join/merge_join_state.rs +++ b/src/query/service/src/pipelines/processors/transforms/range_join/merge_join_state.rs @@ -13,13 +13,13 @@ // limitations under the License. use databend_common_exception::Result; -use databend_common_expression::types::NumberScalar; use databend_common_expression::DataBlock; use databend_common_expression::ScalarRef; use databend_common_expression::SortColumnDescription; +use databend_common_expression::types::NumberScalar; -use crate::pipelines::processors::transforms::range_join::filter_block; use crate::pipelines::processors::transforms::range_join::RangeJoinState; +use crate::pipelines::processors::transforms::range_join::filter_block; impl RangeJoinState { pub fn range_join(&self, task_id: usize) -> Result> { diff --git a/src/query/service/src/pipelines/processors/transforms/range_join/range_join_state.rs b/src/query/service/src/pipelines/processors/transforms/range_join/range_join_state.rs index 45b5acea21e5e..ea59ea3607122 100644 --- a/src/query/service/src/pipelines/processors/transforms/range_join/range_join_state.rs +++ b/src/query/service/src/pipelines/processors/transforms/range_join/range_join_state.rs @@ -12,23 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::Arc; use std::sync::atomic; use std::sync::atomic::AtomicU64; -use std::sync::Arc; use databend_common_catalog::table_context::TableContext; use databend_common_column::bitmap::Bitmap; use databend_common_column::bitmap::MutableBitmap; use databend_common_exception::Result; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::NumberScalar; use databend_common_expression::ColumnBuilder; use databend_common_expression::DataBlock; use databend_common_expression::Evaluator; use databend_common_expression::FunctionContext; use databend_common_expression::RemoteExpr; use databend_common_expression::ScalarRef; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::NumberScalar; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_sql::plans::JoinType; use parking_lot::Mutex; diff --git a/src/query/service/src/pipelines/processors/transforms/sort/sort_builder.rs b/src/query/service/src/pipelines/processors/transforms/sort/sort_builder.rs index d4c6bb0d9e955..e22e91901bc33 100644 --- a/src/query/service/src/pipelines/processors/transforms/sort/sort_builder.rs +++ b/src/query/service/src/pipelines/processors/transforms/sort/sort_builder.rs @@ -25,15 +25,7 @@ use databend_common_pipeline::core::PipeItem; use databend_common_pipeline::core::Pipeline; use databend_common_pipeline::core::Processor; use databend_common_pipeline::core::ProcessorPtr; -use databend_common_pipeline_transforms::sorts::core::algorithm::HeapSort; -use databend_common_pipeline_transforms::sorts::core::algorithm::LoserTreeSort; -use databend_common_pipeline_transforms::sorts::core::algorithm::SortAlgorithm; -use databend_common_pipeline_transforms::sorts::core::select_row_type; -use databend_common_pipeline_transforms::sorts::core::RowConverter; -use databend_common_pipeline_transforms::sorts::core::Rows; -use databend_common_pipeline_transforms::sorts::core::RowsTypeVisitor; -use databend_common_pipeline_transforms::sorts::utils::add_order_field; -use databend_common_pipeline_transforms::sorts::utils::ORDER_COL_NAME; +use databend_common_pipeline_transforms::MemorySettings; use databend_common_pipeline_transforms::sorts::Base; use databend_common_pipeline_transforms::sorts::BoundedMultiSortMergeProcessor; use databend_common_pipeline_transforms::sorts::BroadcastChannel; @@ -45,8 +37,16 @@ use databend_common_pipeline_transforms::sorts::TransformSortBoundBroadcast; use databend_common_pipeline_transforms::sorts::TransformSortCollect; use databend_common_pipeline_transforms::sorts::TransformSortRestore; use databend_common_pipeline_transforms::sorts::TransformSortRoute; +use databend_common_pipeline_transforms::sorts::core::RowConverter; +use databend_common_pipeline_transforms::sorts::core::Rows; +use databend_common_pipeline_transforms::sorts::core::RowsTypeVisitor; +use databend_common_pipeline_transforms::sorts::core::algorithm::HeapSort; +use databend_common_pipeline_transforms::sorts::core::algorithm::LoserTreeSort; +use databend_common_pipeline_transforms::sorts::core::algorithm::SortAlgorithm; +use databend_common_pipeline_transforms::sorts::core::select_row_type; +use databend_common_pipeline_transforms::sorts::utils::ORDER_COL_NAME; +use databend_common_pipeline_transforms::sorts::utils::add_order_field; use databend_common_pipeline_transforms::traits::DataBlockSpill; -use databend_common_pipeline_transforms::MemorySettings; use super::*; use crate::servers::flight::v1::exchange::ExchangeInjector; diff --git a/src/query/service/src/pipelines/processors/transforms/transform_async_function.rs b/src/query/service/src/pipelines/processors/transforms/transform_async_function.rs index 84e80bd560514..58aa309e80a8b 100644 --- a/src/query/service/src/pipelines/processors/transforms/transform_async_function.rs +++ b/src/query/service/src/pipelines/processors/transforms/transform_async_function.rs @@ -13,17 +13,17 @@ // limitations under the License. use std::collections::BTreeMap; +use std::sync::Arc; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; -use std::sync::Arc; use databend_common_base::base::tokio::sync::RwLock; use databend_common_catalog::catalog::Catalog; use databend_common_exception::Result; -use databend_common_expression::types::UInt64Type; use databend_common_expression::AutoIncrementExpr; use databend_common_expression::DataBlock; use databend_common_expression::FromData; +use databend_common_expression::types::UInt64Type; use databend_common_meta_app::principal::AutoIncrementKey; use databend_common_meta_app::schema::GetAutoIncrementNextValueReq; use databend_common_meta_app::schema::GetSequenceNextValueReq; diff --git a/src/query/service/src/pipelines/processors/transforms/transform_branched_async_function.rs b/src/query/service/src/pipelines/processors/transforms/transform_branched_async_function.rs index e8ec40ce6c332..6dd26bf6e8316 100644 --- a/src/query/service/src/pipelines/processors/transforms/transform_branched_async_function.rs +++ b/src/query/service/src/pipelines/processors/transforms/transform_branched_async_function.rs @@ -24,10 +24,10 @@ use databend_common_meta_app::schema::SequenceIdent; use databend_common_pipeline_transforms::processors::AsyncTransform; use databend_common_sql::binder::AsyncFunctionDesc; +use crate::pipelines::processors::transforms::TransformAsyncFunction; use crate::pipelines::processors::transforms::transform_async_function::AutoIncrementNextValFetcher; use crate::pipelines::processors::transforms::transform_async_function::SequenceCounters; use crate::pipelines::processors::transforms::transform_async_function::SequenceNextValFetcher; -use crate::pipelines::processors::transforms::TransformAsyncFunction; use crate::sessions::QueryContext; use crate::sql::plans::AsyncFunctionArgument; diff --git a/src/query/service/src/pipelines/processors/transforms/transform_cache_scan.rs b/src/query/service/src/pipelines/processors/transforms/transform_cache_scan.rs index 3c60061721aef..09fc6aef05270 100644 --- a/src/query/service/src/pipelines/processors/transforms/transform_cache_scan.rs +++ b/src/query/service/src/pipelines/processors/transforms/transform_cache_scan.rs @@ -24,8 +24,8 @@ use databend_common_pipeline::core::ProcessorPtr; use databend_common_pipeline::sources::AsyncSource; use databend_common_pipeline::sources::AsyncSourcer; -use crate::pipelines::processors::transforms::BasicHashJoinState; use crate::pipelines::processors::HashJoinState; +use crate::pipelines::processors::transforms::BasicHashJoinState; use crate::sessions::QueryContext; #[derive(Clone)] diff --git a/src/query/service/src/pipelines/processors/transforms/transform_dictionary.rs b/src/query/service/src/pipelines/processors/transforms/transform_dictionary.rs index 2a7bce3d5876c..140186ee9c22a 100644 --- a/src/query/service/src/pipelines/processors/transforms/transform_dictionary.rs +++ b/src/query/service/src/pipelines/processors/transforms/transform_dictionary.rs @@ -21,8 +21,13 @@ use std::sync::Arc; use databend_common_column::bitmap::Bitmap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::date::date_to_string; -use databend_common_expression::types::timestamp::timestamp_to_string; +use databend_common_expression::BlockEntry; +use databend_common_expression::Column; +use databend_common_expression::ColumnBuilder; +use databend_common_expression::DataBlock; +use databend_common_expression::Scalar; +use databend_common_expression::ScalarRef; +use databend_common_expression::Value; use databend_common_expression::types::AccessType; use databend_common_expression::types::AnyType; use databend_common_expression::types::DataType; @@ -31,30 +36,25 @@ use databend_common_expression::types::NumberDataType; use databend_common_expression::types::NumberScalar; use databend_common_expression::types::StringColumn; use databend_common_expression::types::StringType; +use databend_common_expression::types::date::date_to_string; +use databend_common_expression::types::timestamp::timestamp_to_string; use databend_common_expression::with_integer_mapped_type; -use databend_common_expression::BlockEntry; -use databend_common_expression::Column; -use databend_common_expression::ColumnBuilder; -use databend_common_expression::DataBlock; -use databend_common_expression::Scalar; -use databend_common_expression::ScalarRef; -use databend_common_expression::Value; use databend_common_sql::binder::AsyncFunctionDesc; use jiff::tz::TimeZone; -use redis::aio::ConnectionManager; use redis::AsyncCommands; use redis::Client; use redis::ConnectionInfo; use redis::ProtocolVersion; use redis::RedisConnectionInfo; +use redis::aio::ConnectionManager; use sqlx::MySqlPool; use crate::pipelines::processors::transforms::TransformAsyncFunction; +use crate::sql::IndexType; use crate::sql::plans::AsyncFunctionArgument; use crate::sql::plans::DictGetFunctionArgument; use crate::sql::plans::DictionarySource; use crate::sql::plans::SqlSource; -use crate::sql::IndexType; macro_rules! sqlx_fetch_optional { ($pool:expr, $sql:expr, $key_type:ty, $val_type:ty, $format_val_fn:expr) => {{ diff --git a/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs b/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs index 0a5f9574c5e70..5886ea2f4e51e 100644 --- a/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs +++ b/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs @@ -22,7 +22,6 @@ use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::infer_schema_type; use databend_common_expression::BlockEntry; use databend_common_expression::ColumnBuilder; use databend_common_expression::DataBlock; @@ -32,6 +31,7 @@ use databend_common_expression::Expr; use databend_common_expression::FunctionContext; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::infer_schema_type; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::schema::CreateOption; use databend_common_meta_app::tenant::Tenant; @@ -39,9 +39,9 @@ use databend_common_pipeline::core::OutputPort; use databend_common_pipeline::core::ProcessorPtr; use databend_common_pipeline::sources::AsyncSource; use databend_common_pipeline::sources::AsyncSourcer; +use databend_common_sql::IndexType; use databend_common_sql::plans::CreateTablePlan; use databend_common_sql::plans::DropTablePlan; -use databend_common_sql::IndexType; use databend_common_storages_basic::MemoryTable; use futures_util::TryStreamExt; diff --git a/src/query/service/src/pipelines/processors/transforms/transform_resort_addon_without_source_schema.rs b/src/query/service/src/pipelines/processors/transforms/transform_resort_addon_without_source_schema.rs index 078ffac53db9c..af67794bf1974 100644 --- a/src/query/service/src/pipelines/processors/transforms/transform_resort_addon_without_source_schema.rs +++ b/src/query/service/src/pipelines/processors/transforms/transform_resort_addon_without_source_schema.rs @@ -18,7 +18,6 @@ use std::sync::Arc; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::type_check::check_cast; use databend_common_expression::BlockMetaInfoDowncast; use databend_common_expression::ColumnRef; use databend_common_expression::Constant; @@ -26,6 +25,7 @@ use databend_common_expression::DataBlock; use databend_common_expression::DataSchemaRef; use databend_common_expression::Scalar; use databend_common_expression::SourceSchemaIndex; +use databend_common_expression::type_check::check_cast; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_pipeline_transforms::blocks::CompoundBlockOperator; use databend_common_pipeline_transforms::processors::Transform; diff --git a/src/query/service/src/pipelines/processors/transforms/transform_srf.rs b/src/query/service/src/pipelines/processors/transforms/transform_srf.rs index 0758c42740f5d..db2647d1d3e15 100644 --- a/src/query/service/src/pipelines/processors/transforms/transform_srf.rs +++ b/src/query/service/src/pipelines/processors/transforms/transform_srf.rs @@ -16,13 +16,6 @@ use std::collections::VecDeque; use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::types::nullable::NullableColumnBuilder; -use databend_common_expression::types::AnyType; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberColumn; -use databend_common_expression::types::NumberType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::VariantType; use databend_common_expression::BlockEntry; use databend_common_expression::Column; use databend_common_expression::ColumnBuilder; @@ -33,6 +26,13 @@ use databend_common_expression::FunctionCall; use databend_common_expression::FunctionContext; use databend_common_expression::ScalarRef; use databend_common_expression::Value; +use databend_common_expression::types::AnyType; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberColumn; +use databend_common_expression::types::NumberType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::VariantType; +use databend_common_expression::types::nullable::NullableColumnBuilder; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_pipeline::core::InputPort; use databend_common_pipeline::core::OutputPort; diff --git a/src/query/service/src/pipelines/processors/transforms/transform_udf_script.rs b/src/query/service/src/pipelines/processors/transforms/transform_udf_script.rs index d79ddab826e0a..6d80e48ee1e5d 100644 --- a/src/query/service/src/pipelines/processors/transforms/transform_udf_script.rs +++ b/src/query/service/src/pipelines/processors/transforms/transform_udf_script.rs @@ -12,12 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::btree_map::Entry; use std::collections::BTreeMap; +use std::collections::btree_map::Entry; use std::str; -use std::sync::atomic::AtomicUsize; use std::sync::Arc; use std::sync::LazyLock; +use std::sync::atomic::AtomicUsize; use arrow_array::RecordBatch; use arrow_udf_runtime::javascript::FunctionOptions; @@ -25,17 +25,17 @@ use databend_common_base::runtime::GlobalIORuntime; use databend_common_cache::Cache; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::converts::arrow::ARROW_EXT_TYPE_VARIANT; -use databend_common_expression::converts::arrow::EXTENSION_KEY; -use databend_common_expression::types::DataType; -use databend_common_expression::variant_transform::contains_variant; -use databend_common_expression::variant_transform::transform_variant; use databend_common_expression::BlockEntry; use databend_common_expression::DataBlock; use databend_common_expression::DataField; use databend_common_expression::DataSchema; use databend_common_expression::FunctionContext; use databend_common_expression::Value; +use databend_common_expression::converts::arrow::ARROW_EXT_TYPE_VARIANT; +use databend_common_expression::converts::arrow::EXTENSION_KEY; +use databend_common_expression::types::DataType; +use databend_common_expression::variant_transform::contains_variant; +use databend_common_expression::variant_transform::transform_variant; #[cfg_attr(not(feature = "python-udf"), allow(unused_imports))] use databend_common_meta_app::principal::StageInfo; use databend_common_pipeline_transforms::processors::Transform; @@ -880,9 +880,9 @@ mod venv { use parking_lot::RwLock; use uuid::Uuid; use walkdir::WalkDir; - use zip::write::FileOptions; use zip::ZipArchive; use zip::ZipWriter; + use zip::write::FileOptions; static PY_VENV_ARCHIVE_DIR: LazyLock = LazyLock::new(|| { let base = std::env::var("DATABEND_PY_UDF_CACHE_DIR") diff --git a/src/query/service/src/pipelines/processors/transforms/transform_udf_server.rs b/src/query/service/src/pipelines/processors/transforms/transform_udf_server.rs index ea7f8dfbd4903..7ceb7b2544787 100644 --- a/src/query/service/src/pipelines/processors/transforms/transform_udf_server.rs +++ b/src/query/service/src/pipelines/processors/transforms/transform_udf_server.rs @@ -23,12 +23,12 @@ use databend_common_base::runtime::profile::ProfileStatisticsName; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::udf_client::error_kind; -use databend_common_expression::udf_client::is_transport_error_message; -use databend_common_expression::udf_client::UDFFlightClient; use databend_common_expression::BlockEntry; use databend_common_expression::ColumnBuilder; use databend_common_expression::DataBlock; +use databend_common_expression::udf_client::UDFFlightClient; +use databend_common_expression::udf_client::error_kind; +use databend_common_expression::udf_client::is_transport_error_message; use databend_common_metrics::external_server::record_error_external; use databend_common_metrics::external_server::record_retry_external; use databend_common_pipeline_transforms::processors::AsyncTransform; diff --git a/src/query/service/src/pipelines/processors/transforms/window/partition/transform_window_partition_collect.rs b/src/query/service/src/pipelines/processors/transforms/window/partition/transform_window_partition_collect.rs index f3d49ebde23ed..de879703201c7 100644 --- a/src/query/service/src/pipelines/processors/transforms/window/partition/transform_window_partition_collect.rs +++ b/src/query/service/src/pipelines/processors/transforms/window/partition/transform_window_partition_collect.rs @@ -32,9 +32,9 @@ use databend_common_settings::Settings; use databend_common_storage::DataOperator; use either::Either; -use super::window_partition_buffer_v2::WindowPartitionBufferV2; use super::WindowPartitionBuffer; use super::WindowPartitionMeta; +use super::window_partition_buffer_v2::WindowPartitionBufferV2; use crate::pipelines::processors::transforms::DataProcessorStrategy; use crate::sessions::QueryContext; use crate::spillers::BackpressureSpiller; diff --git a/src/query/service/src/pipelines/processors/transforms/window/partition/window_partition_exchange.rs b/src/query/service/src/pipelines/processors/transforms/window/partition/window_partition_exchange.rs index 935d6d100c822..7336d6a24bd57 100644 --- a/src/query/service/src/pipelines/processors/transforms/window/partition/window_partition_exchange.rs +++ b/src/query/service/src/pipelines/processors/transforms/window/partition/window_partition_exchange.rs @@ -15,9 +15,9 @@ use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::group_hash_entries; use databend_common_expression::DataBlock; use databend_common_expression::ProjectedBlock; +use databend_common_expression::group_hash_entries; use databend_common_pipeline::basic::Exchange; use super::WindowPartitionMeta; diff --git a/src/query/service/src/pipelines/processors/transforms/window/partition/window_partition_meta.rs b/src/query/service/src/pipelines/processors/transforms/window/partition/window_partition_meta.rs index 4de4ba5f52518..bfa2f4b11a24a 100644 --- a/src/query/service/src/pipelines/processors/transforms/window/partition/window_partition_meta.rs +++ b/src/query/service/src/pipelines/processors/transforms/window/partition/window_partition_meta.rs @@ -15,10 +15,10 @@ use std::fmt::Debug; use std::fmt::Formatter; -use databend_common_expression::local_block_meta_serde; use databend_common_expression::BlockMetaInfo; use databend_common_expression::BlockMetaInfoPtr; use databend_common_expression::DataBlock; +use databend_common_expression::local_block_meta_serde; pub struct WindowPartitionMeta { // Each element in `partitioned_data` is (partition_id, data_block). diff --git a/src/query/service/src/pipelines/processors/transforms/window/partition/window_partition_partial_top_n_exchange.rs b/src/query/service/src/pipelines/processors/transforms/window/partition/window_partition_partial_top_n_exchange.rs index 1af9ed374d0dc..b3d840283e3a6 100644 --- a/src/query/service/src/pipelines/processors/transforms/window/partition/window_partition_partial_top_n_exchange.rs +++ b/src/query/service/src/pipelines/processors/transforms/window/partition/window_partition_partial_top_n_exchange.rs @@ -15,11 +15,11 @@ use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::group_hash_value_spread; -use databend_common_expression::visitor::ValueVisitor; use databend_common_expression::DataBlock; use databend_common_expression::SortColumnDescription; use databend_common_expression::SortCompare; +use databend_common_expression::group_hash_value_spread; +use databend_common_expression::visitor::ValueVisitor; use databend_common_pipeline::basic::Exchange; use super::WindowPartitionMeta; @@ -170,10 +170,10 @@ impl WindowPartitionTopNExchange { #[cfg(test)] mod tests { - use databend_common_expression::types::Int32Type; - use databend_common_expression::types::StringType; use databend_common_expression::BlockEntry; use databend_common_expression::FromData; + use databend_common_expression::types::Int32Type; + use databend_common_expression::types::StringType; use super::*; diff --git a/src/query/service/src/pipelines/processors/transforms/window/transform_window.rs b/src/query/service/src/pipelines/processors/transforms/window/transform_window.rs index 384d7e551de2a..7972e27efb0f6 100644 --- a/src/query/service/src/pipelines/processors/transforms/window/transform_window.rs +++ b/src/query/service/src/pipelines/processors/transforms/window/transform_window.rs @@ -24,9 +24,17 @@ use std::sync::Arc; use databend_common_column::types::months_days_micros; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::BlockEntry; +use databend_common_expression::Column; +use databend_common_expression::ColumnBuilder; +use databend_common_expression::DataBlock; +use databend_common_expression::FunctionContext; +use databend_common_expression::Scalar; +use databend_common_expression::ScalarRef; +use databend_common_expression::SortColumnDescription; use databend_common_expression::arithmetics_type::ResultTypeOfUnary; -use databend_common_expression::date_helper::calc_date_to_timestamp; use databend_common_expression::date_helper::EvalMonthsImpl; +use databend_common_expression::date_helper::calc_date_to_timestamp; use databend_common_expression::types::AccessType; use databend_common_expression::types::DataType; use databend_common_expression::types::DateType; @@ -36,24 +44,16 @@ use databend_common_expression::types::NumberScalar; use databend_common_expression::types::NumberType; use databend_common_expression::types::TimestampType; use databend_common_expression::with_number_mapped_type; -use databend_common_expression::BlockEntry; -use databend_common_expression::Column; -use databend_common_expression::ColumnBuilder; -use databend_common_expression::DataBlock; -use databend_common_expression::FunctionContext; -use databend_common_expression::Scalar; -use databend_common_expression::ScalarRef; -use databend_common_expression::SortColumnDescription; use databend_common_pipeline::core::Event; use databend_common_pipeline::core::InputPort; use databend_common_pipeline::core::OutputPort; use databend_common_pipeline::core::Processor; use databend_common_sql::plans::WindowFuncFrameUnits; +use super::WindowFunctionInfo; use super::frame_bound::FrameBound; use super::window_function::WindowFuncAggImpl; use super::window_function::WindowFunctionImpl; -use super::WindowFunctionInfo; use crate::physical_plans::LagLeadDefault; #[derive(Debug, Clone)] @@ -983,7 +983,7 @@ impl TransformWindow { cmp_v_timestamp.cmp(&ref_v_timestamp) } else { return Err(ErrorCode::IllegalDataType( - "window function `RANGE BETWEEN` for DATE `ORDER BY` must use numeric or interval" + "window function `RANGE BETWEEN` for DATE `ORDER BY` must use numeric or interval", )); } } @@ -993,7 +993,7 @@ impl TransformWindow { let Some(n) = offset.as_interval() else { return Err(ErrorCode::IllegalDataType( - "window function `RANGE BETWEEN` for DATE `ORDER BY` must use interval" + "window function `RANGE BETWEEN` for DATE `ORDER BY` must use interval", )); }; let func_ctx = FunctionContext::default(); @@ -1004,7 +1004,11 @@ impl TransformWindow { }; cmp_v.cmp(&ref_v) } - _ => return Err(ErrorCode::IllegalDataType("window function `ORDER BY` must be of numeric, date, or timestamp type when `RANGE BETWEEN`")), + _ => { + return Err(ErrorCode::IllegalDataType( + "window function `ORDER BY` must be of numeric, date, or timestamp type when `RANGE BETWEEN`", + )); + } }; Ok(ordering) } @@ -1485,30 +1489,30 @@ mod tests { use databend_common_base::base::tokio; use databend_common_exception::Result; - use databend_common_expression::block_debug::assert_blocks_eq; - use databend_common_expression::types::DataType; - use databend_common_expression::types::Int32Type; - use databend_common_expression::types::NumberDataType; - use databend_common_expression::types::NumberScalar; use databend_common_expression::Column; use databend_common_expression::ColumnBuilder; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::Scalar; + use databend_common_expression::block_debug::assert_blocks_eq; + use databend_common_expression::types::DataType; + use databend_common_expression::types::Int32Type; + use databend_common_expression::types::NumberDataType; + use databend_common_expression::types::NumberScalar; use databend_common_functions::aggregates::AggregateFunctionFactory; - use databend_common_pipeline::core::port::connect; use databend_common_pipeline::core::Event; use databend_common_pipeline::core::InputPort; use databend_common_pipeline::core::OutputPort; use databend_common_pipeline::core::Processor; + use databend_common_pipeline::core::port::connect; use databend_common_sql::plans::WindowFuncFrameUnits; use super::TransformWindow; use super::WindowBlock; use super::WindowSortDesc; - use crate::pipelines::processors::transforms::window::transform_window::RowPtr; use crate::pipelines::processors::transforms::window::FrameBound; use crate::pipelines::processors::transforms::window::WindowFunctionInfo; + use crate::pipelines::processors::transforms::window::transform_window::RowPtr; fn get_ranking_transform_window(bounds: (FrameBound, FrameBound)) -> Result { let func = WindowFunctionInfo::DenseRank; diff --git a/src/query/service/src/pipelines/processors/transforms/window/window_function.rs b/src/query/service/src/pipelines/processors/transforms/window/window_function.rs index 3dbbaa4f2fe18..779c7b69e7759 100644 --- a/src/query/service/src/pipelines/processors/transforms/window/window_function.rs +++ b/src/query/service/src/pipelines/processors/transforms/window/window_function.rs @@ -16,9 +16,6 @@ use std::sync::Arc; use databend_common_base::runtime::drop_guard; use databend_common_exception::Result; -use databend_common_expression::get_states_layout; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; use databend_common_expression::AggrState; use databend_common_expression::AggrStateLoc; use databend_common_expression::ColumnBuilder; @@ -26,6 +23,9 @@ use databend_common_expression::DataBlock; use databend_common_expression::DataSchema; use databend_common_expression::ProjectedBlock; use databend_common_expression::StateAddr; +use databend_common_expression::get_states_layout; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; use databend_common_functions::aggregates::AggregateFunction; use databend_common_functions::aggregates::AggregateFunctionFactory; use databend_common_functions::aggregates::AggregateFunctionSortDesc; diff --git a/src/query/service/src/schedulers/fragments/fragmenter.rs b/src/query/service/src/schedulers/fragments/fragmenter.rs index 3c8d61c202373..8f56fe1be7e84 100644 --- a/src/query/service/src/schedulers/fragments/fragmenter.rs +++ b/src/query/service/src/schedulers/fragments/fragmenter.rs @@ -44,8 +44,8 @@ use crate::physical_plans::ReplaceInto; use crate::physical_plans::Sequence; use crate::physical_plans::TableScan; use crate::physical_plans::VisitorCast; -use crate::schedulers::fragments::plan_fragment::FragmentType; use crate::schedulers::PlanFragment; +use crate::schedulers::fragments::plan_fragment::FragmentType; use crate::servers::flight::v1::exchange::BroadcastExchange; use crate::servers::flight::v1::exchange::DataExchange; use crate::servers::flight::v1::exchange::MergeExchange; diff --git a/src/query/service/src/schedulers/fragments/query_fragment_actions.rs b/src/query/service/src/schedulers/fragments/query_fragment_actions.rs index 97ca84618f5ac..854767137cb9a 100644 --- a/src/query/service/src/schedulers/fragments/query_fragment_actions.rs +++ b/src/query/service/src/schedulers/fragments/query_fragment_actions.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::hash_map::Entry; use std::collections::HashMap; +use std::collections::hash_map::Entry; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; diff --git a/src/query/service/src/schedulers/scheduler.rs b/src/query/service/src/schedulers/scheduler.rs index 78e47a2714e12..7c4585dfc43f1 100644 --- a/src/query/service/src/schedulers/scheduler.rs +++ b/src/query/service/src/schedulers/scheduler.rs @@ -17,18 +17,18 @@ use std::sync::Arc; use async_trait::async_trait; use databend_common_exception::Result; use databend_common_expression::DataBlock; -use databend_common_sql::planner::QueryExecutor; use databend_common_sql::Planner; +use databend_common_sql::planner::QueryExecutor; use futures_util::TryStreamExt; use crate::interpreters::InterpreterFactory; -use crate::physical_plans::build_broadcast_plans; use crate::physical_plans::PhysicalPlan; +use crate::physical_plans::build_broadcast_plans; +use crate::pipelines::PipelineBuildResult; +use crate::pipelines::PipelineBuilder; use crate::pipelines::attach_runtime_filter_logger; use crate::pipelines::executor::ExecutorSettings; use crate::pipelines::executor::PipelinePullingExecutor; -use crate::pipelines::PipelineBuildResult; -use crate::pipelines::PipelineBuilder; use crate::schedulers::Fragmenter; use crate::schedulers::QueryFragmentsActions; use crate::sessions::QueryContext; diff --git a/src/query/service/src/servers/admin/admin_service.rs b/src/query/service/src/servers/admin/admin_service.rs index c251c3acc1bf7..dc29726cdfcf2 100644 --- a/src/query/service/src/servers/admin/admin_service.rs +++ b/src/query/service/src/servers/admin/admin_service.rs @@ -17,22 +17,22 @@ use std::time::Duration; use databend_common_config::InnerConfig; use databend_common_exception::ErrorCode; +use databend_common_http::HttpError; +use databend_common_http::HttpShutdownHandler; use databend_common_http::health_handler; use databend_common_http::home::debug_home_handler; #[cfg(feature = "memory-profiling")] use databend_common_http::jeprof::debug_jeprof_dump_handler; use databend_common_http::pprof::debug_pprof_handler; use databend_common_http::stack::debug_dump_stack; -use databend_common_http::HttpError; -use databend_common_http::HttpShutdownHandler; use databend_common_meta_types::anyerror::AnyError; use log::info; use log::warn; +use poem::Endpoint; +use poem::Route; use poem::get; use poem::listener::OpensslTlsConfig; use poem::post; -use poem::Endpoint; -use poem::Route; use crate::servers::Server; @@ -49,7 +49,7 @@ impl AdminService { }) } - fn build_router(&self) -> impl Endpoint { + fn build_router(&self) -> impl Endpoint + use<> { #[cfg_attr(not(feature = "memory-profiling"), allow(unused_mut))] let mut route = Route::new() .at("/v1/health", get(health_handler)) diff --git a/src/query/service/src/servers/admin/v1/config.rs b/src/query/service/src/servers/admin/v1/config.rs index e0d7cb2e44517..0336139c80022 100644 --- a/src/query/service/src/servers/admin/v1/config.rs +++ b/src/query/service/src/servers/admin/v1/config.rs @@ -13,8 +13,8 @@ // limitations under the License. use databend_common_config::GlobalConfig; -use poem::web::Json; use poem::IntoResponse; +use poem::web::Json; #[poem::handler] #[async_backtrace::framed] diff --git a/src/query/service/src/servers/admin/v1/instance_status.rs b/src/query/service/src/servers/admin/v1/instance_status.rs index 231e549a54855..86b05308604f7 100644 --- a/src/query/service/src/servers/admin/v1/instance_status.rs +++ b/src/query/service/src/servers/admin/v1/instance_status.rs @@ -14,8 +14,8 @@ use std::time::SystemTime; -use poem::web::Json; use poem::IntoResponse; +use poem::web::Json; use serde::Deserialize; use serde::Serialize; diff --git a/src/query/service/src/servers/admin/v1/processes.rs b/src/query/service/src/servers/admin/v1/processes.rs index a99d67aba1f77..782fdd3c0e0c6 100644 --- a/src/query/service/src/servers/admin/v1/processes.rs +++ b/src/query/service/src/servers/admin/v1/processes.rs @@ -14,8 +14,8 @@ use std::time::SystemTime; -use poem::web::Json; use poem::IntoResponse; +use poem::web::Json; use serde::Deserialize; use serde::Serialize; diff --git a/src/query/service/src/servers/admin/v1/query_dump.rs b/src/query/service/src/servers/admin/v1/query_dump.rs index c8cf88b964b3e..fdd0f46085c68 100644 --- a/src/query/service/src/servers/admin/v1/query_dump.rs +++ b/src/query/service/src/servers/admin/v1/query_dump.rs @@ -17,9 +17,9 @@ use std::collections::HashMap; use databend_common_config::GlobalConfig; use databend_common_exception::Result; use http::StatusCode; +use poem::IntoResponse; use poem::web::Json; use poem::web::Path; -use poem::IntoResponse; use crate::clusters::ClusterDiscovery; use crate::clusters::ClusterHelper; @@ -41,7 +41,7 @@ pub async fn running_query_dump(Path(query_id): Path) -> poem::Result { request: Request, diff --git a/src/query/service/src/servers/flight/v1/actions/flight_actions.rs b/src/query/service/src/servers/flight/v1/actions/flight_actions.rs index 1e54b10056d74..cf128a4dacd6b 100644 --- a/src/query/service/src/servers/flight/v1/actions/flight_actions.rs +++ b/src/query/service/src/servers/flight/v1/actions/flight_actions.rs @@ -16,8 +16,8 @@ use std::collections::HashMap; use std::future::Future; use std::sync::Arc; -use databend_common_base::runtime::catch_unwind; use databend_common_base::runtime::CatchUnwindFuture; +use databend_common_base::runtime::catch_unwind; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_exception::ResultExt; @@ -25,22 +25,22 @@ use futures_util::future::BoxFuture; use serde::Deserialize; use serde::Serialize; +use crate::servers::flight::v1::actions::GET_RUNNING_QUERY_DUMP; +use crate::servers::flight::v1::actions::INIT_QUERY_FRAGMENTS; +use crate::servers::flight::v1::actions::KILL_QUERY; +use crate::servers::flight::v1::actions::START_PREPARED_QUERY; +use crate::servers::flight::v1::actions::SYSTEM_ACTION; use crate::servers::flight::v1::actions::get_running_query_dump::get_running_query_dump; -use crate::servers::flight::v1::actions::init_query_env::init_query_env; use crate::servers::flight::v1::actions::init_query_env::INIT_QUERY_ENV; +use crate::servers::flight::v1::actions::init_query_env::init_query_env; use crate::servers::flight::v1::actions::init_query_fragments::init_query_fragments; use crate::servers::flight::v1::actions::kill_query::kill_query; -use crate::servers::flight::v1::actions::set_priority::set_priority; use crate::servers::flight::v1::actions::set_priority::SET_PRIORITY; +use crate::servers::flight::v1::actions::set_priority::set_priority; use crate::servers::flight::v1::actions::start_prepared_query::start_prepared_query; use crate::servers::flight::v1::actions::system_action::system_action; -use crate::servers::flight::v1::actions::truncate_table::truncate_table; use crate::servers::flight::v1::actions::truncate_table::TRUNCATE_TABLE; -use crate::servers::flight::v1::actions::GET_RUNNING_QUERY_DUMP; -use crate::servers::flight::v1::actions::INIT_QUERY_FRAGMENTS; -use crate::servers::flight::v1::actions::KILL_QUERY; -use crate::servers::flight::v1::actions::START_PREPARED_QUERY; -use crate::servers::flight::v1::actions::SYSTEM_ACTION; +use crate::servers::flight::v1::actions::truncate_table::truncate_table; pub struct FlightActions { #[allow(clippy::type_complexity)] diff --git a/src/query/service/src/servers/flight/v1/actions/init_query_env.rs b/src/query/service/src/servers/flight/v1/actions/init_query_env.rs index c64119b54c934..4817ae48d0a38 100644 --- a/src/query/service/src/servers/flight/v1/actions/init_query_env.rs +++ b/src/query/service/src/servers/flight/v1/actions/init_query_env.rs @@ -15,10 +15,10 @@ use std::sync::Arc; use databend_common_base::base::GlobalInstance; +use databend_common_base::runtime::GLOBAL_MEM_STAT; use databend_common_base::runtime::MemStat; use databend_common_base::runtime::ParentMemStat; use databend_common_base::runtime::ThreadTracker; -use databend_common_base::runtime::GLOBAL_MEM_STAT; use databend_common_config::GlobalConfig; use databend_common_exception::Result; use databend_common_management::WorkloadGroupResourceManager; diff --git a/src/query/service/src/servers/flight/v1/actions/mod.rs b/src/query/service/src/servers/flight/v1/actions/mod.rs index 97f8db0d67f47..ee8f2473d231f 100644 --- a/src/query/service/src/servers/flight/v1/actions/mod.rs +++ b/src/query/service/src/servers/flight/v1/actions/mod.rs @@ -28,12 +28,12 @@ use databend_common_catalog::session_type::SessionType; use databend_common_config::GlobalConfig; use databend_common_exception::Result; use databend_common_settings::Settings; -pub use flight_actions::flight_actions; pub use flight_actions::FlightActions; +pub use flight_actions::flight_actions; pub use get_running_query_dump::GET_RUNNING_QUERY_DUMP; pub use init_query_env::INIT_QUERY_ENV; -pub use init_query_fragments::init_query_fragments; pub use init_query_fragments::INIT_QUERY_FRAGMENTS; +pub use init_query_fragments::init_query_fragments; pub use kill_query::KILL_QUERY; pub use set_priority::SET_PRIORITY; pub use start_prepared_query::START_PREPARED_QUERY; diff --git a/src/query/service/src/servers/flight/v1/exchange/exchange_injector.rs b/src/query/service/src/servers/flight/v1/exchange/exchange_injector.rs index d84adad6a4087..5bd2ce2927b4e 100644 --- a/src/query/service/src/servers/flight/v1/exchange/exchange_injector.rs +++ b/src/query/service/src/servers/flight/v1/exchange/exchange_injector.rs @@ -20,12 +20,12 @@ use databend_common_pipeline::core::Pipeline; use databend_common_settings::FlightCompression; use super::exchange_params::MergeExchangeParams; -use crate::servers::flight::v1::exchange::serde::TransformExchangeDeserializer; -use crate::servers::flight::v1::exchange::serde::TransformExchangeSerializer; -use crate::servers::flight::v1::exchange::serde::TransformScatterExchangeSerializer; use crate::servers::flight::v1::exchange::DataExchange; use crate::servers::flight::v1::exchange::ExchangeSorting; use crate::servers::flight::v1::exchange::ShuffleExchangeParams; +use crate::servers::flight::v1::exchange::serde::TransformExchangeDeserializer; +use crate::servers::flight::v1::exchange::serde::TransformExchangeSerializer; +use crate::servers::flight::v1::exchange::serde::TransformScatterExchangeSerializer; use crate::servers::flight::v1::scatter::BroadcastFlightScatter; use crate::servers::flight::v1::scatter::FlightScatter; use crate::servers::flight::v1::scatter::HashFlightScatter; diff --git a/src/query/service/src/servers/flight/v1/exchange/exchange_manager.rs b/src/query/service/src/servers/flight/v1/exchange/exchange_manager.rs index 3bc53b2c4e044..815f26072c065 100644 --- a/src/query/service/src/servers/flight/v1/exchange/exchange_manager.rs +++ b/src/query/service/src/servers/flight/v1/exchange/exchange_manager.rs @@ -13,38 +13,38 @@ // limitations under the License. use std::cell::SyncUnsafeCell; -use std::collections::hash_map::Entry; use std::collections::HashMap; +use std::collections::hash_map::Entry; use std::ops::Deref; +use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; -use std::sync::Arc; use std::time::Duration; -use arrow_flight::flight_service_client::FlightServiceClient; use arrow_flight::FlightData; +use arrow_flight::flight_service_client::FlightServiceClient; use async_channel::Receiver; -use databend_common_base::base::tokio::sync::oneshot; +use databend_common_base::JoinHandle; use databend_common_base::base::GlobalInstance; +use databend_common_base::base::tokio::sync::oneshot; use databend_common_base::runtime::ExecutorStatsSnapshot; use databend_common_base::runtime::GlobalIORuntime; use databend_common_base::runtime::QueryPerf; use databend_common_base::runtime::Thread; use databend_common_base::runtime::TrySpawn; -use databend_common_base::JoinHandle; use databend_common_config::GlobalConfig; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_grpc::ConnectionFactory; +use databend_common_pipeline::core::ExecutionInfo; use databend_common_pipeline::core::always_callback; use databend_common_pipeline::core::basic_callback; -use databend_common_pipeline::core::ExecutionInfo; use fastrace::prelude::*; use log::warn; use parking_lot::Mutex; use parking_lot::ReentrantMutex; -use petgraph::prelude::EdgeRef; use petgraph::Direction; +use petgraph::prelude::EdgeRef; use tonic::Status; use super::exchange_params::ExchangeParams; @@ -57,15 +57,19 @@ use super::statistics_sender::StatisticsSender; use crate::clusters::ClusterHelper; use crate::clusters::FlightParams; use crate::physical_plans::PhysicalPlan; +use crate::pipelines::PipelineBuildResult; +use crate::pipelines::PipelineBuilder; use crate::pipelines::attach_runtime_filter_logger; use crate::pipelines::executor::ExecutorSettings; use crate::pipelines::executor::PipelineCompleteExecutor; -use crate::pipelines::PipelineBuildResult; -use crate::pipelines::PipelineBuilder; use crate::schedulers::QueryFragmentsActions; -use crate::servers::flight::v1::actions::init_query_fragments; +use crate::servers::flight::FlightClient; +use crate::servers::flight::FlightExchange; +use crate::servers::flight::FlightReceiver; +use crate::servers::flight::FlightSender; use crate::servers::flight::v1::actions::INIT_QUERY_FRAGMENTS; use crate::servers::flight::v1::actions::START_PREPARED_QUERY; +use crate::servers::flight::v1::actions::init_query_fragments; use crate::servers::flight::v1::exchange::DataExchange; use crate::servers::flight::v1::exchange::DefaultExchangeInjector; use crate::servers::flight::v1::exchange::ExchangeInjector; @@ -73,10 +77,6 @@ use crate::servers::flight::v1::packets::Edge; use crate::servers::flight::v1::packets::QueryEnv; use crate::servers::flight::v1::packets::QueryFragment; use crate::servers::flight::v1::packets::QueryFragments; -use crate::servers::flight::FlightClient; -use crate::servers::flight::FlightExchange; -use crate::servers::flight::FlightReceiver; -use crate::servers::flight::FlightSender; use crate::sessions::QueryContext; use crate::sessions::TableContext; diff --git a/src/query/service/src/servers/flight/v1/exchange/exchange_sink_writer.rs b/src/query/service/src/servers/flight/v1/exchange/exchange_sink_writer.rs index 8cfcbf624ac54..72d484e12c702 100644 --- a/src/query/service/src/servers/flight/v1/exchange/exchange_sink_writer.rs +++ b/src/query/service/src/servers/flight/v1/exchange/exchange_sink_writer.rs @@ -29,8 +29,8 @@ use databend_common_pipeline::sinks::AsyncSinker; use databend_common_pipeline::sinks::Sink; use databend_common_pipeline::sinks::Sinker; -use crate::servers::flight::v1::exchange::serde::ExchangeSerializeMeta; use crate::servers::flight::FlightSender; +use crate::servers::flight::v1::exchange::serde::ExchangeSerializeMeta; pub struct ExchangeWriterSink { flight_sender: FlightSender, diff --git a/src/query/service/src/servers/flight/v1/exchange/exchange_source_reader.rs b/src/query/service/src/servers/flight/v1/exchange/exchange_source_reader.rs index 85525459b1548..67e0fa9a57bb6 100644 --- a/src/query/service/src/servers/flight/v1/exchange/exchange_source_reader.rs +++ b/src/query/service/src/servers/flight/v1/exchange/exchange_source_reader.rs @@ -13,9 +13,9 @@ // limitations under the License. use std::any::Any; +use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; -use std::sync::Arc; use databend_common_exception::Result; use databend_common_expression::DataBlock; @@ -27,9 +27,9 @@ use databend_common_pipeline::core::Processor; use databend_common_pipeline::core::ProcessorPtr; use log::info; +use crate::servers::flight::FlightReceiver; use crate::servers::flight::v1::exchange::serde::ExchangeDeserializeMeta; use crate::servers::flight::v1::packets::DataPacket; -use crate::servers::flight::FlightReceiver; pub struct ExchangeSourceReader { finished: AtomicBool, diff --git a/src/query/service/src/servers/flight/v1/exchange/exchange_transform_shuffle.rs b/src/query/service/src/servers/flight/v1/exchange/exchange_transform_shuffle.rs index 01ae9fec15ef1..0db515509391c 100644 --- a/src/query/service/src/servers/flight/v1/exchange/exchange_transform_shuffle.rs +++ b/src/query/service/src/servers/flight/v1/exchange/exchange_transform_shuffle.rs @@ -21,11 +21,11 @@ use std::sync::Arc; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::local_block_meta_serde; use databend_common_expression::BlockMetaInfo; use databend_common_expression::BlockMetaInfoDowncast; use databend_common_expression::BlockMetaInfoPtr; use databend_common_expression::DataBlock; +use databend_common_expression::local_block_meta_serde; use databend_common_pipeline::core::Event; use databend_common_pipeline::core::EventCause; use databend_common_pipeline::core::InputPort; diff --git a/src/query/service/src/servers/flight/v1/exchange/serde/exchange_deserializer.rs b/src/query/service/src/servers/flight/v1/exchange/serde/exchange_deserializer.rs index 6b450788928e8..a4882273e3039 100644 --- a/src/query/service/src/servers/flight/v1/exchange/serde/exchange_deserializer.rs +++ b/src/query/service/src/servers/flight/v1/exchange/serde/exchange_deserializer.rs @@ -23,14 +23,14 @@ use arrow_ipc::root_as_message; use arrow_schema::Schema as ArrowSchema; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::local_block_meta_serde; use databend_common_expression::BlockMetaInfo; use databend_common_expression::BlockMetaInfoPtr; use databend_common_expression::DataBlock; use databend_common_expression::DataSchema; use databend_common_expression::DataSchemaRef; -use databend_common_io::prelude::bincode_deserialize_from_slice; +use databend_common_expression::local_block_meta_serde; use databend_common_io::prelude::BinaryRead; +use databend_common_io::prelude::bincode_deserialize_from_slice; use databend_common_pipeline::core::InputPort; use databend_common_pipeline::core::OutputPort; use databend_common_pipeline::core::ProcessorPtr; diff --git a/src/query/service/src/servers/flight/v1/exchange/serde/exchange_serializer.rs b/src/query/service/src/servers/flight/v1/exchange/serde/exchange_serializer.rs index 08f89f2fa6808..e5301e19d7d72 100644 --- a/src/query/service/src/servers/flight/v1/exchange/serde/exchange_serializer.rs +++ b/src/query/service/src/servers/flight/v1/exchange/serde/exchange_serializer.rs @@ -20,10 +20,10 @@ use arrow_array::RecordBatch; use arrow_array::RecordBatchOptions; use arrow_flight::FlightData; use arrow_flight::SchemaAsIpc; +use arrow_ipc::CompressionType; use arrow_ipc::writer::DictionaryTracker; use arrow_ipc::writer::IpcDataGenerator; use arrow_ipc::writer::IpcWriteOptions; -use arrow_ipc::CompressionType; use arrow_schema::ArrowError; use arrow_schema::Schema as ArrowSchema; use bytes::Bytes; @@ -31,12 +31,12 @@ use databend_common_base::runtime::profile::Profile; use databend_common_base::runtime::profile::ProfileStatisticsName; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::local_block_meta_serde; use databend_common_expression::BlockMetaInfo; use databend_common_expression::BlockMetaInfoPtr; use databend_common_expression::DataBlock; -use databend_common_io::prelude::bincode_serialize_into_buf; +use databend_common_expression::local_block_meta_serde; use databend_common_io::prelude::BinaryWrite; +use databend_common_io::prelude::bincode_serialize_into_buf; use databend_common_pipeline::core::InputPort; use databend_common_pipeline::core::OutputPort; use databend_common_pipeline::core::ProcessorPtr; @@ -191,12 +191,14 @@ pub fn serialize_block( let (_, dict, values) = match data_block.is_empty() { true => batches_to_flight_data_with_options( &ArrowSchema::empty(), - vec![RecordBatch::try_new_with_options( - Arc::new(ArrowSchema::empty()), - vec![], - &RecordBatchOptions::new().with_row_count(Some(0)), - ) - .unwrap()], + vec![ + RecordBatch::try_new_with_options( + Arc::new(ArrowSchema::empty()), + vec![], + &RecordBatchOptions::new().with_row_count(Some(0)), + ) + .unwrap(), + ], options, )?, false => { diff --git a/src/query/service/src/servers/flight/v1/exchange/serde/mod.rs b/src/query/service/src/servers/flight/v1/exchange/serde/mod.rs index 7349b2f46b0c1..931b5906afcf9 100644 --- a/src/query/service/src/servers/flight/v1/exchange/serde/mod.rs +++ b/src/query/service/src/servers/flight/v1/exchange/serde/mod.rs @@ -15,10 +15,10 @@ mod exchange_deserializer; mod exchange_serializer; -pub use exchange_deserializer::deserialize_block; pub use exchange_deserializer::ExchangeDeserializeMeta; pub use exchange_deserializer::TransformExchangeDeserializer; -pub use exchange_serializer::serialize_block; +pub use exchange_deserializer::deserialize_block; pub use exchange_serializer::ExchangeSerializeMeta; pub use exchange_serializer::TransformExchangeSerializer; pub use exchange_serializer::TransformScatterExchangeSerializer; +pub use exchange_serializer::serialize_block; diff --git a/src/query/service/src/servers/flight/v1/exchange/statistics_receiver.rs b/src/query/service/src/servers/flight/v1/exchange/statistics_receiver.rs index 784029542933f..d55cf56c273e2 100644 --- a/src/query/service/src/servers/flight/v1/exchange/statistics_receiver.rs +++ b/src/query/service/src/servers/flight/v1/exchange/statistics_receiver.rs @@ -13,22 +13,22 @@ // limitations under the License. use std::collections::HashMap; -use std::sync::atomic::Ordering; use std::sync::Arc; +use std::sync::atomic::Ordering; -use databend_common_base::base::tokio::sync::broadcast::channel; +use databend_common_base::JoinHandle; use databend_common_base::base::tokio::sync::broadcast::Sender; +use databend_common_base::base::tokio::sync::broadcast::channel; use databend_common_base::runtime::Runtime; use databend_common_base::runtime::TrySpawn; -use databend_common_base::JoinHandle; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use futures_util::future::select; use futures_util::future::Either; +use futures_util::future::select; +use crate::servers::flight::FlightExchange; use crate::servers::flight::v1::packets::DataPacket; use crate::servers::flight::v1::packets::ProgressInfo; -use crate::servers::flight::FlightExchange; use crate::sessions::MemoryUpdater; use crate::sessions::QueryContext; diff --git a/src/query/service/src/servers/flight/v1/exchange/statistics_sender.rs b/src/query/service/src/servers/flight/v1/exchange/statistics_sender.rs index 9b0b62e8c46f6..69a87333fee11 100644 --- a/src/query/service/src/servers/flight/v1/exchange/statistics_sender.rs +++ b/src/query/service/src/servers/flight/v1/exchange/statistics_sender.rs @@ -17,13 +17,13 @@ use std::sync::Arc; use std::time::Duration; use async_channel::Sender; +use databend_common_base::JoinHandle; use databend_common_base::base::tokio::sync::oneshot; use databend_common_base::base::tokio::time::sleep; use databend_common_base::runtime::MemStat; use databend_common_base::runtime::QueryPerf; use databend_common_base::runtime::QueryPerfGuard; use databend_common_base::runtime::TrySpawn; -use databend_common_base::JoinHandle; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -33,10 +33,10 @@ use futures_util::future::Either; use log::warn; use crate::pipelines::executor::PipelineExecutor; -use crate::servers::flight::v1::packets::DataPacket; -use crate::servers::flight::v1::packets::ProgressInfo; use crate::servers::flight::FlightExchange; use crate::servers::flight::FlightSender; +use crate::servers::flight::v1::packets::DataPacket; +use crate::servers::flight::v1::packets::ProgressInfo; use crate::sessions::QueryContext; pub struct StatisticsSender { diff --git a/src/query/service/src/servers/flight/v1/flight_service.rs b/src/query/service/src/servers/flight/v1/flight_service.rs index 4bb372e2bc593..4ac0297a65749 100644 --- a/src/query/service/src/servers/flight/v1/flight_service.rs +++ b/src/query/service/src/servers/flight/v1/flight_service.rs @@ -14,7 +14,6 @@ use std::pin::Pin; -use arrow_flight::flight_service_server::FlightService; use arrow_flight::Action; use arrow_flight::ActionType; use arrow_flight::Criteria; @@ -29,6 +28,7 @@ use arrow_flight::PutResult; use arrow_flight::Result as FlightResult; use arrow_flight::SchemaResult; use arrow_flight::Ticket; +use arrow_flight::flight_service_server::FlightService; use databend_common_config::GlobalConfig; use databend_common_exception::ErrorCode; use fastrace::func_path; @@ -41,8 +41,8 @@ use tonic::Status; use tonic::Streaming; use crate::servers::flight::request_builder::RequestGetter; -use crate::servers::flight::v1::actions::flight_actions; use crate::servers::flight::v1::actions::FlightActions; +use crate::servers::flight::v1::actions::flight_actions; use crate::servers::flight::v1::exchange::DataExchangeManager; pub type FlightStream = diff --git a/src/query/service/src/servers/flight/v1/packets/packet_fragment.rs b/src/query/service/src/servers/flight/v1/packets/packet_fragment.rs index 1185e175fd29d..963363e7267d1 100644 --- a/src/query/service/src/servers/flight/v1/packets/packet_fragment.rs +++ b/src/query/service/src/servers/flight/v1/packets/packet_fragment.rs @@ -18,9 +18,9 @@ use std::collections::VecDeque; use std::fmt::Debug; use std::fmt::Formatter; -use serde::de::Error; use serde::Deserializer; use serde::Serializer; +use serde::de::Error; use crate::physical_plans::IPhysicalPlan; use crate::physical_plans::PhysicalPlan; diff --git a/src/query/service/src/servers/flight/v1/packets/packet_publisher.rs b/src/query/service/src/servers/flight/v1/packets/packet_publisher.rs index cdd4819da0a32..4355fcb5c4db8 100644 --- a/src/query/service/src/servers/flight/v1/packets/packet_publisher.rs +++ b/src/query/service/src/servers/flight/v1/packets/packet_publisher.rs @@ -29,9 +29,9 @@ use databend_common_meta_app::principal::UserInfo; use databend_common_meta_types::NodeInfo; use databend_common_settings::Settings; use log::debug; +use petgraph::Graph; use petgraph::dot::Dot; use petgraph::graph::NodeIndex; -use petgraph::Graph; use serde::Deserialize; use serde::Serialize; diff --git a/src/query/service/src/servers/flight/v1/scatter/flight_scatter_hash.rs b/src/query/service/src/servers/flight/v1/scatter/flight_scatter_hash.rs index 35e6656bd94b9..7969ed2451b10 100644 --- a/src/query/service/src/servers/flight/v1/scatter/flight_scatter_hash.rs +++ b/src/query/service/src/servers/flight/v1/scatter/flight_scatter_hash.rs @@ -17,15 +17,6 @@ use std::hash::Hasher; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::type_check::check_function; -use databend_common_expression::types::number::NumberScalar; -use databend_common_expression::types::AccessType; -use databend_common_expression::types::AnyType; -use databend_common_expression::types::Buffer; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NullableType; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::NumberType; use databend_common_expression::DataBlock; use databend_common_expression::Evaluator; use databend_common_expression::Expr; @@ -34,6 +25,15 @@ use databend_common_expression::FunctionID; use databend_common_expression::RemoteExpr; use databend_common_expression::Scalar; use databend_common_expression::Value; +use databend_common_expression::type_check::check_function; +use databend_common_expression::types::AccessType; +use databend_common_expression::types::AnyType; +use databend_common_expression::types::Buffer; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NullableType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::NumberType; +use databend_common_expression::types::number::NumberScalar; use databend_common_functions::BUILTIN_FUNCTIONS; use crate::servers::flight::v1::scatter::flight_scatter::FlightScatter; diff --git a/src/query/service/src/servers/flight_sql/flight_sql_server.rs b/src/query/service/src/servers/flight_sql/flight_sql_server.rs index 1872935508673..0682f23e2807f 100644 --- a/src/query/service/src/servers/flight_sql/flight_sql_server.rs +++ b/src/query/service/src/servers/flight_sql/flight_sql_server.rs @@ -23,13 +23,13 @@ use databend_common_config::InnerConfig; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use log::info; -use tonic::transport::server::TcpIncoming; use tonic::transport::Identity; use tonic::transport::Server; use tonic::transport::ServerTlsConfig; +use tonic::transport::server::TcpIncoming; -use crate::servers::flight_sql::flight_sql_service::FlightSqlServiceImpl; use crate::servers::Server as DatabendQueryServer; +use crate::servers::flight_sql::flight_sql_service::FlightSqlServiceImpl; use crate::sessions::BuildInfoRef; pub struct FlightSQLServer { diff --git a/src/query/service/src/servers/flight_sql/flight_sql_service/catalog.rs b/src/query/service/src/servers/flight_sql/flight_sql_service/catalog.rs index 4335c4cc4f058..4cc10c1027ed2 100644 --- a/src/query/service/src/servers/flight_sql/flight_sql_service/catalog.rs +++ b/src/query/service/src/servers/flight_sql/flight_sql_service/catalog.rs @@ -14,9 +14,9 @@ use std::sync::Arc; -use arrow_array::builder::StringBuilder; use arrow_array::ArrayRef; use arrow_array::RecordBatch; +use arrow_array::builder::StringBuilder; use arrow_flight::utils::batches_to_flight_data; use arrow_schema::DataType; use arrow_schema::Field; diff --git a/src/query/service/src/servers/flight_sql/flight_sql_service/mod.rs b/src/query/service/src/servers/flight_sql/flight_sql_service/mod.rs index 28e4ea8ec0c0f..457a77ad60585 100644 --- a/src/query/service/src/servers/flight_sql/flight_sql_service/mod.rs +++ b/src/query/service/src/servers/flight_sql/flight_sql_service/mod.rs @@ -26,8 +26,8 @@ use std::sync::Arc; use arrow_flight::FlightData; use catalog::CatalogInfoProvider; use dashmap::DashMap; -use databend_common_sql::plans::Plan; use databend_common_sql::PlanExtras; +use databend_common_sql::plans::Plan; use futures::Stream; use parking_lot::Mutex; use sql_info::SqlInfoProvider; diff --git a/src/query/service/src/servers/flight_sql/flight_sql_service/query.rs b/src/query/service/src/servers/flight_sql/flight_sql_service/query.rs index 297b3d96310c2..21dae4849721b 100644 --- a/src/query/service/src/servers/flight_sql/flight_sql_service/query.rs +++ b/src/query/service/src/servers/flight_sql/flight_sql_service/query.rs @@ -14,17 +14,17 @@ use std::ops::Deref; use std::pin::pin; -use std::sync::atomic::AtomicBool; -use std::sync::atomic::Ordering; use std::sync::Arc; use std::sync::LazyLock; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; use arrow_flight::FlightData; use arrow_flight::SchemaAsIpc; -use arrow_ipc::writer; use arrow_ipc::MessageBuilder; use arrow_ipc::MessageHeader; use arrow_ipc::MetadataVersion; +use arrow_ipc::writer; use arrow_schema::Schema as ArrowSchema; use async_stream::stream; use bytes::Bytes; @@ -33,10 +33,10 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::DataBlock; use databend_common_expression::DataSchema; -use databend_common_sql::get_query_kind; -use databend_common_sql::plans::Plan; use databend_common_sql::PlanExtras; use databend_common_sql::Planner; +use databend_common_sql::get_query_kind; +use databend_common_sql::plans::Plan; use databend_common_storages_fuse::TableContext; use futures::Stream; use futures::StreamExt; @@ -45,9 +45,9 @@ use serde::Deserialize; use serde::Serialize; use tonic::Status; -use super::status; use super::DoGetStream; use super::FlightSqlServiceImpl; +use super::status; use crate::interpreters::InterpreterFactory; use crate::sessions::QueryContext; use crate::sessions::Session; diff --git a/src/query/service/src/servers/flight_sql/flight_sql_service/service.rs b/src/query/service/src/servers/flight_sql/flight_sql_service/service.rs index da45a24584eb9..31cbb4a1f1c9e 100644 --- a/src/query/service/src/servers/flight_sql/flight_sql_service/service.rs +++ b/src/query/service/src/servers/flight_sql/flight_sql_service/service.rs @@ -16,10 +16,19 @@ use std::pin::Pin; use std::sync::Arc; use std::time::Duration; +use arrow_flight::Action; +use arrow_flight::FlightDescriptor; +use arrow_flight::FlightEndpoint; +use arrow_flight::FlightInfo; +use arrow_flight::HandshakeRequest; +use arrow_flight::HandshakeResponse; +use arrow_flight::IpcMessage; +use arrow_flight::Location; +use arrow_flight::PutResult; +use arrow_flight::SchemaAsIpc; +use arrow_flight::Ticket; use arrow_flight::flight_descriptor::DescriptorType; use arrow_flight::flight_service_server::FlightService; -use arrow_flight::sql::server::FlightSqlService; -use arrow_flight::sql::server::PeekableFlightDataStream; use arrow_flight::sql::ActionBeginSavepointRequest; use arrow_flight::sql::ActionBeginSavepointResult; use arrow_flight::sql::ActionBeginTransactionRequest; @@ -53,29 +62,20 @@ use arrow_flight::sql::DoPutUpdateResult; use arrow_flight::sql::ProstMessageExt; use arrow_flight::sql::SqlInfo; use arrow_flight::sql::TicketStatementQuery; -use arrow_flight::Action; -use arrow_flight::FlightDescriptor; -use arrow_flight::FlightEndpoint; -use arrow_flight::FlightInfo; -use arrow_flight::HandshakeRequest; -use arrow_flight::HandshakeResponse; -use arrow_flight::IpcMessage; -use arrow_flight::Location; -use arrow_flight::PutResult; -use arrow_flight::SchemaAsIpc; -use arrow_flight::Ticket; +use arrow_flight::sql::server::FlightSqlService; +use arrow_flight::sql::server::PeekableFlightDataStream; use arrow_ipc::writer::IpcWriteOptions; use databend_common_base::base::uuid::Uuid; use databend_common_expression::DataSchema; use futures::Stream; use log::info; use prost::Message; -use tonic::metadata::MetadataValue; -use tonic::server::NamedService; use tonic::Request; use tonic::Response; use tonic::Status; use tonic::Streaming; +use tonic::metadata::MetadataValue; +use tonic::server::NamedService; use super::status; use crate::servers::flight_sql::flight_sql_service::FlightSqlServiceImpl; diff --git a/src/query/service/src/servers/flight_sql/flight_sql_service/session.rs b/src/query/service/src/servers/flight_sql/flight_sql_service/session.rs index 4464b5d3a75e9..92276727c041e 100644 --- a/src/query/service/src/servers/flight_sql/flight_sql_service/session.rs +++ b/src/query/service/src/servers/flight_sql/flight_sql_service/session.rs @@ -14,15 +14,15 @@ use std::sync::Arc; -use base64::prelude::BASE64_STANDARD; use base64::Engine; +use base64::prelude::BASE64_STANDARD; use databend_common_catalog::session_type::SessionType; use databend_common_meta_app::principal::AuthInfo; use databend_common_meta_app::principal::UserIdentity; use databend_common_users::UserApiProvider; -use tonic::metadata::MetadataMap; use tonic::Request; use tonic::Status; +use tonic::metadata::MetadataMap; use super::status; use crate::servers::flight_sql::flight_sql_service::FlightSqlServiceImpl; diff --git a/src/query/service/src/servers/flight_sql/flight_sql_service/sql_info.rs b/src/query/service/src/servers/flight_sql/flight_sql_service/sql_info.rs index be362b6f28964..adc8f5258cb59 100644 --- a/src/query/service/src/servers/flight_sql/flight_sql_service/sql_info.rs +++ b/src/query/service/src/servers/flight_sql/flight_sql_service/sql_info.rs @@ -14,13 +14,13 @@ use std::sync::Arc; -use arrow_array::builder::Int32Builder; -use arrow_array::builder::StringBuilder; use arrow_array::ArrayRef; use arrow_array::RecordBatch; +use arrow_array::builder::Int32Builder; +use arrow_array::builder::StringBuilder; +use arrow_flight::FlightData; use arrow_flight::sql::SqlInfo; use arrow_flight::utils::batches_to_flight_data; -use arrow_flight::FlightData; use arrow_schema::ArrowError; use arrow_schema::DataType; use arrow_schema::Field; diff --git a/src/query/service/src/servers/http/clickhouse_handler.rs b/src/query/service/src/servers/http/clickhouse_handler.rs index fdc6e3f219acc..6b391e31837b0 100644 --- a/src/query/service/src/servers/http/clickhouse_handler.rs +++ b/src/query/service/src/servers/http/clickhouse_handler.rs @@ -23,8 +23,8 @@ use databend_common_catalog::session_type::SessionType; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_exception::ToErrorCode; -use databend_common_expression::infer_table_schema; use databend_common_expression::DataSchemaRef; +use databend_common_expression::infer_table_schema; use databend_common_formats::ClickhouseFormatType; use databend_common_formats::FileFormatOptionsExt; use databend_common_formats::FileFormatTypeExt; @@ -36,6 +36,11 @@ use http::HeaderMap; use http::StatusCode; use log::info; use naive_cityhash::cityhash128; +use poem::Body; +use poem::Endpoint; +use poem::EndpointExt; +use poem::IntoResponse; +use poem::Route; use poem::error::BadRequest; use poem::error::InternalServerError; use poem::error::Result as PoemResult; @@ -43,17 +48,12 @@ use poem::get; use poem::post; use poem::web::Query; use poem::web::WithContentType; -use poem::Body; -use poem::Endpoint; -use poem::EndpointExt; -use poem::IntoResponse; -use poem::Route; use serde::Deserialize; use serde::Serialize; -use crate::interpreters::interpreter_plan_sql; use crate::interpreters::InterpreterFactory; use crate::interpreters::InterpreterPtr; +use crate::interpreters::interpreter_plan_sql; use crate::servers::http::middleware::sanitize_request_headers; use crate::servers::http::v1::HttpQueryContext; use crate::sessions::QueryContext; @@ -373,7 +373,7 @@ pub async fn clickhouse_ping_handler() -> String { "OK.\n".to_string() } -pub fn clickhouse_router() -> impl Endpoint { +pub fn clickhouse_router() -> impl Endpoint + use<> { Route::new() .at( "/", @@ -423,9 +423,10 @@ fn get_default_format( let name = match ¶ms.default_format { None => match headers.get("X-CLICKHOUSE-FORMAT") { None => "TSV", - Some(v) => v.to_str().map_err_to_code(ErrorCode::BadBytes, || { - "value of X-CLICKHOUSE-FORMAT is not string" - })?, + Some(v) => v.to_str().map_err_to_code( + ErrorCode::BadBytes, + || "value of X-CLICKHOUSE-FORMAT is not string", + )?, }, Some(s) => s, }; diff --git a/src/query/service/src/servers/http/error.rs b/src/query/service/src/servers/http/error.rs index d7da09586cef9..f47a3ad832ba6 100644 --- a/src/query/service/src/servers/http/error.rs +++ b/src/query/service/src/servers/http/error.rs @@ -19,12 +19,12 @@ use std::fmt::Formatter; use databend_common_exception::ErrorCode; use http::StatusCode; use jwt_simple::prelude::Deserialize; -use poem::error::ResponseError; -use poem::web::Json; use poem::IntoResponse; use poem::Response; -use serde::de::StdError; +use poem::error::ResponseError; +use poem::web::Json; use serde::Serialize; +use serde::de::StdError; #[derive(Serialize, Deserialize, Debug, Clone)] pub struct QueryError { diff --git a/src/query/service/src/servers/http/http_services.rs b/src/query/service/src/servers/http/http_services.rs index 6f3c8dc415856..003a32370ed30 100644 --- a/src/query/service/src/servers/http/http_services.rs +++ b/src/query/service/src/servers/http/http_services.rs @@ -23,25 +23,25 @@ use databend_common_http::HttpShutdownHandler; use databend_common_meta_types::anyerror::AnyError; use http::StatusCode; use log::info; +use poem::Endpoint; +use poem::EndpointExt; +use poem::IntoResponse; +use poem::Route; use poem::get; use poem::listener::OpensslTlsConfig; use poem::middleware::CatchPanic; use poem::middleware::CookieJarManager; use poem::middleware::NormalizePath; use poem::middleware::TrailingSlash; -use poem::Endpoint; -use poem::EndpointExt; -use poem::IntoResponse; -use poem::Route; use super::v1::HttpQueryContext; -use crate::servers::http::middleware::json_response; +use crate::servers::Server; use crate::servers::http::middleware::EndpointKind; use crate::servers::http::middleware::HTTPSessionMiddleware; use crate::servers::http::middleware::PanicHandler; +use crate::servers::http::middleware::json_response; use crate::servers::http::v1::clickhouse_router; use crate::servers::http::v1::query_route; -use crate::servers::Server; #[derive(Copy, Clone)] pub enum HttpHandlerKind { @@ -92,7 +92,7 @@ impl HttpHandler { #[allow(clippy::let_with_type_underscore)] #[async_backtrace::framed] - async fn build_router(&self, sock: SocketAddr) -> impl Endpoint { + async fn build_router(&self, sock: SocketAddr) -> impl Endpoint + use<> { let ep_clickhouse = Route::new() .nest("/", clickhouse_router()) .with(HTTPSessionMiddleware::create( diff --git a/src/query/service/src/servers/http/middleware/json_header.rs b/src/query/service/src/servers/http/middleware/json_header.rs index fc2b3f4c04103..159e4828aec46 100644 --- a/src/query/service/src/servers/http/middleware/json_header.rs +++ b/src/query/service/src/servers/http/middleware/json_header.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use base64::engine::general_purpose::URL_SAFE; use base64::Engine; -use serde::de; +use base64::engine::general_purpose::URL_SAFE; use serde::Serialize; +use serde::de; pub fn encode_json_header(v: &T) -> String where T: ?Sized + Serialize { diff --git a/src/query/service/src/servers/http/middleware/mod.rs b/src/query/service/src/servers/http/middleware/mod.rs index f9d7aad445d14..e34a8fe6626f4 100644 --- a/src/query/service/src/servers/http/middleware/mod.rs +++ b/src/query/service/src/servers/http/middleware/mod.rs @@ -22,10 +22,10 @@ pub mod session_header; pub(crate) use client_capabilities::ClientCapabilities; pub(crate) use metrics::MetricsMiddleware; pub(crate) use panic_handler::PanicHandler; -pub(crate) use session::forward_request_with_body; -pub use session::json_response; -pub(crate) use session::sanitize_request_headers; pub use session::EndpointKind; // for it tests only pub use session::HTTPSessionEndpoint; pub use session::HTTPSessionMiddleware; +pub(crate) use session::forward_request_with_body; +pub use session::json_response; +pub(crate) use session::sanitize_request_headers; diff --git a/src/query/service/src/servers/http/middleware/session.rs b/src/query/service/src/servers/http/middleware/session.rs index 584646cafff6c..c855497ded55e 100644 --- a/src/query/service/src/servers/http/middleware/session.rs +++ b/src/query/service/src/servers/http/middleware/session.rs @@ -40,11 +40,11 @@ use fastrace::func_name; use headers::authorization::Basic; use headers::authorization::Bearer; use headers::authorization::Credentials; -use http::header::AUTHORIZATION; use http::HeaderMap; use http::HeaderValue; use http::Method; use http::StatusCode; +use http::header::AUTHORIZATION; use log::error; use log::info; use log::warn; @@ -52,9 +52,6 @@ use opentelemetry::baggage::BaggageExt; use opentelemetry::propagation::Extractor; use opentelemetry::propagation::TextMapPropagator; use opentelemetry_sdk::propagation::BaggagePropagator; -use poem::error::ResponseError; -use poem::error::Result as PoemResult; -use poem::web::Json; use poem::Addr; use poem::Endpoint; use poem::Error; @@ -62,23 +59,26 @@ use poem::IntoResponse; use poem::Middleware; use poem::Request; use poem::Response; +use poem::error::ResponseError; +use poem::error::Result as PoemResult; +use poem::web::Json; use uuid::Uuid; use crate::auth::AuthMgr; use crate::auth::Credential; use crate::clusters::ClusterDiscovery; +use crate::servers::HttpHandlerKind; use crate::servers::http::error::HttpErrorCode; use crate::servers::http::error::JsonErrorOnly; use crate::servers::http::error::QueryError; +use crate::servers::http::middleware::ClientCapabilities; use crate::servers::http::middleware::session_header::ClientSession; use crate::servers::http::middleware::session_header::ClientSessionType; -use crate::servers::http::middleware::ClientCapabilities; use crate::servers::http::v1::HttpQueryContext; use crate::servers::http::v1::SessionClaim; use crate::servers::login_history::LoginEventType; use crate::servers::login_history::LoginHandler; use crate::servers::login_history::LoginHistory; -use crate::servers::HttpHandlerKind; use crate::sessions::SessionManager; const USER_AGENT: &str = "User-Agent"; const TRACE_PARENT: &str = "traceparent"; @@ -645,7 +645,11 @@ impl Endpoint for HTTPSessionEndpoint { .into()); } Ok(None) => { - let msg = format!("Not find the '{}' warehouse; it is possible that all nodes of the warehouse have gone offline. Please exit the client and reconnect, or use `use warehouse `. request = {}.", warehouse, get_request_info(req)); + let msg = format!( + "Not find the '{}' warehouse; it is possible that all nodes of the warehouse have gone offline. Please exit the client and reconnect, or use `use warehouse `. request = {}.", + warehouse, + get_request_info(req) + ); warn!("{}", msg); return Err(Error::from(HttpErrorCode::bad_request( ErrorCode::UnknownWarehouse(msg), diff --git a/src/query/service/src/servers/http/middleware/session_header.rs b/src/query/service/src/servers/http/middleware/session_header.rs index e1198c818a703..e14841d5383bd 100644 --- a/src/query/service/src/servers/http/middleware/session_header.rs +++ b/src/query/service/src/servers/http/middleware/session_header.rs @@ -27,19 +27,19 @@ use jwt_simple::prelude::Deserialize; use jwt_simple::prelude::Serialize; use log::info; use log::warn; -use poem::web::cookie::Cookie; -use poem::web::cookie::CookieJar; use poem::Request; use poem::Response; +use poem::web::cookie::Cookie; +use poem::web::cookie::CookieJar; use serde::Deserializer; use serde::Serializer; use uuid::Uuid; +use crate::servers::http::middleware::ClientCapabilities; use crate::servers::http::middleware::json_header::decode_json_header; use crate::servers::http::middleware::json_header::encode_json_header; -use crate::servers::http::middleware::ClientCapabilities; -use crate::servers::http::v1::unix_ts; use crate::servers::http::v1::ClientSessionManager; +use crate::servers::http::v1::unix_ts; const COOKIE_LAST_REFRESH_TIME: &str = "last_refresh_time"; const COOKIE_SESSION_ID: &str = "session_id"; @@ -232,9 +232,9 @@ impl ClientSession { } Err(err) => { warn!( - "Invalid last_refresh_time: detected clock drift or incorrect timestamp, difference: {:?}", - err.duration() - ); + "Invalid last_refresh_time: detected clock drift or incorrect timestamp, difference: {:?}", + err.duration() + ); } } Ok(()) diff --git a/src/query/service/src/servers/http/v1/catalog/get_database_table.rs b/src/query/service/src/servers/http/v1/catalog/get_database_table.rs index f913a13354a83..2117e761219e7 100644 --- a/src/query/service/src/servers/http/v1/catalog/get_database_table.rs +++ b/src/query/service/src/servers/http/v1/catalog/get_database_table.rs @@ -19,12 +19,12 @@ use databend_common_catalog::catalog::CatalogManager; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_users::Object; +use poem::IntoResponse; use poem::error::InternalServerError; use poem::error::NotFound; use poem::error::Result as PoemResult; use poem::web::Json; use poem::web::Path; -use poem::IntoResponse; use serde::Deserialize; use serde::Serialize; diff --git a/src/query/service/src/servers/http/v1/catalog/list_database_streams.rs b/src/query/service/src/servers/http/v1/catalog/list_database_streams.rs index 24a38589de8ee..cdb7390762bf4 100644 --- a/src/query/service/src/servers/http/v1/catalog/list_database_streams.rs +++ b/src/query/service/src/servers/http/v1/catalog/list_database_streams.rs @@ -22,12 +22,12 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_storages_stream::stream_table::StreamTable; use databend_common_users::Object; +use poem::IntoResponse; use poem::error::InternalServerError; use poem::error::NotFound; use poem::error::Result as PoemResult; use poem::web::Json; use poem::web::Path; -use poem::IntoResponse; use serde::Deserialize; use serde::Serialize; diff --git a/src/query/service/src/servers/http/v1/catalog/list_database_table_fields.rs b/src/query/service/src/servers/http/v1/catalog/list_database_table_fields.rs index 0767dff5c0d18..bea2c1d32342c 100644 --- a/src/query/service/src/servers/http/v1/catalog/list_database_table_fields.rs +++ b/src/query/service/src/servers/http/v1/catalog/list_database_table_fields.rs @@ -16,12 +16,12 @@ use databend_common_catalog::catalog::CatalogManager; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_users::Object; +use poem::IntoResponse; use poem::error::InternalServerError; use poem::error::NotFound; use poem::error::Result as PoemResult; use poem::web::Json; use poem::web::Path; -use poem::IntoResponse; use serde::Deserialize; use serde::Serialize; diff --git a/src/query/service/src/servers/http/v1/catalog/list_database_tables.rs b/src/query/service/src/servers/http/v1/catalog/list_database_tables.rs index e0f7f0cd6960d..1468a71adb160 100644 --- a/src/query/service/src/servers/http/v1/catalog/list_database_tables.rs +++ b/src/query/service/src/servers/http/v1/catalog/list_database_tables.rs @@ -17,19 +17,19 @@ use databend_common_catalog::catalog::CatalogManager; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_users::Object; +use poem::IntoResponse; use poem::error::InternalServerError; use poem::error::NotFound; use poem::error::Result as PoemResult; use poem::web::Json; use poem::web::Path; -use poem::IntoResponse; use serde::Deserialize; use serde::Serialize; use crate::interpreters::ShowCreateQuerySettings; use crate::interpreters::ShowCreateTableInterpreter; -use crate::servers::http::v1::catalog::get_database_table::TableDetail; use crate::servers::http::v1::HttpQueryContext; +use crate::servers::http::v1::catalog::get_database_table::TableDetail; #[derive(Serialize, Deserialize, Eq, PartialEq, Debug, Default)] pub struct ListDatabaseTablesResponse { diff --git a/src/query/service/src/servers/http/v1/catalog/list_databases.rs b/src/query/service/src/servers/http/v1/catalog/list_databases.rs index f8055eac2b0ad..b79508343a279 100644 --- a/src/query/service/src/servers/http/v1/catalog/list_databases.rs +++ b/src/query/service/src/servers/http/v1/catalog/list_databases.rs @@ -15,10 +15,10 @@ use databend_common_catalog::catalog::CatalogManager; use databend_common_exception::Result; use databend_common_users::Object; +use poem::IntoResponse; use poem::error::InternalServerError; use poem::error::Result as PoemResult; use poem::web::Json; -use poem::IntoResponse; use serde::Deserialize; use serde::Serialize; diff --git a/src/query/service/src/servers/http/v1/catalog/search_databases.rs b/src/query/service/src/servers/http/v1/catalog/search_databases.rs index 09e9438fd7a76..a3f9bd4500817 100644 --- a/src/query/service/src/servers/http/v1/catalog/search_databases.rs +++ b/src/query/service/src/servers/http/v1/catalog/search_databases.rs @@ -15,10 +15,10 @@ use databend_common_catalog::catalog::CatalogManager; use databend_common_exception::Result; use databend_common_users::Object; +use poem::IntoResponse; use poem::error::InternalServerError; use poem::error::Result as PoemResult; use poem::web::Json; -use poem::IntoResponse; use serde::Deserialize; use serde::Serialize; diff --git a/src/query/service/src/servers/http/v1/catalog/search_tables.rs b/src/query/service/src/servers/http/v1/catalog/search_tables.rs index c2ad3b8ea519f..eb3978aa22e0e 100644 --- a/src/query/service/src/servers/http/v1/catalog/search_tables.rs +++ b/src/query/service/src/servers/http/v1/catalog/search_tables.rs @@ -17,10 +17,10 @@ use chrono::Utc; use databend_common_catalog::catalog::CatalogManager; use databend_common_exception::Result; use databend_common_users::Object; +use poem::IntoResponse; use poem::error::InternalServerError; use poem::error::Result as PoemResult; use poem::web::Json; -use poem::IntoResponse; use serde::Deserialize; use serde::Serialize; diff --git a/src/query/service/src/servers/http/v1/catalog/stats.rs b/src/query/service/src/servers/http/v1/catalog/stats.rs index 6f3192bc1d527..77bb1287bc088 100644 --- a/src/query/service/src/servers/http/v1/catalog/stats.rs +++ b/src/query/service/src/servers/http/v1/catalog/stats.rs @@ -15,10 +15,10 @@ use databend_common_catalog::catalog::CatalogManager; use databend_common_exception::Result; use databend_common_users::Object; +use poem::IntoResponse; use poem::error::InternalServerError; use poem::error::Result as PoemResult; use poem::web::Json; -use poem::IntoResponse; use serde::Deserialize; use serde::Serialize; diff --git a/src/query/service/src/servers/http/v1/discovery.rs b/src/query/service/src/servers/http/v1/discovery.rs index 055b608598908..bda083b4a4d69 100644 --- a/src/query/service/src/servers/http/v1/discovery.rs +++ b/src/query/service/src/servers/http/v1/discovery.rs @@ -13,10 +13,10 @@ // limitations under the License. use databend_common_config::GlobalConfig; +use poem::Request; use poem::error::InternalServerError; use poem::error::Result as PoemResult; use poem::web::Json; -use poem::Request; use crate::clusters::ClusterDiscovery; use crate::clusters::ClusterHelper; diff --git a/src/query/service/src/servers/http/v1/http_query_handlers.rs b/src/query/service/src/servers/http/v1/http_query_handlers.rs index 27254059956f0..86c373f6508ce 100644 --- a/src/query/service/src/servers/http/v1/http_query_handlers.rs +++ b/src/query/service/src/servers/http/v1/http_query_handlers.rs @@ -15,17 +15,17 @@ use std::collections::HashMap; use std::sync::Arc; -use databend_common_base::base::mask_connection_info; use databend_common_base::base::GlobalInstance; +use databend_common_base::base::mask_connection_info; use databend_common_base::headers::HEADER_QUERY_ID; use databend_common_base::headers::HEADER_QUERY_PAGE_ROWS; use databend_common_base::headers::HEADER_QUERY_STATE; -use databend_common_base::runtime::drop_guard; -use databend_common_base::runtime::execute_futures_in_parallel; +use databend_common_base::runtime::GLOBAL_MEM_STAT; use databend_common_base::runtime::MemStat; use databend_common_base::runtime::ParentMemStat; use databend_common_base::runtime::ThreadTracker; -use databend_common_base::runtime::GLOBAL_MEM_STAT; +use databend_common_base::runtime::drop_guard; +use databend_common_base::runtime::execute_futures_in_parallel; use databend_common_config::GlobalConfig; use databend_common_exception::ErrorCode; use databend_common_expression::DataSchema; @@ -40,6 +40,13 @@ use http::StatusCode; use log::error; use log::info; use log::warn; +use poem::EndpointExt; +use poem::FromRequest; +use poem::IntoResponse; +use poem::Request; +use poem::RequestBody; +use poem::Response; +use poem::Route; use poem::error::Error as PoemError; use poem::error::ResponseError; use poem::error::Result as PoemResult; @@ -49,13 +56,6 @@ use poem::post; use poem::put; use poem::web::Json; use poem::web::Path; -use poem::EndpointExt; -use poem::FromRequest; -use poem::IntoResponse; -use poem::Request; -use poem::RequestBody; -use poem::Response; -use poem::Route; use serde::Deserialize; use serde::Serialize; use uuid::Uuid; @@ -67,12 +67,16 @@ use super::query::HttpQueryRequest; use super::query::HttpQueryResponseInternal; use super::query::ResponseState; use crate::clusters::ClusterDiscovery; +use crate::servers::HttpHandlerKind; use crate::servers::http::error::HttpErrorCode; use crate::servers::http::error::QueryError; -use crate::servers::http::middleware::forward_request_with_body; use crate::servers::http::middleware::EndpointKind; use crate::servers::http::middleware::HTTPSessionMiddleware; use crate::servers::http::middleware::MetricsMiddleware; +use crate::servers::http::middleware::forward_request_with_body; +use crate::servers::http::v1::HttpQueryContext; +use crate::servers::http::v1::HttpQueryManager; +use crate::servers::http::v1::HttpSessionConf; use crate::servers::http::v1::catalog::catalog_stats_handler; use crate::servers::http::v1::catalog::get_database_table_handler; use crate::servers::http::v1::catalog::list_database_streams_handler; @@ -84,8 +88,8 @@ use crate::servers::http::v1::catalog::search_tables_handler; use crate::servers::http::v1::discovery_nodes; use crate::servers::http::v1::login_handler; use crate::servers::http::v1::logout_handler; -use crate::servers::http::v1::query::blocks_serializer::BlocksSerializer; use crate::servers::http::v1::query::Progresses; +use crate::servers::http::v1::query::blocks_serializer::BlocksSerializer; use crate::servers::http::v1::refresh_handler; use crate::servers::http::v1::roles::list_roles_handler; use crate::servers::http::v1::streaming_load_handler; @@ -93,10 +97,6 @@ use crate::servers::http::v1::upload_to_stage; use crate::servers::http::v1::users::create_user_handler; use crate::servers::http::v1::users::list_users_handler; use crate::servers::http::v1::verify_handler; -use crate::servers::http::v1::HttpQueryContext; -use crate::servers::http::v1::HttpQueryManager; -use crate::servers::http::v1::HttpSessionConf; -use crate::servers::HttpHandlerKind; use crate::sessions::QueryAffect; pub fn make_page_uri(query_id: &str, page_no: usize) -> String { @@ -276,7 +276,7 @@ impl QueryResponse { BodyFormat::Arrow if !schema.fields.is_empty() && !data.is_empty() => { let buf: Result<_, ErrorCode> = try { const META_KEY: &str = "response_header"; - let json_res = serde_json::to_string(&res)?; + let json_res = serde_json::to_string(&res).map_err(ErrorCode::from)?; data.to_arrow_ipc(&schema, vec![(META_KEY.to_string(), json_res)])? }; @@ -616,9 +616,14 @@ pub(crate) async fn query_handler( None => (0, None), Some(p) => (p.page.data.num_rows(), p.next_page_no), }; - info!("Initial response for query_id={}, state={:?}, rows={}, next_page={:?}, sql='{}'", - &query.id, &resp.state, rows, next_page, mask_connection_info(&sql) - ); + info!( + "Initial response for query_id={}, state={:?}, rows={}, next_page={:?}, sql='{}'", + &query.id, + &resp.state, + rows, + next_page, + mask_connection_info(&sql) + ); query .update_expire_time(false, resp.is_data_drained()) .await; diff --git a/src/query/service/src/servers/http/v1/mod.rs b/src/query/service/src/servers/http/v1/mod.rs index 5ad6d628319b9..870d1c23b56fa 100644 --- a/src/query/service/src/servers/http/v1/mod.rs +++ b/src/query/service/src/servers/http/v1/mod.rs @@ -24,15 +24,13 @@ pub mod users; mod verify; pub use discovery::discovery_nodes; +pub use http_query_handlers::QueryResponse; +pub use http_query_handlers::QueryResponseField; +pub use http_query_handlers::QueryStats; pub use http_query_handlers::make_final_uri; pub use http_query_handlers::make_page_uri; pub use http_query_handlers::make_state_uri; pub use http_query_handlers::query_route; -pub use http_query_handlers::QueryResponse; -pub use http_query_handlers::QueryResponseField; -pub use http_query_handlers::QueryStats; -pub use query::blocks_serializer::BlocksCollector; -pub use query::blocks_serializer::BlocksSerializer; pub use query::ExecuteStateKind; pub use query::ExpiringMap; pub use query::ExpiringState; @@ -40,19 +38,21 @@ pub use query::HttpQueryContext; pub use query::HttpQueryManager; pub use query::HttpSessionConf; pub use query::HttpSessionStateInternal; +pub use query::blocks_serializer::BlocksCollector; +pub use query::blocks_serializer::BlocksSerializer; pub use roles::list_roles_handler; -pub use session::login_handler::login_handler; +pub use session::ClientSessionManager; +pub(crate) use session::SessionClaim; pub use session::login_handler::LoginResponse; +pub use session::login_handler::login_handler; pub use session::logout_handler::logout_handler; -pub use session::refresh_handler::refresh_handler; pub use session::refresh_handler::RefreshResponse; +pub use session::refresh_handler::refresh_handler; pub(crate) use session::unix_ts; -pub use session::ClientSessionManager; -pub(crate) use session::SessionClaim; -pub use stage::upload_to_stage; pub use stage::UploadToStageResponse; -pub use streaming_load::streaming_load_handler; +pub use stage::upload_to_stage; pub use streaming_load::LoadResponse; +pub use streaming_load::streaming_load_handler; pub use users::create_user_handler; pub use users::list_users_handler; pub use verify::verify_handler; diff --git a/src/query/service/src/servers/http/v1/query/blocks_serializer.rs b/src/query/service/src/servers/http/v1/query/blocks_serializer.rs index 2052feabd5727..b500edf1a83ef 100644 --- a/src/query/service/src/servers/http/v1/query/blocks_serializer.rs +++ b/src/query/service/src/servers/http/v1/query/blocks_serializer.rs @@ -15,19 +15,20 @@ use std::cell::RefCell; use std::ops::DerefMut; -use arrow_ipc::writer::IpcWriteOptions; -use arrow_ipc::writer::StreamWriter; use arrow_ipc::CompressionType; use arrow_ipc::MetadataVersion; +use arrow_ipc::writer::IpcWriteOptions; +use arrow_ipc::writer::StreamWriter; use databend_common_exception::Result; -use databend_common_expression::types::date::date_to_string; -use databend_common_expression::types::interval::interval_to_string; -use databend_common_expression::types::timestamp::timestamp_to_string; use databend_common_expression::BlockEntry; use databend_common_expression::Column; use databend_common_expression::DataBlock; use databend_common_expression::DataSchema; +use databend_common_expression::types::date::date_to_string; +use databend_common_expression::types::interval::interval_to_string; +use databend_common_expression::types::timestamp::timestamp_to_string; use databend_common_formats::field_encoder::FieldEncoderValues; +use databend_common_io::GeometryDataType; use databend_common_io::ewkb_to_geo; use databend_common_io::geo_to_ewkb; use databend_common_io::geo_to_ewkt; @@ -35,7 +36,6 @@ use databend_common_io::geo_to_json; use databend_common_io::geo_to_wkb; use databend_common_io::geo_to_wkt; use databend_common_io::prelude::FormatSettings; -use databend_common_io::GeometryDataType; use geozero::wkb::Ewkb; use jsonb::RawJsonb; use serde::ser::SerializeSeq; diff --git a/src/query/service/src/servers/http/v1/query/execute_state.rs b/src/query/service/src/servers/http/v1/query/execute_state.rs index 44fed5883b7fd..abcca46562812 100644 --- a/src/query/service/src/servers/http/v1/query/execute_state.rs +++ b/src/query/service/src/servers/http/v1/query/execute_state.rs @@ -16,6 +16,7 @@ use std::collections::HashMap; use std::sync::Arc; use std::time::SystemTime; +use ExecuteState::*; use databend_common_base::base::ProgressValues; use databend_common_base::base::SpillProgress; use databend_common_base::runtime::CatchUnwindFuture; @@ -37,14 +38,13 @@ use log::info; use parking_lot::Mutex; use serde::Deserialize; use serde::Serialize; -use ExecuteState::*; use super::http_query::ResponseState; use super::sized_spsc::SizedChannelSender; -use crate::interpreters::interpreter_plan_sql; use crate::interpreters::Interpreter; use crate::interpreters::InterpreterFactory; use crate::interpreters::InterpreterQueryLog; +use crate::interpreters::interpreter_plan_sql; use crate::servers::http::v1::http_query_handlers::ResultFormatSettings; use crate::sessions::AcquireQueueGuard; use crate::sessions::QueryAffect; diff --git a/src/query/service/src/servers/http/v1/query/http_query.rs b/src/query/service/src/servers/http/v1/query/http_query.rs index 480398ada9698..5d6246c72b29a 100644 --- a/src/query/service/src/servers/http/v1/query/http_query.rs +++ b/src/query/service/src/servers/http/v1/query/http_query.rs @@ -15,9 +15,9 @@ use std::collections::BTreeMap; use std::collections::HashMap; use std::fmt::Debug; +use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; -use std::sync::Arc; use std::time::Duration; use std::time::Instant; @@ -45,19 +45,13 @@ use log::error; use log::info; use log::warn; use parking_lot::Mutex; -use poem::web::Json; use poem::IntoResponse; +use poem::web::Json; use serde::Deserialize; use serde::Deserializer; use serde::Serialize; use serde::Serializer; -use super::blocks_serializer::BlocksSerializer; -use super::execute_state::ExecuteStarting; -use super::execute_state::ExecuteStopped; -use super::execute_state::ExecutionError; -use super::execute_state::ExecutorSessionState; -use super::execute_state::Progresses; use super::CloseReason; use super::ExecuteState; use super::ExecuteStateKind; @@ -66,12 +60,18 @@ use super::HttpQueryContext; use super::PageManager; use super::ResponseData; use super::Wait; +use super::blocks_serializer::BlocksSerializer; +use super::execute_state::ExecuteStarting; +use super::execute_state::ExecuteStopped; +use super::execute_state::ExecutionError; +use super::execute_state::ExecutorSessionState; +use super::execute_state::Progresses; use crate::servers::http::error::QueryError; -use crate::servers::http::v1::http_query_handlers::ResultFormatSettings; use crate::servers::http::v1::ClientSessionManager; use crate::servers::http::v1::HttpQueryManager; use crate::servers::http::v1::QueryResponse; use crate::servers::http::v1::QueryStats; +use crate::servers::http::v1::http_query_handlers::ResultFormatSettings; use crate::sessions::QueryAffect; use crate::sessions::Session; @@ -438,7 +438,9 @@ impl HttpSessionConf { return Err(ErrorCode::SessionLost(format!( "Temp table lost due to server restart (at {}) or route error: node_id={} (expected {}); session_id={}, query_id={}, is_sticky_node={}", http_query_manager.server_info.start_time, - http_query_manager.server_info.id, id, http_ctx.client_session_id.as_deref().unwrap_or("None"), + http_query_manager.server_info.id, + id, + http_ctx.client_session_id.as_deref().unwrap_or("None"), query_id, http_ctx.is_sticky_node ))); @@ -457,9 +459,11 @@ impl HttpSessionConf { )); } Some(s) => { - return Err(ErrorCode::SessionTimeout(format!( + return Err(ErrorCode::SessionTimeout(format!( "temporary tables in session {} expired after idle for more than {} seconds, when starting query {}", - s.header.id, ClientSessionManager::instance().max_idle_time.as_secs(), query_id, + s.header.id, + ClientSessionManager::instance().max_idle_time.as_secs(), + query_id, ))); } } diff --git a/src/query/service/src/servers/http/v1/query/http_query_context.rs b/src/query/service/src/servers/http/v1/query/http_query_context.rs index 35c066ad31041..c8a4730261d58 100644 --- a/src/query/service/src/servers/http/v1/query/http_query_context.rs +++ b/src/query/service/src/servers/http/v1/query/http_query_context.rs @@ -26,9 +26,9 @@ use poem::Request; use poem::RequestBody; use crate::auth::Credential; +use crate::servers::http::middleware::ClientCapabilities; use crate::servers::http::middleware::session_header::ClientSession; use crate::servers::http::middleware::session_header::ClientSessionType; -use crate::servers::http::middleware::ClientCapabilities; use crate::servers::http::v1::ClientSessionManager; use crate::servers::http::v1::HttpQueryManager; use crate::servers::http::v1::HttpSessionConf; diff --git a/src/query/service/src/servers/http/v1/query/http_query_manager.rs b/src/query/service/src/servers/http/v1/query/http_query_manager.rs index f5c1529c1fb1c..b580375d1c890 100644 --- a/src/query/service/src/servers/http/v1/query/http_query_manager.rs +++ b/src/query/service/src/servers/http/v1/query/http_query_manager.rs @@ -22,11 +22,11 @@ use std::time::Instant; use std::time::SystemTime; use chrono::SecondsFormat; -use databend_common_base::base::tokio::time::sleep; +use databend_common_base::JoinHandle; use databend_common_base::base::GlobalInstance; +use databend_common_base::base::tokio::time::sleep; use databend_common_base::runtime::GlobalIORuntime; use databend_common_base::runtime::TrySpawn; -use databend_common_base::JoinHandle; use databend_common_config::InnerConfig; use databend_common_exception::ErrorCode; use databend_common_exception::Result; diff --git a/src/query/service/src/servers/http/v1/query/page_manager.rs b/src/query/service/src/servers/http/v1/query/page_manager.rs index a5eac36d5ac60..8880d719979a8 100644 --- a/src/query/service/src/servers/http/v1/query/page_manager.rs +++ b/src/query/service/src/servers/http/v1/query/page_manager.rs @@ -17,12 +17,12 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use super::Wait; use super::blocks_serializer::BlocksSerializer; use super::http_query::PaginationConf; -use super::sized_spsc::sized_spsc; use super::sized_spsc::SizedChannelReceiver; use super::sized_spsc::SizedChannelSender; -use super::Wait; +use super::sized_spsc::sized_spsc; use crate::spillers::LiteSpiller; #[derive(Clone)] diff --git a/src/query/service/src/servers/http/v1/query/sized_spsc.rs b/src/query/service/src/servers/http/v1/query/sized_spsc.rs index 07a470355ce36..4c0eeb29ea9a4 100644 --- a/src/query/service/src/servers/http/v1/query/sized_spsc.rs +++ b/src/query/service/src/servers/http/v1/query/sized_spsc.rs @@ -20,9 +20,9 @@ use std::sync::Arc; use std::sync::Mutex; use std::time::Instant; +use databend_common_base::base::WatchNotify; use databend_common_base::base::tokio; use databend_common_base::base::tokio::sync::Notify; -use databend_common_base::base::WatchNotify; use databend_common_exception::Result; use databend_common_expression::BlockEntry; use databend_common_expression::DataBlock; @@ -552,10 +552,10 @@ mod tests { use std::time::Duration; use databend_common_exception::ErrorCode; + use databend_common_expression::FromData; use databend_common_expression::types::Int32Type; use databend_common_expression::types::Number; use databend_common_expression::types::NumberType; - use databend_common_expression::FromData; use databend_common_io::prelude::FormatSettings; use databend_common_pipeline_transforms::traits::DataBlockSpill; use databend_common_pipeline_transforms::traits::Location; diff --git a/src/query/service/src/servers/http/v1/roles.rs b/src/query/service/src/servers/http/v1/roles.rs index 983818ec434ca..a97f178491b95 100644 --- a/src/query/service/src/servers/http/v1/roles.rs +++ b/src/query/service/src/servers/http/v1/roles.rs @@ -14,10 +14,10 @@ use databend_common_exception::Result; use databend_common_users::BUILTIN_ROLE_PUBLIC; +use poem::IntoResponse; use poem::error::InternalServerError; use poem::error::Result as PoemResult; use poem::web::Json; -use poem::IntoResponse; use serde::Deserialize; use serde::Serialize; diff --git a/src/query/service/src/servers/http/v1/session/client_session_manager.rs b/src/query/service/src/servers/http/v1/session/client_session_manager.rs index 8021ec31f332f..b8f4319783b7a 100644 --- a/src/query/service/src/servers/http/v1/session/client_session_manager.rs +++ b/src/query/service/src/servers/http/v1/session/client_session_manager.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::btree_map::Entry; use std::collections::BTreeMap; +use std::collections::btree_map::Entry; use std::sync::Arc; use std::time::Duration; use std::time::SystemTime; @@ -30,8 +30,8 @@ use databend_common_meta_app::principal::user_token::QueryTokenInfo; use databend_common_meta_app::principal::user_token::TokenType; use databend_common_meta_app::tenant::Tenant; use databend_common_users::UserApiProvider; -use databend_storages_common_session::drop_all_temp_tables; use databend_storages_common_session::TempTblMgrRef; +use databend_storages_common_session::drop_all_temp_tables; use log::error; use log::info; use log::warn; @@ -41,10 +41,10 @@ use sha2::Digest; use sha2::Sha256; use tokio::time::Instant; +use crate::servers::http::v1::SessionClaim; use crate::servers::http::v1::session::consts::TOMBSTONE_TTL; use crate::servers::http::v1::session::consts::TTL_GRACE_PERIOD_META; use crate::servers::http::v1::session::consts::TTL_GRACE_PERIOD_QUERY; -use crate::servers::http::v1::SessionClaim; use crate::sessions::Session; use crate::sessions::SessionPrivilegeManager; diff --git a/src/query/service/src/servers/http/v1/session/login_handler.rs b/src/query/service/src/servers/http/v1/session/login_handler.rs index dcc876a7f274f..df01f90e8a685 100644 --- a/src/query/service/src/servers/http/v1/session/login_handler.rs +++ b/src/query/service/src/servers/http/v1/session/login_handler.rs @@ -17,15 +17,15 @@ use std::collections::BTreeMap; use databend_common_storages_fuse::TableContext; use jwt_simple::prelude::Deserialize; use jwt_simple::prelude::Serialize; +use poem::IntoResponse; use poem::error::Result as PoemResult; use poem::web::Json; use poem::web::Query; -use poem::IntoResponse; use crate::auth::Credential; use crate::servers::http::error::HttpErrorCode; -use crate::servers::http::v1::session::client_session_manager::ClientSessionManager; use crate::servers::http::v1::HttpQueryContext; +use crate::servers::http::v1::session::client_session_manager::ClientSessionManager; #[derive(Deserialize, Clone)] struct LoginRequest { diff --git a/src/query/service/src/servers/http/v1/session/logout_handler.rs b/src/query/service/src/servers/http/v1/session/logout_handler.rs index d163fc5de675d..ad3c843575825 100644 --- a/src/query/service/src/servers/http/v1/session/logout_handler.rs +++ b/src/query/service/src/servers/http/v1/session/logout_handler.rs @@ -14,15 +14,15 @@ use jwt_simple::prelude::Serialize; use log::info; +use poem::IntoResponse; use poem::error::Result as PoemResult; use poem::web::Json; -use poem::IntoResponse; use crate::auth::Credential; use crate::servers::http::error::HttpErrorCode; use crate::servers::http::error::QueryError; -use crate::servers::http::v1::session::client_session_manager::ClientSessionManager; use crate::servers::http::v1::HttpQueryContext; +use crate::servers::http::v1::session::client_session_manager::ClientSessionManager; #[derive(Serialize, Debug, Clone)] pub struct LogoutResponse { diff --git a/src/query/service/src/servers/http/v1/session/mod.rs b/src/query/service/src/servers/http/v1/session/mod.rs index 89c42a74858a7..6321971995879 100644 --- a/src/query/service/src/servers/http/v1/session/mod.rs +++ b/src/query/service/src/servers/http/v1/session/mod.rs @@ -23,5 +23,5 @@ pub mod refresh_handler; mod token; pub use client_session_manager::ClientSessionManager; -pub(crate) use token::unix_ts; pub(crate) use token::SessionClaim; +pub(crate) use token::unix_ts; diff --git a/src/query/service/src/servers/http/v1/session/refresh_handler.rs b/src/query/service/src/servers/http/v1/session/refresh_handler.rs index 1997c56b697ef..1173bb1030d61 100644 --- a/src/query/service/src/servers/http/v1/session/refresh_handler.rs +++ b/src/query/service/src/servers/http/v1/session/refresh_handler.rs @@ -14,15 +14,15 @@ use jwt_simple::prelude::Deserialize; use jwt_simple::prelude::Serialize; +use poem::IntoResponse; use poem::error::Result as PoemResult; use poem::web::Json; -use poem::IntoResponse; use crate::auth::Credential; use crate::servers::http::error::HttpErrorCode; +use crate::servers::http::v1::HttpQueryContext; use crate::servers::http::v1::session::client_session_manager::ClientSessionManager; use crate::servers::http::v1::session::login_handler::TokensInfo; -use crate::servers::http::v1::HttpQueryContext; #[derive(Deserialize, Clone)] struct RefreshRequest { diff --git a/src/query/service/src/servers/http/v1/session/token.rs b/src/query/service/src/servers/http/v1/session/token.rs index d757f6c425262..59e901bcae45f 100644 --- a/src/query/service/src/servers/http/v1/session/token.rs +++ b/src/query/service/src/servers/http/v1/session/token.rs @@ -19,8 +19,8 @@ use base64::prelude::*; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_meta_app::principal::user_token::TokenType; -use rand::rngs::OsRng; use rand::RngCore; +use rand::rngs::OsRng; use serde::Deserialize; use serde::Serialize; diff --git a/src/query/service/src/servers/http/v1/stage.rs b/src/query/service/src/servers/http/v1/stage.rs index add5afa257bc7..a4bfcc386b0b7 100644 --- a/src/query/service/src/servers/http/v1/stage.rs +++ b/src/query/service/src/servers/http/v1/stage.rs @@ -17,14 +17,14 @@ use databend_common_catalog::session_type::SessionType; use databend_common_meta_app::principal::StageInfo; use databend_common_storages_stage::StageTable; use databend_common_users::UserApiProvider; -use futures_util::io; use futures_util::AsyncWriteExt; +use futures_util::io; use http::StatusCode; +use poem::Request; use poem::error::InternalServerError; use poem::error::Result as PoemResult; use poem::web::Json; use poem::web::Multipart; -use poem::Request; use serde::Deserialize; use serde::Serialize; diff --git a/src/query/service/src/servers/http/v1/streaming_load.rs b/src/query/service/src/servers/http/v1/streaming_load.rs index fdd649dff3456..8152ed92d3117 100644 --- a/src/query/service/src/servers/http/v1/streaming_load.rs +++ b/src/query/service/src/servers/http/v1/streaming_load.rs @@ -18,9 +18,9 @@ databend_common_tracing::register_module_tag!("[HTTP-STREAMING-LOAD]"); use std::future::Future; use std::sync::Arc; +use databend_common_base::base::ProgressValues; use databend_common_base::base::tokio; use databend_common_base::base::tokio::io::AsyncReadExt; -use databend_common_base::base::ProgressValues; use databend_common_base::headers::HEADER_QUERY_CONTEXT; use databend_common_base::headers::HEADER_SQL; use databend_common_base::runtime::MemStat; @@ -31,9 +31,9 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::DataBlock; use databend_common_meta_app::principal::FileFormatParams; +use databend_common_sql::Planner; use databend_common_sql::plans::InsertInputSource; use databend_common_sql::plans::Plan; -use databend_common_sql::Planner; use databend_common_storages_stage::BytesBatch; use databend_storages_common_session::TxnState; use fastrace::future::FutureExt; @@ -42,14 +42,14 @@ use http::StatusCode; use log::debug; use log::info; use log::warn; +use poem::IntoResponse; +use poem::Request; +use poem::Response; use poem::error::BadRequest; use poem::error::InternalServerError; use poem::error::Result as PoemResult; use poem::web::Json; use poem::web::Multipart; -use poem::IntoResponse; -use poem::Request; -use poem::Response; use serde::Deserialize; use serde::Serialize; use tokio::sync::mpsc::Sender; @@ -113,24 +113,23 @@ pub async fn streaming_load_handler( tracking_payload.mem_stat = Some(query_mem_stat.clone()); let _tracking_guard = ThreadTracker::tracking(tracking_payload); let root = get_http_tracing_span("http::streaming_load_handler", ctx, &ctx.query_id); - let mut session_conf: Option = match req.headers().get(HEADER_QUERY_CONTEXT) { - Some(v) => { - let s = v.to_str().unwrap().to_string(); - match serde_json::from_str(s.trim()) { - Ok(s) => Some(s), - Err(e) => { - return poem::Error::from_string( + let mut session_conf: Option = + match req.headers().get(HEADER_QUERY_CONTEXT) { + Some(v) => { + let s = v.to_str().unwrap().to_string(); + match serde_json::from_str(s.trim()) { + Ok(s) => Some(s), + Err(e) => return poem::Error::from_string( format!( "invalid value for header {HEADER_QUERY_CONTEXT}({s}) in request: {e}" ), StatusCode::BAD_REQUEST, ) - .into_response() + .into_response(), } } - } - None => None, - }; + None => None, + }; let res = ThreadTracker::tracking_future( streaming_load_handler_inner(ctx, req, multipart, query_mem_stat, &session_conf) .in_span(root), @@ -227,52 +226,61 @@ async fn streaming_load_handler_inner( as usize; match &mut plan { - Plan::Insert(insert) => { - match &mut insert.source { - InsertInputSource::StreamingLoad(streaming_load) - => { - if !streaming_load.file_format.support_streaming_load() { - return Err(poem::Error::from_string( format!( "Unsupported file format: {}", streaming_load.file_format.get_type() ), StatusCode::BAD_REQUEST)); - } - let (tx, rx) = tokio::sync::mpsc::channel(1); - *streaming_load.receiver.lock() = Some(rx); + Plan::Insert(insert) => match &mut insert.source { + InsertInputSource::StreamingLoad(streaming_load) => { + if !streaming_load.file_format.support_streaming_load() { + return Err(poem::Error::from_string( + format!( + "Unsupported file format: {}", + streaming_load.file_format.get_type() + ), + StatusCode::BAD_REQUEST, + )); + } + let (tx, rx) = tokio::sync::mpsc::channel(1); + *streaming_load.receiver.lock() = Some(rx); - let format = streaming_load.file_format.clone(); - let handler = query_context.spawn(execute_query(http_context.clone(), query_context.clone(), plan, mem_stat)); - read_multi_part(multipart, &format, tx, input_read_buffer_size).await?; + let format = streaming_load.file_format.clone(); + let handler = query_context.spawn(execute_query( + http_context.clone(), + query_context.clone(), + plan, + mem_stat, + )); + read_multi_part(multipart, &format, tx, input_read_buffer_size).await?; - match handler.await { - Ok(Ok(_)) => Ok(Json(LoadResponse { - id: http_context.query_id.clone(), - stats: query_context.get_write_progress().get_values(), - })), - Ok(Err(cause)) => { - info!("Query execution failed: {:?}", cause); - Err(poem::Error::from_string( + match handler.await { + Ok(Ok(_)) => Ok(Json(LoadResponse { + id: http_context.query_id.clone(), + stats: query_context.get_write_progress().get_values(), + })), + Ok(Err(cause)) => { + info!("Query execution failed: {:?}", cause); + Err(poem::Error::from_string( format!( "Query execution failed: {}", cause.display_with_sql(sql).message() ), StatusCode::BAD_REQUEST, - ))}, - Err(err) => { - info!("Internal server error: {:?}", err); - Err(poem::Error::from_string( - "Internal server error: execution thread panicked", - StatusCode::INTERNAL_SERVER_ERROR, - )) - }, + )) + } + Err(err) => { + info!("Internal server error: {:?}", err); + Err(poem::Error::from_string( + "Internal server error: execution thread panicked", + StatusCode::INTERNAL_SERVER_ERROR, + )) } } - _non_supported_source => Err(poem::Error::from_string( - format!( - "Unsupported INSERT source. Streaming upload only supports 'INSERT INTO $table FILE_FORMAT = (type = ...)'. Got: {}", - plan - ), - StatusCode::BAD_REQUEST, - )), } - } + _non_supported_source => Err(poem::Error::from_string( + format!( + "Unsupported INSERT source. Streaming upload only supports 'INSERT INTO $table FILE_FORMAT = (type = ...)'. Got: {}", + plan + ), + StatusCode::BAD_REQUEST, + )), + }, non_insert_plan => Err(poem::Error::from_string( format!( "Only INSERT statements are supported in streaming load. Got: {}", diff --git a/src/query/service/src/servers/http/v1/users.rs b/src/query/service/src/servers/http/v1/users.rs index caf91258265b5..71355b6853609 100644 --- a/src/query/service/src/servers/http/v1/users.rs +++ b/src/query/service/src/servers/http/v1/users.rs @@ -20,12 +20,12 @@ use databend_common_meta_app::principal::UserInfo; use databend_common_meta_app::principal::UserPrivilegeSet; use databend_common_meta_app::schema::CreateOption; use databend_common_users::UserApiProvider; +use poem::IntoResponse; use poem::error::BadRequest; use poem::error::Forbidden; use poem::error::InternalServerError; use poem::error::Result as PoemResult; use poem::web::Json; -use poem::IntoResponse; use serde::Deserialize; use serde::Serialize; diff --git a/src/query/service/src/servers/http/v1/verify.rs b/src/query/service/src/servers/http/v1/verify.rs index 7443932ae49fc..37672210ab713 100644 --- a/src/query/service/src/servers/http/v1/verify.rs +++ b/src/query/service/src/servers/http/v1/verify.rs @@ -14,9 +14,9 @@ use databend_common_users::UserApiProvider; use jwt_simple::prelude::Serialize; +use poem::IntoResponse; use poem::error::Result as PoemResult; use poem::web::Json; -use poem::IntoResponse; use crate::servers::http::error::HttpErrorCode; use crate::servers::http::v1::HttpQueryContext; diff --git a/src/query/service/src/servers/mod.rs b/src/query/service/src/servers/mod.rs index 5070ebead70cb..fa99d1bf495ff 100644 --- a/src/query/service/src/servers/mod.rs +++ b/src/query/service/src/servers/mod.rs @@ -18,9 +18,9 @@ pub use server::Server; pub use server::ShutdownHandle; pub use self::flight_sql::FlightSQLServer; -pub use self::http::v1::HttpQueryManager; pub use self::http::HttpHandler; pub use self::http::HttpHandlerKind; +pub use self::http::v1::HttpQueryManager; pub use self::mysql::MySQLConnection; pub use self::mysql::MySQLFederated; pub use self::mysql::MySQLHandler; diff --git a/src/query/service/src/servers/mysql/mysql_federated.rs b/src/query/service/src/servers/mysql/mysql_federated.rs index c4aead33eb8dc..f5815f1f02ef7 100644 --- a/src/query/service/src/servers/mysql/mysql_federated.rs +++ b/src/query/service/src/servers/mysql/mysql_federated.rs @@ -16,8 +16,6 @@ use std::collections::HashMap; use std::sync::Arc; use std::sync::LazyLock; -use databend_common_expression::types::StringType; -use databend_common_expression::utils::FromData; use databend_common_expression::DataBlock; use databend_common_expression::DataSchema; use databend_common_expression::DataSchemaRef; @@ -25,6 +23,8 @@ use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::StringType; +use databend_common_expression::utils::FromData; use regex::Regex; use crate::servers::federated_helper::FederatedHelper; diff --git a/src/query/service/src/servers/mysql/mysql_handler.rs b/src/query/service/src/servers/mysql/mysql_handler.rs index 2328da095ae31..4e1dde8568ef4 100644 --- a/src/query/service/src/servers/mysql/mysql_handler.rs +++ b/src/query/service/src/servers/mysql/mysql_handler.rs @@ -23,10 +23,10 @@ use databend_common_base::runtime::Runtime; use databend_common_base::runtime::TrySpawn; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use futures::StreamExt; use futures::future::AbortHandle; use futures::future::AbortRegistration; use futures::future::Abortable; -use futures::StreamExt; use log::error; use rustls::ServerConfig; use socket2::TcpKeepalive; @@ -76,7 +76,11 @@ impl MySQLHandler { Ok((TcpListenerStream::new(listener), listener_addr)) } - fn listen_loop(&self, stream: ListeningStream, rt: Arc) -> impl Future { + fn listen_loop( + &self, + stream: ListeningStream, + rt: Arc, + ) -> impl Future + use<> { let keepalive = self.keepalive.clone(); let tls = self.tls.clone(); diff --git a/src/query/service/src/servers/mysql/mysql_interactive_worker.rs b/src/query/service/src/servers/mysql/mysql_interactive_worker.rs index bc14b05eb27da..45cb4a2ed2993 100644 --- a/src/query/service/src/servers/mysql/mysql_interactive_worker.rs +++ b/src/query/service/src/servers/mysql/mysql_interactive_worker.rs @@ -16,10 +16,10 @@ use std::sync::Arc; use std::time::Duration; use std::time::Instant; +use databend_common_base::base::BuildInfoRef; use databend_common_base::base::convert_byte_size; use databend_common_base::base::convert_number_size; use databend_common_base::base::tokio::io::AsyncWrite; -use databend_common_base::base::BuildInfoRef; use databend_common_base::runtime::MemStat; use databend_common_base::runtime::ThreadTracker; use databend_common_base::runtime::TrySpawn; @@ -46,23 +46,23 @@ use opensrv_mysql::InitWriter; use opensrv_mysql::ParamParser; use opensrv_mysql::QueryResultWriter; use opensrv_mysql::StatementMetaWriter; -use rand::thread_rng; use rand::Rng as _; +use rand::thread_rng; use uuid::Uuid; use crate::auth::CredentialType; -use crate::interpreters::interpreter_plan_sql; use crate::interpreters::Interpreter; use crate::interpreters::InterpreterFactory; +use crate::interpreters::interpreter_plan_sql; use crate::servers::login_history::LoginEventType; use crate::servers::login_history::LoginHandler; use crate::servers::login_history::LoginHistory; +use crate::servers::mysql::MYSQL_VERSION; +use crate::servers::mysql::MySQLFederated; use crate::servers::mysql::writers::DFInitResultWriter; use crate::servers::mysql::writers::DFQueryResultWriter; use crate::servers::mysql::writers::ProgressReporter; use crate::servers::mysql::writers::QueryResult; -use crate::servers::mysql::MySQLFederated; -use crate::servers::mysql::MYSQL_VERSION; use crate::sessions::QueryContext; use crate::sessions::Session; use crate::sessions::TableContext; @@ -481,11 +481,10 @@ impl InteractiveWorkerBase { None, )?; - let query_result = query_result - .await - .map_err_to_code(ErrorCode::TokioError, || { - "Cannot join handle from context's runtime" - })?; + let query_result = query_result.await.map_err_to_code( + ErrorCode::TokioError, + || "Cannot join handle from context's runtime", + )?; let reporter = Box::new(ContextProgressReporter::new(context.clone(), instant)) as Box; query_result.map(|data| (data, Some(reporter))) diff --git a/src/query/service/src/servers/mysql/mysql_session.rs b/src/query/service/src/servers/mysql/mysql_session.rs index 2dae5e5460149..5daac29c5c242 100644 --- a/src/query/service/src/servers/mysql/mysql_session.rs +++ b/src/query/service/src/servers/mysql/mysql_session.rs @@ -18,9 +18,9 @@ use std::sync::Arc; use databend_common_base::base::tokio::io::AsyncWriteExt; use databend_common_base::base::tokio::io::BufWriter; +use databend_common_base::base::tokio::net::TcpStream; use databend_common_base::base::tokio::net::tcp::OwnedReadHalf; use databend_common_base::base::tokio::net::tcp::OwnedWriteHalf; -use databend_common_base::base::tokio::net::TcpStream; use databend_common_base::runtime::Runtime; use databend_common_base::runtime::Thread; use databend_common_base::runtime::TrySpawn; @@ -33,19 +33,19 @@ use databend_storages_common_session::drop_all_temp_tables; use log::error; use log::info; use log::warn; -use opensrv_mysql::plain_run_with_options; -use opensrv_mysql::secure_run_with_options; use opensrv_mysql::AsyncMysqlIntermediary; use opensrv_mysql::ErrorKind; use opensrv_mysql::IntermediaryOptions; use opensrv_mysql::ServerHandshakeConfig; +use opensrv_mysql::plain_run_with_options; +use opensrv_mysql::secure_run_with_options; use rand::RngCore; use rustls::ServerConfig; use socket2::SockRef; use socket2::TcpKeepalive; -use crate::servers::mysql::mysql_interactive_worker::InteractiveWorker; use crate::servers::mysql::MYSQL_VERSION; +use crate::servers::mysql::mysql_interactive_worker::InteractiveWorker; use crate::sessions::Session; use crate::sessions::SessionManager; @@ -107,7 +107,7 @@ impl MySQLConnection { Err(error) => { return Err(ErrorCode::TokioError(format!( "Handshaking mysql connection failed: {error}" - ))) + ))); } }; @@ -278,11 +278,10 @@ impl MySQLConnection { // TODO: move to ToBlockingStream trait fn convert_stream(stream: TcpStream) -> Result { - let stream = stream - .into_std() - .map_err_to_code(ErrorCode::TokioError, || { - "Cannot to convert Tokio TcpStream to Std TcpStream" - })?; + let stream = stream.into_std().map_err_to_code( + ErrorCode::TokioError, + || "Cannot to convert Tokio TcpStream to Std TcpStream", + )?; Ok(stream) } diff --git a/src/query/service/src/servers/mysql/writers/query_result_writer.rs b/src/query/service/src/servers/mysql/writers/query_result_writer.rs index f6c32c027d820..876feadaa72ce 100644 --- a/src/query/service/src/servers/mysql/writers/query_result_writer.rs +++ b/src/query/service/src/servers/mysql/writers/query_result_writer.rs @@ -17,14 +17,14 @@ use std::sync::Arc; use databend_common_base::base::tokio::io::AsyncWrite; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::number::NumberScalar; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; use databend_common_expression::Column as ExprColumn; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRef; use databend_common_expression::ScalarRef; use databend_common_expression::SendableDataBlockStream; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::number::NumberScalar; use databend_common_formats::field_encoder::FieldEncoderValues; use databend_common_io::prelude::FormatSettings; use futures_util::StreamExt; diff --git a/src/query/service/src/servers/server.rs b/src/query/service/src/servers/server.rs index 792846fccb06c..337e84dc89dac 100644 --- a/src/query/service/src/servers/server.rs +++ b/src/query/service/src/servers/server.rs @@ -13,20 +13,20 @@ // limitations under the License. use std::net::SocketAddr; +use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; -use std::sync::Arc; use std::time::Duration; -use databend_common_base::base::signal_stream; use databend_common_base::base::DummySignalStream; use databend_common_base::base::SignalStream; use databend_common_base::base::SignalType; +use databend_common_base::base::signal_stream; use databend_common_base::runtime::drop_guard; use databend_common_config::GlobalConfig; use databend_common_exception::Result; -use futures::stream::Abortable; use futures::StreamExt; +use futures::stream::Abortable; use log::error; use log::info; use tokio_stream::wrappers::TcpListenerStream; diff --git a/src/query/service/src/sessions/mod.rs b/src/query/service/src/sessions/mod.rs index 399ebfbe53b7a..f3a00f8a3e4fe 100644 --- a/src/query/service/src/sessions/mod.rs +++ b/src/query/service/src/sessions/mod.rs @@ -30,8 +30,8 @@ mod session_status; pub use databend_common_base::base::BuildInfoRef; pub use databend_common_catalog::table_context::TableContext; pub use query_affect::QueryAffect; -pub use query_ctx::convert_query_log_timestamp; pub use query_ctx::QueryContext; +pub use query_ctx::convert_query_log_timestamp; pub use query_ctx_shared::MemoryUpdater; pub use query_ctx_shared::QueryContextShared; pub use queue_mgr::AcquireQueueGuard; diff --git a/src/query/service/src/sessions/query_ctx.rs b/src/query/service/src/sessions/query_ctx.rs index 525729e5a27e8..aab5b283837a5 100644 --- a/src/query/service/src/sessions/query_ctx.rs +++ b/src/query/service/src/sessions/query_ctx.rs @@ -17,15 +17,15 @@ databend_common_tracing::register_module_tag!("[QUERY-CTX]"); use std::any::Any; use std::cmp::min; -use std::collections::hash_map::Entry; use std::collections::HashMap; use std::collections::HashSet; use std::collections::VecDeque; +use std::collections::hash_map::Entry; use std::future::Future; use std::str::FromStr; +use std::sync::Arc; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; -use std::sync::Arc; use std::time::Duration; use std::time::Instant; use std::time::SystemTime; @@ -34,21 +34,21 @@ use std::time::UNIX_EPOCH; use async_channel::Receiver; use async_channel::Sender; use chrono_tz::Tz; -use dashmap::mapref::multiple::RefMulti; use dashmap::DashMap; +use dashmap::mapref::multiple::RefMulti; use databend_common_ast::ast::FormatTreeNode; +use databend_common_base::JoinHandle; use databend_common_base::base::Progress; use databend_common_base::base::ProgressValues; use databend_common_base::base::SpillProgress; use databend_common_base::base::WatchNotify; -use databend_common_base::runtime::profile::Profile; -use databend_common_base::runtime::profile::ProfileStatisticsName; use databend_common_base::runtime::ExecutorStatsSnapshot; use databend_common_base::runtime::GlobalIORuntime; use databend_common_base::runtime::MemStat; use databend_common_base::runtime::ThreadTracker; use databend_common_base::runtime::TrySpawn; -use databend_common_base::JoinHandle; +use databend_common_base::runtime::profile::Profile; +use databend_common_base::runtime::profile::ProfileStatisticsName; use databend_common_catalog::catalog::CATALOG_DEFAULT; use databend_common_catalog::database::Database; use databend_common_catalog::lock::LockTableOption; @@ -88,6 +88,8 @@ use databend_common_expression::TableSchema; use databend_common_io::prelude::FormatSettings; use databend_common_license::license::Feature; use databend_common_license::license_manager::LicenseManagerSwitch; +use databend_common_meta_app::principal::COPY_MAX_FILES_COMMIT_MSG; +use databend_common_meta_app::principal::COPY_MAX_FILES_PER_COMMIT; use databend_common_meta_app::principal::FileFormatParams; use databend_common_meta_app::principal::GrantObject; use databend_common_meta_app::principal::OnErrorMode; @@ -97,8 +99,6 @@ use databend_common_meta_app::principal::StageInfo; use databend_common_meta_app::principal::UserDefinedConnection; use databend_common_meta_app::principal::UserInfo; use databend_common_meta_app::principal::UserPrivilegeType; -use databend_common_meta_app::principal::COPY_MAX_FILES_COMMIT_MSG; -use databend_common_meta_app::principal::COPY_MAX_FILES_PER_COMMIT; use databend_common_meta_app::schema::CatalogType; use databend_common_meta_app::schema::DropTableByIdReq; use databend_common_meta_app::schema::GetTableCopiedFileReq; @@ -111,7 +111,6 @@ use databend_common_pipeline::core::LockGuard; use databend_common_pipeline::core::PlanProfile; use databend_common_settings::Settings; use databend_common_sql::IndexType; -use databend_common_storage::init_stage_operator; use databend_common_storage::CopyStatus; use databend_common_storage::DataOperator; use databend_common_storage::FileStatus; @@ -120,6 +119,7 @@ use databend_common_storage::MutationStatus; use databend_common_storage::StageFileInfo; use databend_common_storage::StageFilesInfo; use databend_common_storage::StorageMetrics; +use databend_common_storage::init_stage_operator; use databend_common_storages_basic::ResultScan; use databend_common_storages_delta::DeltaTable; use databend_common_storages_fuse::FuseTable; @@ -132,16 +132,16 @@ use databend_common_storages_stream::stream_table::StreamTable; use databend_common_users::GrantObjectVisibilityChecker; use databend_common_users::Object; use databend_common_users::UserApiProvider; -use databend_storages_common_session::drop_table_by_id; use databend_storages_common_session::SessionState; use databend_storages_common_session::TxnManagerRef; +use databend_storages_common_session::drop_table_by_id; use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::SnapshotTimestampValidationContext; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use databend_storages_common_table_meta::meta::TableSnapshot; use databend_storages_common_table_meta::table::OPT_KEY_TEMP_PREFIX; -use jiff::tz::TimeZone; use jiff::Zoned; +use jiff::tz::TimeZone; use log::debug; use log::info; use parking_lot::Mutex; @@ -154,14 +154,14 @@ use crate::locks::LockManager; use crate::pipelines::executor::PipelineExecutor; use crate::servers::flight::v1::exchange::DataExchangeManager; use crate::servers::http::v1::ClientSessionManager; -use crate::sessions::query_affect::QueryAffect; -use crate::sessions::query_ctx_shared::MemoryUpdater; use crate::sessions::BuildInfoRef; use crate::sessions::ProcessInfo; use crate::sessions::QueriesQueueManager; use crate::sessions::QueryContextShared; use crate::sessions::Session; use crate::sessions::SessionManager; +use crate::sessions::query_affect::QueryAffect; +use crate::sessions::query_ctx_shared::MemoryUpdater; use crate::spillers; use crate::sql::binder::get_storage_params_from_options; use crate::storages::Table; @@ -1479,12 +1479,10 @@ impl TableContext for QueryContext { let stream = StreamTable::try_from_table(table.as_ref())?; let actual_batch_limit = stream.max_batch_size(); if actual_batch_limit != final_batch_size { - return Err(ErrorCode::StorageUnsupported( - format!( - "Stream batch size must be consistent within transaction: actual={:?}, requested={:?}", - actual_batch_limit, final_batch_size - ) - )); + return Err(ErrorCode::StorageUnsupported(format!( + "Stream batch size must be consistent within transaction: actual={:?}, requested={:?}", + actual_batch_limit, final_batch_size + ))); } } else if max_batch_size.is_some() { return Err(ErrorCode::StorageUnsupported( @@ -2040,7 +2038,7 @@ impl TableContext for QueryContext { _ => { return Err(ErrorCode::SemanticError( "Query from ORC file only support $1 as column position", - )) + )); } }; let schema = Arc::new(TableSchema::empty()); diff --git a/src/query/service/src/sessions/query_ctx_shared.rs b/src/query/service/src/sessions/query_ctx_shared.rs index eb8d331e0ce52..c0b372b1c8e44 100644 --- a/src/query/service/src/sessions/query_ctx_shared.rs +++ b/src/query/service/src/sessions/query_ctx_shared.rs @@ -12,29 +12,29 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::hash_map::Entry; use std::collections::HashMap; use std::collections::HashSet; +use std::collections::hash_map::Entry; +use std::sync::Arc; +use std::sync::Weak; use std::sync::atomic::AtomicBool; use std::sync::atomic::AtomicU32; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; -use std::sync::Arc; -use std::sync::Weak; use std::time::Duration; use std::time::SystemTime; use async_channel::Receiver; use async_channel::Sender; use dashmap::DashMap; -use databend_common_base::base::short_sql; use databend_common_base::base::Progress; use databend_common_base::base::SpillProgress; use databend_common_base::base::WatchNotify; -use databend_common_base::runtime::drop_guard; +use databend_common_base::base::short_sql; use databend_common_base::runtime::ExecutorStatsSnapshot; use databend_common_base::runtime::MemStat; use databend_common_base::runtime::Runtime; +use databend_common_base::runtime::drop_guard; use databend_common_catalog::catalog::Catalog; use databend_common_catalog::catalog::CatalogManager; use databend_common_catalog::merge_into_join::MergeIntoJoin; @@ -75,9 +75,9 @@ use uuid::Uuid; use crate::clusters::Cluster; use crate::clusters::ClusterDiscovery; use crate::pipelines::executor::PipelineExecutor; -use crate::sessions::query_affect::QueryAffect; use crate::sessions::BuildInfoRef; use crate::sessions::Session; +use crate::sessions::query_affect::QueryAffect; use crate::spillers; use crate::storages::Table; diff --git a/src/query/service/src/sessions/queue_mgr.rs b/src/query/service/src/sessions/queue_mgr.rs index a1858b00e3a70..88e4bd8aac0cc 100644 --- a/src/query/service/src/sessions/queue_mgr.rs +++ b/src/query/service/src/sessions/queue_mgr.rs @@ -20,9 +20,9 @@ use std::fmt::Display; use std::future::Future; use std::hash::Hash; use std::pin::Pin; +use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; -use std::sync::Arc; use std::task::Context; use std::task::Poll; use std::task::Waker; @@ -31,14 +31,14 @@ use std::time::Instant; use std::time::SystemTime; use databend_common_ast::ast::ExplainKind; -use databend_common_base::base::escape_for_key; -use databend_common_base::base::tokio::sync::Mutex as TokioMutex; use databend_common_base::base::GlobalInstance; use databend_common_base::base::WatchNotify; -use databend_common_base::runtime::workload_group::QuotaValue; +use databend_common_base::base::escape_for_key; +use databend_common_base::base::tokio::sync::Mutex as TokioMutex; +use databend_common_base::runtime::ThreadTracker; use databend_common_base::runtime::workload_group::MAX_CONCURRENCY_QUOTA_KEY; use databend_common_base::runtime::workload_group::QUERY_QUEUED_TIMEOUT_QUOTA_KEY; -use databend_common_base::runtime::ThreadTracker; +use databend_common_base::runtime::workload_group::QuotaValue; use databend_common_catalog::table_context::TableContext; use databend_common_config::InnerConfig; use databend_common_exception::ErrorCode; @@ -56,10 +56,10 @@ use databend_common_metrics::session::incr_session_queue_acquire_error_count; use databend_common_metrics::session::incr_session_queue_acquire_timeout_count; use databend_common_metrics::session::record_session_queue_acquire_duration_ms; use databend_common_metrics::session::set_session_queued_queries; +use databend_common_sql::PlanExtras; use databend_common_sql::plans::ModifyColumnAction; use databend_common_sql::plans::ModifyTableColumnPlan; use databend_common_sql::plans::Plan; -use databend_common_sql::PlanExtras; use futures_util::future::Either; use log::info; use parking_lot::Mutex; @@ -246,7 +246,11 @@ impl QueueManager { let current_used_after = permits.saturating_sub(available_permits_after); info!( "[ACQUIRED] Workload group '{}' local (max_concurrency={}): {}/{} slots used (waited {:.2}s)", - workload_group.meta.name, permits, current_used_after, permits, instant.elapsed().as_secs_f64() + workload_group.meta.name, + permits, + current_used_after, + permits, + instant.elapsed().as_secs_f64() ); timeout -= instant.elapsed(); @@ -376,7 +380,11 @@ impl QueueManager { let used_slots = self.permits - self.semaphore.available_permits(); info!( "[ACQUIRED] Warehouse resource (max_running_queries={}): {}/{} slots used, {} queries waiting (waited {:.2}s)", - self.permits, used_slots, self.permits, queue_length, instant.elapsed().as_secs_f64() + self.permits, + used_slots, + self.permits, + queue_length, + instant.elapsed().as_secs_f64() ); Ok(guards) diff --git a/src/query/service/src/sessions/session.rs b/src/query/service/src/sessions/session.rs index 3272d8c728e87..7bf2868e6b132 100644 --- a/src/query/service/src/sessions/session.rs +++ b/src/query/service/src/sessions/session.rs @@ -16,11 +16,11 @@ use std::collections::HashMap; use std::net::SocketAddr; use std::sync::Arc; +use databend_common_base::runtime::ThreadTracker; use databend_common_base::runtime::drop_guard; -use databend_common_base::runtime::workload_group::QuotaValue; use databend_common_base::runtime::workload_group::CPU_QUOTA_KEY; use databend_common_base::runtime::workload_group::QUERY_TIMEOUT_QUOTA_KEY; -use databend_common_base::runtime::ThreadTracker; +use databend_common_base::runtime::workload_group::QuotaValue; use databend_common_catalog::cluster_info::Cluster; use databend_common_catalog::session_type::SessionType; use databend_common_config::GlobalConfig; @@ -37,23 +37,23 @@ use databend_common_meta_app::tenant::Tenant; use databend_common_pipeline::core::PlanProfile; use databend_common_settings::OutofMemoryBehavior; use databend_common_settings::Settings; +use databend_common_users::BUILTIN_ROLE_PUBLIC; use databend_common_users::GrantObjectVisibilityChecker; use databend_common_users::Object; -use databend_common_users::BUILTIN_ROLE_PUBLIC; use databend_storages_common_session::TempTblMgrRef; use databend_storages_common_session::TxnManagerRef; use log::debug; use parking_lot::RwLock; use crate::clusters::ClusterDiscovery; -use crate::sessions::session_privilege_mgr::SessionPrivilegeManager; -use crate::sessions::session_privilege_mgr::SessionPrivilegeManagerImpl; use crate::sessions::BuildInfoRef; use crate::sessions::QueryContext; use crate::sessions::QueryContextShared; use crate::sessions::SessionContext; use crate::sessions::SessionManager; use crate::sessions::SessionStatus; +use crate::sessions::session_privilege_mgr::SessionPrivilegeManager; +use crate::sessions::session_privilege_mgr::SessionPrivilegeManagerImpl; pub struct Session { pub(in crate::sessions) id: String, diff --git a/src/query/service/src/sessions/session_ctx.rs b/src/query/service/src/sessions/session_ctx.rs index a2418d523962f..59285a0a2814e 100644 --- a/src/query/service/src/sessions/session_ctx.rs +++ b/src/query/service/src/sessions/session_ctx.rs @@ -14,10 +14,10 @@ use std::collections::HashMap; use std::collections::HashSet; -use std::sync::atomic::AtomicBool; -use std::sync::atomic::Ordering; use std::sync::Arc; use std::sync::Weak; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; use databend_common_catalog::session_type::SessionType; use databend_common_config::GlobalConfig; diff --git a/src/query/service/src/sessions/session_mgr.rs b/src/query/service/src/sessions/session_mgr.rs index ac4b207f8507d..ac64bf78c3092 100644 --- a/src/query/service/src/sessions/session_mgr.rs +++ b/src/query/service/src/sessions/session_mgr.rs @@ -15,18 +15,18 @@ use std::collections::HashMap; use std::future::Future; use std::ops::DerefMut; -use std::sync::atomic::AtomicU32; -use std::sync::atomic::Ordering; use std::sync::Arc; use std::sync::Weak; +use std::sync::atomic::AtomicU32; +use std::sync::atomic::Ordering; use std::time::Duration; -use databend_common_base::base::tokio; use databend_common_base::base::GlobalInstance; use databend_common_base::base::SignalStream; -use databend_common_base::runtime::metrics::GLOBAL_METRICS_REGISTRY; +use databend_common_base::base::tokio; use databend_common_base::runtime::ExecutorStatsSnapshot; use databend_common_base::runtime::LimitMemGuard; +use databend_common_base::runtime::metrics::GLOBAL_METRICS_REGISTRY; use databend_common_catalog::session_type::SessionType; use databend_common_catalog::table_context::ProcessInfoState; use databend_common_config::GlobalConfig; @@ -37,16 +37,16 @@ use databend_common_meta_app::principal::UserInfo; use databend_common_metrics::session::*; use databend_common_pipeline::core::PlanProfile; use databend_common_settings::Settings; -use futures::future::Either; use futures::StreamExt; +use futures::future::Either; use log::info; use parking_lot::RwLock; -use crate::sessions::session::Session; -use crate::sessions::session_mgr_metrics::SessionManagerMetricsCollector; use crate::sessions::ProcessInfo; use crate::sessions::SessionContext; use crate::sessions::SessionManagerStatus; +use crate::sessions::session::Session; +use crate::sessions::session_mgr_metrics::SessionManagerMetricsCollector; pub struct SessionManager { pub(in crate::sessions) max_sessions: usize, diff --git a/src/query/service/src/sessions/session_privilege_mgr.rs b/src/query/service/src/sessions/session_privilege_mgr.rs index 15411672da8ab..c2cc09795d837 100644 --- a/src/query/service/src/sessions/session_privilege_mgr.rs +++ b/src/query/service/src/sessions/session_privilege_mgr.rs @@ -25,12 +25,12 @@ use databend_common_meta_app::principal::OwnershipObject; use databend_common_meta_app::principal::RoleInfo; use databend_common_meta_app::principal::UserInfo; use databend_common_meta_app::principal::UserPrivilegeType; +use databend_common_users::BUILTIN_ROLE_ACCOUNT_ADMIN; +use databend_common_users::BUILTIN_ROLE_PUBLIC; use databend_common_users::GrantObjectVisibilityChecker; use databend_common_users::Object; use databend_common_users::RoleCacheManager; use databend_common_users::UserApiProvider; -use databend_common_users::BUILTIN_ROLE_ACCOUNT_ADMIN; -use databend_common_users::BUILTIN_ROLE_PUBLIC; use databend_enterprise_resources_management::ResourcesManagement; use crate::sessions::SessionContext; diff --git a/src/query/service/src/spillers/adapter.rs b/src/query/service/src/spillers/adapter.rs index e52d47da6d69e..c33877f6dba9a 100644 --- a/src/query/service/src/spillers/adapter.rs +++ b/src/query/service/src/spillers/adapter.rs @@ -12,18 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::hash_map::Entry; use std::collections::HashMap; use std::collections::HashSet; +use std::collections::hash_map::Entry; use std::ops::DerefMut; use std::ops::Range; use std::sync::Arc; use std::sync::RwLock; use std::time::Instant; +use databend_common_base::base::ProgressValues; use databend_common_base::base::dma_buffer_to_bytes; use databend_common_base::base::dma_read_file_range; -use databend_common_base::base::ProgressValues; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; use databend_common_expression::DataBlock; @@ -36,14 +36,14 @@ use opendal::Buffer; use opendal::Operator; use parquet::file::metadata::RowGroupMetaDataPtr; +use super::Location; +use super::SpillsBufferPool; use super::async_buffer::SpillTarget; use super::block_reader::BlocksReader; use super::block_writer::BlocksWriter; use super::inner::*; use super::row_group_encoder::*; use super::serialize::*; -use super::Location; -use super::SpillsBufferPool; use crate::sessions::QueryContext; #[derive(Clone)] @@ -297,7 +297,7 @@ impl Spiller { let data = match location { Location::Local(path) => match &self.local_operator { - Some(ref local) => { + Some(local) => { local .read_with(path.file_name().unwrap().to_str().unwrap()) .range(data_range) diff --git a/src/query/service/src/spillers/async_buffer.rs b/src/query/service/src/spillers/async_buffer.rs index 976ba679b0c53..f7f58afe1b71c 100644 --- a/src/query/service/src/spillers/async_buffer.rs +++ b/src/query/service/src/spillers/async_buffer.rs @@ -32,27 +32,27 @@ use databend_common_base::runtime::Runtime; use databend_common_base::runtime::TrySpawn; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::infer_table_schema; use databend_common_expression::DataBlock; use databend_common_expression::DataSchema; use databend_common_expression::DataSchemaRef; use databend_common_expression::TableSchemaRef; +use databend_common_expression::infer_table_schema; use databend_common_meta_app::storage::StorageParams; -use databend_common_storages_parquet::parquet_reader::row_group::get_ranges; -use databend_common_storages_parquet::parquet_reader::RowGroupCore; use databend_common_storages_parquet::ReadSettings; -use fastrace::future::FutureExt; +use databend_common_storages_parquet::parquet_reader::RowGroupCore; +use databend_common_storages_parquet::parquet_reader::row_group::get_ranges; use fastrace::Span; +use fastrace::future::FutureExt; use opendal::Metadata; use opendal::Operator; use opendal::Writer; +use parquet::arrow::ArrowWriter; +use parquet::arrow::FieldLevels; +use parquet::arrow::ProjectionMask; use parquet::arrow::arrow_reader::ParquetRecordBatchReader; use parquet::arrow::arrow_reader::RowGroups; use parquet::arrow::parquet_to_arrow_field_levels; use parquet::arrow::parquet_to_arrow_schema; -use parquet::arrow::ArrowWriter; -use parquet::arrow::FieldLevels; -use parquet::arrow::ProjectionMask; use parquet::basic::Compression; use parquet::file::metadata::RowGroupMetaData; use parquet::file::properties::EnabledStatistics; @@ -832,9 +832,9 @@ impl Background { #[cfg(test)] mod tests { use std::io::Write; + use std::sync::Arc; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; - use std::sync::Arc; use databend_common_base::runtime::spawn; use opendal::Operator; diff --git a/src/query/service/src/spillers/block_reader.rs b/src/query/service/src/spillers/block_reader.rs index 2883c74ef2d9d..98154ebb6b713 100644 --- a/src/query/service/src/spillers/block_reader.rs +++ b/src/query/service/src/spillers/block_reader.rs @@ -22,8 +22,8 @@ use databend_common_expression::DataBlock; use databend_common_pipeline_transforms::traits::Location; use opendal::Operator; -use super::serialize::deserialize_block; use super::Layout; +use super::serialize::deserialize_block; pub struct BlocksReader<'a> { read_bytes: u64, diff --git a/src/query/service/src/spillers/block_writer.rs b/src/query/service/src/spillers/block_writer.rs index 031748ffc4e02..05508e6d8baf9 100644 --- a/src/query/service/src/spillers/block_writer.rs +++ b/src/query/service/src/spillers/block_writer.rs @@ -16,8 +16,8 @@ use byteorder::BigEndian; use byteorder::WriteBytesExt; use bytes::BufMut; use bytes::BytesMut; -use databend_common_base::base::dma_buffer_to_bytes; use databend_common_base::base::Alignment; +use databend_common_base::base::dma_buffer_to_bytes; use databend_common_exception::Result; use databend_common_expression::DataBlock; use databend_common_pipeline_transforms::traits::Location; diff --git a/src/query/service/src/spillers/inner.rs b/src/query/service/src/spillers/inner.rs index 9b4fe414424e7..4fa84bb4b899f 100644 --- a/src/query/service/src/spillers/inner.rs +++ b/src/query/service/src/spillers/inner.rs @@ -17,23 +17,23 @@ use std::fmt::Formatter; use std::sync::Arc; use std::time::Instant; -use databend_common_base::base::dma_buffer_to_bytes; -use databend_common_base::base::dma_read_file_range; use databend_common_base::base::Alignment; use databend_common_base::base::DmaWriteBuf; use databend_common_base::base::GlobalUniqName; +use databend_common_base::base::dma_buffer_to_bytes; +use databend_common_base::base::dma_read_file_range; use databend_common_base::runtime::profile::Profile; use databend_common_base::runtime::profile::ProfileStatisticsName; use databend_common_exception::Result; use databend_common_expression::DataBlock; use databend_storages_common_cache::TempDir; -use opendal::services::Fs; use opendal::Buffer; use opendal::Operator; +use opendal::services::Fs; +use super::Location; use super::async_buffer::SpillTarget; use super::serialize::*; -use super::Location; /// Spiller type, currently only supports HashJoin #[derive(Clone, Debug, Eq, PartialEq)] diff --git a/src/query/service/src/spillers/row_group_encoder.rs b/src/query/service/src/spillers/row_group_encoder.rs index f832c69e5865d..6f7b7c6b3dc20 100644 --- a/src/query/service/src/spillers/row_group_encoder.rs +++ b/src/query/service/src/spillers/row_group_encoder.rs @@ -19,30 +19,30 @@ use std::sync::Arc; use arrow_schema::Schema; use bytes::Bytes; -use databend_common_base::base::dma_buffer_to_bytes; use databend_common_base::base::DmaWriteBuf; use databend_common_base::base::SyncDmaFile; +use databend_common_base::base::dma_buffer_to_bytes; use databend_common_base::rangemap::RangeMerger; use databend_common_exception::Result; use databend_common_expression::BlockEntry; use databend_common_expression::DataBlock; use databend_common_expression::DataSchema; use databend_common_expression::Value; -use databend_common_storages_parquet::parquet_reader::RowGroupCore; use databend_common_storages_parquet::ReadSettings; +use databend_common_storages_parquet::parquet_reader::RowGroupCore; use databend_storages_common_cache::ParquetMetaData; use databend_storages_common_cache::TempDir; use databend_storages_common_cache::TempPath; use either::Either; use opendal::Operator; +use parquet::arrow::ArrowSchemaConverter; +use parquet::arrow::FieldLevels; +use parquet::arrow::ProjectionMask; use parquet::arrow::arrow_reader::ParquetRecordBatchReader; +use parquet::arrow::arrow_writer::ArrowColumnWriter; use parquet::arrow::arrow_writer::compute_leaves; use parquet::arrow::arrow_writer::get_column_writers; -use parquet::arrow::arrow_writer::ArrowColumnWriter; use parquet::arrow::parquet_to_arrow_field_levels; -use parquet::arrow::ArrowSchemaConverter; -use parquet::arrow::FieldLevels; -use parquet::arrow::ProjectionMask; use parquet::errors; use parquet::file::metadata::RowGroupMetaData; use parquet::file::metadata::RowGroupMetaDataPtr; @@ -53,11 +53,11 @@ use parquet::file::writer::SerializedFileWriter; use parquet::file::writer::SerializedRowGroupWriter; use parquet::schema::types::SchemaDescriptor; -use super::async_buffer::BufferWriter; -use super::async_buffer::SpillTarget; use super::Location; use super::SpillerInner; use super::SpillsBufferPool; +use super::async_buffer::BufferWriter; +use super::async_buffer::SpillTarget; pub struct Properties { schema: Arc, @@ -534,13 +534,13 @@ impl SpillerInner { #[cfg(test)] mod tests { use databend_common_exception::Result; - use databend_common_expression::types::array::ArrayColumnBuilder; - use databend_common_expression::types::number::Int32Type; + use databend_common_expression::Column; + use databend_common_expression::FromData; use databend_common_expression::types::ArgType; use databend_common_expression::types::DataType; use databend_common_expression::types::StringType; - use databend_common_expression::Column; - use databend_common_expression::FromData; + use databend_common_expression::types::array::ArrayColumnBuilder; + use databend_common_expression::types::number::Int32Type; use super::*; diff --git a/src/query/service/src/spillers/serialize.rs b/src/query/service/src/spillers/serialize.rs index 681e6e217367c..386bb731c60d9 100644 --- a/src/query/service/src/spillers/serialize.rs +++ b/src/query/service/src/spillers/serialize.rs @@ -25,18 +25,18 @@ use databend_common_base::base::Alignment; use databend_common_base::base::DmaWriteBuf; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::arrow::write_column; -use databend_common_expression::infer_table_schema; -use databend_common_expression::types::DataType; use databend_common_expression::BlockEntry; use databend_common_expression::DataBlock; use databend_common_expression::DataField; use databend_common_expression::DataSchema; use databend_common_expression::Value; +use databend_common_expression::arrow::write_column; +use databend_common_expression::infer_table_schema; +use databend_common_expression::types::DataType; use databend_storages_common_io::BufferReader; use opendal::Buffer; -use parquet::arrow::arrow_reader::ParquetRecordBatchReader; use parquet::arrow::ArrowWriter; +use parquet::arrow::arrow_reader::ParquetRecordBatchReader; use parquet::basic::Compression; use parquet::file::properties::EnabledStatistics; use parquet::file::properties::WriterProperties; @@ -199,10 +199,10 @@ fn bare_blocks_to_parquet( #[cfg(test)] mod tests { use bytes::Bytes; + use databend_common_expression::FromData; use databend_common_expression::block_debug::assert_block_value_eq; use databend_common_expression::types::Int64Type; use databend_common_expression::types::StringType; - use databend_common_expression::FromData; use super::*; diff --git a/src/query/service/src/spillers/test_memory.rs b/src/query/service/src/spillers/test_memory.rs index bacf0bdf9f5a8..e5143a2690b6d 100644 --- a/src/query/service/src/spillers/test_memory.rs +++ b/src/query/service/src/spillers/test_memory.rs @@ -15,19 +15,19 @@ use bytesize::ByteSize; use databend_common_base::base::GlobalInstance; use databend_common_base::mem_allocator::TrackingGlobalAllocator; -use databend_common_base::runtime::GlobalIORuntime; use databend_common_base::runtime::GLOBAL_MEM_STAT; +use databend_common_base::runtime::GlobalIORuntime; use databend_common_config::SpillConfig; use databend_common_exception::Result; -use databend_common_expression::types::decimal::DecimalSize; -use databend_common_expression::types::number::Int32Type; -use databend_common_expression::types::number::Int64Type; -use databend_common_expression::types::DateType; -use databend_common_expression::types::Decimal128Type; -use databend_common_expression::types::StringType; use databend_common_expression::Column; use databend_common_expression::DataBlock; use databend_common_expression::FromData; +use databend_common_expression::types::DateType; +use databend_common_expression::types::Decimal128Type; +use databend_common_expression::types::StringType; +use databend_common_expression::types::decimal::DecimalSize; +use databend_common_expression::types::number::Int32Type; +use databend_common_expression::types::number::Int64Type; use databend_common_pipeline_transforms::traits::DataBlockSpill; use databend_common_pipeline_transforms::traits::Location; use databend_common_storage::DataOperator; diff --git a/src/query/service/src/table_functions/async_crash_me.rs b/src/query/service/src/table_functions/async_crash_me.rs index 901b522da2c62..ca5d6ed1d20a7 100644 --- a/src/query/service/src/table_functions/async_crash_me.rs +++ b/src/query/service/src/table_functions/async_crash_me.rs @@ -13,10 +13,7 @@ // limitations under the License. use std::any::Any; -use std::pin::Pin; use std::sync::Arc; -use std::task::Context; -use std::task::Poll; use chrono::DateTime; use databend_common_catalog::plan::DataSourcePlan; @@ -38,7 +35,6 @@ use databend_common_pipeline::core::Pipeline; use databend_common_pipeline::core::ProcessorPtr; use databend_common_pipeline::sources::AsyncSource; use databend_common_pipeline::sources::AsyncSourcer; -use futures::Stream; use crate::sessions::TableContext; use crate::storages::Table; @@ -172,18 +168,3 @@ impl TableFunction for AsyncCrashMeTable { self } } - -struct AsyncCrashMeStream { - message: Option, -} - -impl Stream for AsyncCrashMeStream { - type Item = Result; - - fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - match &self.message { - None => panic!("async crash me panic"), - Some(message) => panic!("{}", message), - } - } -} diff --git a/src/query/service/src/table_functions/cloud/task_dependents.rs b/src/query/service/src/table_functions/cloud/task_dependents.rs index 768649ee669f9..c6d21ac830ea0 100644 --- a/src/query/service/src/table_functions/cloud/task_dependents.rs +++ b/src/query/service/src/table_functions/cloud/task_dependents.rs @@ -32,18 +32,18 @@ use databend_common_cloud_control::pb::Task; use databend_common_cloud_control::task_utils; use databend_common_config::GlobalConfig; use databend_common_exception::ErrorCode; -use databend_common_expression::types::ArrayType; -use databend_common_expression::types::DataType; -use databend_common_expression::types::ReturnType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::TimestampType; -use databend_common_expression::types::ValueType; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::ArrayType; +use databend_common_expression::types::DataType; +use databend_common_expression::types::ReturnType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::TimestampType; +use databend_common_expression::types::ValueType; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; diff --git a/src/query/service/src/table_functions/cloud/task_history.rs b/src/query/service/src/table_functions/cloud/task_history.rs index 55f8b052dd91f..d46821c047b6b 100644 --- a/src/query/service/src/table_functions/cloud/task_history.rs +++ b/src/query/service/src/table_functions/cloud/task_history.rs @@ -29,10 +29,10 @@ use databend_common_cloud_control::pb::ShowTaskRunsRequest; use databend_common_config::GlobalConfig; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::date_helper::DateConverter; -use databend_common_expression::infer_table_schema; use databend_common_expression::DataBlock; use databend_common_expression::Scalar; +use databend_common_expression::date_helper::DateConverter; +use databend_common_expression::infer_table_schema; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; diff --git a/src/query/service/src/table_functions/copy_history/copy_history_table.rs b/src/query/service/src/table_functions/copy_history/copy_history_table.rs index 1f456c71d8881..f98890f26fe28 100644 --- a/src/query/service/src/table_functions/copy_history/copy_history_table.rs +++ b/src/query/service/src/table_functions/copy_history/copy_history_table.rs @@ -25,10 +25,6 @@ use databend_common_catalog::table_args::TableArgs; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::TimestampType; -use databend_common_expression::types::UInt64Type; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::Scalar; @@ -36,6 +32,10 @@ use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::TimestampType; +use databend_common_expression::types::UInt64Type; use databend_common_meta_app::schema::TableCopiedFileInfo; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_types::MetaId; diff --git a/src/query/service/src/table_functions/fuse_vacuum2/fuse_vacuum2_table.rs b/src/query/service/src/table_functions/fuse_vacuum2/fuse_vacuum2_table.rs index be84dff9d7ae2..3547bea37b41a 100644 --- a/src/query/service/src/table_functions/fuse_vacuum2/fuse_vacuum2_table.rs +++ b/src/query/service/src/table_functions/fuse_vacuum2/fuse_vacuum2_table.rs @@ -22,24 +22,24 @@ use databend_common_catalog::table::TableExt; use databend_common_catalog::table_args::TableArgs; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::StringType; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::StringType; use databend_common_license::license::Feature::Vacuum; use databend_common_license::license_manager::LicenseManagerSwitch; +use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::table_functions::SimpleTableFunc; use databend_common_storages_fuse::table_functions::bool_literal; use databend_common_storages_fuse::table_functions::bool_value; use databend_common_storages_fuse::table_functions::parse_db_tb_args; use databend_common_storages_fuse::table_functions::string_literal; use databend_common_storages_fuse::table_functions::string_value; -use databend_common_storages_fuse::table_functions::SimpleTableFunc; -use databend_common_storages_fuse::FuseTable; -use databend_enterprise_vacuum_handler::get_vacuum_handler; use databend_enterprise_vacuum_handler::VacuumHandlerWrapper; +use databend_enterprise_vacuum_handler::get_vacuum_handler; use log::info; use log::warn; diff --git a/src/query/service/src/table_functions/infer_schema/infer_schema_table.rs b/src/query/service/src/table_functions/infer_schema/infer_schema_table.rs index 7642693c97693..1c9113707d35d 100644 --- a/src/query/service/src/table_functions/infer_schema/infer_schema_table.rs +++ b/src/query/service/src/table_functions/infer_schema/infer_schema_table.rs @@ -30,12 +30,12 @@ use databend_common_catalog::table_args::TableArgs; use databend_common_compress::CompressAlgorithm; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::NumberDataType; use databend_common_expression::BlockThresholds; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::NumberDataType; use databend_common_meta_app::principal::FileFormatParams; use databend_common_meta_app::principal::StageInfo; use databend_common_meta_app::principal::StageType; @@ -46,21 +46,21 @@ use databend_common_pipeline::core::Pipeline; use databend_common_pipeline::sources::PrefetchAsyncSourcer; use databend_common_pipeline_transforms::TransformPipelineHelper; use databend_common_sql::binder::resolve_file_location; -use databend_common_storage::init_stage_operator; +use databend_common_storage::Scheme; use databend_common_storage::StageFilesInfo; +use databend_common_storage::init_stage_operator; use databend_common_storages_stage::BytesReader; use databend_common_storages_stage::Decompressor; use databend_common_storages_stage::InferSchemaPartInfo; use databend_common_storages_stage::LoadContext; use databend_common_users::Object; use databend_storages_common_stage::SingleFilePartition; -use opendal::Scheme; use super::parquet::ParquetInferSchemaSource; use crate::sessions::TableContext; +use crate::table_functions::TableFunction; use crate::table_functions::infer_schema::separator::InferSchemaSeparator; use crate::table_functions::infer_schema::table_args::InferSchemaArgsParsed; -use crate::table_functions::TableFunction; pub(crate) const INFER_SCHEMA: &str = "infer_schema"; diff --git a/src/query/service/src/table_functions/infer_schema/merge.rs b/src/query/service/src/table_functions/infer_schema/merge.rs index 5aa78e263a7cb..708d155ee2116 100644 --- a/src/query/service/src/table_functions/infer_schema/merge.rs +++ b/src/query/service/src/table_functions/infer_schema/merge.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_expression::types::NumberDataType; use databend_common_expression::TableDataType; use databend_common_expression::TableSchema; +use databend_common_expression::types::NumberDataType; const UNSIGNED_TYPES: [NumberDataType; 4] = [ NumberDataType::UInt8, @@ -31,11 +31,7 @@ const SIGNED_TYPES: [NumberDataType; 4] = [ const FLOAT_TYPES: [NumberDataType; 2] = [NumberDataType::Float32, NumberDataType::Float64]; fn wrap_nullable(ty: TableDataType, is_nullable: bool) -> TableDataType { - if is_nullable { - ty.wrap_nullable() - } else { - ty - } + if is_nullable { ty.wrap_nullable() } else { ty } } pub fn merge_type( @@ -115,10 +111,10 @@ pub fn merge_schema(defined: TableSchema, guess: TableSchema) -> TableSchema { #[cfg(test)] mod tests { - use databend_common_expression::types::NumberDataType; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; + use databend_common_expression::types::NumberDataType; use crate::table_functions::infer_schema::merge::merge_schema; use crate::table_functions::infer_schema::merge::merge_type; diff --git a/src/query/service/src/table_functions/infer_schema/parquet.rs b/src/query/service/src/table_functions/infer_schema/parquet.rs index 49dd4cd54efcf..0734464a46cd4 100644 --- a/src/query/service/src/table_functions/infer_schema/parquet.rs +++ b/src/query/service/src/table_functions/infer_schema/parquet.rs @@ -17,20 +17,20 @@ use std::sync::Arc; use arrow_schema::Schema; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::BooleanType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::UInt64Type; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::TableSchema; +use databend_common_expression::types::BooleanType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::UInt64Type; use databend_common_meta_app::principal::StageInfo; use databend_common_pipeline::core::OutputPort; use databend_common_pipeline::core::ProcessorPtr; use databend_common_pipeline::sources::AsyncSource; use databend_common_pipeline::sources::AsyncSourcer; +use databend_common_storage::StageFileInfo; use databend_common_storage::init_stage_operator; use databend_common_storage::read_parquet_schema_async_rs; -use databend_common_storage::StageFileInfo; use futures_util::future::try_join_all; use itertools::Itertools; diff --git a/src/query/service/src/table_functions/infer_schema/separator.rs b/src/query/service/src/table_functions/infer_schema/separator.rs index b5607f0a3b9f1..22bf8759f4adb 100644 --- a/src/query/service/src/table_functions/infer_schema/separator.rs +++ b/src/query/service/src/table_functions/infer_schema/separator.rs @@ -16,19 +16,19 @@ use std::collections::HashMap; use std::io::Cursor; use arrow_csv::reader::Format; -use arrow_json::reader::infer_json_schema_from_iterator; use arrow_json::reader::ValueIter; +use arrow_json::reader::infer_json_schema_from_iterator; use arrow_schema::ArrowError; use arrow_schema::Schema; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::BooleanType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::UInt64Type; use databend_common_expression::BlockMetaInfoDowncast; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::TableSchema; +use databend_common_expression::types::BooleanType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::UInt64Type; use databend_common_meta_app::principal::FileFormatParams; use databend_common_pipeline_transforms::AccumulatingTransform; use databend_common_storages_stage::BytesBatch; diff --git a/src/query/service/src/table_functions/infer_schema/table_args.rs b/src/query/service/src/table_functions/infer_schema/table_args.rs index 9781bc742ee4b..6a99b717dc17e 100644 --- a/src/query/service/src/table_functions/infer_schema/table_args.rs +++ b/src/query/service/src/table_functions/infer_schema/table_args.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_catalog::table_args::i64_value; use databend_common_catalog::table_args::TableArgs; +use databend_common_catalog::table_args::i64_value; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_storage::StageFilesInfo; diff --git a/src/query/service/src/table_functions/inspect_parquet/inspect_parquet_table.rs b/src/query/service/src/table_functions/inspect_parquet/inspect_parquet_table.rs index 448c10c541437..64c27961d3a22 100644 --- a/src/query/service/src/table_functions/inspect_parquet/inspect_parquet_table.rs +++ b/src/query/service/src/table_functions/inspect_parquet/inspect_parquet_table.rs @@ -24,16 +24,16 @@ use databend_common_catalog::table::Table; use databend_common_catalog::table_args::TableArgs; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::Int64Type; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::UInt64Type; use databend_common_expression::BlockEntry; use databend_common_expression::DataBlock; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::Int64Type; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::UInt64Type; use databend_common_meta_app::principal::StageType; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; @@ -44,9 +44,9 @@ use databend_common_pipeline::core::ProcessorPtr; use databend_common_pipeline::sources::AsyncSource; use databend_common_pipeline::sources::AsyncSourcer; use databend_common_sql::binder::resolve_stage_location; +use databend_common_storage::StageFilesInfo; use databend_common_storage::init_stage_operator; use databend_common_storage::read_metadata_async; -use databend_common_storage::StageFilesInfo; use databend_common_storages_fuse::table_functions::string_literal; use databend_common_users::Object; diff --git a/src/query/service/src/table_functions/list_stage/list_stage_table.rs b/src/query/service/src/table_functions/list_stage/list_stage_table.rs index ecfdce67eb4f2..bcd2fbe52c15c 100644 --- a/src/query/service/src/table_functions/list_stage/list_stage_table.rs +++ b/src/query/service/src/table_functions/list_stage/list_stage_table.rs @@ -25,15 +25,15 @@ use databend_common_catalog::table_context::TableContext; use databend_common_catalog::table_function::TableFunction; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::UInt64Type; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::UInt64Type; use databend_common_meta_app::principal::StageType; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; @@ -49,8 +49,8 @@ use databend_common_storage::StageFileInfoStream; use databend_common_storage::StageFilesInfo; use databend_common_storages_stage::StageTable; use databend_common_users::Object; -use futures_util::stream::Chunks; use futures_util::StreamExt; +use futures_util::stream::Chunks; use crate::table_functions::list_stage::table_args::ListStageArgsParsed; diff --git a/src/query/service/src/table_functions/mod.rs b/src/query/service/src/table_functions/mod.rs index dbf09caea6666..dd344471970d1 100644 --- a/src/query/service/src/table_functions/mod.rs +++ b/src/query/service/src/table_functions/mod.rs @@ -35,15 +35,15 @@ mod temporary_tables_table; mod udf_table; pub use copy_history::CopyHistoryTable; -pub use numbers::generate_numbers_parts; pub use numbers::NumbersPartInfo; pub use numbers::NumbersTable; +pub use numbers::generate_numbers_parts; pub use others::LicenseInfoTable; pub use others::TenantQuotaTable; pub use policy_references::PolicyReferencesTable; +pub use system::TableStatisticsFunc; pub use system::get_fuse_table_snapshot; pub use system::get_fuse_table_statistics; -pub use system::TableStatisticsFunc; pub use table_function::TableFunction; pub use table_function_factory::TableFunctionFactory; pub use temporary_tables_table::TemporaryTablesTable; diff --git a/src/query/service/src/table_functions/numbers/mod.rs b/src/query/service/src/table_functions/numbers/mod.rs index 571b92e9fee55..9cb9f72eada21 100644 --- a/src/query/service/src/table_functions/numbers/mod.rs +++ b/src/query/service/src/table_functions/numbers/mod.rs @@ -15,6 +15,6 @@ mod numbers_part; mod numbers_table; -pub use numbers_part::generate_numbers_parts; pub use numbers_part::NumbersPartInfo; +pub use numbers_part::generate_numbers_parts; pub use numbers_table::NumbersTable; diff --git a/src/query/service/src/table_functions/numbers/numbers_table.rs b/src/query/service/src/table_functions/numbers/numbers_table.rs index 9b013862db54b..0c2ab5432bf96 100644 --- a/src/query/service/src/table_functions/numbers/numbers_table.rs +++ b/src/query/service/src/table_functions/numbers/numbers_table.rs @@ -26,11 +26,6 @@ use databend_common_catalog::table::DistributionLevel; use databend_common_catalog::table::TableStatistics; use databend_common_catalog::table_args::TableArgs; use databend_common_exception::Result; -use databend_common_expression::type_check::check_number; -use databend_common_expression::types::number::NumberScalar; -use databend_common_expression::types::number::UInt64Type; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::utils::FromData; use databend_common_expression::DataBlock; use databend_common_expression::Expr; use databend_common_expression::FunctionContext; @@ -38,6 +33,11 @@ use databend_common_expression::Scalar; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::type_check::check_number; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::number::NumberScalar; +use databend_common_expression::types::number::UInt64Type; +use databend_common_expression::utils::FromData; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; @@ -51,8 +51,8 @@ use databend_common_pipeline::sources::SyncSource; use databend_common_pipeline::sources::SyncSourcer; use databend_storages_common_table_meta::table::ChangeType; -use super::numbers_part::generate_numbers_parts; use super::NumbersPartInfo; +use super::numbers_part::generate_numbers_parts; use crate::sessions::TableContext; use crate::storages::Table; use crate::table_functions::TableFunction; diff --git a/src/query/service/src/table_functions/others/license_info.rs b/src/query/service/src/table_functions/others/license_info.rs index 2a09534e36543..e8a8955e626f1 100644 --- a/src/query/service/src/table_functions/others/license_info.rs +++ b/src/query/service/src/table_functions/others/license_info.rs @@ -28,14 +28,14 @@ use databend_common_catalog::table_function::TableFunction; use databend_common_exception::ErrorCode; pub use databend_common_exception::Result; use databend_common_exception::ToErrorCode; -use databend_common_expression::types::StringType; -use databend_common_expression::types::TimestampType; use databend_common_expression::BlockEntry; use databend_common_expression::DataBlock; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::StringType; +use databend_common_expression::types::TimestampType; use databend_common_license::license::Feature; use databend_common_license::license::LicenseInfo; use databend_common_license::license_manager::LicenseManagerSwitch; diff --git a/src/query/service/src/table_functions/others/tenant_quota.rs b/src/query/service/src/table_functions/others/tenant_quota.rs index a722644fb2931..6d56313a21647 100644 --- a/src/query/service/src/table_functions/others/tenant_quota.rs +++ b/src/query/service/src/table_functions/others/tenant_quota.rs @@ -25,8 +25,6 @@ use databend_common_catalog::table_context::TableContext; use databend_common_catalog::table_function::TableFunction; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::UInt32Type; use databend_common_expression::BlockEntry; use databend_common_expression::DataBlock; use databend_common_expression::Scalar; @@ -34,6 +32,8 @@ use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::UInt32Type; use databend_common_meta_app::principal::UserOptionFlag; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; diff --git a/src/query/service/src/table_functions/others/udf.rs b/src/query/service/src/table_functions/others/udf.rs index 6412b30dce85e..35390653c657f 100644 --- a/src/query/service/src/table_functions/others/udf.rs +++ b/src/query/service/src/table_functions/others/udf.rs @@ -33,9 +33,6 @@ use databend_common_config::GlobalConfig; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_exception::ToErrorCode; -use databend_common_expression::types::DataType; -use databend_common_expression::types::StringType; -use databend_common_expression::udf_client::UDFFlightClient; use databend_common_expression::BlockEntry; use databend_common_expression::DataBlock; use databend_common_expression::FromData; @@ -44,6 +41,9 @@ use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::DataType; +use databend_common_expression::types::StringType; +use databend_common_expression::udf_client::UDFFlightClient; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; diff --git a/src/query/service/src/table_functions/policy_references/policy_references_table.rs b/src/query/service/src/table_functions/policy_references/policy_references_table.rs index c01ae67908daf..94419a7e166eb 100644 --- a/src/query/service/src/table_functions/policy_references/policy_references_table.rs +++ b/src/query/service/src/table_functions/policy_references/policy_references_table.rs @@ -21,13 +21,12 @@ use databend_common_catalog::plan::PartStatistics; use databend_common_catalog::plan::Partitions; use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table::Table; -use databend_common_catalog::table_args::string_value; use databend_common_catalog::table_args::TableArgs; +use databend_common_catalog::table_args::string_value; use databend_common_catalog::table_context::TableContext; use databend_common_catalog::table_function::TableFunction; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::StringType; use databend_common_expression::DataBlock; use databend_common_expression::DataSchema; use databend_common_expression::DataSchemaRef; @@ -36,23 +35,24 @@ use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::StringType; use databend_common_meta_api::kv_pb_api::KVPbApi; use databend_common_meta_api::name_id_value_api::NameIdValueApi; use databend_common_meta_app::data_mask::DataMaskNameIdent; use databend_common_meta_app::data_mask::MaskPolicyIdTableId; use databend_common_meta_app::data_mask::MaskPolicyTableIdIdent; -use databend_common_meta_app::row_access_policy::row_access_policy_table_id_ident::RowAccessPolicyIdTableId; use databend_common_meta_app::row_access_policy::RowAccessPolicyNameIdent; use databend_common_meta_app::row_access_policy::RowAccessPolicyTableIdIdent; +use databend_common_meta_app::row_access_policy::row_access_policy_table_id_ident::RowAccessPolicyIdTableId; use databend_common_meta_app::schema::SecurityPolicyColumnMap; use databend_common_meta_app::schema::TableIdToName; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; use databend_common_meta_kvapi::kvapi::DirName; -use databend_common_pipeline::core::processor::ProcessorPtr; use databend_common_pipeline::core::OutputPort; use databend_common_pipeline::core::Pipeline; +use databend_common_pipeline::core::processor::ProcessorPtr; use databend_common_pipeline::sources::AsyncSource; use databend_common_pipeline::sources::AsyncSourcer; use databend_common_users::UserApiProvider; @@ -289,7 +289,7 @@ async fn collect_policy_reference_rows( return Err(ErrorCode::BadArguments(format!( "Invalid REF_ENTITY_DOMAIN '{}'. Expected TABLE or VIEW", raw - ))) + ))); } } } diff --git a/src/query/service/src/table_functions/show_grants/show_grants_table.rs b/src/query/service/src/table_functions/show_grants/show_grants_table.rs index 08fed04a67d56..d10c48bf2a792 100644 --- a/src/query/service/src/table_functions/show_grants/show_grants_table.rs +++ b/src/query/service/src/table_functions/show_grants/show_grants_table.rs @@ -28,7 +28,6 @@ use databend_common_catalog::table_context::TableContext; use databend_common_catalog::table_function::TableFunction; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::StringType; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::Scalar; @@ -36,6 +35,7 @@ use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::StringType; use databend_common_management::RoleApi; use databend_common_management::UserApi; use databend_common_management::WarehouseInfo; @@ -56,10 +56,10 @@ use databend_common_pipeline::core::ProcessorPtr; use databend_common_pipeline::sources::AsyncSource; use databend_common_pipeline::sources::AsyncSourcer; use databend_common_sql::validate_function_arg; +use databend_common_users::BUILTIN_ROLE_ACCOUNT_ADMIN; use databend_common_users::Object; use databend_common_users::RoleCacheManager; use databend_common_users::UserApiProvider; -use databend_common_users::BUILTIN_ROLE_ACCOUNT_ADMIN; use databend_enterprise_resources_management::ResourcesManagement; use itertools::Itertools; diff --git a/src/query/service/src/table_functions/show_roles/show_roles_table.rs b/src/query/service/src/table_functions/show_roles/show_roles_table.rs index 94eb190d3a7d7..b6c8b456febd8 100644 --- a/src/query/service/src/table_functions/show_roles/show_roles_table.rs +++ b/src/query/service/src/table_functions/show_roles/show_roles_table.rs @@ -24,16 +24,16 @@ use databend_common_catalog::table_args::TableArgs; use databend_common_catalog::table_context::TableContext; use databend_common_catalog::table_function::TableFunction; use databend_common_exception::Result; -use databend_common_expression::types::BooleanType; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::UInt64Type; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::BooleanType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::UInt64Type; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; diff --git a/src/query/service/src/table_functions/show_sequences/show_sequences_table.rs b/src/query/service/src/table_functions/show_sequences/show_sequences_table.rs index b4e29376d24d4..0558a18762b2f 100644 --- a/src/query/service/src/table_functions/show_sequences/show_sequences_table.rs +++ b/src/query/service/src/table_functions/show_sequences/show_sequences_table.rs @@ -24,17 +24,17 @@ use databend_common_catalog::table_args::TableArgs; use databend_common_catalog::table_context::TableContext; use databend_common_catalog::table_function::TableFunction; use databend_common_exception::Result; -use databend_common_expression::types::Int64Type; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::TimestampType; -use databend_common_expression::types::UInt64Type; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::Int64Type; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::TimestampType; +use databend_common_expression::types::UInt64Type; use databend_common_meta_app::schema::ListSequencesReq; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; diff --git a/src/query/service/src/table_functions/show_variables/show_variables_table.rs b/src/query/service/src/table_functions/show_variables/show_variables_table.rs index b865676e3a389..f2c5c865704d1 100644 --- a/src/query/service/src/table_functions/show_variables/show_variables_table.rs +++ b/src/query/service/src/table_functions/show_variables/show_variables_table.rs @@ -24,13 +24,13 @@ use databend_common_catalog::table_args::TableArgs; use databend_common_catalog::table_context::TableContext; use databend_common_catalog::table_function::TableFunction; use databend_common_exception::Result; -use databend_common_expression::types::StringType; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::StringType; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; diff --git a/src/query/service/src/table_functions/srf/range.rs b/src/query/service/src/table_functions/srf/range.rs index e3e4f31dc965e..f75b25ef58d00 100644 --- a/src/query/service/src/table_functions/srf/range.rs +++ b/src/query/service/src/table_functions/srf/range.rs @@ -25,9 +25,6 @@ use databend_common_catalog::table_args::TableArgs; use databend_common_catalog::table_function::TableFunction; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::infer_schema_type; -use databend_common_expression::type_check::check_number; -use databend_common_expression::types::*; use databend_common_expression::DataBlock; use databend_common_expression::Expr; use databend_common_expression::FromData; @@ -35,6 +32,9 @@ use databend_common_expression::FunctionContext; use databend_common_expression::Scalar; use databend_common_expression::TableField; use databend_common_expression::TableSchema; +use databend_common_expression::infer_schema_type; +use databend_common_expression::type_check::check_number; +use databend_common_expression::types::*; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; diff --git a/src/query/service/src/table_functions/sync_crash_me.rs b/src/query/service/src/table_functions/sync_crash_me.rs index febb5488aa1a4..ea72d2cded4e6 100644 --- a/src/query/service/src/table_functions/sync_crash_me.rs +++ b/src/query/service/src/table_functions/sync_crash_me.rs @@ -13,10 +13,7 @@ // limitations under the License. use std::any::Any; -use std::pin::Pin; use std::sync::Arc; -use std::task::Context; -use std::task::Poll; use chrono::DateTime; use databend_common_base::base::Progress; @@ -38,7 +35,6 @@ use databend_common_pipeline::core::Pipeline; use databend_common_pipeline::core::ProcessorPtr; use databend_common_pipeline::sources::SyncSource; use databend_common_pipeline::sources::SyncSourcer; -use futures::Stream; use crate::sessions::TableContext; use crate::storages::Table; @@ -175,18 +171,3 @@ impl TableFunction for SyncCrashMeTable { self } } - -struct SyncCrashMeStream { - message: Option, -} - -impl Stream for SyncCrashMeStream { - type Item = Result; - - fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - match &self.message { - None => panic!("sync crash me panic"), - Some(message) => panic!("{}", message), - } - } -} diff --git a/src/query/service/src/table_functions/system/mod.rs b/src/query/service/src/table_functions/system/mod.rs index 7ed125b59678d..160bd5abc7bb8 100644 --- a/src/query/service/src/table_functions/system/mod.rs +++ b/src/query/service/src/table_functions/system/mod.rs @@ -13,6 +13,6 @@ // limitations under the License. mod table_statistics; +pub use table_statistics::TableStatisticsFunc; pub use table_statistics::get_fuse_table_snapshot; pub use table_statistics::get_fuse_table_statistics; -pub use table_statistics::TableStatisticsFunc; diff --git a/src/query/service/src/table_functions/system/table_statistics.rs b/src/query/service/src/table_functions/system/table_statistics.rs index 67a809e56f9a4..de206f306434c 100644 --- a/src/query/service/src/table_functions/system/table_statistics.rs +++ b/src/query/service/src/table_functions/system/table_statistics.rs @@ -16,23 +16,23 @@ use std::sync::Arc; use databend_common_catalog::catalog_kind::CATALOG_DEFAULT; use databend_common_catalog::plan::DataSourcePlan; -use databend_common_catalog::table_args::string_value; use databend_common_catalog::table_args::TableArgs; +use databend_common_catalog::table_args::string_value; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::StringType; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRefExt; -use databend_common_storages_fuse::io::read::SnapshotHistoryReader; +use databend_common_expression::types::StringType; +use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::io::MetaReaders; use databend_common_storages_fuse::io::TableMetaLocationGenerator; -use databend_common_storages_fuse::table_functions::string_literal; +use databend_common_storages_fuse::io::read::SnapshotHistoryReader; use databend_common_storages_fuse::table_functions::SimpleTableFunc; -use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::table_functions::string_literal; use databend_storages_common_table_meta::meta::TableSnapshot; use futures::stream::StreamExt; use log::warn; diff --git a/src/query/service/src/table_functions/table_function_factory.rs b/src/query/service/src/table_functions/table_function_factory.rs index 0c2f6ae6f9731..53a9f7a860aef 100644 --- a/src/query/service/src/table_functions/table_function_factory.rs +++ b/src/query/service/src/table_functions/table_function_factory.rs @@ -41,12 +41,13 @@ use databend_storages_common_table_meta::table_id_ranges::SYS_TBL_FUNC_ID_BEGIN; use itertools::Itertools; use parking_lot::RwLock; -use super::others::UdfEchoTable; use super::LicenseInfoTable; use super::TenantQuotaTable; +use super::others::UdfEchoTable; use crate::storages::fuse::table_functions::ClusteringInformationFunc; use crate::storages::fuse::table_functions::FuseSegmentFunc; use crate::storages::fuse::table_functions::FuseSnapshotFunc; +use crate::table_functions::TableFunction; use crate::table_functions::async_crash_me::AsyncCrashMeTable; use crate::table_functions::cloud::TaskDependentsEnableTable; use crate::table_functions::cloud::TaskDependentsTable; @@ -65,7 +66,6 @@ use crate::table_functions::show_variables::ShowVariables; use crate::table_functions::srf::RangeTable; use crate::table_functions::sync_crash_me::SyncCrashMeTable; use crate::table_functions::system::TableStatisticsFunc; -use crate::table_functions::TableFunction; type TableFunctionCreators = RwLock)>>; pub trait TableFunctionCreator: Send + Sync { diff --git a/src/query/service/src/table_functions/temporary_tables_table.rs b/src/query/service/src/table_functions/temporary_tables_table.rs index 0d768880009ac..b92be36b92056 100644 --- a/src/query/service/src/table_functions/temporary_tables_table.rs +++ b/src/query/service/src/table_functions/temporary_tables_table.rs @@ -19,18 +19,18 @@ use databend_common_catalog::table::DistributionLevel; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::BooleanType; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::TimestampType; -use databend_common_expression::types::UInt64Type; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::Scalar; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::BooleanType; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::TimestampType; +use databend_common_expression::types::UInt64Type; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; diff --git a/src/query/service/src/table_functions/udf_table.rs b/src/query/service/src/table_functions/udf_table.rs index 854720f1aec08..45aeff1379924 100644 --- a/src/query/service/src/table_functions/udf_table.rs +++ b/src/query/service/src/table_functions/udf_table.rs @@ -26,12 +26,12 @@ use databend_common_catalog::table_context::TableContext; use databend_common_catalog::table_function::TableFunction; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::cast_scalar; -use databend_common_expression::infer_schema_type; -use databend_common_expression::types::DataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::cast_scalar; +use databend_common_expression::infer_schema_type; +use databend_common_expression::types::DataType; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::principal::StageType; use databend_common_meta_app::principal::UDTFServer; @@ -39,8 +39,8 @@ use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::storage::StorageParams; use databend_common_pipeline::core::Pipeline; use databend_common_pipeline::sources::AsyncSourcer; -use databend_common_sql::binder::resolve_stage_location; use databend_common_sql::StageLocationParam; +use databend_common_sql::binder::resolve_stage_location; use crate::pipelines::builders::UdtfFunctionDesc; use crate::pipelines::builders::UdtfServerSource; diff --git a/src/query/service/src/task/meta.rs b/src/query/service/src/task/meta.rs index 7271a8774341d..5672b9f4d3139 100644 --- a/src/query/service/src/task/meta.rs +++ b/src/query/service/src/task/meta.rs @@ -20,8 +20,8 @@ use databend_common_exception::Result; use databend_common_meta_client::ClientHandle; use databend_common_meta_kvapi::kvapi::KVApi; use databend_common_meta_kvapi::kvapi::KvApiExt; -use databend_common_meta_semaphore::acquirer::Permit; use databend_common_meta_semaphore::Semaphore; +use databend_common_meta_semaphore::acquirer::Permit; use databend_common_meta_types::MatchSeq; use databend_common_meta_types::Operation; use databend_common_meta_types::UpsertKV; diff --git a/src/query/service/src/task/service.rs b/src/query/service/src/task/service.rs index 4fc406540a283..aac7a07a3f1d5 100644 --- a/src/query/service/src/task/service.rs +++ b/src/query/service/src/task/service.rs @@ -19,9 +19,9 @@ use std::collections::BTreeMap; use std::collections::HashMap; use std::ops::Deref; use std::str::FromStr; +use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; -use std::sync::Arc; use std::time::Duration; use async_stream::stream; @@ -40,10 +40,6 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::DataBlock; use databend_common_meta_api::kv_pb_api::decode_seqv; -use databend_common_meta_app::principal::task::TaskMessage; -use databend_common_meta_app::principal::task::TaskMessageType; -use databend_common_meta_app::principal::task::EMPTY_TASK_ID; -use databend_common_meta_app::principal::task_message_ident::TaskMessageIdent; use databend_common_meta_app::principal::ScheduleOptions; use databend_common_meta_app::principal::ScheduleType; use databend_common_meta_app::principal::State; @@ -53,19 +49,23 @@ use databend_common_meta_app::principal::TaskRun; use databend_common_meta_app::principal::UserIdentity; use databend_common_meta_app::principal::UserInfo; use databend_common_meta_app::principal::WarehouseOptions; +use databend_common_meta_app::principal::task::EMPTY_TASK_ID; +use databend_common_meta_app::principal::task::TaskMessage; +use databend_common_meta_app::principal::task::TaskMessageType; +use databend_common_meta_app::principal::task_message_ident::TaskMessageIdent; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_kvapi::kvapi::Key; use databend_common_meta_store::MetaStoreProvider; +use databend_common_meta_types::MetaError; use databend_common_meta_types::protobuf::WatchRequest; use databend_common_meta_types::protobuf::WatchResponse; -use databend_common_meta_types::MetaError; use databend_common_sql::Planner; -use databend_common_users::UserApiProvider; use databend_common_users::BUILTIN_ROLE_ACCOUNT_ADMIN; +use databend_common_users::UserApiProvider; use databend_common_version::BUILD_INFO; use futures::Stream; -use futures_util::stream::BoxStream; use futures_util::TryStreamExt; +use futures_util::stream::BoxStream; use itertools::Itertools; use log::error; use tokio::time::sleep; diff --git a/src/query/service/src/test_kits/block_writer.rs b/src/query/service/src/test_kits/block_writer.rs index f9a7ee9f12918..a11e8e9d55b36 100644 --- a/src/query/service/src/test_kits/block_writer.rs +++ b/src/query/service/src/test_kits/block_writer.rs @@ -21,16 +21,15 @@ use databend_common_io::constants::DEFAULT_BLOCK_BUFFER_SIZE; use databend_common_io::constants::DEFAULT_BLOCK_INDEX_BUFFER_SIZE; use databend_common_sql::ApproxDistinctColumns; use databend_common_sql::BloomIndexColumns; -use databend_common_storages_fuse::io::build_column_hlls; -use databend_common_storages_fuse::io::serialize_block; +use databend_common_storages_fuse::FuseStorageFormat; use databend_common_storages_fuse::io::TableMetaLocationGenerator; use databend_common_storages_fuse::io::WriteSettings; -use databend_common_storages_fuse::FuseStorageFormat; +use databend_common_storages_fuse::io::build_column_hlls; +use databend_common_storages_fuse::io::serialize_block; use databend_storages_common_blocks::blocks_to_parquet; use databend_storages_common_index::BloomIndex; use databend_storages_common_index::BloomIndexBuilder; use databend_storages_common_index::RangeIndex; -use databend_storages_common_table_meta::meta::encode_column_hll; use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::ClusterStatistics; use databend_storages_common_table_meta::meta::Compression; @@ -38,6 +37,7 @@ use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::RawBlockHLL; use databend_storages_common_table_meta::meta::StatisticsOfColumns; use databend_storages_common_table_meta::meta::TableMetaTimestamps; +use databend_storages_common_table_meta::meta::encode_column_hll; use databend_storages_common_table_meta::table::TableCompression; use opendal::Operator; use parquet::format::FileMetaData; diff --git a/src/query/service/src/test_kits/check.rs b/src/query/service/src/test_kits/check.rs index 418c9a22d917d..5812529a70749 100644 --- a/src/query/service/src/test_kits/check.rs +++ b/src/query/service/src/test_kits/check.rs @@ -17,18 +17,18 @@ use std::str; use databend_common_catalog::table::Table; use databend_common_config::GlobalConfig; use databend_common_exception::Result; -use databend_common_expression::block_debug::assert_blocks_sorted_eq_with_name; use databend_common_expression::DataBlock; use databend_common_expression::SendableDataBlockStream; +use databend_common_expression::block_debug::assert_blocks_sorted_eq_with_name; use databend_common_meta_app::storage::StorageParams; -use databend_common_storages_fuse::operations::load_last_snapshot_hint; -use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::FUSE_TBL_BLOCK_PREFIX; use databend_common_storages_fuse::FUSE_TBL_SEGMENT_PREFIX; use databend_common_storages_fuse::FUSE_TBL_SEGMENT_STATISTICS_PREFIX; use databend_common_storages_fuse::FUSE_TBL_SNAPSHOT_PREFIX; use databend_common_storages_fuse::FUSE_TBL_SNAPSHOT_STATISTICS_PREFIX; use databend_common_storages_fuse::FUSE_TBL_XOR_BLOOM_INDEX_PREFIX; +use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::operations::load_last_snapshot_hint; use futures::TryStreamExt; use walkdir::WalkDir; @@ -192,9 +192,11 @@ pub async fn check_data_dir( "ts_location_opt: {:?}, table_statistic_files: {:?}", ts_location, table_statistic_files ); - assert!(table_statistic_files - .iter() - .any(|e| e.contains(&ts_location))); + assert!( + table_statistic_files + .iter() + .any(|e| e.contains(&ts_location)) + ); } Ok(()) diff --git a/src/query/service/src/test_kits/context.rs b/src/query/service/src/test_kits/context.rs index f95ea84e9094c..9d0bb98a1ff4c 100644 --- a/src/query/service/src/test_kits/context.rs +++ b/src/query/service/src/test_kits/context.rs @@ -22,9 +22,9 @@ use databend_common_expression::SendableDataBlockStream; use futures::TryStreamExt; use crate::interpreters::InterpreterFactory; +use crate::pipelines::PipelineBuildResult; use crate::pipelines::executor::ExecutorSettings; use crate::pipelines::executor::PipelineCompleteExecutor; -use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; use crate::sql::Planner; diff --git a/src/query/service/src/test_kits/fixture.rs b/src/query/service/src/test_kits/fixture.rs index 9c14641aefbd0..cf7af262f6c6b 100644 --- a/src/query/service/src/test_kits/fixture.rs +++ b/src/query/service/src/test_kits/fixture.rs @@ -24,14 +24,6 @@ use databend_common_catalog::cluster_info::Cluster; use databend_common_catalog::session_type::SessionType; use databend_common_config::InnerConfig; use databend_common_exception::Result; -use databend_common_expression::infer_table_schema; -use databend_common_expression::types::binary::BinaryColumnBuilder; -use databend_common_expression::types::number::Int32Type; -use databend_common_expression::types::number::Int64Type; -use databend_common_expression::types::string::StringColumnBuilder; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; use databend_common_expression::Column; use databend_common_expression::ComputedExpr; use databend_common_expression::DataBlock; @@ -44,6 +36,14 @@ use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::infer_table_schema; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::binary::BinaryColumnBuilder; +use databend_common_expression::types::number::Int32Type; +use databend_common_expression::types::number::Int64Type; +use databend_common_expression::types::string::StringColumnBuilder; use databend_common_license::license_manager::LicenseManager; use databend_common_license::license_manager::OssLicenseManager; use databend_common_meta_app::principal::AuthInfo; @@ -72,6 +72,7 @@ use jsonb::Value as JsonbValue; use log::info; use uuid::Uuid; +use crate::GlobalServices; use crate::clusters::ClusterDiscovery; use crate::clusters::ClusterHelper; use crate::interpreters::CreateTableInterpreter; @@ -86,10 +87,9 @@ use crate::sessions::SessionManager; use crate::sessions::TableContext; use crate::sql::Planner; use crate::storages::Table; -use crate::test_kits::execute_pipeline; use crate::test_kits::ClusterDescriptor; use crate::test_kits::ConfigBuilder; -use crate::GlobalServices; +use crate::test_kits::execute_pipeline; pub struct TestFixture { pub(crate) default_ctx: Arc, @@ -264,7 +264,7 @@ impl TestFixture { async fn init_global_with_config(config: &InnerConfig) -> Result<()> { let version = &BUILD_INFO; set_panic_hook(version.commit_detail.clone()); - std::env::set_var("UNIT_TEST", "TRUE"); + unsafe { std::env::set_var("UNIT_TEST", "TRUE") }; #[cfg(debug_assertions)] { diff --git a/src/query/service/src/test_kits/fuse.rs b/src/query/service/src/test_kits/fuse.rs index 9ec4d6eb57827..f78714a52a077 100644 --- a/src/query/service/src/test_kits/fuse.rs +++ b/src/query/service/src/test_kits/fuse.rs @@ -20,27 +20,24 @@ use chrono::DateTime; use chrono::Duration; use chrono::Utc; use databend_common_exception::Result; -use databend_common_expression::types::NumberScalar; use databend_common_expression::BlockThresholds; use databend_common_expression::DataBlock; use databend_common_expression::DataSchemaRef; use databend_common_expression::ScalarRef; use databend_common_expression::SendableDataBlockStream; +use databend_common_expression::types::NumberScalar; use databend_common_sql::optimizer::ir::SExpr; use databend_common_sql::plans::Mutation; use databend_common_storages_factory::Table; +use databend_common_storages_fuse::FUSE_TBL_SEGMENT_PREFIX; +use databend_common_storages_fuse::FuseStorageFormat; +use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::io::MetaWriter; use databend_common_storages_fuse::io::TableMetaLocationGenerator; use databend_common_storages_fuse::statistics::gen_columns_statistics; use databend_common_storages_fuse::statistics::merge_statistics; use databend_common_storages_fuse::statistics::reducers::reduce_block_metas; -use databend_common_storages_fuse::FuseStorageFormat; -use databend_common_storages_fuse::FuseTable; -use databend_common_storages_fuse::FUSE_TBL_SEGMENT_PREFIX; use databend_storages_common_cache::SegmentStatistics; -use databend_storages_common_table_meta::meta::testing::SegmentInfoV2; -use databend_storages_common_table_meta::meta::testing::TableSnapshotV2; -use databend_storages_common_table_meta::meta::testing::TableSnapshotV4; use databend_storages_common_table_meta::meta::AdditionalStatsMeta; use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::Location; @@ -49,14 +46,17 @@ use databend_storages_common_table_meta::meta::Statistics; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use databend_storages_common_table_meta::meta::TableSnapshot; use databend_storages_common_table_meta::meta::Versioned; +use databend_storages_common_table_meta::meta::testing::SegmentInfoV2; +use databend_storages_common_table_meta::meta::testing::TableSnapshotV2; +use databend_storages_common_table_meta::meta::testing::TableSnapshotV4; use futures_util::TryStreamExt; use opendal::Operator; use serde::Serialize; use uuid::Uuid; +use super::TestFixture; use super::block_writer::BlockWriter; use super::old_version_generator; -use super::TestFixture; use crate::interpreters::Interpreter; use crate::interpreters::MutationInterpreter; use crate::sessions::QueryContext; diff --git a/src/query/service/src/test_kits/mod.rs b/src/query/service/src/test_kits/mod.rs index c9fe82f456992..d4679b553968c 100644 --- a/src/query/service/src/test_kits/mod.rs +++ b/src/query/service/src/test_kits/mod.rs @@ -26,8 +26,8 @@ mod old_version_generator; pub use block_writer::BlockWriter; pub use check::*; pub use cluster::ClusterDescriptor; -pub use config::config_with_spill; pub use config::ConfigBuilder; +pub use config::config_with_spill; pub use context::*; pub use fixture::*; pub use fuse::*; diff --git a/src/query/service/src/test_kits/old_version_generator.rs b/src/query/service/src/test_kits/old_version_generator.rs index f56988a9e9516..9b333c8378ea2 100644 --- a/src/query/service/src/test_kits/old_version_generator.rs +++ b/src/query/service/src/test_kits/old_version_generator.rs @@ -15,12 +15,12 @@ use databend_common_expression::DataBlock; use databend_common_storages_fuse::FUSE_TBL_BLOCK_PREFIX; use databend_common_storages_fuse::FUSE_TBL_SEGMENT_PREFIX; -use databend_storages_common_table_meta::meta::uuid_from_date_time; use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::SegmentInfo; use databend_storages_common_table_meta::meta::TableMetaTimestamps; -use databend_storages_common_table_meta::meta::Versioned; use databend_storages_common_table_meta::meta::VACUUM2_OBJECT_KEY_PREFIX; +use databend_storages_common_table_meta::meta::Versioned; +use databend_storages_common_table_meta::meta::uuid_from_date_time; use uuid::Uuid; #[derive(Clone)] pub struct TableMetaLocationGenerator { diff --git a/src/query/service/tests/it/auth.rs b/src/query/service/tests/it/auth.rs index 34b2381a3c49d..fe1d8d5087970 100644 --- a/src/query/service/tests/it/auth.rs +++ b/src/query/service/tests/it/auth.rs @@ -27,11 +27,11 @@ use databend_query::auth::Credential; use databend_query::test_kits::*; use jwt_simple::prelude::*; use p256::EncodedPoint; -use wiremock::matchers::method; -use wiremock::matchers::path; use wiremock::Mock; use wiremock::MockServer; use wiremock::ResponseTemplate; +use wiremock::matchers::method; +use wiremock::matchers::path; #[derive(Serialize, Deserialize)] struct NonCustomClaims { @@ -202,11 +202,12 @@ async fn test_auth_mgr_with_jwt_multi_sources() -> Result<()> { ) .await; assert!(res3.is_err()); - assert!(res3 - .err() - .unwrap() - .to_string() - .contains("could not decode token from all available jwt key stores")); + assert!( + res3.err() + .unwrap() + .to_string() + .contains("could not decode token from all available jwt key stores") + ); } Ok(()) @@ -262,11 +263,12 @@ async fn test_auth_mgr_with_jwt() -> Result<()> { .await; assert!(res.is_err()); - assert!(res - .err() - .unwrap() - .to_string() - .contains("missing field `subject` in jwt")); + assert!( + res.err() + .unwrap() + .to_string() + .contains("missing field `subject` in jwt") + ); } // without custom claims @@ -285,11 +287,12 @@ async fn test_auth_mgr_with_jwt() -> Result<()> { ) .await; assert!(res.is_err()); - assert!(res - .err() - .unwrap() - .message() - .contains("User 'test'@'%' does not exist")); + assert!( + res.err() + .unwrap() + .message() + .contains("User 'test'@'%' does not exist") + ); } // with custom claims @@ -310,11 +313,12 @@ async fn test_auth_mgr_with_jwt() -> Result<()> { ) .await; assert!(res.is_err()); - assert!(res - .err() - .unwrap() - .message() - .contains("User 'test'@'%' does not exist")); + assert!( + res.err() + .unwrap() + .message() + .contains("User 'test'@'%' does not exist") + ); } // with create user @@ -498,11 +502,12 @@ async fn test_auth_mgr_with_jwt_es256() -> Result<()> { ) .await; assert!(res.is_err()); - assert!(res - .err() - .unwrap() - .to_string() - .contains("missing field `subject` in jwt")); + assert!( + res.err() + .unwrap() + .to_string() + .contains("missing field `subject` in jwt") + ); } // without custom claims @@ -521,11 +526,12 @@ async fn test_auth_mgr_with_jwt_es256() -> Result<()> { ) .await; assert!(res.is_err()); - assert!(res - .err() - .unwrap() - .message() - .contains("User 'test'@'%' does not exist")); + assert!( + res.err() + .unwrap() + .message() + .contains("User 'test'@'%' does not exist") + ); } // with custom claims @@ -546,11 +552,12 @@ async fn test_auth_mgr_with_jwt_es256() -> Result<()> { ) .await; assert!(res.is_err()); - assert!(res - .err() - .unwrap() - .message() - .contains("User 'test'@'%' does not exist")); + assert!( + res.err() + .unwrap() + .message() + .contains("User 'test'@'%' does not exist") + ); } // with create user diff --git a/src/query/service/tests/it/catalogs/database_catalog.rs b/src/query/service/tests/it/catalogs/database_catalog.rs index aad9ba3d0c6b4..35a49e79d2e78 100644 --- a/src/query/service/tests/it/catalogs/database_catalog.rs +++ b/src/query/service/tests/it/catalogs/database_catalog.rs @@ -17,11 +17,10 @@ use std::sync::Arc; use chrono::Utc; use databend_common_base::base::tokio; use databend_common_exception::Result; -use databend_common_expression::types::NumberDataType; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; -use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; +use databend_common_expression::types::NumberDataType; use databend_common_meta_app::schema::CreateDatabaseReq; use databend_common_meta_app::schema::CreateOption; use databend_common_meta_app::schema::CreateTableReq; @@ -31,6 +30,7 @@ use databend_common_meta_app::schema::DropTableByIdReq; use databend_common_meta_app::schema::RenameDatabaseReq; use databend_common_meta_app::schema::TableMeta; use databend_common_meta_app::schema::TableNameIdent; +use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_app::tenant::Tenant; use databend_query::catalogs::Catalog; diff --git a/src/query/service/tests/it/catalogs/immutable_catalogs.rs b/src/query/service/tests/it/catalogs/immutable_catalogs.rs index ac41c7de9c2e3..5b1447a0669e2 100644 --- a/src/query/service/tests/it/catalogs/immutable_catalogs.rs +++ b/src/query/service/tests/it/catalogs/immutable_catalogs.rs @@ -14,14 +14,14 @@ use databend_common_base::base::tokio; use databend_common_exception::Result; -use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_app::schema::CreateDatabaseReq; use databend_common_meta_app::schema::CreateOption; use databend_common_meta_app::schema::DropDatabaseReq; use databend_common_meta_app::schema::RenameDatabaseReq; +use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_app::tenant::Tenant; -use databend_query::catalogs::default::ImmutableCatalog; use databend_query::catalogs::Catalog; +use databend_query::catalogs::default::ImmutableCatalog; use crate::tests::create_catalog; diff --git a/src/query/service/tests/it/catalogs/test_error_handling.rs b/src/query/service/tests/it/catalogs/test_error_handling.rs index bd235f8ba0d72..17354b641b09c 100644 --- a/src/query/service/tests/it/catalogs/test_error_handling.rs +++ b/src/query/service/tests/it/catalogs/test_error_handling.rs @@ -22,17 +22,17 @@ use std::sync::Arc; use databend_common_base::base::tokio; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::NumberDataType; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; -use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; +use databend_common_expression::types::NumberDataType; use databend_common_meta_app::schema::CreateDatabaseReq; use databend_common_meta_app::schema::CreateOption; use databend_common_meta_app::schema::CreateTableReq; use databend_common_meta_app::schema::DatabaseMeta; use databend_common_meta_app::schema::TableMeta; use databend_common_meta_app::schema::TableNameIdent; +use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_app::tenant::Tenant; use databend_query::catalogs::Catalog; diff --git a/src/query/service/tests/it/indexes/aggregating_index/index_refresh.rs b/src/query/service/tests/it/indexes/aggregating_index/index_refresh.rs index bbba3cbadcccf..4f37ed1c77b48 100644 --- a/src/query/service/tests/it/indexes/aggregating_index/index_refresh.rs +++ b/src/query/service/tests/it/indexes/aggregating_index/index_refresh.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![allow(clippy::collapsible_if)] + use std::fs; use std::path::Path; use std::path::PathBuf; @@ -25,16 +27,16 @@ use databend_common_base::base::tokio; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::block_debug::assert_two_blocks_sorted_eq_with_name; use databend_common_expression::DataBlock; use databend_common_expression::SendableDataBlockStream; +use databend_common_expression::block_debug::assert_two_blocks_sorted_eq_with_name; use databend_common_meta_app::schema::CreateIndexReq; use databend_common_meta_app::schema::IndexMeta; use databend_common_meta_app::schema::IndexNameIdent; use databend_common_meta_app::schema::IndexType; -use databend_common_sql::plans::Plan; use databend_common_sql::AggregatingIndexRewriter; use databend_common_sql::Planner; +use databend_common_sql::plans::Plan; use databend_query::interpreters::InterpreterFactory; use databend_query::sessions::QueryContext; use databend_query::test_kits::*; diff --git a/src/query/service/tests/it/indexes/inverted_index/index_refresh.rs b/src/query/service/tests/it/indexes/inverted_index/index_refresh.rs index 4d5cc17a7d276..aad4f62664024 100644 --- a/src/query/service/tests/it/indexes/inverted_index/index_refresh.rs +++ b/src/query/service/tests/it/indexes/inverted_index/index_refresh.rs @@ -21,19 +21,19 @@ use databend_common_catalog::plan::InvertedIndexInfo; use databend_common_catalog::table::Table; use databend_common_catalog::table::TableExt; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::DataField; use databend_common_expression::DataSchema; +use databend_common_expression::types::DataType; use databend_common_meta_app::schema::CreateOption; use databend_common_meta_app::schema::CreateTableIndexReq; use databend_common_meta_app::schema::TableIndexType; use databend_common_sql::plans::RefreshTableIndexPlan; -use databend_common_storages_fuse::io::read::InvertedIndexReader; +use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::TableContext; use databend_common_storages_fuse::io::MetaReaders; use databend_common_storages_fuse::io::TableMetaLocationGenerator; +use databend_common_storages_fuse::io::read::InvertedIndexReader; use databend_common_storages_fuse::pruning::create_inverted_index_query; -use databend_common_storages_fuse::FuseTable; -use databend_common_storages_fuse::TableContext; use databend_query::interpreters::Interpreter; use databend_query::interpreters::RefreshTableIndexInterpreter; use databend_query::test_kits::append_string_sample_data; diff --git a/src/query/service/tests/it/indexes/inverted_index/pruning.rs b/src/query/service/tests/it/indexes/inverted_index/pruning.rs index a67586213e25e..5ef10a6e80e28 100644 --- a/src/query/service/tests/it/indexes/inverted_index/pruning.rs +++ b/src/query/service/tests/it/indexes/inverted_index/pruning.rs @@ -21,11 +21,6 @@ use databend_common_catalog::plan::InvertedIndexInfo; use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table::TableExt; use databend_common_exception::Result; -use databend_common_expression::types::number::UInt64Type; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::VariantType; -use databend_common_expression::types::F32; use databend_common_expression::DataBlock; use databend_common_expression::DataSchema; use databend_common_expression::FromData; @@ -33,15 +28,20 @@ use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::F32; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::VariantType; +use databend_common_expression::types::number::UInt64Type; use databend_common_meta_app::schema::CreateOption; use databend_common_meta_app::schema::CreateTableIndexReq; use databend_common_meta_app::schema::TableIndexType; +use databend_common_sql::BloomIndexColumns; use databend_common_sql::plans::CreateTablePlan; use databend_common_sql::plans::RefreshTableIndexPlan; -use databend_common_sql::BloomIndexColumns; -use databend_common_storages_fuse::pruning::create_segment_location_vector; -use databend_common_storages_fuse::pruning::FusePruner; use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::pruning::FusePruner; +use databend_common_storages_fuse::pruning::create_segment_location_vector; use databend_query::interpreters::CreateTableInterpreter; use databend_query::interpreters::Interpreter; use databend_query::interpreters::RefreshTableIndexInterpreter; diff --git a/src/query/service/tests/it/indexes/vector_index/index_refresh.rs b/src/query/service/tests/it/indexes/vector_index/index_refresh.rs index bd8e93b7fc41d..2eb4edfdbc062 100644 --- a/src/query/service/tests/it/indexes/vector_index/index_refresh.rs +++ b/src/query/service/tests/it/indexes/vector_index/index_refresh.rs @@ -25,9 +25,9 @@ use databend_common_meta_app::schema::CreateTableIndexReq; use databend_common_meta_app::schema::TableIndexType; use databend_common_sql::plans::RefreshTableIndexPlan; use databend_common_storage::read_parquet_schema_async_rs; -use databend_common_storages_fuse::io::MetaReaders; use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::TableContext; +use databend_common_storages_fuse::io::MetaReaders; use databend_query::interpreters::Interpreter; use databend_query::interpreters::RefreshTableIndexInterpreter; use databend_query::sessions::QueryContext; diff --git a/src/query/service/tests/it/indexes/vector_index/pruning.rs b/src/query/service/tests/it/indexes/vector_index/pruning.rs index c2f748a6367a9..8b572e9401458 100644 --- a/src/query/service/tests/it/indexes/vector_index/pruning.rs +++ b/src/query/service/tests/it/indexes/vector_index/pruning.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![allow(clippy::cloned_ref_to_slice_refs)] + use std::collections::BTreeMap; use std::sync::Arc; @@ -21,14 +23,6 @@ use databend_common_catalog::plan::Filters; use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::plan::VectorIndexInfo; use databend_common_exception::Result; -use databend_common_expression::type_check::check_function; -use databend_common_expression::types::number::UInt64Type; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::NumberScalar; -use databend_common_expression::types::VectorColumn; -use databend_common_expression::types::VectorDataType; -use databend_common_expression::types::F32; use databend_common_expression::Column; use databend_common_expression::ColumnRef; use databend_common_expression::Constant; @@ -41,15 +35,23 @@ use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::type_check::check_function; +use databend_common_expression::types::DataType; +use databend_common_expression::types::F32; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::NumberScalar; +use databend_common_expression::types::VectorColumn; +use databend_common_expression::types::VectorDataType; +use databend_common_expression::types::number::UInt64Type; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::schema::CreateOption; use databend_common_meta_app::schema::TableIndex; use databend_common_meta_app::schema::TableIndexType; -use databend_common_sql::plans::CreateTablePlan; use databend_common_sql::BloomIndexColumns; -use databend_common_storages_fuse::pruning::create_segment_location_vector; -use databend_common_storages_fuse::pruning::FusePruner; +use databend_common_sql::plans::CreateTablePlan; use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::pruning::FusePruner; +use databend_common_storages_fuse::pruning::create_segment_location_vector; use databend_query::interpreters::CreateTableInterpreter; use databend_query::interpreters::Interpreter; use databend_query::sessions::QueryContext; diff --git a/src/query/service/tests/it/metrics.rs b/src/query/service/tests/it/metrics.rs index 2eb85ea1b2700..194a0b91b6747 100644 --- a/src/query/service/tests/it/metrics.rs +++ b/src/query/service/tests/it/metrics.rs @@ -18,8 +18,8 @@ use databend_common_base::base::tokio; #[cfg(target_os = "linux")] use databend_common_base::runtime::metrics::dump_process_stat; use databend_common_base::runtime::metrics::register_counter; -use databend_query::servers::metrics::MetricService; use databend_query::servers::Server; +use databend_query::servers::metrics::MetricService; #[tokio::test(flavor = "multi_thread")] async fn test_metric_server() -> databend_common_exception::Result<()> { diff --git a/src/query/service/tests/it/parquet_rs/prune_pages.rs b/src/query/service/tests/it/parquet_rs/prune_pages.rs index 90ff4071680a6..98560df1eeea1 100644 --- a/src/query/service/tests/it/parquet_rs/prune_pages.rs +++ b/src/query/service/tests/it/parquet_rs/prune_pages.rs @@ -24,8 +24,8 @@ use parquet::arrow::arrow_reader::ArrowReaderOptions; use parquet::arrow::arrow_reader::RowSelection; use parquet::arrow::arrow_reader::RowSelector; -use crate::parquet_rs::data::make_test_file_page; use crate::parquet_rs::data::Scenario; +use crate::parquet_rs::data::make_test_file_page; use crate::parquet_rs::utils::create_parquet_test_fixture; use crate::parquet_rs::utils::get_data_source_plan; diff --git a/src/query/service/tests/it/parquet_rs/prune_row_groups.rs b/src/query/service/tests/it/parquet_rs/prune_row_groups.rs index c420fc67abd53..e6bbbad22492e 100644 --- a/src/query/service/tests/it/parquet_rs/prune_row_groups.rs +++ b/src/query/service/tests/it/parquet_rs/prune_row_groups.rs @@ -21,8 +21,8 @@ use databend_common_expression::TableSchema; use databend_common_storages_parquet::ParquetPruner; use parquet::file::metadata::ParquetMetaDataReader; -use super::data::make_test_file_rg; use super::data::Scenario; +use super::data::make_test_file_rg; use super::utils::get_data_source_plan; use crate::parquet_rs::utils::create_parquet_test_fixture; diff --git a/src/query/service/tests/it/parquet_rs/utils.rs b/src/query/service/tests/it/parquet_rs/utils.rs index 636e9baeeb517..3c1c3317bca9f 100644 --- a/src/query/service/tests/it/parquet_rs/utils.rs +++ b/src/query/service/tests/it/parquet_rs/utils.rs @@ -17,8 +17,8 @@ use std::sync::Arc; use databend_common_catalog::plan::DataSourcePlan; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_sql::plans::Plan; use databend_common_sql::Planner; +use databend_common_sql::plans::Plan; use databend_query::physical_plans::PhysicalPlanBuilder; use databend_query::test_kits::ConfigBuilder; use databend_query::test_kits::TestFixture; diff --git a/src/query/service/tests/it/pipelines/executor/executor_graph.rs b/src/query/service/tests/it/pipelines/executor/executor_graph.rs index 13aaa4bd054e4..32cc4a9feab1b 100644 --- a/src/query/service/tests/it/pipelines/executor/executor_graph.rs +++ b/src/query/service/tests/it/pipelines/executor/executor_graph.rs @@ -17,9 +17,9 @@ use std::sync::Arc; use std::sync::Mutex; use databend_common_base::base::tokio; -use databend_common_base::base::tokio::sync::mpsc::channel; use databend_common_base::base::tokio::sync::mpsc::Receiver; use databend_common_base::base::tokio::sync::mpsc::Sender; +use databend_common_base::base::tokio::sync::mpsc::channel; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; use databend_common_expression::DataBlock; diff --git a/src/query/service/tests/it/pipelines/executor/pipeline_executor.rs b/src/query/service/tests/it/pipelines/executor/pipeline_executor.rs index 4a1630ed34214..dcc43bea2b280 100644 --- a/src/query/service/tests/it/pipelines/executor/pipeline_executor.rs +++ b/src/query/service/tests/it/pipelines/executor/pipeline_executor.rs @@ -12,14 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; -use std::sync::Arc; use databend_common_base::base::tokio; -use databend_common_base::base::tokio::sync::mpsc::channel; use databend_common_base::base::tokio::sync::mpsc::Receiver; use databend_common_base::base::tokio::sync::mpsc::Sender; +use databend_common_base::base::tokio::sync::mpsc::channel; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; diff --git a/src/query/service/tests/it/pipelines/filter/filter_executor.rs b/src/query/service/tests/it/pipelines/filter/filter_executor.rs index 54d241018799e..5b37124dda9a0 100644 --- a/src/query/service/tests/it/pipelines/filter/filter_executor.rs +++ b/src/query/service/tests/it/pipelines/filter/filter_executor.rs @@ -12,15 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_expression::Column; +use databend_common_expression::DataBlock; +use databend_common_expression::Evaluator; +use databend_common_expression::FunctionContext; use databend_common_expression::filter::FilterExecutor; use databend_common_expression::types::BooleanType; use databend_common_expression::types::DataType; use databend_common_expression::types::DecimalSize; use databend_common_expression::types::NumberDataType; -use databend_common_expression::Column; -use databend_common_expression::DataBlock; -use databend_common_expression::Evaluator; -use databend_common_expression::FunctionContext; use databend_common_functions::BUILTIN_FUNCTIONS; use itertools::Itertools; use rand::Rng; diff --git a/src/query/service/tests/it/pipelines/filter/random_filter_expr.rs b/src/query/service/tests/it/pipelines/filter/random_filter_expr.rs index 8ddac9dea5e9d..92d41af03e554 100644 --- a/src/query/service/tests/it/pipelines/filter/random_filter_expr.rs +++ b/src/query/service/tests/it/pipelines/filter/random_filter_expr.rs @@ -15,20 +15,20 @@ use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::Column; use databend_common_expression::DataField; use databend_common_expression::DataSchema; use databend_common_expression::Expr; use databend_common_expression::RemoteExpr; +use databend_common_expression::types::DataType; use databend_common_functions::BUILTIN_FUNCTIONS; -use databend_common_sql::executor::cast_expr_to_non_null_boolean; -use databend_common_sql::plans::BoundColumnRef; -use databend_common_sql::plans::FunctionCall; use databend_common_sql::ColumnBinding; use databend_common_sql::ScalarExpr; use databend_common_sql::TypeCheck; use databend_common_sql::Visibility; +use databend_common_sql::executor::cast_expr_to_non_null_boolean; +use databend_common_sql::plans::BoundColumnRef; +use databend_common_sql::plans::FunctionCall; use itertools::Itertools; use rand::Rng; diff --git a/src/query/service/tests/it/pipelines/transforms/sort.rs b/src/query/service/tests/it/pipelines/transforms/sort.rs index 735e7658cac89..045520cb5106f 100644 --- a/src/query/service/tests/it/pipelines/transforms/sort.rs +++ b/src/query/service/tests/it/pipelines/transforms/sort.rs @@ -16,16 +16,16 @@ use std::collections::VecDeque; use std::sync::Arc; use databend_common_base::base::tokio; -use databend_common_base::base::tokio::sync::mpsc::channel; use databend_common_base::base::tokio::sync::mpsc::Receiver; +use databend_common_base::base::tokio::sync::mpsc::channel; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::Int32Type; use databend_common_expression::DataBlock; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRefExt; use databend_common_expression::FromData; use databend_common_expression::SortColumnDescription; +use databend_common_expression::types::Int32Type; use databend_common_pipeline::core::InputPort; use databend_common_pipeline::core::OutputPort; use databend_common_pipeline::core::Pipe; @@ -38,8 +38,8 @@ use databend_query::pipelines::executor::ExecutorSettings; use databend_query::pipelines::executor::QueryPipelineExecutor; use databend_query::sessions::QueryContext; use databend_query::test_kits::TestFixture; -use rand::rngs::ThreadRng; use rand::Rng; +use rand::rngs::ThreadRng; fn create_source_pipe(ctx: Arc, data: Vec>) -> Result { use std::sync::Mutex; diff --git a/src/query/service/tests/it/pipelines/udf_transport.rs b/src/query/service/tests/it/pipelines/udf_transport.rs index 312f871b09b7d..8f54156378e20 100644 --- a/src/query/service/tests/it/pipelines/udf_transport.rs +++ b/src/query/service/tests/it/pipelines/udf_transport.rs @@ -17,9 +17,6 @@ use std::sync::Arc; use arrow_array::RecordBatch; use arrow_array::StringArray; -use arrow_flight::encode::FlightDataEncoderBuilder; -use arrow_flight::flight_service_server::FlightService; -use arrow_flight::flight_service_server::FlightServiceServer; use arrow_flight::Action; use arrow_flight::ActionType; use arrow_flight::Criteria; @@ -34,24 +31,27 @@ use arrow_flight::PollInfo; use arrow_flight::PutResult; use arrow_flight::SchemaResult; use arrow_flight::Ticket; +use arrow_flight::encode::FlightDataEncoderBuilder; +use arrow_flight::flight_service_server::FlightService; +use arrow_flight::flight_service_server::FlightServiceServer; use arrow_schema::DataType as ArrowDataType; use arrow_schema::Field; use arrow_schema::Schema; use databend_common_base::base::tokio; use databend_common_exception::Result; -use databend_common_expression::types::DataType; -use databend_common_expression::udf_client::UDFFlightClient; use databend_common_expression::BlockEntry; use databend_common_expression::Column; -use futures::stream; +use databend_common_expression::types::DataType; +use databend_common_expression::udf_client::UDFFlightClient; use futures::Stream; use futures::StreamExt; +use futures::stream; use tokio::time::timeout; use tokio_stream::wrappers::TcpListenerStream; -use tonic::transport::Server; use tonic::Request; use tonic::Response; use tonic::Status; +use tonic::transport::Server; #[derive(Clone)] enum MockMode { diff --git a/src/query/service/tests/it/servers/admin/admin_service.rs b/src/query/service/tests/it/servers/admin/admin_service.rs index 61144b2b666d6..d101f15f3cbac 100644 --- a/src/query/service/tests/it/servers/admin/admin_service.rs +++ b/src/query/service/tests/it/servers/admin/admin_service.rs @@ -18,8 +18,8 @@ use std::io::Read; use databend_common_base::base::get_free_tcp_port; use databend_common_base::base::tokio; use databend_common_exception::Result; -use databend_query::servers::admin::AdminService; use databend_query::servers::Server; +use databend_query::servers::admin::AdminService; use databend_query::test_kits::*; use crate::tests::tls_constants::*; diff --git a/src/query/service/tests/it/servers/admin/v1/cluster.rs b/src/query/service/tests/it/servers/admin/v1/cluster.rs index 12aa6209329e2..f8aea57205f7f 100644 --- a/src/query/service/tests/it/servers/admin/v1/cluster.rs +++ b/src/query/service/tests/it/servers/admin/v1/cluster.rs @@ -17,14 +17,14 @@ use databend_common_exception::Result; use databend_common_meta_types::NodeInfo; use databend_query::servers::admin::v1::cluster::*; use databend_query::test_kits::*; -use http::header; use http::Method; use http::StatusCode; use http::Uri; -use poem::get; +use http::header; use poem::Endpoint; use poem::Request; use poem::Route; +use poem::get; use pretty_assertions::assert_eq; #[tokio::test(flavor = "multi_thread")] diff --git a/src/query/service/tests/it/servers/admin/v1/config.rs b/src/query/service/tests/it/servers/admin/v1/config.rs index f5f7b9b95c91f..4cd1aa0c2fb0f 100644 --- a/src/query/service/tests/it/servers/admin/v1/config.rs +++ b/src/query/service/tests/it/servers/admin/v1/config.rs @@ -18,10 +18,10 @@ use databend_query::test_kits::*; use http::Method; use http::StatusCode; use http::Uri; -use poem::get; use poem::Endpoint; use poem::Request; use poem::Route; +use poem::get; use pretty_assertions::assert_eq; // for `app.oneshot()` #[tokio::test(flavor = "multi_thread")] diff --git a/src/query/service/tests/it/servers/admin/v1/status.rs b/src/query/service/tests/it/servers/admin/v1/status.rs index 93138ae0fb959..a346788937f07 100644 --- a/src/query/service/tests/it/servers/admin/v1/status.rs +++ b/src/query/service/tests/it/servers/admin/v1/status.rs @@ -20,22 +20,22 @@ use databend_common_exception::Result; use databend_common_meta_app::principal::UserIdentity; use databend_common_meta_app::tenant::Tenant; use databend_common_users::UserApiProvider; -use databend_query::interpreters::interpreter_plan_sql; use databend_query::interpreters::Interpreter; use databend_query::interpreters::InterpreterFactory; -use databend_query::servers::admin::v1::instance_status::instance_status_handler; +use databend_query::interpreters::interpreter_plan_sql; use databend_query::servers::admin::v1::instance_status::InstanceStatus; +use databend_query::servers::admin::v1::instance_status::instance_status_handler; use databend_query::sessions::QueryContext; use databend_query::sessions::SessionManager; use databend_query::test_kits::*; -use http::header; use http::Method; use http::StatusCode; use http::Uri; -use poem::get; +use http::header; use poem::Endpoint; use poem::Request; use poem::Route; +use poem::get; use pretty_assertions::assert_eq; use tokio_stream::StreamExt; diff --git a/src/query/service/tests/it/servers/flight/flight_service.rs b/src/query/service/tests/it/servers/flight/flight_service.rs index a589de4fdc85b..3f4613eade52e 100644 --- a/src/query/service/tests/it/servers/flight/flight_service.rs +++ b/src/query/service/tests/it/servers/flight/flight_service.rs @@ -16,8 +16,8 @@ use std::net::SocketAddr; use std::net::TcpListener; use std::str::FromStr; -use arrow_flight::flight_service_client::FlightServiceClient; use arrow_flight::Empty; +use arrow_flight::flight_service_client::FlightServiceClient; use databend_common_base::base::tokio; use databend_common_exception::ErrorCode; use databend_common_exception::Result; diff --git a/src/query/service/tests/it/servers/http/clickhouse_handler.rs b/src/query/service/tests/it/servers/http/clickhouse_handler.rs index a65d20a70f6dd..514a6cde301b7 100644 --- a/src/query/service/tests/it/servers/http/clickhouse_handler.rs +++ b/src/query/service/tests/it/servers/http/clickhouse_handler.rs @@ -15,25 +15,25 @@ use std::collections::HashMap; use databend_common_base::base::tokio; +use databend_query::servers::HttpHandlerKind; +use databend_query::servers::http::CLICKHOUSE_VERSION; use databend_query::servers::http::middleware::EndpointKind; use databend_query::servers::http::middleware::HTTPSessionEndpoint; use databend_query::servers::http::middleware::HTTPSessionMiddleware; use databend_query::servers::http::v1::clickhouse_router; -use databend_query::servers::http::CLICKHOUSE_VERSION; -use databend_query::servers::HttpHandlerKind; use databend_query::test_kits::TestFixture; use http::Method; use http::StatusCode; use http::Uri; -use poem::error::Result as PoemResult; -use poem::middleware::CookieJarManager; -use poem::middleware::CookieJarManagerEndpoint; -use poem::web::headers::Authorization; use poem::Body; use poem::Endpoint; use poem::EndpointExt; use poem::Request; use poem::Route; +use poem::error::Result as PoemResult; +use poem::middleware::CookieJarManager; +use poem::middleware::CookieJarManagerEndpoint; +use poem::web::headers::Authorization; use pretty_assertions::assert_eq; macro_rules! assert_error { diff --git a/src/query/service/tests/it/servers/http/http_query_handlers.rs b/src/query/service/tests/it/servers/http/http_query_handlers.rs index 12f5319748254..4ef0e0ccc6c96 100644 --- a/src/query/service/tests/it/servers/http/http_query_handlers.rs +++ b/src/query/service/tests/it/servers/http/http_query_handlers.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![allow(clippy::unnecessary_unwrap)] use std::collections::BTreeMap; use std::fs::File; use std::io::Read; @@ -27,27 +28,27 @@ use databend_common_config::UserConfig; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_meta_app::principal::PasswordHashMethod; -use databend_common_users::CustomClaims; -use databend_common_users::EnsureUser; use databend_common_users::BUILTIN_ROLE_ACCOUNT_ADMIN; use databend_common_users::BUILTIN_ROLE_PUBLIC; +use databend_common_users::CustomClaims; +use databend_common_users::EnsureUser; use databend_common_version::DATABEND_SEMVER; -use databend_query::servers::admin::v1::instance_status::instance_status_handler; +use databend_query::servers::HttpHandler; +use databend_query::servers::HttpHandlerKind; use databend_query::servers::admin::v1::instance_status::InstanceStatus; +use databend_query::servers::admin::v1::instance_status::instance_status_handler; use databend_query::servers::http::error::QueryError; use databend_query::servers::http::middleware::json_response; +use databend_query::servers::http::v1::ExecuteStateKind; +use databend_query::servers::http::v1::HttpSessionConf; +use databend_query::servers::http::v1::QueryResponseField; +use databend_query::servers::http::v1::QueryStats; use databend_query::servers::http::v1::catalog; use databend_query::servers::http::v1::make_page_uri; use databend_query::servers::http::v1::query_route; use databend_query::servers::http::v1::roles::ListRolesResponse; use databend_query::servers::http::v1::users::CreateUserRequest; use databend_query::servers::http::v1::users::ListUsersResponse; -use databend_query::servers::http::v1::ExecuteStateKind; -use databend_query::servers::http::v1::HttpSessionConf; -use databend_query::servers::http::v1::QueryResponseField; -use databend_query::servers::http::v1::QueryStats; -use databend_query::servers::HttpHandler; -use databend_query::servers::HttpHandlerKind; use databend_query::sessions::QueryAffect; use databend_query::test_kits::ConfigBuilder; use databend_query::test_kits::TestFixture; @@ -55,30 +56,30 @@ use databend_storages_common_session::TxnState; use futures_util::future::try_join_all; use headers::Header; use headers::HeaderMapExt; -use http::header; use http::HeaderMap; use http::HeaderValue; use http::Method; use http::StatusCode; +use http::header; use jwt_simple::algorithms::RS256KeyPair; use jwt_simple::algorithms::RSAKeyPairLike; use jwt_simple::claims::JWTClaims; use jwt_simple::claims::NoCustomClaims; use jwt_simple::prelude::Clock; -use poem::get; use poem::Endpoint; use poem::EndpointExt; use poem::Request; use poem::Response; use poem::Route; +use poem::get; use pretty_assertions::assert_eq; use serde::Deserialize; use tokio::time::sleep; -use wiremock::matchers::method; -use wiremock::matchers::path; use wiremock::Mock; use wiremock::MockServer; use wiremock::ResponseTemplate; +use wiremock::matchers::method; +use wiremock::matchers::path; use crate::tests::tls_constants::*; @@ -551,7 +552,7 @@ async fn test_active_sessions() -> Result<()> { let mut results = try_join_all(handlers) .await? .into_iter() - .map(|(_status, resp)| (resp.error.map(|e| e.message).unwrap_or_default())) + .map(|(_status, resp)| resp.error.map(|e| e.message).unwrap_or_default()) .collect::>(); results.sort(); let msg = "Failed to upgrade session: Current active sessions (2) has exceeded the max_active_sessions limit (2)"; diff --git a/src/query/service/tests/it/servers/http/json_block.rs b/src/query/service/tests/it/servers/http/json_block.rs index ae6ddc4f61b95..5b928b38e8ba5 100644 --- a/src/query/service/tests/it/servers/http/json_block.rs +++ b/src/query/service/tests/it/servers/http/json_block.rs @@ -14,13 +14,13 @@ use databend_common_column::bitmap::Bitmap; use databend_common_exception::Result; -use databend_common_expression::types::nullable::NullableColumn; -use databend_common_expression::types::number::Float64Type; -use databend_common_expression::types::number::Int32Type; +use databend_common_expression::FromData; use databend_common_expression::types::BooleanType; use databend_common_expression::types::DateType; use databend_common_expression::types::StringType; -use databend_common_expression::FromData; +use databend_common_expression::types::nullable::NullableColumn; +use databend_common_expression::types::number::Float64Type; +use databend_common_expression::types::number::Int32Type; use databend_common_io::prelude::FormatSettings; use databend_query::servers::http::v1::BlocksCollector; use pretty_assertions::assert_eq; diff --git a/src/query/service/tests/it/servers/mysql/mysql_handler.rs b/src/query/service/tests/it/servers/mysql/mysql_handler.rs index 9be86ecb34a46..93d4111326229 100644 --- a/src/query/service/tests/it/servers/mysql/mysql_handler.rs +++ b/src/query/service/tests/it/servers/mysql/mysql_handler.rs @@ -27,11 +27,11 @@ use databend_query::servers::MySQLHandler; use databend_query::servers::MySQLTlsConfig; use databend_query::test_kits::ConfigBuilder; use databend_query::test_kits::TestFixture; -use mysql_async::prelude::FromRow; -use mysql_async::prelude::Queryable; use mysql_async::FromRowError; use mysql_async::Row; use mysql_async::SslOpts; +use mysql_async::prelude::FromRow; +use mysql_async::prelude::Queryable; use tokio::sync::Barrier; use crate::tests::tls_constants::*; @@ -205,6 +205,7 @@ async fn create_connection(port: u16, with_tls: bool) -> Result Result<()> { diff --git a/src/query/service/tests/it/sessions/queue_mgr.rs b/src/query/service/tests/it/sessions/queue_mgr.rs index 94a03677b7cb5..71a1ec6cfff1a 100644 --- a/src/query/service/tests/it/sessions/queue_mgr.rs +++ b/src/query/service/tests/it/sessions/queue_mgr.rs @@ -19,14 +19,14 @@ use std::time::Instant; use std::time::SystemTime; use std::time::UNIX_EPOCH; -use databend_common_base::base::tokio::sync::Mutex; use databend_common_base::base::WatchNotify; +use databend_common_base::base::tokio::sync::Mutex; +use databend_common_base::runtime::MemStat; +use databend_common_base::runtime::ThreadTracker; +use databend_common_base::runtime::workload_group::MAX_CONCURRENCY_QUOTA_KEY; use databend_common_base::runtime::workload_group::QuotaValue; use databend_common_base::runtime::workload_group::WorkloadGroup; use databend_common_base::runtime::workload_group::WorkloadGroupResource; -use databend_common_base::runtime::workload_group::MAX_CONCURRENCY_QUOTA_KEY; -use databend_common_base::runtime::MemStat; -use databend_common_base::runtime::ThreadTracker; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -436,7 +436,7 @@ async fn test_heavy_actions() -> Result<()> { Query { sql: "CREATE TABLE test_heavy_create AS SELECT 1", add_to_queue: true, - } + }, ]; let fixture = TestFixture::setup().await?; diff --git a/src/query/service/tests/it/spillers/spill_fallback.rs b/src/query/service/tests/it/spillers/spill_fallback.rs index c6a7a34da9bdc..1c019bd38c5e0 100644 --- a/src/query/service/tests/it/spillers/spill_fallback.rs +++ b/src/query/service/tests/it/spillers/spill_fallback.rs @@ -15,17 +15,17 @@ use databend_common_base::base::tokio; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::Int32Type; use databend_common_expression::DataBlock; use databend_common_expression::FromData; +use databend_common_expression::types::Int32Type; use databend_common_storage::DataOperator; use databend_query::spillers::Location; use databend_query::spillers::Spiller; use databend_query::spillers::SpillerConfig; use databend_query::spillers::SpillerDiskConfig; use databend_query::spillers::SpillerType; -use databend_query::test_kits::config_with_spill; use databend_query::test_kits::TestFixture; +use databend_query::test_kits::config_with_spill; use databend_storages_common_cache::TempDirManager; /// ASCII flow of the test (data view): @@ -104,10 +104,7 @@ async fn test_spill_fallback_to_remote_when_local_full() -> Result<()> { assert!( saw_remote, "should fallback to remote when local quota is exhausted (used_local_bytes={}, limit={}, first_block_bytes={}, attempts={})", - used_local_bytes, - limit, - first_block_bytes, - max_attempts + used_local_bytes, limit, first_block_bytes, max_attempts ); // Cleanup the temp directory for this query. diff --git a/src/query/service/tests/it/spillers/spill_profile.rs b/src/query/service/tests/it/spillers/spill_profile.rs index d097e7f273908..80cf6c97d9d97 100644 --- a/src/query/service/tests/it/spillers/spill_profile.rs +++ b/src/query/service/tests/it/spillers/spill_profile.rs @@ -17,12 +17,12 @@ use std::sync::Arc; use std::time::Instant; +use databend_common_base::runtime::ThreadTracker; use databend_common_base::runtime::profile::Profile; use databend_common_base::runtime::profile::ProfileStatisticsName; -use databend_common_base::runtime::ThreadTracker; +use databend_query::spillers::SpillTarget; use databend_query::spillers::record_read_profile; use databend_query::spillers::record_write_profile; -use databend_query::spillers::SpillTarget; fn create_test_profile() -> Arc { Arc::new(Profile::create( diff --git a/src/query/service/tests/it/spillers/spiller.rs b/src/query/service/tests/it/spillers/spiller.rs index eae8d1da9cb8c..65e08b1b78257 100644 --- a/src/query/service/tests/it/spillers/spiller.rs +++ b/src/query/service/tests/it/spillers/spiller.rs @@ -17,11 +17,11 @@ use std::assert_matches::assert_matches; use databend_common_base::base::tokio; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::Int32Type; -use databend_common_expression::types::NumberScalar; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::ScalarRef; +use databend_common_expression::types::Int32Type; +use databend_common_expression::types::NumberScalar; use databend_common_storage::DataOperator; use databend_query::spillers::Location; use databend_query::spillers::Spiller; diff --git a/src/query/service/tests/it/sql/exec/get_table_bind_test.rs b/src/query/service/tests/it/sql/exec/get_table_bind_test.rs index 61f09377b0d27..22036d26ea21d 100644 --- a/src/query/service/tests/it/sql/exec/get_table_bind_test.rs +++ b/src/query/service/tests/it/sql/exec/get_table_bind_test.rs @@ -15,15 +15,15 @@ use std::any::Any; use std::collections::HashMap; use std::collections::HashSet; -use std::sync::atomic::AtomicUsize; use std::sync::Arc; +use std::sync::atomic::AtomicUsize; use std::time::Duration; use dashmap::DashMap; -use databend_common_base::base::tokio; use databend_common_base::base::Progress; use databend_common_base::base::ProgressValues; use databend_common_base::base::WatchNotify; +use databend_common_base::base::tokio; use databend_common_catalog::catalog::Catalog; use databend_common_catalog::cluster_info::Cluster; use databend_common_catalog::database::Database; @@ -62,8 +62,6 @@ use databend_common_meta_app::principal::UDTFServer; use databend_common_meta_app::principal::UserDefinedConnection; use databend_common_meta_app::principal::UserInfo; use databend_common_meta_app::principal::UserPrivilegeType; -use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; -use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; use databend_common_meta_app::schema::CatalogInfo; use databend_common_meta_app::schema::CommitTableMetaReply; use databend_common_meta_app::schema::CommitTableMetaReq; @@ -136,6 +134,8 @@ use databend_common_meta_app::schema::UpdateIndexReply; use databend_common_meta_app::schema::UpdateIndexReq; use databend_common_meta_app::schema::UpsertTableOptionReply; use databend_common_meta_app::schema::UpsertTableOptionReq; +use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; +use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_types::MetaId; use databend_common_meta_types::SeqV; diff --git a/src/query/service/tests/it/sql/exec/mod.rs b/src/query/service/tests/it/sql/exec/mod.rs index 1dac047fc0a8d..92d78fdd86900 100644 --- a/src/query/service/tests/it/sql/exec/mod.rs +++ b/src/query/service/tests/it/sql/exec/mod.rs @@ -18,8 +18,8 @@ use databend_common_base::runtime::TrySpawn; use databend_common_catalog::lock::LockTableOption; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_sql::plans::Plan; use databend_common_sql::Planner; +use databend_common_sql::plans::Plan; use databend_common_storages_fuse::FuseTable; use databend_query::interpreters::Interpreter; use databend_query::interpreters::OptimizeCompactBlockInterpreter; diff --git a/src/query/service/tests/it/sql/planner/optimizer/ir/expr/visitor.rs b/src/query/service/tests/it/sql/planner/optimizer/ir/expr/visitor.rs index 83ffa53968766..fe429d90a4c62 100644 --- a/src/query/service/tests/it/sql/planner/optimizer/ir/expr/visitor.rs +++ b/src/query/service/tests/it/sql/planner/optimizer/ir/expr/visitor.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![allow(clippy::collapsible_if)] + use std::sync::Arc; use databend_common_exception::Result; diff --git a/src/query/service/tests/it/sql/planner/optimizer/optimizer_test.rs b/src/query/service/tests/it/sql/planner/optimizer/optimizer_test.rs index bc9318d9518f9..2fad4a779248a 100644 --- a/src/query/service/tests/it/sql/planner/optimizer/optimizer_test.rs +++ b/src/query/service/tests/it/sql/planner/optimizer/optimizer_test.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![allow(clippy::replace_box)] + use std::collections::HashMap; use std::fs; use std::io::Write; @@ -20,32 +22,32 @@ use std::path::PathBuf; use std::sync::Arc; use databend_common_base::base::GlobalUniqName; -use databend_common_catalog::cluster_info::Cluster; -use databend_common_catalog::table_context::TableContext; use databend_common_catalog::BasicColumnStatistics; use databend_common_catalog::TableStatistics; +use databend_common_catalog::cluster_info::Cluster; +use databend_common_catalog::table_context::TableContext; use databend_common_column::binview::ViewType; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::Scalar; +use databend_common_expression::types::F64; use databend_common_expression::types::Number; use databend_common_expression::types::NumberScalar; -use databend_common_expression::types::F64; -use databend_common_expression::Scalar; use databend_common_meta_types::NodeInfo; +use databend_common_sql::BaseTableColumn; +use databend_common_sql::ColumnEntry; +use databend_common_sql::FormatOptions; +use databend_common_sql::IndexType; +use databend_common_sql::Metadata; +use databend_common_sql::MetadataRef; use databend_common_sql::optimize; +use databend_common_sql::optimizer::OptimizerContext; use databend_common_sql::optimizer::ir::SExpr; use databend_common_sql::optimizer::ir::SExprVisitor; use databend_common_sql::optimizer::ir::VisitAction; -use databend_common_sql::optimizer::OptimizerContext; use databend_common_sql::plans::Plan; use databend_common_sql::plans::RelOperator; use databend_common_sql::plans::Statistics; -use databend_common_sql::BaseTableColumn; -use databend_common_sql::ColumnEntry; -use databend_common_sql::FormatOptions; -use databend_common_sql::IndexType; -use databend_common_sql::Metadata; -use databend_common_sql::MetadataRef; use databend_common_storage::Datum; use databend_query::clusters::ClusterHelper; use databend_query::physical_plans::PhysicalPlanBuilder; diff --git a/src/query/service/tests/it/sql/planner/optimizer/optimizers/operator/filter/equivalent_constants_visitor.rs b/src/query/service/tests/it/sql/planner/optimizer/optimizers/operator/filter/equivalent_constants_visitor.rs index 1e060cb993347..9acd37ada40bd 100644 --- a/src/query/service/tests/it/sql/planner/optimizer/optimizers/operator/filter/equivalent_constants_visitor.rs +++ b/src/query/service/tests/it/sql/planner/optimizer/optimizers/operator/filter/equivalent_constants_visitor.rs @@ -14,14 +14,12 @@ use std::sync::Arc; +use databend_common_ast::parser::Dialect; use databend_common_ast::parser::parse_expr; use databend_common_ast::parser::tokenize_sql; -use databend_common_ast::parser::Dialect; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; use databend_common_expression::types::DataType; -use databend_common_sql::optimizer::optimizers::operator::EquivalentConstantsVisitor; -use databend_common_sql::plans::VisitorMut; use databend_common_sql::BindContext; use databend_common_sql::ColumnBinding; use databend_common_sql::Metadata; @@ -29,6 +27,8 @@ use databend_common_sql::NameResolutionContext; use databend_common_sql::ScalarExpr; use databend_common_sql::TypeChecker; use databend_common_sql::Visibility; +use databend_common_sql::optimizer::optimizers::operator::EquivalentConstantsVisitor; +use databend_common_sql::plans::VisitorMut; use databend_query::sessions::QueryContext; use databend_query::test_kits::TestFixture; use parking_lot::RwLock; diff --git a/src/query/service/tests/it/sql/planner/optimizer/optimizers/operator/filter/normalize_disjunctive_filter_test.rs b/src/query/service/tests/it/sql/planner/optimizer/optimizers/operator/filter/normalize_disjunctive_filter_test.rs index 6b610d345a98d..6073ba94869e0 100644 --- a/src/query/service/tests/it/sql/planner/optimizer/optimizers/operator/filter/normalize_disjunctive_filter_test.rs +++ b/src/query/service/tests/it/sql/planner/optimizer/optimizers/operator/filter/normalize_disjunctive_filter_test.rs @@ -12,11 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![allow(clippy::collapsible_if)] + use databend_common_exception::Result; +use databend_common_expression::Scalar; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::NumberScalar; -use databend_common_expression::Scalar; use databend_common_sql::optimizer::optimizers::operator::NormalizeDisjunctiveFilterOptimizer; use databend_common_sql::planner::plans::FunctionCall; use databend_common_sql::planner::plans::ScalarExpr; @@ -220,7 +222,7 @@ fn test_different_table_columns() -> Result<()> { ); // Check if the first expression is t1.a - if let ScalarExpr::BoundColumnRef(ref col_ref) = &after[0] { + if let ScalarExpr::BoundColumnRef(col_ref) = &after[0] { assert_eq!(col_ref.column.column_name, "a"); assert_eq!(col_ref.column.table_name, Some("t1".to_string())); } else { @@ -228,12 +230,12 @@ fn test_different_table_columns() -> Result<()> { } // Check if the second expression is (t2.b OR t2.c) - if let ScalarExpr::FunctionCall(ref func) = &after[1] { + if let ScalarExpr::FunctionCall(func) = &after[1] { assert_eq!(func.func_name, "or"); assert_eq!(func.arguments.len(), 2); // Check t2.b - if let ScalarExpr::BoundColumnRef(ref col_ref) = &func.arguments[0] { + if let ScalarExpr::BoundColumnRef(col_ref) = &func.arguments[0] { assert_eq!(col_ref.column.column_name, "b"); assert_eq!(col_ref.column.table_name, Some("t2".to_string())); } else { @@ -241,7 +243,7 @@ fn test_different_table_columns() -> Result<()> { } // Check t2.c - if let ScalarExpr::BoundColumnRef(ref col_ref) = &func.arguments[1] { + if let ScalarExpr::BoundColumnRef(col_ref) = &func.arguments[1] { assert_eq!(col_ref.column.column_name, "c"); assert_eq!(col_ref.column.table_name, Some("t2".to_string())); } else { diff --git a/src/query/service/tests/it/sql/planner/optimizer/optimizers/rule/agg_rules/agg_index_query_rewrite.rs b/src/query/service/tests/it/sql/planner/optimizer/optimizers/rule/agg_rules/agg_index_query_rewrite.rs index dda3fc161f61a..f055ce6f3d63b 100644 --- a/src/query/service/tests/it/sql/planner/optimizer/optimizers/rule/agg_rules/agg_index_query_rewrite.rs +++ b/src/query/service/tests/it/sql/planner/optimizer/optimizers/rule/agg_rules/agg_index_query_rewrite.rs @@ -17,22 +17,22 @@ use std::sync::Arc; use databend_common_ast::ast::Engine; use databend_common_base::base::tokio; use databend_common_exception::Result; -use databend_common_expression::types::NumberDataType; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::NumberDataType; use databend_common_meta_app::schema::CreateOption; +use databend_common_sql::BindContext; +use databend_common_sql::MetadataRef; +use databend_common_sql::optimizer::OptimizerContext; use databend_common_sql::optimizer::ir::SExpr; use databend_common_sql::optimizer::optimizers::recursive::RecursiveRuleOptimizer; -use databend_common_sql::optimizer::optimizers::rule::RuleID; use databend_common_sql::optimizer::optimizers::rule::DEFAULT_REWRITE_RULES; -use databend_common_sql::optimizer::OptimizerContext; +use databend_common_sql::optimizer::optimizers::rule::RuleID; use databend_common_sql::plans::AggIndexInfo; use databend_common_sql::plans::CreateTablePlan; use databend_common_sql::plans::Plan; use databend_common_sql::plans::RelOperator; -use databend_common_sql::BindContext; -use databend_common_sql::MetadataRef; use databend_query::interpreters::CreateTableInterpreter; use databend_query::interpreters::Interpreter; use databend_query::sessions::QueryContext; diff --git a/src/query/service/tests/it/sql/planner/optimizer/optimizers/rule/agg_rules/eager_aggregation.rs b/src/query/service/tests/it/sql/planner/optimizer/optimizers/rule/agg_rules/eager_aggregation.rs index 8ccc963f0f2e8..8275d96ecf0e8 100644 --- a/src/query/service/tests/it/sql/planner/optimizer/optimizers/rule/agg_rules/eager_aggregation.rs +++ b/src/query/service/tests/it/sql/planner/optimizer/optimizers/rule/agg_rules/eager_aggregation.rs @@ -17,6 +17,7 @@ use std::io::Write; use std::sync::Arc; use databend_common_exception::Result; +use databend_common_sql::optimizer::OptimizerContext; use databend_common_sql::optimizer::ir::Matcher; use databend_common_sql::optimizer::ir::SExpr; use databend_common_sql::optimizer::ir::SExprVisitor; @@ -25,12 +26,11 @@ use databend_common_sql::optimizer::optimizers::operator::PullUpFilterOptimizer; use databend_common_sql::optimizer::optimizers::operator::RuleNormalizeAggregateOptimizer; use databend_common_sql::optimizer::optimizers::operator::RuleStatsAggregateOptimizer; use databend_common_sql::optimizer::optimizers::recursive::RecursiveRuleOptimizer; +use databend_common_sql::optimizer::optimizers::rule::DEFAULT_REWRITE_RULES; use databend_common_sql::optimizer::optimizers::rule::Rule; use databend_common_sql::optimizer::optimizers::rule::RuleEagerAggregation; use databend_common_sql::optimizer::optimizers::rule::RuleID; use databend_common_sql::optimizer::optimizers::rule::TransformResult; -use databend_common_sql::optimizer::optimizers::rule::DEFAULT_REWRITE_RULES; -use databend_common_sql::optimizer::OptimizerContext; use databend_common_sql::plans::Plan; use databend_common_storages_fuse::TableContext; use databend_query::sessions::QueryContext; diff --git a/src/query/service/tests/it/sql/planner/optimizer/optimizers/rule/filter_rules/push_down_filter_join_test.rs b/src/query/service/tests/it/sql/planner/optimizer/optimizers/rule/filter_rules/push_down_filter_join_test.rs index b6ced07c4f0f3..4500d4b72d89f 100644 --- a/src/query/service/tests/it/sql/planner/optimizer/optimizers/rule/filter_rules/push_down_filter_join_test.rs +++ b/src/query/service/tests/it/sql/planner/optimizer/optimizers/rule/filter_rules/push_down_filter_join_test.rs @@ -13,9 +13,12 @@ // limitations under the License. use databend_common_exception::Result; +use databend_common_expression::Scalar; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; -use databend_common_expression::Scalar; +use databend_common_sql::ColumnSet; +use databend_common_sql::MetadataRef; +use databend_common_sql::ScalarExpr; use databend_common_sql::optimizer::ir::SExpr; use databend_common_sql::optimizer::optimizers::rule::Rule; use databend_common_sql::optimizer::optimizers::rule::RulePushDownFilterJoin; @@ -23,9 +26,6 @@ use databend_common_sql::optimizer::optimizers::rule::TransformResult; use databend_common_sql::planner::plans::JoinEquiCondition; use databend_common_sql::planner::plans::JoinType; use databend_common_sql::planner::plans::RelOperator; -use databend_common_sql::ColumnSet; -use databend_common_sql::MetadataRef; -use databend_common_sql::ScalarExpr; use pretty_assertions::assert_eq; use crate::sql::planner::optimizer::test_utils::ExprBuilder; @@ -297,7 +297,9 @@ fn run_join_filter_test(test_case: &JoinFilterTestCase, metadata: &MetadataRef) normalized_after_expected.contains("Filter") && normalized_after.contains("Filter"); if normalized_after.contains("Inner Join") && expected_filter_pushed { - println!("Note: Optimizer converted RIGHT JOIN to INNER JOIN, but filter was correctly pushed down."); + println!( + "Note: Optimizer converted RIGHT JOIN to INNER JOIN, but filter was correctly pushed down." + ); } else { assert_eq!( normalized_after, normalized_after_expected, diff --git a/src/query/service/tests/it/sql/planner/optimizer/test_utils.rs b/src/query/service/tests/it/sql/planner/optimizer/test_utils.rs index ab4017ea828f6..b40981a5c9727 100644 --- a/src/query/service/tests/it/sql/planner/optimizer/test_utils.rs +++ b/src/query/service/tests/it/sql/planner/optimizer/test_utils.rs @@ -18,13 +18,19 @@ use std::sync::Arc; use databend_common_base::base::OrderedFloat; use databend_common_catalog::catalog::CatalogManager; use databend_common_exception::Result; +use databend_common_expression::Scalar; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberScalar; -use databend_common_expression::Scalar; +use databend_common_sql::ColumnBinding; +use databend_common_sql::ColumnSet; +use databend_common_sql::IndexType; +use databend_common_sql::NameResolutionContext; +use databend_common_sql::Planner; +use databend_common_sql::Visibility; use databend_common_sql::optimizer::ir::SExpr; -use databend_common_sql::planner::plans::Filter; use databend_common_sql::planner::Binder; use databend_common_sql::planner::Metadata; +use databend_common_sql::planner::plans::Filter; use databend_common_sql::plans::Aggregate; use databend_common_sql::plans::AggregateMode; use databend_common_sql::plans::BoundColumnRef; @@ -40,12 +46,6 @@ use databend_common_sql::plans::RelOperator; use databend_common_sql::plans::ScalarExpr; use databend_common_sql::plans::ScalarItem; use databend_common_sql::plans::Scan; -use databend_common_sql::ColumnBinding; -use databend_common_sql::ColumnSet; -use databend_common_sql::IndexType; -use databend_common_sql::NameResolutionContext; -use databend_common_sql::Planner; -use databend_common_sql::Visibility; use databend_common_storages_fuse::TableContext; use databend_query::interpreters::InterpreterFactory; use databend_query::sessions::QueryContext; diff --git a/src/query/service/tests/it/sql/planner/semantic/name_resolution.rs b/src/query/service/tests/it/sql/planner/semantic/name_resolution.rs index 6a0ba0ab879f8..b5bdb02b666f3 100644 --- a/src/query/service/tests/it/sql/planner/semantic/name_resolution.rs +++ b/src/query/service/tests/it/sql/planner/semantic/name_resolution.rs @@ -13,12 +13,12 @@ // limitations under the License. use databend_common_ast::ast::Identifier; +use databend_common_ast::parser::Dialect; use databend_common_ast::parser::parse_expr; use databend_common_ast::parser::tokenize_sql; -use databend_common_ast::parser::Dialect; -use databend_query::sql::normalize_identifier; use databend_query::sql::IdentifierNormalizer; use databend_query::sql::NameResolutionContext; +use databend_query::sql::normalize_identifier; use derive_visitor::DriveMut; // typos:off diff --git a/src/query/service/tests/it/storages/fuse/bloom_index_meta_size.rs b/src/query/service/tests/it/storages/fuse/bloom_index_meta_size.rs index a6ef2e8d06600..ec26082b6d230 100644 --- a/src/query/service/tests/it/storages/fuse/bloom_index_meta_size.rs +++ b/src/query/service/tests/it/storages/fuse/bloom_index_meta_size.rs @@ -18,9 +18,6 @@ use std::sync::Arc; use chrono::Utc; use databend_common_base::base::tokio; -use databend_common_expression::types::Int32Type; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::NumberScalar; use databend_common_expression::ColumnId; use databend_common_expression::DataBlock; use databend_common_expression::FromData; @@ -28,10 +25,13 @@ use databend_common_expression::Scalar; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::Int32Type; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::NumberScalar; +use databend_common_storages_fuse::FuseStorageFormat; use databend_common_storages_fuse::io::TableMetaLocationGenerator; -use databend_common_storages_fuse::statistics::gen_columns_statistics; use databend_common_storages_fuse::statistics::STATS_STRING_PREFIX_LEN; -use databend_common_storages_fuse::FuseStorageFormat; +use databend_common_storages_fuse::statistics::gen_columns_statistics; use databend_query::test_kits::*; use databend_storages_common_cache::CacheAccessor; use databend_storages_common_cache::CacheValue; @@ -48,8 +48,8 @@ use databend_storages_common_table_meta::meta::Statistics; use databend_storages_common_table_meta::meta::Versioned; use opendal::Operator; use parquet::format::FileMetaData; -use sysinfo::get_current_pid; use sysinfo::System; +use sysinfo::get_current_pid; use uuid::Uuid; // NOTE: diff --git a/src/query/service/tests/it/storages/fuse/meta/column_oriented.rs b/src/query/service/tests/it/storages/fuse/meta/column_oriented.rs index 28dd954cafcaf..d566bbf6c4088 100644 --- a/src/query/service/tests/it/storages/fuse/meta/column_oriented.rs +++ b/src/query/service/tests/it/storages/fuse/meta/column_oriented.rs @@ -15,9 +15,6 @@ use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::types::BinaryType; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::UInt64Type; use databend_common_expression::Column; use databend_common_expression::DataBlock; use databend_common_expression::FromData; @@ -26,25 +23,28 @@ use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRef; -use databend_common_storages_fuse::io::read::read_column_oriented_segment; +use databend_common_expression::types::BinaryType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::UInt64Type; +use databend_common_storages_fuse::FuseStorageFormat; use databend_common_storages_fuse::io::TableMetaLocationGenerator; +use databend_common_storages_fuse::io::read::read_column_oriented_segment; use databend_common_storages_fuse::statistics::gen_columns_statistics; use databend_common_storages_fuse::statistics::reduce_block_metas; -use databend_common_storages_fuse::FuseStorageFormat; use databend_query::test_kits::BlockWriter; use databend_query::test_kits::TestFixture; use databend_storages_common_cache::CacheAccessor; use databend_storages_common_cache::CacheManager; -use databend_storages_common_table_meta::meta::column_oriented_segment::*; -use databend_storages_common_table_meta::meta::decode; -use databend_storages_common_table_meta::meta::testing::MetaEncoding; use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::ClusterStatistics; use databend_storages_common_table_meta::meta::Compression; +use databend_storages_common_table_meta::meta::column_oriented_segment::*; +use databend_storages_common_table_meta::meta::decode; +use databend_storages_common_table_meta::meta::testing::MetaEncoding; use opendal::Operator; -async fn generate_column_oriented_segment( -) -> Result<(ColumnOrientedSegment, Vec, TableSchemaRef)> { +async fn generate_column_oriented_segment() +-> Result<(ColumnOrientedSegment, Vec, TableSchemaRef)> { let field_1 = TableField::new("u64", TableDataType::Number(NumberDataType::UInt64)); let field_2 = TableField::new( "nullable_u64", diff --git a/src/query/service/tests/it/storages/fuse/meta/snapshot.rs b/src/query/service/tests/it/storages/fuse/meta/snapshot.rs index 97bf5ad373cc2..db8cf84cbd940 100644 --- a/src/query/service/tests/it/storages/fuse/meta/snapshot.rs +++ b/src/query/service/tests/it/storages/fuse/meta/snapshot.rs @@ -18,11 +18,11 @@ use std::sync::Arc; use databend_common_expression::TableSchema; use databend_query::test_kits::TestFixture; +use databend_storages_common_table_meta::meta::TableMetaTimestamps; +use databend_storages_common_table_meta::meta::TableSnapshot; use databend_storages_common_table_meta::meta::testing::StatisticsV0; use databend_storages_common_table_meta::meta::testing::TableSnapshotV1; use databend_storages_common_table_meta::meta::testing::TableSnapshotV2; -use databend_storages_common_table_meta::meta::TableMetaTimestamps; -use databend_storages_common_table_meta::meta::TableSnapshot; use uuid::Uuid; fn default_snapshot() -> TableSnapshot { diff --git a/src/query/service/tests/it/storages/fuse/operations/alter_table.rs b/src/query/service/tests/it/storages/fuse/operations/alter_table.rs index 8b69fac3acaa3..32c2de2c0fadf 100644 --- a/src/query/service/tests/it/storages/fuse/operations/alter_table.rs +++ b/src/query/service/tests/it/storages/fuse/operations/alter_table.rs @@ -14,13 +14,9 @@ use std::collections::HashSet; -use databend_common_base::base::tokio; use databend_common_base::base::OrderedFloat; +use databend_common_base::base::tokio; use databend_common_exception::Result; -use databend_common_expression::types::Float64Type; -use databend_common_expression::types::Int32Type; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::UInt64Type; use databend_common_expression::Column; use databend_common_expression::ColumnId; use databend_common_expression::DataBlock; @@ -28,13 +24,17 @@ use databend_common_expression::FromData; use databend_common_expression::Scalar; use databend_common_expression::TableDataType; use databend_common_expression::TableField; +use databend_common_expression::types::Float64Type; +use databend_common_expression::types::Int32Type; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::UInt64Type; +use databend_common_sql::Planner; use databend_common_sql::plans::AddColumnOption; use databend_common_sql::plans::AddTableColumnPlan; use databend_common_sql::plans::DropTableColumnPlan; -use databend_common_sql::Planner; -use databend_common_storages_fuse::io::MetaReaders; use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::TableContext; +use databend_common_storages_fuse::io::MetaReaders; use databend_query::interpreters::AddTableColumnInterpreter; use databend_query::interpreters::DropTableColumnInterpreter; use databend_query::interpreters::Interpreter; diff --git a/src/query/service/tests/it/storages/fuse/operations/commit.rs b/src/query/service/tests/it/storages/fuse/operations/commit.rs index 7c30b06b2f4bb..8003886199cc9 100644 --- a/src/query/service/tests/it/storages/fuse/operations/commit.rs +++ b/src/query/service/tests/it/storages/fuse/operations/commit.rs @@ -19,10 +19,10 @@ use std::sync::Arc; use std::time::Duration; use dashmap::DashMap; -use databend_common_base::base::tokio; use databend_common_base::base::Progress; use databend_common_base::base::ProgressValues; use databend_common_base::base::WatchNotify; +use databend_common_base::base::tokio; use databend_common_catalog::catalog::Catalog; use databend_common_catalog::cluster_info::Cluster; use databend_common_catalog::database::Database; @@ -61,8 +61,6 @@ use databend_common_meta_app::principal::UDTFServer; use databend_common_meta_app::principal::UserDefinedConnection; use databend_common_meta_app::principal::UserInfo; use databend_common_meta_app::principal::UserPrivilegeType; -use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; -use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; use databend_common_meta_app::schema::CatalogInfo; use databend_common_meta_app::schema::CommitTableMetaReply; use databend_common_meta_app::schema::CommitTableMetaReq; @@ -134,6 +132,8 @@ use databend_common_meta_app::schema::UpdateMultiTableMetaReq; use databend_common_meta_app::schema::UpdateMultiTableMetaResult; use databend_common_meta_app::schema::UpsertTableOptionReply; use databend_common_meta_app::schema::UpsertTableOptionReq; +use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; +use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_types::MetaId; use databend_common_meta_types::SeqV; @@ -147,9 +147,9 @@ use databend_common_storage::FileStatus; use databend_common_storage::MultiTableInsertStatus; use databend_common_storage::MutationStatus; use databend_common_storage::StageFileInfo; -use databend_common_storages_fuse::operations::load_last_snapshot_hint; -use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::FUSE_TBL_SNAPSHOT_PREFIX; +use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::operations::load_last_snapshot_hint; use databend_common_users::GrantObjectVisibilityChecker; use databend_common_users::Object; use databend_query::sessions::BuildInfoRef; diff --git a/src/query/service/tests/it/storages/fuse/operations/gc.rs b/src/query/service/tests/it/storages/fuse/operations/gc.rs index 4d73008d6f25d..31da6942c4f04 100644 --- a/src/query/service/tests/it/storages/fuse/operations/gc.rs +++ b/src/query/service/tests/it/storages/fuse/operations/gc.rs @@ -19,8 +19,8 @@ use chrono::Utc; use databend_common_base::base::tokio; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_storages_fuse::io::MetaWriter; use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::io::MetaWriter; use databend_query::test_kits::*; use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::TableSnapshot; diff --git a/src/query/service/tests/it/storages/fuse/operations/internal_column.rs b/src/query/service/tests/it/storages/fuse/operations/internal_column.rs index 6bb7a4c78d90f..a28c761c849c4 100644 --- a/src/query/service/tests/it/storages/fuse/operations/internal_column.rs +++ b/src/query/service/tests/it/storages/fuse/operations/internal_column.rs @@ -20,20 +20,20 @@ use databend_common_catalog::plan::InternalColumn; use databend_common_catalog::plan::InternalColumnMeta; use databend_common_catalog::plan::PartInfoPtr; use databend_common_exception::Result; -use databend_common_expression::block_debug::pretty_format_blocks; +use databend_common_expression::BLOCK_NAME_COL_NAME; use databend_common_expression::DataBlock; use databend_common_expression::FieldIndex; -use databend_common_expression::BLOCK_NAME_COL_NAME; use databend_common_expression::ROW_ID_COL_NAME; use databend_common_expression::SEGMENT_NAME_COL_NAME; use databend_common_expression::SNAPSHOT_NAME_COL_NAME; +use databend_common_expression::block_debug::pretty_format_blocks; use databend_common_pipeline::core::Pipeline; +use databend_common_sql::Planner; use databend_common_sql::binder::INTERNAL_COLUMN_FACTORY; use databend_common_sql::executor::table_read_plan::ToReadDataSourcePlan; -use databend_common_sql::Planner; -use databend_common_storages_fuse::io::MetaReaders; use databend_common_storages_fuse::FuseBlockPartInfo; use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::io::MetaReaders; use databend_query::interpreters::InterpreterFactory; use databend_query::pipelines::executor::ExecutorSettings; use databend_query::pipelines::executor::QueryPipelineExecutor; diff --git a/src/query/service/tests/it/storages/fuse/operations/mutation/block_compact_mutator.rs b/src/query/service/tests/it/storages/fuse/operations/mutation/block_compact_mutator.rs index 4b518baa9e50f..a150932d32003 100644 --- a/src/query/service/tests/it/storages/fuse/operations/mutation/block_compact_mutator.rs +++ b/src/query/service/tests/it/storages/fuse/operations/mutation/block_compact_mutator.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![allow(clippy::manual_is_multiple_of)] + use std::collections::HashSet; use std::sync::Arc; @@ -44,8 +46,8 @@ use databend_storages_common_table_meta::meta::SegmentInfo; use databend_storages_common_table_meta::meta::Statistics; use databend_storages_common_table_meta::meta::TableSnapshot; use opendal::Operator; -use rand::thread_rng; use rand::Rng; +use rand::thread_rng; use crate::storages::fuse::operations::mutation::segments_compact_mutator::CompactSegmentTestFixture; diff --git a/src/query/service/tests/it/storages/fuse/operations/mutation/mod.rs b/src/query/service/tests/it/storages/fuse/operations/mutation/mod.rs index 154198361f62b..e55cba91bad0e 100644 --- a/src/query/service/tests/it/storages/fuse/operations/mutation/mod.rs +++ b/src/query/service/tests/it/storages/fuse/operations/mutation/mod.rs @@ -18,5 +18,5 @@ mod recluster_mutator; mod segments_compact_mutator; pub use block_compact_mutator::verify_compact_tasks; -pub use segments_compact_mutator::compact_segment; pub use segments_compact_mutator::CompactSegmentTestFixture; +pub use segments_compact_mutator::compact_segment; diff --git a/src/query/service/tests/it/storages/fuse/operations/mutation/recluster_mutator.rs b/src/query/service/tests/it/storages/fuse/operations/mutation/recluster_mutator.rs index c70c37a235cb8..248b284138411 100644 --- a/src/query/service/tests/it/storages/fuse/operations/mutation/recluster_mutator.rs +++ b/src/query/service/tests/it/storages/fuse/operations/mutation/recluster_mutator.rs @@ -21,13 +21,15 @@ use databend_common_base::base::tokio; use databend_common_catalog::plan::ReclusterParts; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; use databend_common_expression::BlockThresholds; use databend_common_expression::DataBlock; use databend_common_expression::Scalar; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRef; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; +use databend_common_storages_fuse::FuseBlockPartInfo; +use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::io::MetaWriter; use databend_common_storages_fuse::io::TableMetaLocationGenerator; use databend_common_storages_fuse::operations::ReclusterMode; @@ -35,8 +37,6 @@ use databend_common_storages_fuse::operations::ReclusterMutator; use databend_common_storages_fuse::pruning::create_segment_location_vector; use databend_common_storages_fuse::statistics::reducers::merge_statistics_mut; use databend_common_storages_fuse::statistics::reducers::reduce_block_metas; -use databend_common_storages_fuse::FuseBlockPartInfo; -use databend_common_storages_fuse::FuseTable; use databend_query::sessions::TableContext; use databend_query::test_kits::*; use databend_storages_common_table_meta::meta; @@ -46,12 +46,12 @@ use databend_storages_common_table_meta::meta::SegmentInfo; use databend_storages_common_table_meta::meta::Statistics; use databend_storages_common_table_meta::meta::TableSnapshot; use databend_storages_common_table_meta::meta::Versioned; -use rand::thread_rng; use rand::Rng; +use rand::thread_rng; use uuid::Uuid; -use crate::storages::fuse::operations::mutation::verify_compact_tasks; use crate::storages::fuse::operations::mutation::CompactSegmentTestFixture; +use crate::storages::fuse::operations::mutation::verify_compact_tasks; use crate::storages::fuse::utils::new_empty_snapshot; #[tokio::test(flavor = "multi_thread")] @@ -217,14 +217,14 @@ async fn test_safety_for_recluster() -> Result<()> { number_of_segments, number_of_blocks, ); - let unclustered: bool = rand.gen(); + let unclustered: bool = rand.r#gen(); let mut unclustered_segment_indices = HashSet::new(); if unclustered { unclustered_segment_indices = block_number_of_segments .iter() .rev() .enumerate() - .filter(|(_, &num)| num % 4 == 0) + .filter(|(_, num)| *num % 4 == 0) .map(|(index, _)| index) .collect(); } diff --git a/src/query/service/tests/it/storages/fuse/operations/mutation/segments_compact_mutator.rs b/src/query/service/tests/it/storages/fuse/operations/mutation/segments_compact_mutator.rs index df406a607f94c..202a57c95451c 100644 --- a/src/query/service/tests/it/storages/fuse/operations/mutation/segments_compact_mutator.rs +++ b/src/query/service/tests/it/storages/fuse/operations/mutation/segments_compact_mutator.rs @@ -22,38 +22,37 @@ use databend_common_catalog::table::Table; use databend_common_catalog::table::TableExt; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::number::NumberColumn; -use databend_common_expression::types::number::NumberScalar; use databend_common_expression::BlockThresholds; use databend_common_expression::Column; use databend_common_expression::DataBlock; use databend_common_expression::Scalar; use databend_common_expression::SendableDataBlockStream; use databend_common_expression::Value; +use databend_common_expression::types::number::NumberColumn; +use databend_common_expression::types::number::NumberScalar; use databend_common_io::constants::DEFAULT_BLOCK_BUFFER_SIZE; use databend_common_storage::DataOperator; -use databend_common_storages_fuse::io::serialize_block; +use databend_common_storages_fuse::FuseStorageFormat; +use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::io::CompactSegmentInfoReader; use databend_common_storages_fuse::io::MetaReaders; use databend_common_storages_fuse::io::MetaWriter; use databend_common_storages_fuse::io::SegmentsIO; use databend_common_storages_fuse::io::TableMetaLocationGenerator; use databend_common_storages_fuse::io::WriteSettings; +use databend_common_storages_fuse::io::serialize_block; use databend_common_storages_fuse::operations::CompactOptions; use databend_common_storages_fuse::operations::SegmentCompactMutator; use databend_common_storages_fuse::operations::SegmentCompactionState; use databend_common_storages_fuse::operations::SegmentCompactor; +use databend_common_storages_fuse::statistics::RowOrientedSegmentBuilder; use databend_common_storages_fuse::statistics::gen_columns_statistics; use databend_common_storages_fuse::statistics::reducers::merge_statistics_mut; use databend_common_storages_fuse::statistics::sort_by_cluster_stats; -use databend_common_storages_fuse::statistics::RowOrientedSegmentBuilder; -use databend_common_storages_fuse::FuseStorageFormat; -use databend_common_storages_fuse::FuseTable; use databend_query::sessions::QueryContext; use databend_query::sessions::TableContext; use databend_query::test_kits::*; use databend_storages_common_cache::LoadParams; -use databend_storages_common_table_meta::meta::column_oriented_segment::SegmentBuilder; use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::ClusterStatistics; use databend_storages_common_table_meta::meta::Compression; @@ -61,9 +60,10 @@ use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::SegmentInfo; use databend_storages_common_table_meta::meta::Statistics; use databend_storages_common_table_meta::meta::Versioned; +use databend_storages_common_table_meta::meta::column_oriented_segment::SegmentBuilder; use futures_util::TryStreamExt; -use rand::thread_rng; use rand::Rng; +use rand::thread_rng; #[tokio::test(flavor = "multi_thread")] async fn test_compact_segment_normal_case() -> Result<()> { @@ -735,10 +735,11 @@ impl CompactSegmentTestFixture { } else { cluster_key_id.map(|v| { let val = block.get_by_offset(0); - let left = vec![unsafe { val.value().index_unchecked(0) }.to_owned()]; - let right = - vec![unsafe { val.index_unchecked(val.value().len() - 1) } - .to_owned()]; + let left_value = unsafe { val.index_unchecked(0) }.to_owned(); + let right_value = + unsafe { val.index_unchecked(val.value().len() - 1) }.to_owned(); + let left = vec![left_value]; + let right = vec![right_value]; let level = if left.eq(&right) && block.num_rows() >= thresholds.block_per_segment { diff --git a/src/query/service/tests/it/storages/fuse/operations/navigate.rs b/src/query/service/tests/it/storages/fuse/operations/navigate.rs index 3cd8d1a90d551..5e7f7c7877e32 100644 --- a/src/query/service/tests/it/storages/fuse/operations/navigate.rs +++ b/src/query/service/tests/it/storages/fuse/operations/navigate.rs @@ -12,18 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::ops::Sub; use std::time::Duration; +use chrono::Utc; use databend_common_base::base::tokio; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::DataBlock; use databend_common_storages_fuse::io::SnapshotHistoryReader; +use databend_query::storages::fuse::FuseTable; use databend_query::storages::fuse::io::MetaReaders; use databend_query::storages::fuse::io::TableMetaLocationGenerator; -use databend_query::storages::fuse::FuseTable; use databend_query::test_kits::*; use futures::TryStreamExt; @@ -97,10 +97,7 @@ async fn test_fuse_navigate() -> Result<()> { // 4. navigate to the first snapshot // history is order by timestamp DESC let (latest, _ver) = &snapshots[0]; - let instant = latest - .timestamp - .unwrap() - .sub(chrono::Duration::milliseconds(1)); + let instant = latest.timestamp.unwrap() - chrono::Duration::milliseconds(1); let ctx = fixture.new_query_ctx().await?; let tbl_ctx: std::sync::Arc = ctx.clone(); @@ -114,10 +111,7 @@ async fn test_fuse_navigate() -> Result<()> { // 4. navigate beyond the first snapshot let (first_insertion, _ver) = &snapshots[1]; - let instant = first_insertion - .timestamp - .unwrap() - .sub(chrono::Duration::milliseconds(1)); + let instant = first_insertion.timestamp.unwrap() - chrono::Duration::milliseconds(1); // navigate from the instant that is just one ms before the timestamp of the last insertion let res = fuse_table .navigate_to_time_point(&tbl_ctx, loc.clone(), instant) @@ -225,7 +219,12 @@ async fn test_navigate_for_purge() -> Result<()> { let meta = fuse_table.get_operator().stat(&loc).await?; let modified = meta.last_modified(); assert!(modified.is_some()); - let time_point = modified.unwrap().sub(chrono::Duration::milliseconds(1)); + let millis = modified.unwrap().into_inner().as_millisecond(); + let seconds = millis / 1000; + let nanos = ((millis % 1000) * 1_000_000) as u32; + let base_time = chrono::DateTime::::from_timestamp(seconds as i64, nanos) + .expect("valid timestamp from operator metadata"); + let time_point = base_time - chrono::Duration::milliseconds(1); // navigate from the instant that is just one ms before the timestamp of the latest snapshot. let (navigate, files) = fuse_table.list_by_time_point(time_point).await?; assert_eq!(2, files.len()); diff --git a/src/query/service/tests/it/storages/fuse/operations/table_analyze.rs b/src/query/service/tests/it/storages/fuse/operations/table_analyze.rs index 22e9f27352617..5ae8b6ba243f1 100644 --- a/src/query/service/tests/it/storages/fuse/operations/table_analyze.rs +++ b/src/query/service/tests/it/storages/fuse/operations/table_analyze.rs @@ -19,27 +19,27 @@ use databend_common_base::base::tokio; use databend_common_catalog::table::Table; use databend_common_catalog::table::TableExt; use databend_common_exception::Result; -use databend_common_expression::types::number::NumberScalar; use databend_common_expression::ColumnId; use databend_common_expression::Scalar; +use databend_common_expression::types::number::NumberScalar; use databend_common_io::prelude::borsh_deserialize_from_slice; use databend_common_storage::MetaHLL12; +use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::io::MetaReaders; use databend_common_storages_fuse::io::MetaWriter; use databend_common_storages_fuse::statistics::reducers::merge_statistics_mut; -use databend_common_storages_fuse::FuseTable; use databend_query::sessions::QueryContext; use databend_query::sessions::TableContext; -use databend_query::sql::plans::Plan; use databend_query::sql::Planner; +use databend_query::sql::plans::Plan; use databend_query::test_kits::*; use databend_storages_common_cache::LoadParams; -use databend_storages_common_table_meta::meta::testing::TableSnapshotStatisticsV3; use databend_storages_common_table_meta::meta::SegmentInfo; use databend_storages_common_table_meta::meta::Statistics; use databend_storages_common_table_meta::meta::TableSnapshot; use databend_storages_common_table_meta::meta::TableSnapshotStatistics; use databend_storages_common_table_meta::meta::Versioned; +use databend_storages_common_table_meta::meta::testing::TableSnapshotStatisticsV3; #[tokio::test(flavor = "multi_thread")] async fn test_table_modify_column_ndv_statistics() -> Result<()> { diff --git a/src/query/service/tests/it/storages/fuse/pruning.rs b/src/query/service/tests/it/storages/fuse/pruning.rs index 110ec9331a86b..741b67502e534 100644 --- a/src/query/service/tests/it/storages/fuse/pruning.rs +++ b/src/query/service/tests/it/storages/fuse/pruning.rs @@ -18,10 +18,6 @@ use databend_common_ast::ast::Engine; use databend_common_base::base::tokio; use databend_common_catalog::plan::PushDownInfo; use databend_common_exception::Result; -use databend_common_expression::types::number::Int64Type; -use databend_common_expression::types::number::UInt64Type; -use databend_common_expression::types::ArgType; -use databend_common_expression::types::NumberDataType; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::RemoteExpr; @@ -29,20 +25,24 @@ use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::ArgType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::number::Int64Type; +use databend_common_expression::types::number::UInt64Type; use databend_common_meta_app::schema::CreateOption; +use databend_common_sql::BloomIndexColumns; use databend_common_sql::parse_to_filters; use databend_common_sql::plans::CreateTablePlan; -use databend_common_sql::BloomIndexColumns; -use databend_common_storages_fuse::pruning::create_segment_location_vector; -use databend_common_storages_fuse::pruning::FusePruner; use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::pruning::FusePruner; +use databend_common_storages_fuse::pruning::create_segment_location_vector; use databend_query::interpreters::CreateTableInterpreter; use databend_query::interpreters::Interpreter; use databend_query::sessions::QueryContext; use databend_query::sessions::TableContext; -use databend_query::storages::fuse::io::MetaReaders; use databend_query::storages::fuse::FUSE_OPT_KEY_BLOCK_PER_SEGMENT; use databend_query::storages::fuse::FUSE_OPT_KEY_ROW_PER_BLOCK; +use databend_query::storages::fuse::io::MetaReaders; use databend_query::test_kits::*; use databend_storages_common_cache::LoadParams; use databend_storages_common_table_meta::meta::BlockMeta; diff --git a/src/query/service/tests/it/storages/fuse/pruning_column_oriented_segment.rs b/src/query/service/tests/it/storages/fuse/pruning_column_oriented_segment.rs index bc72a874c4561..09041c89c7ed3 100644 --- a/src/query/service/tests/it/storages/fuse/pruning_column_oriented_segment.rs +++ b/src/query/service/tests/it/storages/fuse/pruning_column_oriented_segment.rs @@ -23,10 +23,6 @@ use databend_common_catalog::plan::PartInfoPtr; use databend_common_catalog::plan::PushDownInfo; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::number::Int64Type; -use databend_common_expression::types::number::UInt64Type; -use databend_common_expression::types::ArgType; -use databend_common_expression::types::NumberDataType; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::RemoteExpr; @@ -34,24 +30,28 @@ use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::ArgType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::number::Int64Type; +use databend_common_expression::types::number::UInt64Type; use databend_common_meta_app::schema::CreateOption; use databend_common_pipeline::core::Pipeline; +use databend_common_sql::BloomIndexColumns; use databend_common_sql::parse_to_filters; use databend_common_sql::plans::CreateTablePlan; -use databend_common_sql::BloomIndexColumns; -use databend_common_storages_fuse::pruning::create_segment_location_vector; -use databend_common_storages_fuse::pruning::FusePruner; use databend_common_storages_fuse::FuseBlockPartInfo; use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::pruning::FusePruner; +use databend_common_storages_fuse::pruning::create_segment_location_vector; use databend_query::interpreters::CreateTableInterpreter; use databend_query::interpreters::Interpreter; use databend_query::pipelines::executor::ExecutorSettings; use databend_query::pipelines::executor::QueryPipelineExecutor; use databend_query::sessions::QueryContext; use databend_query::sessions::TableContext; -use databend_query::storages::fuse::io::MetaReaders; use databend_query::storages::fuse::FUSE_OPT_KEY_BLOCK_PER_SEGMENT; use databend_query::storages::fuse::FUSE_OPT_KEY_ROW_PER_BLOCK; +use databend_query::storages::fuse::io::MetaReaders; use databend_query::test_kits::*; use databend_storages_common_cache::LoadParams; use databend_storages_common_table_meta::meta::TableSnapshot; diff --git a/src/query/service/tests/it/storages/fuse/pruning_pipeline.rs b/src/query/service/tests/it/storages/fuse/pruning_pipeline.rs index b48ce3ec94edf..5fbc42c62e434 100644 --- a/src/query/service/tests/it/storages/fuse/pruning_pipeline.rs +++ b/src/query/service/tests/it/storages/fuse/pruning_pipeline.rs @@ -24,10 +24,6 @@ use databend_common_catalog::plan::PartStatistics; use databend_common_catalog::plan::PushDownInfo; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::number::Int64Type; -use databend_common_expression::types::number::UInt64Type; -use databend_common_expression::types::ArgType; -use databend_common_expression::types::NumberDataType; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::RemoteExpr; @@ -35,24 +31,28 @@ use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::ArgType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::number::Int64Type; +use databend_common_expression::types::number::UInt64Type; use databend_common_meta_app::schema::CreateOption; use databend_common_pipeline::core::Pipeline; +use databend_common_sql::BloomIndexColumns; use databend_common_sql::parse_to_filters; use databend_common_sql::plans::CreateTablePlan; -use databend_common_sql::BloomIndexColumns; -use databend_common_storages_fuse::pruning::create_segment_location_vector; -use databend_common_storages_fuse::pruning::FusePruner; use databend_common_storages_fuse::FuseBlockPartInfo; use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::pruning::FusePruner; +use databend_common_storages_fuse::pruning::create_segment_location_vector; use databend_query::interpreters::CreateTableInterpreter; use databend_query::interpreters::Interpreter; use databend_query::pipelines::executor::ExecutorSettings; use databend_query::pipelines::executor::QueryPipelineExecutor; use databend_query::sessions::QueryContext; use databend_query::sessions::TableContext; -use databend_query::storages::fuse::io::MetaReaders; use databend_query::storages::fuse::FUSE_OPT_KEY_BLOCK_PER_SEGMENT; use databend_query::storages::fuse::FUSE_OPT_KEY_ROW_PER_BLOCK; +use databend_query::storages::fuse::io::MetaReaders; use databend_query::test_kits::*; use databend_storages_common_cache::LoadParams; use databend_storages_common_table_meta::meta::TableSnapshot; diff --git a/src/query/service/tests/it/storages/fuse/statistics.rs b/src/query/service/tests/it/storages/fuse/statistics.rs index 83fc87e974878..a533ff7314ac3 100644 --- a/src/query/service/tests/it/storages/fuse/statistics.rs +++ b/src/query/service/tests/it/storages/fuse/statistics.rs @@ -18,12 +18,6 @@ use std::sync::Arc; use chrono::Utc; use databend_common_base::base::tokio; use databend_common_exception::Result; -use databend_common_expression::type_check::check; -use databend_common_expression::types::number::Int32Type; -use databend_common_expression::types::number::NumberScalar; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; use databend_common_expression::BlockThresholds; use databend_common_expression::Column; use databend_common_expression::DataBlock; @@ -36,32 +30,38 @@ use databend_common_expression::Scalar; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; -use databend_common_functions::aggregates::eval_aggr; +use databend_common_expression::type_check::check; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::number::Int32Type; +use databend_common_expression::types::number::NumberScalar; use databend_common_functions::BUILTIN_FUNCTIONS; -use databend_common_sql::evaluator::BlockOperator; +use databend_common_functions::aggregates::eval_aggr; use databend_common_sql::ApproxDistinctColumns; +use databend_common_sql::evaluator::BlockOperator; +use databend_common_storages_fuse::FuseStorageFormat; use databend_common_storages_fuse::io::build_column_hlls; -use databend_common_storages_fuse::statistics::reducers::reduce_block_metas; -use databend_common_storages_fuse::statistics::Trim; use databend_common_storages_fuse::statistics::END_OF_UNICODE_RANGE; use databend_common_storages_fuse::statistics::STATS_STRING_PREFIX_LEN; -use databend_common_storages_fuse::FuseStorageFormat; +use databend_common_storages_fuse::statistics::Trim; +use databend_common_storages_fuse::statistics::reducers::reduce_block_metas; use databend_query::storages::fuse::io::TableMetaLocationGenerator; -use databend_query::storages::fuse::statistics::gen_columns_statistics; -use databend_query::storages::fuse::statistics::reducers; use databend_query::storages::fuse::statistics::ClusterStatsGenerator; use databend_query::storages::fuse::statistics::RowOrientedSegmentBuilder; +use databend_query::storages::fuse::statistics::gen_columns_statistics; +use databend_query::storages::fuse::statistics::reducers; use databend_query::test_kits::*; use databend_storages_common_index::RangeIndex; -use databend_storages_common_table_meta::meta::column_oriented_segment::SegmentBuilder; -use databend_storages_common_table_meta::meta::decode_column_hll; -use databend_storages_common_table_meta::meta::encode_column_hll; use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::ClusterStatistics; use databend_storages_common_table_meta::meta::ColumnStatistics; use databend_storages_common_table_meta::meta::Compression; use databend_storages_common_table_meta::meta::Statistics; use databend_storages_common_table_meta::meta::VirtualColumnMeta; +use databend_storages_common_table_meta::meta::column_oriented_segment::SegmentBuilder; +use databend_storages_common_table_meta::meta::decode_column_hll; +use databend_storages_common_table_meta::meta::encode_column_hll; use opendal::Operator; use rand::Rng; @@ -822,10 +822,10 @@ fn test_reduce_block_meta() -> Result<()> { let mut acc_file_size = 0; let mut acc_bloom_filter_index_size = 0; for _ in 0..size { - let row_count = rng.gen::() / size; - let block_size = rng.gen::() / size; - let file_size = rng.gen::() / size; - let bloom_filter_index_size = rng.gen::() / size; + let row_count = rng.r#gen::() / size; + let block_size = rng.r#gen::() / size; + let file_size = rng.r#gen::() / size; + let bloom_filter_index_size = rng.r#gen::() / size; acc_row_count += row_count; acc_block_size += block_size; acc_file_size += file_size; diff --git a/src/query/service/tests/it/storages/null.rs b/src/query/service/tests/it/storages/null.rs index f8ec8c0fcc195..2871dc0549f6a 100644 --- a/src/query/service/tests/it/storages/null.rs +++ b/src/query/service/tests/it/storages/null.rs @@ -14,10 +14,10 @@ use databend_common_base::base::tokio; use databend_common_exception::Result; -use databend_common_expression::types::NumberDataType; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::NumberDataType; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; use databend_common_sql::executor::table_read_plan::ToReadDataSourcePlan; diff --git a/src/query/service/tests/it/storages/statistics/column_statistics.rs b/src/query/service/tests/it/storages/statistics/column_statistics.rs index 713597ad4d6c0..ecca28d8134ae 100644 --- a/src/query/service/tests/it/storages/statistics/column_statistics.rs +++ b/src/query/service/tests/it/storages/statistics/column_statistics.rs @@ -16,9 +16,6 @@ use std::sync::Arc; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::number::Float64Type; -use databend_common_expression::types::number::Int64Type; -use databend_common_expression::types::NumberDataType; use databend_common_expression::Column; use databend_common_expression::DataBlock; use databend_common_expression::FromData; @@ -27,6 +24,9 @@ use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRef; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::number::Float64Type; +use databend_common_expression::types::number::Int64Type; use databend_query::storages::fuse::statistics::gen_columns_statistics; use databend_query::test_kits::TestFixture; diff --git a/src/query/service/tests/it/storages/system.rs b/src/query/service/tests/it/storages/system.rs index 0f80c2dd109d2..bb22fce3c1cb8 100644 --- a/src/query/service/tests/it/storages/system.rs +++ b/src/query/service/tests/it/storages/system.rs @@ -64,11 +64,11 @@ use databend_query::test_kits::ConfigBuilder; use databend_query::test_kits::TestFixture; use futures::TryStreamExt; use goldenfile::Mint; -use wiremock::matchers::method; -use wiremock::matchers::path; use wiremock::Mock; use wiremock::MockServer; use wiremock::ResponseTemplate; +use wiremock::matchers::method; +use wiremock::matchers::path; async fn run_table_tests( file: &mut impl Write, diff --git a/src/query/service/tests/it/table_functions/numbers_table.rs b/src/query/service/tests/it/table_functions/numbers_table.rs index 4dda49f1e6089..b5b49b17dd3a4 100644 --- a/src/query/service/tests/it/table_functions/numbers_table.rs +++ b/src/query/service/tests/it/table_functions/numbers_table.rs @@ -20,9 +20,9 @@ use databend_common_expression::Scalar; use databend_common_sql::executor::table_read_plan::ToReadDataSourcePlan; use databend_query::sessions::TableContext; use databend_query::stream::ReadDataBlockStream; -use databend_query::table_functions::generate_numbers_parts; use databend_query::table_functions::NumbersPartInfo; use databend_query::table_functions::NumbersTable; +use databend_query::table_functions::generate_numbers_parts; use databend_query::test_kits::TestFixture; use futures::TryStreamExt; use pretty_assertions::assert_eq; diff --git a/src/query/settings/src/settings.rs b/src/query/settings/src/settings.rs index 92abf3e4db4ea..d1fc4dab8ecf5 100644 --- a/src/query/settings/src/settings.rs +++ b/src/query/settings/src/settings.rs @@ -15,9 +15,9 @@ use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; +use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; -use std::sync::Arc; use dashmap::DashMap; use databend_common_config::GlobalConfig; @@ -28,10 +28,10 @@ use itertools::Itertools; use serde::Deserializer; use serde::Serializer; +use crate::SettingMode; use crate::settings_default::DefaultSettingValue; use crate::settings_default::DefaultSettings; use crate::settings_default::SettingRange; -use crate::SettingMode; #[derive(serde::Serialize, serde::Deserialize, PartialEq, Clone)] pub enum ScopeLevel { @@ -278,8 +278,8 @@ impl<'a> IntoIterator for &'a Settings { #[cfg(test)] mod tests { use std::collections::HashMap; - use std::sync::atomic::AtomicBool; use std::sync::Arc; + use std::sync::atomic::AtomicBool; use dashmap::DashMap; use databend_common_exception::Result; diff --git a/src/query/settings/src/settings_default.rs b/src/query/settings/src/settings_default.rs index a2910d927e260..45cf828dc937c 100644 --- a/src/query/settings/src/settings_default.rs +++ b/src/query/settings/src/settings_default.rs @@ -1542,10 +1542,10 @@ impl DefaultSettings { Some(conf) => { let mut num_cpus = num_cpus::get() as u64; - if conf.storage.params.is_fs() { - if let Ok(n) = std::thread::available_parallelism() { - num_cpus = n.get() as u64; - } + if conf.storage.params.is_fs() + && let Ok(n) = std::thread::available_parallelism() + { + num_cpus = n.get() as u64; // Most of x86_64 CPUs have 2-way Hyper-Threading #[cfg(target_arch = "x86_64")] diff --git a/src/query/settings/src/settings_getter_setter.rs b/src/query/settings/src/settings_getter_setter.rs index d4ec3f429dbae..0b2ee123cd5dc 100644 --- a/src/query/settings/src/settings_getter_setter.rs +++ b/src/query/settings/src/settings_getter_setter.rs @@ -22,13 +22,13 @@ use databend_common_io::GeometryDataType; use databend_common_meta_app::principal::UserSettingValue; use databend_common_meta_app::storage::S3StorageClass; -use crate::settings::Settings; -use crate::settings_default::DefaultSettings; use crate::ChangeValue; use crate::ReplaceIntoShuffleStrategy; use crate::ScopeLevel; use crate::SettingMode; use crate::SettingScope; +use crate::settings::Settings; +use crate::settings_default::DefaultSettings; #[derive(Clone, Copy)] pub enum FlightCompression { @@ -348,7 +348,7 @@ impl Settings { /// # Safety pub unsafe fn get_disable_join_reorder(&self) -> Result { - Ok(self.unchecked_try_get_u64("disable_join_reorder")? != 0) + Ok(unsafe { self.unchecked_try_get_u64("disable_join_reorder") }? != 0) } pub fn get_max_push_down_limit(&self) -> Result { @@ -599,7 +599,7 @@ impl Settings { /// # Safety pub unsafe fn get_deduplicate_label(&self) -> Result> { - let deduplicate_label = self.unchecked_try_get_string("deduplicate_label")?; + let deduplicate_label = unsafe { self.unchecked_try_get_string("deduplicate_label")? }; if deduplicate_label.is_empty() { Ok(None) } else { @@ -609,7 +609,7 @@ impl Settings { /// # Safety pub unsafe fn set_deduplicate_label(&self, val: String) -> Result<()> { - self.unchecked_set_setting("deduplicate_label".to_string(), val) + unsafe { self.unchecked_set_setting("deduplicate_label".to_string(), val) } } pub fn get_enable_distributed_copy(&self) -> Result { @@ -942,7 +942,7 @@ impl Settings { /// # Safety pub unsafe fn set_warehouse(&self, warehouse: String) -> Result<()> { - self.unchecked_set_setting(String::from("warehouse"), warehouse) + unsafe { self.unchecked_set_setting(String::from("warehouse"), warehouse) } } pub fn get_hilbert_num_range_ids(&self) -> Result { diff --git a/src/query/settings/src/settings_global.rs b/src/query/settings/src/settings_global.rs index c0f1a788407f7..486f5d1662d1b 100644 --- a/src/query/settings/src/settings_global.rs +++ b/src/query/settings/src/settings_global.rs @@ -22,11 +22,11 @@ use databend_common_meta_app::tenant::Tenant; use databend_common_users::UserApiProvider; use log::warn; +use crate::ScopeLevel; +use crate::SettingScope; use crate::settings::ChangeValue; use crate::settings::Settings; use crate::settings_default::DefaultSettings; -use crate::ScopeLevel; -use crate::SettingScope; impl Settings { #[async_backtrace::framed] diff --git a/src/query/sql/src/evaluator/cse.rs b/src/query/sql/src/evaluator/cse.rs index 84591434e19d1..014bf73c12c72 100644 --- a/src/query/sql/src/evaluator/cse.rs +++ b/src/query/sql/src/evaluator/cse.rs @@ -14,9 +14,9 @@ use std::collections::HashMap; -use databend_common_expression::expr; use databend_common_expression::Cast; use databend_common_expression::Expr; +use databend_common_expression::expr; use databend_common_functions::BUILTIN_FUNCTIONS; use log::debug; diff --git a/src/query/sql/src/executor/physical_plans/common.rs b/src/query/sql/src/executor/physical_plans/common.rs index cf471ffd57608..d95aaec5b91c5 100644 --- a/src/query/sql/src/executor/physical_plans/common.rs +++ b/src/query/sql/src/executor/physical_plans/common.rs @@ -15,15 +15,15 @@ use std::fmt::Display; use std::fmt::Formatter; -use databend_common_expression::types::DataType; use databend_common_expression::FieldIndex; use databend_common_expression::RemoteExpr; use databend_common_expression::Scalar; +use databend_common_expression::types::DataType; use databend_common_functions::aggregates::AggregateFunctionSortDesc; +use crate::IndexType; use crate::plans::UDFField; use crate::plans::UDFType; -use crate::IndexType; pub type MatchExpr = Vec<(Option, Option>)>; diff --git a/src/query/sql/src/executor/util.rs b/src/query/sql/src/executor/util.rs index 749dd8d52730c..fcbd35773b37d 100644 --- a/src/query/sql/src/executor/util.rs +++ b/src/query/sql/src/executor/util.rs @@ -16,10 +16,10 @@ use std::sync::LazyLock; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::type_check; -use databend_common_expression::types::DataType; use databend_common_expression::ColumnIndex; use databend_common_expression::Expr; +use databend_common_expression::type_check; +use databend_common_expression::types::DataType; use databend_common_functions::BUILTIN_FUNCTIONS; use regex::Regex; diff --git a/src/query/sql/src/lib.rs b/src/query/sql/src/lib.rs index bef2223184eee..d5b33d6285e52 100644 --- a/src/query/sql/src/lib.rs +++ b/src/query/sql/src/lib.rs @@ -12,11 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(clippy::uninlined_format_args)] +#![allow( + clippy::collapsible_if, + clippy::derivable_impls, + clippy::iter_kv_map, + clippy::let_and_return, + clippy::replace_box, + clippy::unnecessary_unwrap, + clippy::uninlined_format_args +)] #![allow(non_local_definitions)] #![feature(box_patterns)] #![feature(iterator_try_reduce)] -#![feature(let_chains)] #![feature(trivial_bounds)] #![feature(try_blocks)] #![feature(extend_one)] diff --git a/src/query/sql/src/planner/binder/aggregate.rs b/src/query/sql/src/planner/binder/aggregate.rs index 5007300fa56b7..de6c5841b5ac7 100644 --- a/src/query/sql/src/planner/binder/aggregate.rs +++ b/src/query/sql/src/planner/binder/aggregate.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::hash_map::Entry; use std::collections::HashMap; use std::collections::HashSet; +use std::collections::hash_map::Entry; use std::sync::Arc; use databend_common_ast::ast::ColumnRef; @@ -24,26 +24,28 @@ use databend_common_ast::ast::Literal; use databend_common_ast::ast::SelectTarget; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::Scalar; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::NumberScalar; -use databend_common_expression::Scalar; use indexmap::Equivalent; use itertools::Itertools; -use super::prune_by_children; use super::ExprContext; use super::Finder; -use crate::binder::project_set::SetReturningAnalyzer; -use crate::binder::scalar::ScalarBinder; -use crate::binder::select::SelectList; +use super::prune_by_children; +use crate::BindContext; +use crate::IndexType; +use crate::MetadataRef; use crate::binder::Binder; use crate::binder::ColumnBinding; use crate::binder::ColumnBindingBuilder; use crate::binder::Visibility; +use crate::binder::project_set::SetReturningAnalyzer; +use crate::binder::scalar::ScalarBinder; +use crate::binder::select::SelectList; use crate::normalize_identifier; use crate::optimizer::ir::SExpr; -use crate::plans::walk_expr_mut; use crate::plans::Aggregate; use crate::plans::AggregateFunction; use crate::plans::AggregateFunctionScalarSortDesc; @@ -57,9 +59,7 @@ use crate::plans::ScalarItem; use crate::plans::UDAFCall; use crate::plans::Visitor; use crate::plans::VisitorMut; -use crate::BindContext; -use crate::IndexType; -use crate::MetadataRef; +use crate::plans::walk_expr_mut; /// Information for `GROUPING SETS`. /// diff --git a/src/query/sql/src/planner/binder/async_function_desc.rs b/src/query/sql/src/planner/binder/async_function_desc.rs index 44e1e575b3917..69c013772c9d6 100644 --- a/src/query/sql/src/planner/binder/async_function_desc.rs +++ b/src/query/sql/src/planner/binder/async_function_desc.rs @@ -14,8 +14,8 @@ use databend_common_expression::types::DataType; -use crate::plans::AsyncFunctionArgument; use crate::IndexType; +use crate::plans::AsyncFunctionArgument; #[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)] pub struct AsyncFunctionDesc { diff --git a/src/query/sql/src/planner/binder/bind_context.rs b/src/query/sql/src/planner/binder/bind_context.rs index 105b1e034e96a..e13d4245708e0 100644 --- a/src/query/sql/src/planner/binder/bind_context.rs +++ b/src/query/sql/src/planner/binder/bind_context.rs @@ -12,17 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::btree_map; use std::collections::BTreeMap; use std::collections::BTreeSet; +use std::collections::btree_map; use std::hash::Hash; use dashmap::DashMap; +use databend_common_ast::Span; use databend_common_ast::ast::Identifier; use databend_common_ast::ast::Query; use databend_common_ast::ast::TableAlias; use databend_common_ast::ast::WindowSpec; -use databend_common_ast::Span; use databend_common_catalog::plan::InternalColumn; use databend_common_catalog::plan::InvertedIndexInfo; use databend_common_catalog::plan::VectorIndexInfo; @@ -38,17 +38,17 @@ use itertools::Itertools; use super::AggregateInfo; use super::INTERNAL_COLUMN_FACTORY; +use crate::ColumnSet; +use crate::IndexType; +use crate::MetadataRef; +use crate::NameResolutionContext; +use crate::binder::ColumnBindingBuilder; use crate::binder::column_binding::ColumnBinding; use crate::binder::project_set::SetReturningInfo; use crate::binder::window::WindowInfo; -use crate::binder::ColumnBindingBuilder; use crate::normalize_identifier; use crate::optimizer::ir::SExpr; use crate::plans::ScalarExpr; -use crate::ColumnSet; -use crate::IndexType; -use crate::MetadataRef; -use crate::NameResolutionContext; /// Context of current expression, this is used to check if /// the expression is valid in current context. @@ -366,7 +366,6 @@ impl BindContext { alias: alias.clone(), scalar: scalar.clone(), }); - alias_match_count += 1; } } } diff --git a/src/query/sql/src/planner/binder/bind_mutation/bind.rs b/src/query/sql/src/planner/binder/bind_mutation/bind.rs index 694877e563f68..6a9a01ca8a291 100644 --- a/src/query/sql/src/planner/binder/bind_mutation/bind.rs +++ b/src/query/sql/src/planner/binder/bind_mutation/bind.rs @@ -17,33 +17,33 @@ use std::collections::HashSet; use std::fmt; use std::sync::Arc; +use databend_common_ast::ParseError; use databend_common_ast::ast::Expr; use databend_common_ast::ast::MatchOperation; use databend_common_ast::ast::MatchedClause; use databend_common_ast::ast::UnmatchedClause; -use databend_common_ast::ParseError; use databend_common_catalog::lock::LockTableOption; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::FieldIndex; use databend_common_expression::TableSchemaRef; +use databend_common_expression::types::DataType; +use crate::BindContext; +use crate::ColumnEntry; +use crate::ScalarBinder; +use crate::ScalarExpr; +use crate::binder::Binder; use crate::binder::bind_mutation::mutation_expression::MutationExpression; use crate::binder::bind_mutation::mutation_expression::MutationExpressionBindResult; use crate::binder::util::TableIdentifier; use crate::binder::wrap_cast; -use crate::binder::Binder; use crate::normalize_identifier; use crate::optimizer::ir::SExpr; use crate::plans::MatchedEvaluator; use crate::plans::Plan; use crate::plans::RelOperator; use crate::plans::UnmatchedEvaluator; -use crate::BindContext; -use crate::ColumnEntry; -use crate::ScalarBinder; -use crate::ScalarExpr; #[derive(Default, Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] pub enum MutationType { diff --git a/src/query/sql/src/planner/binder/bind_mutation/delete.rs b/src/query/sql/src/planner/binder/bind_mutation/delete.rs index 0e8fcb8d7d671..b17ef709dc1af 100644 --- a/src/query/sql/src/planner/binder/bind_mutation/delete.rs +++ b/src/query/sql/src/planner/binder/bind_mutation/delete.rs @@ -19,13 +19,13 @@ use databend_common_ast::ast::TableReference; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use crate::BindContext; +use crate::binder::Binder; +use crate::binder::MutationStrategy; use crate::binder::bind_mutation::bind::Mutation; use crate::binder::bind_mutation::mutation_expression::MutationExpression; use crate::binder::util::TableIdentifier; -use crate::binder::Binder; -use crate::binder::MutationStrategy; use crate::plans::Plan; -use crate::BindContext; impl Binder { #[async_backtrace::framed] diff --git a/src/query/sql/src/planner/binder/bind_mutation/merge.rs b/src/query/sql/src/planner/binder/bind_mutation/merge.rs index 593eab1837c5b..1bc2fd4377bc1 100644 --- a/src/query/sql/src/planner/binder/bind_mutation/merge.rs +++ b/src/query/sql/src/planner/binder/bind_mutation/merge.rs @@ -21,13 +21,13 @@ use databend_common_ast::ast::UnmatchedClause; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use crate::BindContext; +use crate::binder::Binder; +use crate::binder::MutationStrategy; use crate::binder::bind_mutation::bind::Mutation; use crate::binder::bind_mutation::mutation_expression::MutationExpression; use crate::binder::util::TableIdentifier; -use crate::binder::Binder; -use crate::binder::MutationStrategy; use crate::plans::Plan; -use crate::BindContext; // Merge into strategies: // 1. Insert only: target right-anti join source. diff --git a/src/query/sql/src/planner/binder/bind_mutation/mutation_expression.rs b/src/query/sql/src/planner/binder/bind_mutation/mutation_expression.rs index 614280877b673..c9b61cd284bba 100644 --- a/src/query/sql/src/planner/binder/bind_mutation/mutation_expression.rs +++ b/src/query/sql/src/planner/binder/bind_mutation/mutation_expression.rs @@ -24,19 +24,27 @@ use databend_common_catalog::plan::InternalColumnType; use databend_common_catalog::table::Table; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::TableSchema; use databend_common_expression::ROW_ID_COL_NAME; +use databend_common_expression::TableSchema; -use crate::binder::split_conjunctions; -use crate::binder::util::TableIdentifier; +use crate::BindContext; +use crate::ColumnBinding; +use crate::ColumnBindingBuilder; +use crate::ColumnSet; +use crate::DUMMY_COLUMN_INDEX; +use crate::ScalarBinder; +use crate::ScalarExpr; +use crate::Visibility; use crate::binder::Binder; use crate::binder::Finder; use crate::binder::InternalColumnBinding; use crate::binder::MutationStrategy; use crate::binder::MutationType; +use crate::binder::split_conjunctions; +use crate::binder::util::TableIdentifier; +use crate::optimizer::OptimizerContext; use crate::optimizer::ir::SExpr; use crate::optimizer::optimizers::operator::SubqueryDecorrelatorOptimizer; -use crate::optimizer::OptimizerContext; use crate::plans::BoundColumnRef; use crate::plans::Filter; use crate::plans::Join; @@ -45,14 +53,6 @@ use crate::plans::MutationSource; use crate::plans::RelOperator; use crate::plans::SubqueryExpr; use crate::plans::Visitor; -use crate::BindContext; -use crate::ColumnBinding; -use crate::ColumnBindingBuilder; -use crate::ColumnSet; -use crate::ScalarBinder; -use crate::ScalarExpr; -use crate::Visibility; -use crate::DUMMY_COLUMN_INDEX; pub enum MutationExpression { Merge { @@ -591,7 +591,7 @@ pub fn target_probe(s_expr: &SExpr, target_table_index: usize) -> Result { } fn contains_target_table(s_expr: &SExpr, target_table_index: usize) -> bool { - if let RelOperator::Scan(ref scan) = s_expr.plan() { + if let RelOperator::Scan(scan) = s_expr.plan() { scan.table_index == target_table_index } else { s_expr diff --git a/src/query/sql/src/planner/binder/bind_mutation/update.rs b/src/query/sql/src/planner/binder/bind_mutation/update.rs index e18e303edc90b..fbf2f597b84fd 100644 --- a/src/query/sql/src/planner/binder/bind_mutation/update.rs +++ b/src/query/sql/src/planner/binder/bind_mutation/update.rs @@ -22,12 +22,18 @@ use databend_common_ast::ast::TableReference; use databend_common_ast::ast::UpdateStmt; use databend_common_exception::Result; +use crate::BindContext; +use crate::ColumnBinding; +use crate::ColumnBindingBuilder; +use crate::IndexType; +use crate::ScalarExpr; +use crate::Visibility; +use crate::binder::Binder; use crate::binder::aggregate::AggregateRewriter; use crate::binder::bind_mutation::bind::Mutation; use crate::binder::bind_mutation::bind::MutationStrategy; use crate::binder::bind_mutation::mutation_expression::MutationExpression; use crate::binder::util::TableIdentifier; -use crate::binder::Binder; use crate::optimizer::ir::Matcher; use crate::plans::AggregateFunction; use crate::plans::BoundColumnRef; @@ -37,12 +43,6 @@ use crate::plans::RelOp; use crate::plans::RelOperator; use crate::plans::ScalarItem; use crate::plans::VisitorMut; -use crate::BindContext; -use crate::ColumnBinding; -use crate::ColumnBindingBuilder; -use crate::IndexType; -use crate::ScalarExpr; -use crate::Visibility; impl Binder { #[async_backtrace::framed] diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index 3606ea6103b4f..5690c16067b15 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -15,6 +15,8 @@ use std::collections::HashMap; use std::sync::Arc; +use databend_common_ast::Span; +use databend_common_ast::ast::CTE; use databend_common_ast::ast::ColumnDefinition; use databend_common_ast::ast::CreateOption; use databend_common_ast::ast::CreateTableSource; @@ -27,8 +29,6 @@ use databend_common_ast::ast::SetExpr; use databend_common_ast::ast::TableReference; use databend_common_ast::ast::TableType; use databend_common_ast::ast::With; -use databend_common_ast::ast::CTE; -use databend_common_ast::Span; use databend_common_catalog::catalog::CATALOG_DEFAULT; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -38,16 +38,16 @@ use derive_visitor::DriveMut; use derive_visitor::Visitor; use derive_visitor::VisitorMut; +use crate::NameResolutionContext; use crate::normalize_identifier; use crate::optimizer::ir::SExpr; -use crate::planner::binder::scalar::ScalarBinder; use crate::planner::binder::BindContext; use crate::planner::binder::Binder; +use crate::planner::binder::scalar::ScalarBinder; use crate::plans::BoundColumnRef; use crate::plans::ScalarExpr; use crate::plans::Sort; use crate::plans::SortItem; -use crate::NameResolutionContext; #[derive(Debug, Default, Visitor)] #[visitor(TableReference(enter))] struct CTERefCounter { @@ -168,7 +168,8 @@ impl Binder { ); let mut order_by_items = Vec::with_capacity(query.order_by.len()); - let default_nulls_first = self.ctx.get_settings().get_nulls_first(); + let settings = self.ctx.get_settings(); + let default_nulls_first = settings.get_nulls_first(); for order in query.order_by.iter() { match order.expr { diff --git a/src/query/sql/src/planner/binder/bind_query/bind_select.rs b/src/query/sql/src/planner/binder/bind_query/bind_select.rs index af63b3b93256f..1a1f54baf4082 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind_select.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind_select.rs @@ -14,6 +14,7 @@ use std::sync::Arc; +use databend_common_ast::Span; use databend_common_ast::ast::BinaryOperator; use databend_common_ast::ast::ColumnID; use databend_common_ast::ast::ColumnPosition; @@ -38,7 +39,6 @@ use databend_common_ast::ast::SetExpr; use databend_common_ast::ast::TableAlias; use databend_common_ast::ast::TableReference; use databend_common_ast::ast::UnpivotName; -use databend_common_ast::Span; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::DataBlock; @@ -49,15 +49,11 @@ use derive_visitor::Drive; use derive_visitor::Visitor; use log::warn; +use crate::AsyncFunctionRewriter; use crate::optimizer::ir::SExpr; +use crate::planner::QueryExecutor; use crate::planner::binder::BindContext; use crate::planner::binder::Binder; -use crate::planner::QueryExecutor; -use crate::AsyncFunctionRewriter; - -// A normalized IR for `SELECT` clause. -#[derive(Debug, Default)] -pub struct SelectList {} impl Binder { #[async_backtrace::framed] @@ -797,7 +793,7 @@ impl SelectRewriter { let columns = unpivot .column_names .iter() - .map(|name| (name.ident.to_owned())) + .map(|name| name.ident.to_owned()) .collect::>(); if let Some(star) = new_select_list.iter_mut().find(|target| target.is_star()) { star.exclude(columns.clone()); diff --git a/src/query/sql/src/planner/binder/bind_query/bind_value.rs b/src/query/sql/src/planner/binder/bind_query/bind_value.rs index 68d8d620fa241..6c06dbfc66676 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind_value.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind_value.rs @@ -16,21 +16,31 @@ use std::collections::HashMap; use std::collections::HashSet; use std::sync::Arc; -use databend_common_ast::ast::Expr as AExpr; use databend_common_ast::Span; +use databend_common_ast::ast::Expr as AExpr; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::type_check::common_super_type; -use databend_common_expression::types::DataType; use databend_common_expression::ColumnBuilder; use databend_common_expression::DataBlock; use databend_common_expression::DataField; use databend_common_expression::DataSchema; use databend_common_expression::DataSchemaRefExt; use databend_common_expression::Evaluator; +use databend_common_expression::type_check::common_super_type; +use databend_common_expression::types::DataType; use databend_common_functions::BUILTIN_FUNCTIONS; +use crate::BindContext; +use crate::Binder; +use crate::ColumnBinding; +use crate::ColumnBindingBuilder; +use crate::ColumnSet; +use crate::MetadataRef; +use crate::NameResolutionContext; +use crate::ScalarBinder; +use crate::ScalarExpr; +use crate::Visibility; use crate::binder::wrap_cast; use crate::optimizer::ir::SExpr; use crate::plans::Aggregate; @@ -43,16 +53,6 @@ use crate::plans::ExpressionScan; use crate::plans::HashJoinBuildCacheInfo; use crate::plans::RelOperator; use crate::plans::ScalarItem; -use crate::BindContext; -use crate::Binder; -use crate::ColumnBinding; -use crate::ColumnBindingBuilder; -use crate::ColumnSet; -use crate::MetadataRef; -use crate::NameResolutionContext; -use crate::ScalarBinder; -use crate::ScalarExpr; -use crate::Visibility; // The `ExpressionScanContext` is used to store the information of // expression scan and hash join build cache. diff --git a/src/query/sql/src/planner/binder/bind_query/mod.rs b/src/query/sql/src/planner/binder/bind_query/mod.rs index a100b232fa511..c5db12f265874 100644 --- a/src/query/sql/src/planner/binder/bind_query/mod.rs +++ b/src/query/sql/src/planner/binder/bind_query/mod.rs @@ -19,5 +19,5 @@ mod bind_set_expr; mod bind_value; pub use bind_select::MaxColumnPosition; -pub use bind_value::bind_values; pub use bind_value::ExpressionScanContext; +pub use bind_value::bind_values; diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind.rs index a3bfaa7c336ad..8395522408357 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind.rs @@ -15,9 +15,9 @@ use databend_common_ast::ast::TableReference; use databend_common_exception::Result; +use crate::BindContext; use crate::binder::Binder; use crate::optimizer::ir::SExpr; -use crate::BindContext; impl Binder { pub(crate) fn bind_table_reference( diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_asof_join.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_asof_join.rs index 0738c28d95b8c..859cbf3b1be37 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_asof_join.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_asof_join.rs @@ -16,13 +16,15 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::NumberScalar; use databend_common_expression::Scalar; +use databend_common_expression::types::NumberScalar; -use crate::binder::bind_window_function_info; -use crate::binder::window::WindowRewriter; +use crate::BindContext; +use crate::ColumnBindingBuilder; use crate::binder::JoinPredicate; use crate::binder::Visibility; +use crate::binder::bind_window_function_info; +use crate::binder::window::WindowRewriter; use crate::optimizer::ir::RelationalProperty; use crate::optimizer::ir::SExpr; use crate::planner::binder::Binder; @@ -38,8 +40,6 @@ use crate::plans::WindowFuncFrameBound; use crate::plans::WindowFuncFrameUnits; use crate::plans::WindowFuncType; use crate::plans::WindowOrderBy; -use crate::BindContext; -use crate::ColumnBindingBuilder; const GT: &str = "gt"; const GTE: &str = "gte"; diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs index 42fcf4cd2c7bb..5feb86f60be52 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs @@ -22,6 +22,7 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use indexmap::IndexMap; +use crate::ColumnBinding; use crate::binder::BindContext; use crate::binder::Binder; use crate::binder::CteContext; @@ -33,7 +34,6 @@ use crate::plans::MaterializedCTE; use crate::plans::MaterializedCTERef; use crate::plans::RelOperator; use crate::plans::Sequence; -use crate::ColumnBinding; impl Binder { pub fn init_cte(&mut self, bind_context: &mut BindContext, with: &Option) -> Result<()> { @@ -124,7 +124,11 @@ impl Binder { "The CTE '{}' has {} columns ({:?}), but {} aliases ({:?}) were provided. Ensure the number of aliases matches the number of columns in the CTE.", table_name, cte_bind_context.columns.len(), - cte_bind_context.columns.iter().map(|c| &c.column_name).collect::>(), + cte_bind_context + .columns + .iter() + .map(|c| &c.column_name) + .collect::>(), column_alias.len(), column_alias, ))); diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_join.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_join.rs index 43bd47bb5e75f..a8e4e9c6bb53a 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_join.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_join.rs @@ -15,28 +15,32 @@ use std::sync::Arc; use async_recursion::async_recursion; -use databend_common_ast::ast::split_conjunctions_expr; -use databend_common_ast::ast::split_equivalent_predicate_expr; +use databend_common_ast::Span; use databend_common_ast::ast::Expr; use databend_common_ast::ast::JoinCondition; use databend_common_ast::ast::JoinOperator; -use databend_common_ast::Span; +use databend_common_ast::ast::split_conjunctions_expr; +use databend_common_ast::ast::split_equivalent_predicate_expr; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use crate::binder::wrap_nullable; +use crate::BindContext; +use crate::ColumnBinding; +use crate::ColumnSet; +use crate::MetadataRef; use crate::binder::Finder; use crate::binder::JoinPredicate; use crate::binder::Visibility; +use crate::binder::wrap_nullable; use crate::normalize_identifier; +use crate::optimizer::OptimizerContext; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::SExpr; use crate::optimizer::optimizers::operator::FlattenInfo; use crate::optimizer::optimizers::operator::SubqueryDecorrelatorOptimizer; -use crate::optimizer::OptimizerContext; -use crate::planner::binder::scalar::ScalarBinder; use crate::planner::binder::Binder; +use crate::planner::binder::scalar::ScalarBinder; use crate::planner::semantic::NameResolutionContext; use crate::plans::BoundColumnRef; use crate::plans::EvalScalar; @@ -48,10 +52,6 @@ use crate::plans::JoinType; use crate::plans::ScalarExpr; use crate::plans::ScalarItem; use crate::plans::Visitor; -use crate::BindContext; -use crate::ColumnBinding; -use crate::ColumnSet; -use crate::MetadataRef; pub struct JoinConditions { pub(crate) left_conditions: Vec, diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_location.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_location.rs index b49e356f41670..826de5305d706 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_location.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_location.rs @@ -24,10 +24,10 @@ use databend_common_meta_app::principal::FileFormatParams; use databend_common_meta_app::principal::StageFileFormatType; use databend_common_storage::StageFilesInfo; -use crate::binder::copy_into_table::resolve_file_location; +use crate::BindContext; use crate::binder::Binder; +use crate::binder::copy_into_table::resolve_file_location; use crate::optimizer::ir::SExpr; -use crate::BindContext; impl Binder { /// Bind a location. diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_obfuscate.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_obfuscate.rs index 9359906187c98..1456b3c7791b4 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_obfuscate.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_obfuscate.rs @@ -28,16 +28,16 @@ use databend_common_ast::ast::TableAlias; use databend_common_ast::ast::TableReference; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::NumberScalar; use databend_common_expression::Scalar; use databend_common_expression::TableDataType; +use databend_common_expression::types::NumberScalar; +use crate::BindContext; +use crate::ScalarBinder; +use crate::binder::Binder; use crate::binder::table_args::bind_table_args; use crate::binder::util::TableIdentifier; -use crate::binder::Binder; use crate::optimizer::ir::SExpr; -use crate::BindContext; -use crate::ScalarBinder; impl Binder { pub(crate) fn bind_obfuscate( diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_subquery.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_subquery.rs index edf42bc6e4c2d..8af08ca922579 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_subquery.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_subquery.rs @@ -17,10 +17,10 @@ use databend_common_ast::ast::TableAlias; use databend_common_exception::Result; use databend_common_expression::is_stream_column; -use crate::binder::Binder; -use crate::optimizer::ir::SExpr; use crate::BindContext; use crate::Visibility; +use crate::binder::Binder; +use crate::optimizer::ir::SExpr; impl Binder { /// Bind a subquery. diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs index bc3a3d6cd8fed..c5cfd3d763628 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_ast::Span; use databend_common_ast::ast::Identifier; use databend_common_ast::ast::SampleConfig; use databend_common_ast::ast::Statement; @@ -20,7 +21,6 @@ use databend_common_ast::ast::TemporalClause; use databend_common_ast::ast::WithOptions; use databend_common_ast::parser::parse_sql; use databend_common_ast::parser::tokenize_sql; -use databend_common_ast::Span; use databend_common_catalog::table::TimeNavigation; use databend_common_catalog::table_with_options::check_with_opt_valid; use databend_common_catalog::table_with_options::get_with_opt_consume; @@ -30,10 +30,10 @@ use databend_common_exception::Result; use databend_common_storages_basic::view_table::QUERY; use databend_storages_common_table_meta::table::get_change_type; -use crate::binder::util::TableIdentifier; +use crate::BindContext; use crate::binder::Binder; +use crate::binder::util::TableIdentifier; use crate::optimizer::ir::SExpr; -use crate::BindContext; impl Binder { /// Bind a base table. /// A base table is a table that is not a view or CTE. diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs index e595074d60fdf..a344c913df4fc 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs @@ -15,6 +15,7 @@ use std::collections::HashMap; use std::sync::Arc; +use databend_common_ast::Span; use databend_common_ast::ast::Expr; use databend_common_ast::ast::FunctionCall as ASTFunctionCall; use databend_common_ast::ast::Identifier; @@ -24,17 +25,16 @@ use databend_common_ast::ast::SelectStmt; use databend_common_ast::ast::SelectTarget; use databend_common_ast::ast::TableAlias; use databend_common_ast::ast::TableReference; -use databend_common_ast::Span; use databend_common_catalog::catalog::CatalogManager; use databend_common_catalog::catalog_kind::CATALOG_DEFAULT; use databend_common_catalog::table_args::TableArgs; use databend_common_catalog::table_function::TableFunction; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::display::scalar_ref_to_string; -use databend_common_expression::types::NumberScalar; use databend_common_expression::FunctionKind; use databend_common_expression::Scalar; +use databend_common_expression::display::scalar_ref_to_string; +use databend_common_expression::types::NumberScalar; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::principal::UDFDefinition; use databend_common_storages_basic::ResultCacheMetaManager; @@ -42,11 +42,15 @@ use databend_common_storages_basic::ResultScan; use databend_common_users::UserApiProvider; use derive_visitor::DriveMut; -use crate::binder::scalar::ScalarBinder; -use crate::binder::table_args::bind_table_args; +use crate::BindContext; +use crate::Planner; +use crate::ScalarExpr; +use crate::UDFArgVisitor; use crate::binder::Binder; use crate::binder::ColumnBindingBuilder; use crate::binder::Visibility; +use crate::binder::scalar::ScalarBinder; +use crate::binder::table_args::bind_table_args; use crate::optimizer::ir::SExpr; use crate::planner::semantic::normalize_identifier; use crate::plans::BoundColumnRef; @@ -56,10 +60,6 @@ use crate::plans::FunctionCall; use crate::plans::Plan; use crate::plans::RelOperator; use crate::plans::ScalarItem; -use crate::BindContext; -use crate::Planner; -use crate::ScalarExpr; -use crate::UDFArgVisitor; impl Binder { /// Bind a table function. diff --git a/src/query/sql/src/planner/binder/binder.rs b/src/query/sql/src/planner/binder/binder.rs index 79e888fa93e1b..7262197920c25 100644 --- a/src/query/sql/src/planner/binder/binder.rs +++ b/src/query/sql/src/planner/binder/binder.rs @@ -23,20 +23,20 @@ use databend_common_ast::ast::Hint; use databend_common_ast::ast::Identifier; use databend_common_ast::ast::Settings; use databend_common_ast::ast::Statement; +use databend_common_ast::parser::Dialect; use databend_common_ast::parser::parse_sql; use databend_common_ast::parser::tokenize_sql; -use databend_common_ast::parser::Dialect; use databend_common_catalog::catalog::CatalogManager; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::Constant; use databend_common_expression::ConstantFolder; use databend_common_expression::Expr; use databend_common_expression::FunctionKind; use databend_common_expression::SEARCH_MATCHED_COLUMN_ID; use databend_common_expression::SEARCH_SCORE_COLUMN_ID; +use databend_common_expression::types::DataType; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::principal::FileFormatOptionsReader; use databend_common_meta_app::principal::FileFormatParams; @@ -45,11 +45,18 @@ use databend_storages_common_table_meta::table::is_stream_name; use log::warn; use super::Finder; +use crate::BindContext; +use crate::ColumnBinding; +use crate::MetadataRef; +use crate::NameResolutionContext; +use crate::ScalarExpr; +use crate::TypeChecker; +use crate::Visibility; +use crate::binder::ColumnBindingBuilder; use crate::binder::bind_query::ExpressionScanContext; use crate::binder::show::get_show_options; use crate::binder::util::illegal_ident_name; use crate::binder::wrap_cast; -use crate::binder::ColumnBindingBuilder; use crate::normalize_identifier; use crate::optimizer::ir::SExpr; use crate::planner::QueryExecutor; @@ -69,13 +76,6 @@ use crate::plans::ShowFileFormatsPlan; use crate::plans::UseCatalogPlan; use crate::plans::UseDatabasePlan; use crate::plans::Visitor; -use crate::BindContext; -use crate::ColumnBinding; -use crate::MetadataRef; -use crate::NameResolutionContext; -use crate::ScalarExpr; -use crate::TypeChecker; -use crate::Visibility; /// Binder is responsible to transform AST of a query into a canonical logical SExpr. /// @@ -395,9 +395,10 @@ impl Binder { comment, } => { if illegal_ident_name(role_name) { - return Err(ErrorCode::IllegalRole( - format!("[SQL-BINDER] Illegal role name [{}]: role names cannot contain quotes (' or \") or control characters (\\b or \\f)", role_name), - )); + return Err(ErrorCode::IllegalRole(format!( + "[SQL-BINDER] Illegal role name [{}]: role names cannot contain quotes (' or \") or control characters (\\b or \\f)", + role_name + ))); } Plan::CreateRole(Box::new(CreateRolePlan { create_option: create_option.clone().into(), @@ -636,7 +637,9 @@ impl Binder { { self.bind_create_row_access(stmt).await? } else { - return Err(ErrorCode::Unimplemented("Experimental Row Access Policy is unstable and may have compatibility issues. To use it, set enable_experimental_row_access_policy=1")); + return Err(ErrorCode::Unimplemented( + "Experimental Row Access Policy is unstable and may have compatibility issues. To use it, set enable_experimental_row_access_policy=1", + )); } } Statement::DropRowAccessPolicy(stmt) => { @@ -647,7 +650,9 @@ impl Binder { { self.bind_drop_row_access(stmt).await? } else { - return Err(ErrorCode::Unimplemented("Experimental Row Access Policy is unstable and may have compatibility issues. To use it, set enable_experimental_row_access_policy=1")); + return Err(ErrorCode::Unimplemented( + "Experimental Row Access Policy is unstable and may have compatibility issues. To use it, set enable_experimental_row_access_policy=1", + )); } } Statement::DescRowAccessPolicy(stmt) => { @@ -658,7 +663,9 @@ impl Binder { { self.bind_desc_row_access(stmt).await? } else { - return Err(ErrorCode::Unimplemented("Experimental Row Access Policy is unstable and may have compatibility issues. To use it, set enable_experimental_row_access_policy=1")); + return Err(ErrorCode::Unimplemented( + "Experimental Row Access Policy is unstable and may have compatibility issues. To use it, set enable_experimental_row_access_policy=1", + )); } } Statement::SetRole { diff --git a/src/query/sql/src/planner/binder/call.rs b/src/query/sql/src/planner/binder/call.rs index 2064fc6eb79ab..26c122e463710 100644 --- a/src/query/sql/src/planner/binder/call.rs +++ b/src/query/sql/src/planner/binder/call.rs @@ -16,9 +16,9 @@ use databend_common_ast::ast::CallStmt; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use crate::BindContext; use crate::binder::Binder; use crate::plans::Plan; -use crate::BindContext; impl Binder { /// Rewrite call stmt to table functions diff --git a/src/query/sql/src/planner/binder/column_binding.rs b/src/query/sql/src/planner/binder/column_binding.rs index 5d3f23d05a628..129ae1da7078a 100644 --- a/src/query/sql/src/planner/binder/column_binding.rs +++ b/src/query/sql/src/planner/binder/column_binding.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_expression::types::DataType; use databend_common_expression::ColumnIndex; +use databend_common_expression::types::DataType; use crate::IndexType; use crate::Visibility; diff --git a/src/query/sql/src/planner/binder/constraint_expr.rs b/src/query/sql/src/planner/binder/constraint_expr.rs index 610d39043d510..863ab5b432ee8 100644 --- a/src/query/sql/src/planner/binder/constraint_expr.rs +++ b/src/query/sql/src/planner/binder/constraint_expr.rs @@ -15,9 +15,9 @@ use std::sync::Arc; use databend_common_ast::ast::Expr as AExpr; +use databend_common_ast::parser::Dialect; use databend_common_ast::parser::parse_expr; use databend_common_ast::parser::tokenize_sql; -use databend_common_ast::parser::Dialect; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; diff --git a/src/query/sql/src/planner/binder/copy_into_location.rs b/src/query/sql/src/planner/binder/copy_into_location.rs index a6dd9e8d83c7a..0d91877404b9a 100644 --- a/src/query/sql/src/planner/binder/copy_into_location.rs +++ b/src/query/sql/src/planner/binder/copy_into_location.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_ast::ast::quote::display_ident; use databend_common_ast::ast::CopyIntoLocationSource; use databend_common_ast::ast::CopyIntoLocationStmt; use databend_common_ast::ast::Statement; +use databend_common_ast::ast::quote::display_ident; use databend_common_ast::parser::parse_sql; use databend_common_ast::parser::tokenize_sql; use databend_common_exception::ErrorCode; @@ -24,11 +24,11 @@ use databend_common_storage::init_stage_operator; use databend_storages_common_stage::CopyIntoLocationInfo; use opendal::ErrorKind; -use crate::binder::copy_into_table::resolve_file_location; +use crate::BindContext; use crate::binder::Binder; +use crate::binder::copy_into_table::resolve_file_location; use crate::plans::CopyIntoLocationPlan; use crate::plans::Plan; -use crate::BindContext; impl Binder { #[async_backtrace::framed] diff --git a/src/query/sql/src/planner/binder/copy_into_table.rs b/src/query/sql/src/planner/binder/copy_into_table.rs index d5ca0ef4a4d55..9cc9ac930cea6 100644 --- a/src/query/sql/src/planner/binder/copy_into_table.rs +++ b/src/query/sql/src/planner/binder/copy_into_table.rs @@ -15,6 +15,7 @@ use std::str::FromStr; use std::sync::Arc; +use databend_common_ast::Span; use databend_common_ast::ast::ColumnID as AstColumnID; use databend_common_ast::ast::ColumnMatchMode; use databend_common_ast::ast::ColumnRef; @@ -33,28 +34,27 @@ use databend_common_ast::ast::TableAlias; use databend_common_ast::ast::TypeName; use databend_common_ast::parser::parse_values; use databend_common_ast::parser::tokenize_sql; -use databend_common_ast::Span; -use databend_common_catalog::plan::list_stage_files; use databend_common_catalog::plan::StageTableInfo; +use databend_common_catalog::plan::list_stage_files; use databend_common_catalog::table_context::StageAttachment; use databend_common_catalog::table_context::TableContext; use databend_common_config::GlobalConfig; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::shrink_scalar; -use databend_common_expression::types::DataType; use databend_common_expression::DataField; use databend_common_expression::DataSchema; use databend_common_expression::DataSchemaRef; use databend_common_expression::Scalar; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRef; +use databend_common_expression::shrink_scalar; +use databend_common_expression::types::DataType; +use databend_common_meta_app::principal::COPY_MAX_FILES_PER_COMMIT; use databend_common_meta_app::principal::EmptyFieldAs; use databend_common_meta_app::principal::FileFormatOptionsReader; use databend_common_meta_app::principal::FileFormatParams; use databend_common_meta_app::principal::NullAs; use databend_common_meta_app::principal::StageInfo; -use databend_common_meta_app::principal::COPY_MAX_FILES_PER_COMMIT; use databend_common_settings::Settings; use databend_common_storage::StageFilesInfo; use databend_common_users::UserApiProvider; @@ -65,18 +65,18 @@ use log::debug; use log::warn; use parking_lot::RwLock; +use crate::BindContext; +use crate::DefaultExprBinder; +use crate::Metadata; +use crate::NameResolutionContext; +use crate::binder::Binder; use crate::binder::bind_query::MaxColumnPosition; use crate::binder::insert::STAGE_PLACEHOLDER; use crate::binder::location::parse_uri_location; -use crate::binder::Binder; use crate::plans::CopyIntoTableMode; use crate::plans::CopyIntoTablePlan; use crate::plans::Plan; use crate::plans::ValidationMode; -use crate::BindContext; -use crate::DefaultExprBinder; -use crate::Metadata; -use crate::NameResolutionContext; impl Binder { #[async_backtrace::framed] @@ -318,7 +318,7 @@ impl Binder { FileFormatOptionsReader::from_map(options.clone()), false, )?; - if let FileFormatParams::Csv(ref mut fmt) = &mut params { + if let FileFormatParams::Csv(fmt) = &mut params { // TODO: remove this after 1. the old server is no longer supported 2. Driver add the option "EmptyFieldAs=FieldDefault" // CSV attachment is mainly used in Drivers for insert. // In the future, client should use EmptyFieldAs=STRING or FieldDefault to distinguish NULL and empty string. @@ -656,7 +656,9 @@ pub async fn resolve_stage_location( // my_named_stage/abc/ let names: Vec<&str> = location.splitn(2, '/').filter(|v| !v.is_empty()).collect(); if names[0] == STAGE_PLACEHOLDER { - return Err(ErrorCode::BadArguments("placeholder @_databend_upload as location: should be used in streaming_load handler or replaced in client.")); + return Err(ErrorCode::BadArguments( + "placeholder @_databend_upload as location: should be used in streaming_load handler or replaced in client.", + )); } let stage = if names[0] == "~" { diff --git a/src/query/sql/src/planner/binder/ddl/account.rs b/src/query/sql/src/planner/binder/ddl/account.rs index c5527ed0262ab..a27c974d628ac 100644 --- a/src/query/sql/src/planner/binder/ddl/account.rs +++ b/src/query/sql/src/planner/binder/ddl/account.rs @@ -31,8 +31,8 @@ use databend_common_base::base::GlobalInstance; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_management::WorkloadMgr; -use databend_common_meta_api::data_mask_api::DatamaskApi; use databend_common_meta_api::RowAccessPolicyApi; +use databend_common_meta_api::data_mask_api::DatamaskApi; use databend_common_meta_app::data_mask::DataMaskNameIdent; use databend_common_meta_app::principal::AuthInfo; use databend_common_meta_app::principal::GetProcedureReq; @@ -46,6 +46,8 @@ use databend_common_meta_app::principal::UserPrivilegeSet; use databend_common_meta_app::row_access_policy::RowAccessPolicyNameIdent; use databend_common_users::UserApiProvider; +use crate::BindContext; +use crate::Binder; use crate::binder::show::get_show_options; use crate::binder::util::illegal_ident_name; use crate::plans::AlterUserPlan; @@ -56,8 +58,6 @@ use crate::plans::Plan; use crate::plans::RevokePrivilegePlan; use crate::plans::RevokeRolePlan; use crate::plans::RewriteKind; -use crate::BindContext; -use crate::Binder; impl Binder { #[async_backtrace::framed] diff --git a/src/query/sql/src/planner/binder/ddl/catalog.rs b/src/query/sql/src/planner/binder/ddl/catalog.rs index 1e2347de35930..994f989637cf6 100644 --- a/src/query/sql/src/planner/binder/ddl/catalog.rs +++ b/src/query/sql/src/planner/binder/ddl/catalog.rs @@ -27,9 +27,9 @@ use databend_common_ast::ast::UriLocation; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRefExt; +use databend_common_expression::types::DataType; use databend_common_meta_app::schema::CatalogMeta; use databend_common_meta_app::schema::CatalogOption; use databend_common_meta_app::schema::CatalogType; @@ -41,6 +41,8 @@ use databend_common_meta_app::schema::IcebergRestCatalogOption; use databend_common_meta_app::schema::IcebergStorageCatalogOption; use databend_common_meta_app::storage::StorageParams; +use crate::BindContext; +use crate::Binder; use crate::binder::parse_storage_params_from_uri; use crate::normalize_identifier; use crate::plans::CreateCatalogPlan; @@ -48,8 +50,6 @@ use crate::plans::DropCatalogPlan; use crate::plans::Plan; use crate::plans::RewriteKind; use crate::plans::ShowCreateCatalogPlan; -use crate::BindContext; -use crate::Binder; impl Binder { #[async_backtrace::framed] diff --git a/src/query/sql/src/planner/binder/ddl/column.rs b/src/query/sql/src/planner/binder/ddl/column.rs index ec8029524a982..ee50e00c154f4 100644 --- a/src/query/sql/src/planner/binder/ddl/column.rs +++ b/src/query/sql/src/planner/binder/ddl/column.rs @@ -17,12 +17,12 @@ use databend_common_ast::ast::ShowLimit; use databend_common_exception::Result; use log::debug; -use crate::normalize_identifier; -use crate::plans::Plan; -use crate::plans::RewriteKind; use crate::BindContext; use crate::Binder; use crate::SelectBuilder; +use crate::normalize_identifier; +use crate::plans::Plan; +use crate::plans::RewriteKind; impl Binder { #[async_backtrace::framed] diff --git a/src/query/sql/src/planner/binder/ddl/connection.rs b/src/query/sql/src/planner/binder/ddl/connection.rs index 7f802f56b7349..4bbb926dfedbe 100644 --- a/src/query/sql/src/planner/binder/ddl/connection.rs +++ b/src/query/sql/src/planner/binder/ddl/connection.rs @@ -16,10 +16,10 @@ use databend_common_ast::ast::CreateConnectionStmt; use databend_common_ast::ast::UriLocation; use databend_common_exception::Result; +use crate::Binder; use crate::binder::parse_storage_params_from_uri; use crate::plans::CreateConnectionPlan; use crate::plans::Plan; -use crate::Binder; impl Binder { #[async_backtrace::framed] diff --git a/src/query/sql/src/planner/binder/ddl/database.rs b/src/query/sql/src/planner/binder/ddl/database.rs index a071a8d39284f..f6cfb3f386ca6 100644 --- a/src/query/sql/src/planner/binder/ddl/database.rs +++ b/src/query/sql/src/planner/binder/ddl/database.rs @@ -28,12 +28,14 @@ use databend_common_ast::ast::ShowLimit; use databend_common_ast::ast::UndropDatabaseStmt; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRefExt; +use databend_common_expression::types::DataType; use databend_common_meta_app::schema::DatabaseMeta; use log::debug; +use crate::BindContext; +use crate::SelectBuilder; use crate::binder::Binder; use crate::planner::semantic::normalize_identifier; use crate::plans::CreateDatabasePlan; @@ -45,8 +47,6 @@ use crate::plans::RenameDatabasePlan; use crate::plans::RewriteKind; use crate::plans::ShowCreateDatabasePlan; use crate::plans::UndropDatabasePlan; -use crate::BindContext; -use crate::SelectBuilder; impl Binder { #[async_backtrace::framed] diff --git a/src/query/sql/src/planner/binder/ddl/dictionary.rs b/src/query/sql/src/planner/binder/ddl/dictionary.rs index be52a193143a6..e9c2e0a9b8047 100644 --- a/src/query/sql/src/planner/binder/ddl/dictionary.rs +++ b/src/query/sql/src/planner/binder/ddl/dictionary.rs @@ -24,24 +24,24 @@ use databend_common_ast::ast::ShowDictionariesStmt; use databend_common_ast::ast::ShowLimit; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRefExt; use databend_common_expression::TableDataType; use databend_common_expression::TableSchema; +use databend_common_expression::types::DataType; use databend_common_meta_app::schema::DictionaryMeta; use itertools::Itertools; use log::debug; +use crate::BindContext; +use crate::Binder; +use crate::SelectBuilder; use crate::plans::CreateDictionaryPlan; use crate::plans::DropDictionaryPlan; use crate::plans::Plan; use crate::plans::RenameDictionaryPlan; use crate::plans::RewriteKind; use crate::plans::ShowCreateDictionaryPlan; -use crate::BindContext; -use crate::Binder; -use crate::SelectBuilder; pub const DICT_OPT_KEY_SQL_HOST: &str = "host"; pub const DICT_OPT_KEY_SQL_PORT: &str = "port"; diff --git a/src/query/sql/src/planner/binder/ddl/dynamic_table.rs b/src/query/sql/src/planner/binder/ddl/dynamic_table.rs index f6953c8cae87a..d19f2fa156c23 100644 --- a/src/query/sql/src/planner/binder/ddl/dynamic_table.rs +++ b/src/query/sql/src/planner/binder/ddl/dynamic_table.rs @@ -20,9 +20,9 @@ use databend_common_ast::ast::TypeName; use databend_common_config::GlobalConfig; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::infer_schema_type; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::infer_schema_type; use databend_common_meta_app::storage::StorageParams; use databend_storages_common_table_meta::table::OPT_KEY_AS_QUERY; use databend_storages_common_table_meta::table::OPT_KEY_CLUSTER_TYPE; @@ -31,11 +31,11 @@ use databend_storages_common_table_meta::table::OPT_KEY_STORAGE_FORMAT; use databend_storages_common_table_meta::table::OPT_KEY_TABLE_COMPRESSION; use databend_storages_common_table_meta::table::OPT_KEY_TARGET_LAG; +use crate::BindContext; +use crate::Binder; use crate::binder::ddl::table::AnalyzeCreateTableResult; use crate::plans::CreateDynamicTablePlan; use crate::plans::Plan; -use crate::BindContext; -use crate::Binder; impl Binder { pub(in crate::planner::binder) async fn bind_create_dynamic_table( diff --git a/src/query/sql/src/planner/binder/ddl/index.rs b/src/query/sql/src/planner/binder/ddl/index.rs index 45975e13ea662..6c3774644bd1d 100644 --- a/src/query/sql/src/planner/binder/ddl/index.rs +++ b/src/query/sql/src/planner/binder/ddl/index.rs @@ -47,9 +47,15 @@ use derive_visitor::Drive; use derive_visitor::DriveMut; use itertools::Itertools; +use crate::AggregatingIndexChecker; +use crate::AggregatingIndexRewriter; +use crate::BindContext; +use crate::MetadataRef; +use crate::RefreshAggregatingIndexRewriter; +use crate::SUPPORTED_AGGREGATING_INDEX_FUNCTIONS; use crate::binder::Binder; -use crate::optimizer::optimize; use crate::optimizer::OptimizerContext; +use crate::optimizer::optimize; use crate::plans::CreateIndexPlan; use crate::plans::CreateTableIndexPlan; use crate::plans::DropIndexPlan; @@ -57,12 +63,6 @@ use crate::plans::DropTableIndexPlan; use crate::plans::Plan; use crate::plans::RefreshIndexPlan; use crate::plans::RefreshTableIndexPlan; -use crate::AggregatingIndexChecker; -use crate::AggregatingIndexRewriter; -use crate::BindContext; -use crate::MetadataRef; -use crate::RefreshAggregatingIndexRewriter; -use crate::SUPPORTED_AGGREGATING_INDEX_FUNCTIONS; const MAXIMUM_BLOOM_SIZE: u64 = 10 * 1024 * 1024; const MINIMUM_BLOOM_SIZE: u64 = 512; diff --git a/src/query/sql/src/planner/binder/ddl/notification.rs b/src/query/sql/src/planner/binder/ddl/notification.rs index 0596ce693c1a1..5ce3f9382b6d7 100644 --- a/src/query/sql/src/planner/binder/ddl/notification.rs +++ b/src/query/sql/src/planner/binder/ddl/notification.rs @@ -21,13 +21,13 @@ use databend_common_ast::ast::NotificationWebhookOptions; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use crate::Binder; use crate::plans::AlterNotificationPlan; use crate::plans::CreateNotificationPlan; use crate::plans::DescNotificationPlan; use crate::plans::DropNotificationPlan; use crate::plans::NotificationType; use crate::plans::Plan; -use crate::Binder; fn verify_notification_type(t: &String) -> Result { match t.to_lowercase().as_str() { diff --git a/src/query/sql/src/planner/binder/ddl/password_policy.rs b/src/query/sql/src/planner/binder/ddl/password_policy.rs index 1afa12d4246bd..627d0f9834cf1 100644 --- a/src/query/sql/src/planner/binder/ddl/password_policy.rs +++ b/src/query/sql/src/planner/binder/ddl/password_policy.rs @@ -15,15 +15,15 @@ use databend_common_ast::ast::*; use databend_common_exception::Result; -use crate::binder::show::get_show_options; +use crate::BindContext; use crate::binder::Binder; +use crate::binder::show::get_show_options; use crate::plans::AlterPasswordPolicyPlan; use crate::plans::CreatePasswordPolicyPlan; use crate::plans::DescPasswordPolicyPlan; use crate::plans::DropPasswordPolicyPlan; use crate::plans::Plan; use crate::plans::RewriteKind; -use crate::BindContext; impl Binder { #[async_backtrace::framed] diff --git a/src/query/sql/src/planner/binder/ddl/procedure.rs b/src/query/sql/src/planner/binder/ddl/procedure.rs index fbb901420cb20..a634f95bb8f4c 100644 --- a/src/query/sql/src/planner/binder/ddl/procedure.rs +++ b/src/query/sql/src/planner/binder/ddl/procedure.rs @@ -24,28 +24,32 @@ use databend_common_ast::ast::ProcedureLanguage; use databend_common_ast::ast::ProcedureType; use databend_common_ast::ast::ShowOptions; use databend_common_ast::ast::TypeName; +use databend_common_ast::parser::Dialect; +use databend_common_ast::parser::ParseMode; use databend_common_ast::parser::expr::type_name as parse_type_name_ast; use databend_common_ast::parser::run_parser; -use databend_common_ast::parser::script::script_block_or_stmt; use databend_common_ast::parser::script::ScriptBlockOrStmt; +use databend_common_ast::parser::script::script_block_or_stmt; use databend_common_ast::parser::tokenize_sql; -use databend_common_ast::parser::Dialect; -use databend_common_ast::parser::ParseMode; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::Scalar; use databend_common_expression::type_check::common_super_type; use databend_common_expression::types::DataType; -use databend_common_expression::Scalar; use databend_common_functions::BUILTIN_FUNCTIONS; -use databend_common_meta_app::principal::procedure::ProcedureInfo; use databend_common_meta_app::principal::GetProcedureReply; use databend_common_meta_app::principal::GetProcedureReq; use databend_common_meta_app::principal::ProcedureIdentity; use databend_common_meta_app::principal::ProcedureMeta; use databend_common_meta_app::principal::ProcedureNameIdent; +use databend_common_meta_app::principal::procedure::ProcedureInfo; use databend_common_meta_app::tenant::Tenant; use databend_common_users::UserApiProvider; +use crate::BindContext; +use crate::Binder; +use crate::ScalarExpr; +use crate::TypeChecker; use crate::binder::show::get_show_options; use crate::plans::CallProcedurePlan; use crate::plans::ConstantExpr; @@ -57,10 +61,6 @@ use crate::plans::Plan; use crate::plans::RewriteKind; use crate::plans::SubqueryType; use crate::resolve_type_name; -use crate::BindContext; -use crate::Binder; -use crate::ScalarExpr; -use crate::TypeChecker; impl Binder { #[async_backtrace::framed] @@ -78,7 +78,7 @@ impl Binder { _ => { return Err(ErrorCode::InvalidArgument( "immediate script must be a string", - )) + )); } }; diff --git a/src/query/sql/src/planner/binder/ddl/role.rs b/src/query/sql/src/planner/binder/ddl/role.rs index 38da0a9c0c62f..dbef6cc1f6d3d 100644 --- a/src/query/sql/src/planner/binder/ddl/role.rs +++ b/src/query/sql/src/planner/binder/ddl/role.rs @@ -17,11 +17,11 @@ use databend_common_ast::ast::SecondaryRolesOption; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use crate::BindContext; +use crate::Binder; use crate::plans::Plan; use crate::plans::SetRolePlan; use crate::plans::SetSecondaryRolesPlan; -use crate::BindContext; -use crate::Binder; impl Binder { #[async_backtrace::framed] diff --git a/src/query/sql/src/planner/binder/ddl/row_access_policy.rs b/src/query/sql/src/planner/binder/ddl/row_access_policy.rs index 05a83c18a46e7..c0cebbc389b52 100644 --- a/src/query/sql/src/planner/binder/ddl/row_access_policy.rs +++ b/src/query/sql/src/planner/binder/ddl/row_access_policy.rs @@ -23,14 +23,14 @@ use derive_visitor::Drive; use derive_visitor::Visitor; use unicase::Ascii; +use crate::Binder; +use crate::NameResolutionContext; +use crate::TypeChecker; use crate::normalize_identifier; use crate::plans::CreateRowAccessPolicyPlan; use crate::plans::DescRowAccessPolicyPlan; use crate::plans::DropRowAccessPolicyPlan; use crate::plans::Plan; -use crate::Binder; -use crate::NameResolutionContext; -use crate::TypeChecker; impl Binder { #[async_backtrace::framed] diff --git a/src/query/sql/src/planner/binder/ddl/sequence.rs b/src/query/sql/src/planner/binder/ddl/sequence.rs index 309b8d86b3bc6..88dd24db78703 100644 --- a/src/query/sql/src/planner/binder/ddl/sequence.rs +++ b/src/query/sql/src/planner/binder/ddl/sequence.rs @@ -20,14 +20,14 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_meta_app::schema::SequenceIdent; +use crate::BindContext; +use crate::Binder; use crate::binder::show::get_show_options; use crate::plans::CreateSequencePlan; use crate::plans::DescSequencePlan; use crate::plans::DropSequencePlan; use crate::plans::Plan; use crate::plans::RewriteKind; -use crate::BindContext; -use crate::Binder; impl Binder { #[async_backtrace::framed] diff --git a/src/query/sql/src/planner/binder/ddl/stage.rs b/src/query/sql/src/planner/binder/ddl/stage.rs index c4d03ca2ccc38..3b726db57c3fd 100644 --- a/src/query/sql/src/planner/binder/ddl/stage.rs +++ b/src/query/sql/src/planner/binder/ddl/stage.rs @@ -23,9 +23,9 @@ use databend_common_meta_app::principal::StageInfo; use databend_common_storage::init_operator; use super::super::copy_into_table::resolve_stage_location; +use crate::binder::Binder; use crate::binder::insert::STAGE_PLACEHOLDER; use crate::binder::location::parse_storage_params_from_uri; -use crate::binder::Binder; use crate::plans::CreateStagePlan; use crate::plans::Plan; use crate::plans::RemoveStagePlan; diff --git a/src/query/sql/src/planner/binder/ddl/stream.rs b/src/query/sql/src/planner/binder/ddl/stream.rs index b6aad6eff4456..17ef1fe1088e9 100644 --- a/src/query/sql/src/planner/binder/ddl/stream.rs +++ b/src/query/sql/src/planner/binder/ddl/stream.rs @@ -22,14 +22,14 @@ use databend_common_license::license::Feature; use databend_common_license::license_manager::LicenseManagerSwitch; use log::debug; +use crate::BindContext; +use crate::SelectBuilder; use crate::binder::Binder; use crate::normalize_identifier; use crate::plans::CreateStreamPlan; use crate::plans::DropStreamPlan; use crate::plans::Plan; use crate::plans::RewriteKind; -use crate::BindContext; -use crate::SelectBuilder; impl Binder { #[async_backtrace::framed] diff --git a/src/query/sql/src/planner/binder/ddl/table.rs b/src/query/sql/src/planner/binder/ddl/table.rs index cdaf63e1748b8..2c81ad5195c6b 100644 --- a/src/query/sql/src/planner/binder/ddl/table.rs +++ b/src/query/sql/src/planner/binder/ddl/table.rs @@ -70,9 +70,6 @@ use databend_common_catalog::table::CompactionLimits; use databend_common_config::GlobalConfig; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::infer_schema_type; -use databend_common_expression::infer_table_schema; -use databend_common_expression::types::DataType; use databend_common_expression::AutoIncrementExpr; use databend_common_expression::ComputedExpr; use databend_common_expression::DataField; @@ -82,6 +79,9 @@ use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::infer_schema_type; +use databend_common_expression::infer_table_schema; +use databend_common_expression::types::DataType; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_license::license::Feature; use databend_common_license::license_manager::LicenseManagerSwitch; @@ -94,8 +94,6 @@ use databend_common_storage::check_operator; use databend_common_storage::init_operator; use databend_common_storages_basic::view_table::QUERY; use databend_common_storages_basic::view_table::VIEW_ENGINE; -use databend_storages_common_table_meta::table::is_reserved_opt_key; -use databend_storages_common_table_meta::table::TableCompression; use databend_storages_common_table_meta::table::OPT_KEY_CLUSTER_TYPE; use databend_storages_common_table_meta::table::OPT_KEY_DATABASE_ID; use databend_storages_common_table_meta::table::OPT_KEY_ENGINE_META; @@ -104,22 +102,28 @@ use databend_storages_common_table_meta::table::OPT_KEY_STORAGE_PREFIX; use databend_storages_common_table_meta::table::OPT_KEY_TABLE_ATTACHED_DATA_URI; use databend_storages_common_table_meta::table::OPT_KEY_TABLE_COMPRESSION; use databend_storages_common_table_meta::table::OPT_KEY_TEMP_PREFIX; +use databend_storages_common_table_meta::table::TableCompression; +use databend_storages_common_table_meta::table::is_reserved_opt_key; use derive_visitor::DriveMut; use log::debug; use opendal::Operator; -use crate::binder::get_storage_params_from_options; -use crate::binder::parse_storage_params_from_uri; -use crate::binder::scalar::ScalarBinder; +use crate::BindContext; +use crate::DefaultExprBinder; +use crate::Planner; +use crate::SelectBuilder; use crate::binder::Binder; use crate::binder::ColumnBindingBuilder; use crate::binder::ConstraintExprBinder; use crate::binder::Visibility; +use crate::binder::get_storage_params_from_options; +use crate::binder::parse_storage_params_from_uri; +use crate::binder::scalar::ScalarBinder; use crate::optimizer::ir::SExpr; use crate::parse_computed_expr_to_string; +use crate::planner::semantic::IdentifierNormalizer; use crate::planner::semantic::normalize_identifier; use crate::planner::semantic::resolve_type_name; -use crate::planner::semantic::IdentifierNormalizer; use crate::plans::AddColumnOption; use crate::plans::AddTableColumnPlan; use crate::plans::AddTableConstraintPlan; @@ -161,10 +165,6 @@ use crate::plans::VacuumDropTablePlan; use crate::plans::VacuumTableOption; use crate::plans::VacuumTablePlan; use crate::plans::VacuumTemporaryFilesPlan; -use crate::BindContext; -use crate::DefaultExprBinder; -use crate::Planner; -use crate::SelectBuilder; pub(in crate::planner::binder) struct AnalyzeCreateTableResult { pub(in crate::planner::binder) schema: TableSchemaRef, @@ -603,7 +603,7 @@ impl Binder { Some(self.ctx.as_ref()), "when create TABLE with external location", ) - .await?; + .await?; // create a temporary op to check if params is correct let op = init_operator(&sp)?; @@ -737,12 +737,15 @@ impl Binder { // since we get it from table options location and connection when load table each time. // we do this in case we change this idea. storage_params = Some(sp); - (AnalyzeCreateTableResult { - schema: Arc::new(table_schema), - field_comments: vec![], - table_indexes: None, - table_constraints: None, - }, as_query_plan) + ( + AnalyzeCreateTableResult { + schema: Arc::new(table_schema), + field_comments: vec![], + table_indexes: None, + table_constraints: None, + }, + as_query_plan, + ) } Engine::Delta => { let sp = @@ -754,12 +757,15 @@ impl Binder { // we do this in case we change this idea. storage_params = Some(sp); engine_options.insert(OPT_KEY_ENGINE_META.to_lowercase().to_string(), meta); - (AnalyzeCreateTableResult { - schema: Arc::new(table_schema), - field_comments: vec![], - table_indexes: None, - table_constraints: None, - }, as_query_plan) + ( + AnalyzeCreateTableResult { + schema: Arc::new(table_schema), + field_comments: vec![], + table_indexes: None, + table_constraints: None, + }, + as_query_plan, + ) } _ => Err(ErrorCode::BadArguments( "Incorrect CREATE query: required list of column descriptions or AS section or SELECT or ICEBERG/DELTA table engine", @@ -1173,7 +1179,9 @@ impl Binder { "Invalid number of arguments for attaching policy '{}' to '{}': \ expected at least 2 arguments (masked column + condition columns), \ got {} argument(s)", - name, table, columns.len() + name, + table, + columns.len() ))); } @@ -1343,7 +1351,9 @@ impl Binder { .get_settings() .get_enable_experimental_row_access_policy()? { - return Err(ErrorCode::Unimplemented("Experimental Row Access Policy is unstable and may have compatibility issues. To use it, set enable_experimental_row_access_policy=1")); + return Err(ErrorCode::Unimplemented( + "Experimental Row Access Policy is unstable and may have compatibility issues. To use it, set enable_experimental_row_access_policy=1", + )); } let columns = columns .iter() @@ -1367,7 +1377,9 @@ impl Binder { .get_settings() .get_enable_experimental_row_access_policy()? { - return Err(ErrorCode::Unimplemented("Experimental Row Access Policy is unstable and may have compatibility issues. To use it, set enable_experimental_row_access_policy=1")); + return Err(ErrorCode::Unimplemented( + "Experimental Row Access Policy is unstable and may have compatibility issues. To use it, set enable_experimental_row_access_policy=1", + )); } let policy = self.normalize_identifier(policy).name; Ok(Plan::DropTableRowAccessPolicy(Box::new( @@ -1386,7 +1398,9 @@ impl Binder { .get_settings() .get_enable_experimental_row_access_policy()? { - return Err(ErrorCode::Unimplemented("Experimental Row Access Policy is unstable and may have compatibility issues. To use it, set enable_experimental_row_access_policy=1")); + return Err(ErrorCode::Unimplemented( + "Experimental Row Access Policy is unstable and may have compatibility issues. To use it, set enable_experimental_row_access_policy=1", + )); } Ok(Plan::DropAllTableRowAccessPolicies(Box::new( DropAllTableRowAccessPoliciesPlan { diff --git a/src/query/sql/src/planner/binder/ddl/task.rs b/src/query/sql/src/planner/binder/ddl/task.rs index 55672b2a8fd03..d0761a0949bd2 100644 --- a/src/query/sql/src/planner/binder/ddl/task.rs +++ b/src/query/sql/src/planner/binder/ddl/task.rs @@ -25,12 +25,13 @@ use databend_common_ast::ast::ExecuteTaskStmt; use databend_common_ast::ast::ScheduleOptions; use databend_common_ast::ast::ShowTasksStmt; use databend_common_ast::ast::TaskSql; +use databend_common_ast::parser::Dialect; use databend_common_ast::parser::parse_sql; use databend_common_ast::parser::tokenize_sql; -use databend_common_ast::parser::Dialect; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use crate::Binder; use crate::plans::AlterTaskPlan; use crate::plans::CreateTaskPlan; use crate::plans::DescribeTaskPlan; @@ -38,7 +39,6 @@ use crate::plans::DropTaskPlan; use crate::plans::ExecuteTaskPlan; use crate::plans::Plan; use crate::plans::ShowTasksPlan; -use crate::Binder; fn verify_single_statement(sql: &String) -> Result<()> { let tokens = tokenize_sql(sql.as_str()).map_err(|e| { diff --git a/src/query/sql/src/planner/binder/ddl/view.rs b/src/query/sql/src/planner/binder/ddl/view.rs index 031f60e63bbee..d35d02b56fea2 100644 --- a/src/query/sql/src/planner/binder/ddl/view.rs +++ b/src/query/sql/src/planner/binder/ddl/view.rs @@ -19,12 +19,15 @@ use databend_common_ast::ast::DropViewStmt; use databend_common_ast::ast::ShowLimit; use databend_common_ast::ast::ShowViewsStmt; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRefExt; +use databend_common_expression::types::DataType; use derive_visitor::DriveMut; use log::debug; +use crate::BindContext; +use crate::SelectBuilder; +use crate::ViewRewriter; use crate::binder::Binder; use crate::planner::semantic::normalize_identifier; use crate::plans::AlterViewPlan; @@ -33,9 +36,6 @@ use crate::plans::DescribeViewPlan; use crate::plans::DropViewPlan; use crate::plans::Plan; use crate::plans::RewriteKind; -use crate::BindContext; -use crate::SelectBuilder; -use crate::ViewRewriter; impl Binder { #[async_backtrace::framed] diff --git a/src/query/sql/src/planner/binder/ddl/warehouse.rs b/src/query/sql/src/planner/binder/ddl/warehouse.rs index fa5bd5e242421..e69ae1b2e4933 100644 --- a/src/query/sql/src/planner/binder/ddl/warehouse.rs +++ b/src/query/sql/src/planner/binder/ddl/warehouse.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::hash_map::Entry; use std::collections::HashMap; +use std::collections::hash_map::Entry; use databend_common_ast::ast::AddWarehouseClusterStmt; use databend_common_ast::ast::AssignWarehouseNodesStmt; @@ -31,6 +31,7 @@ use databend_common_ast::ast::UnassignWarehouseNodesStmt; use databend_common_ast::ast::UseWarehouseStmt; use databend_common_exception::Result; +use crate::Binder; use crate::plans::AddWarehouseClusterPlan; use crate::plans::AssignWarehouseNodesPlan; use crate::plans::CreateWarehousePlan; @@ -44,7 +45,6 @@ use crate::plans::ResumeWarehousePlan; use crate::plans::SuspendWarehousePlan; use crate::plans::UnassignWarehouseNodesPlan; use crate::plans::UseWarehousePlan; -use crate::Binder; impl Binder { pub(in crate::planner::binder) fn bind_show_online_nodes( diff --git a/src/query/sql/src/planner/binder/ddl/workload.rs b/src/query/sql/src/planner/binder/ddl/workload.rs index aa76082222ebf..7431f1398e0e6 100644 --- a/src/query/sql/src/planner/binder/ddl/workload.rs +++ b/src/query/sql/src/planner/binder/ddl/workload.rs @@ -20,13 +20,13 @@ use databend_common_ast::ast::ShowWorkloadGroupsStmt; use databend_common_ast::ast::UnsetWorkloadGroupQuotasStmt; use databend_common_exception::Result; +use crate::Binder; use crate::plans::CreateWorkloadGroupPlan; use crate::plans::DropWorkloadGroupPlan; use crate::plans::Plan; use crate::plans::RenameWorkloadGroupPlan; use crate::plans::SetWorkloadGroupQuotasPlan; use crate::plans::UnsetWorkloadGroupQuotasPlan; -use crate::Binder; impl Binder { pub(in crate::planner::binder) fn bind_show_workload_groups( diff --git a/src/query/sql/src/planner/binder/default_expr.rs b/src/query/sql/src/planner/binder/default_expr.rs index fcff1b6500ad1..b2b8023119203 100644 --- a/src/query/sql/src/planner/binder/default_expr.rs +++ b/src/query/sql/src/planner/binder/default_expr.rs @@ -15,14 +15,12 @@ use std::sync::Arc; use databend_common_ast::ast::Expr as AExpr; +use databend_common_ast::parser::Dialect; use databend_common_ast::parser::parse_expr; use databend_common_ast::parser::tokenize_sql; -use databend_common_ast::parser::Dialect; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; use databend_common_expression::ConstantFolder; use databend_common_expression::DataBlock; use databend_common_expression::DataField; @@ -33,24 +31,26 @@ use databend_common_expression::FunctionContext; use databend_common_expression::RemoteDefaultExpr; use databend_common_expression::Scalar; use databend_common_expression::TableField; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::principal::AutoIncrementKey; use databend_common_meta_types::MetaId; use parking_lot::RwLock; -use crate::binder::wrap_cast; +use crate::Metadata; +use crate::MetadataRef; use crate::binder::AsyncFunctionDesc; +use crate::binder::wrap_cast; use crate::planner::binder::BindContext; use crate::planner::semantic::NameResolutionContext; use crate::planner::semantic::TypeChecker; -use crate::plans::walk_expr_mut; use crate::plans::AsyncFunctionArgument; use crate::plans::AsyncFunctionCall; use crate::plans::ConstantExpr; use crate::plans::ScalarExpr; use crate::plans::VisitorMut; -use crate::Metadata; -use crate::MetadataRef; +use crate::plans::walk_expr_mut; /// Helper for binding scalar expression with `BindContext`. pub struct DefaultExprBinder { @@ -111,7 +111,7 @@ impl DefaultExprBinder { self } - fn evaluator(&self) -> Evaluator { + fn evaluator(&self) -> Evaluator<'_> { Evaluator::new(&self.dummy_block, &self.func_ctx, &BUILTIN_FUNCTIONS) } diff --git a/src/query/sql/src/planner/binder/distinct.rs b/src/query/sql/src/planner/binder/distinct.rs index ae858946ab7d7..9db2c7e0cd27c 100644 --- a/src/query/sql/src/planner/binder/distinct.rs +++ b/src/query/sql/src/planner/binder/distinct.rs @@ -18,6 +18,9 @@ use std::sync::Arc; use databend_common_ast::Span; use databend_common_exception::Result; +use crate::BindContext; +use crate::IndexType; +use crate::WindowChecker; use crate::binder::Binder; use crate::binder::ColumnBinding; use crate::optimizer::ir::SExpr; @@ -29,9 +32,6 @@ use crate::plans::EvalScalar; use crate::plans::ScalarExpr; use crate::plans::ScalarItem; use crate::plans::VisitorMut as _; -use crate::BindContext; -use crate::IndexType; -use crate::WindowChecker; impl Binder { pub fn bind_distinct( diff --git a/src/query/sql/src/planner/binder/explain.rs b/src/query/sql/src/planner/binder/explain.rs index 2a9d2a51dddc8..387cc292ed43e 100644 --- a/src/query/sql/src/planner/binder/explain.rs +++ b/src/query/sql/src/planner/binder/explain.rs @@ -12,16 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_ast::Span; use databend_common_ast::ast::ExplainKind; use databend_common_ast::ast::ExplainOption; use databend_common_ast::ast::Statement; -use databend_common_ast::Span; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use crate::plans::Plan; use crate::BindContext; use crate::Binder; +use crate::plans::Plan; /// Configuration for the EXPLAIN statement. #[derive(Debug, Clone, PartialEq, Default)] diff --git a/src/query/sql/src/planner/binder/expr_values.rs b/src/query/sql/src/planner/binder/expr_values.rs index 40ca77e1c3226..e80b726248221 100644 --- a/src/query/sql/src/planner/binder/expr_values.rs +++ b/src/query/sql/src/planner/binder/expr_values.rs @@ -18,26 +18,26 @@ use databend_common_ast::ast::Expr as AExpr; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::UInt8Type; use databend_common_expression::BlockEntry; use databend_common_expression::DataBlock; use databend_common_expression::DataSchemaRef; use databend_common_expression::FunctionKind; use databend_common_expression::Scalar; +use databend_common_expression::types::UInt8Type; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_users::Object; -use crate::binder::wrap_cast; -use crate::evaluator::BlockOperator; -use crate::plans::walk_expr_mut; -use crate::plans::ConstantExpr; -use crate::plans::VisitorMut; use crate::BindContext; use crate::DefaultExprBinder; use crate::MetadataRef; use crate::NameResolutionContext; use crate::ScalarBinder; use crate::ScalarExpr; +use crate::binder::wrap_cast; +use crate::evaluator::BlockOperator; +use crate::plans::ConstantExpr; +use crate::plans::VisitorMut; +use crate::plans::walk_expr_mut; pub(crate) struct ExprValuesRewriter { ctx: Arc, diff --git a/src/query/sql/src/planner/binder/having.rs b/src/query/sql/src/planner/binder/having.rs index 305da30e9a156..759d22a340b18 100644 --- a/src/query/sql/src/planner/binder/having.rs +++ b/src/query/sql/src/planner/binder/having.rs @@ -19,18 +19,18 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use super::Finder; -use crate::binder::aggregate::AggregateRewriter; -use crate::binder::split_conjunctions; +use crate::BindContext; +use crate::Binder; use crate::binder::ExprContext; use crate::binder::ScalarBinder; +use crate::binder::aggregate::AggregateRewriter; +use crate::binder::split_conjunctions; use crate::optimizer::ir::SExpr; use crate::planner::semantic::GroupingChecker; use crate::plans::Filter; use crate::plans::ScalarExpr; use crate::plans::Visitor; use crate::plans::VisitorMut as _; -use crate::BindContext; -use crate::Binder; impl Binder { /// Analyze aggregates in having clause, this will rewrite aggregate functions. diff --git a/src/query/sql/src/planner/binder/insert.rs b/src/query/sql/src/planner/binder/insert.rs index 0cc84ce8e91d6..194859b8ac133 100644 --- a/src/query/sql/src/planner/binder/insert.rs +++ b/src/query/sql/src/planner/binder/insert.rs @@ -32,8 +32,10 @@ use databend_common_meta_app::principal::FileFormatParams; use databend_common_storage::StageFilesInfo; use super::util::TableIdentifier; -use crate::binder::resolve_stage_location; +use crate::BindContext; +use crate::DefaultExprBinder; use crate::binder::Binder; +use crate::binder::resolve_stage_location; use crate::normalize_identifier; use crate::plans::CopyIntoTableMode; use crate::plans::Insert; @@ -41,8 +43,6 @@ use crate::plans::InsertInputSource; use crate::plans::InsertValue; use crate::plans::Plan; use crate::plans::StreamingLoadPlan; -use crate::BindContext; -use crate::DefaultExprBinder; pub const STAGE_PLACEHOLDER: &str = "_databend_load"; impl Binder { @@ -194,7 +194,9 @@ impl Binder { match location.as_str() { STAGE_PLACEHOLDER => { if self.ctx.get_session_type() != SessionType::HTTPStreamingLoad { - return Err(ErrorCode::BadArguments("placeholder @_databend_upload in query handler: should be used in streaming_load handler or replaced in client.")); + return Err(ErrorCode::BadArguments( + "placeholder @_databend_upload in query handler: should be used in streaming_load handler or replaced in client.", + )); } let (required_source_schema, values_consts) = if let Some(value) = value { self.prepared_values(value, &schema, settings).await? diff --git a/src/query/sql/src/planner/binder/insert_multi_table.rs b/src/query/sql/src/planner/binder/insert_multi_table.rs index 31c7713c63fe6..e8fef0fe1c259 100644 --- a/src/query/sql/src/planner/binder/insert_multi_table.rs +++ b/src/query/sql/src/planner/binder/insert_multi_table.rs @@ -21,18 +21,18 @@ use databend_common_ast::ast::SourceExpr; use databend_common_ast::ast::TableReference; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::DataSchemaRef; use databend_common_expression::TableSchema; +use databend_common_expression::types::DataType; +use crate::BindContext; +use crate::Binder; use crate::binder::ScalarBinder; use crate::plans::Else; use crate::plans::InsertMultiTable; use crate::plans::Into; use crate::plans::Plan; use crate::plans::When; -use crate::BindContext; -use crate::Binder; impl Binder { #[async_backtrace::framed] pub(in crate::planner::binder) async fn bind_insert_multi_table( diff --git a/src/query/sql/src/planner/binder/internal_column_factory.rs b/src/query/sql/src/planner/binder/internal_column_factory.rs index 90fc2a2db5e58..aceda84f94ae6 100644 --- a/src/query/sql/src/planner/binder/internal_column_factory.rs +++ b/src/query/sql/src/planner/binder/internal_column_factory.rs @@ -20,8 +20,8 @@ use databend_common_catalog::plan::InternalColumnType; use databend_common_expression::BASE_BLOCK_IDS_COL_NAME; use databend_common_expression::BASE_ROW_ID_COL_NAME; use databend_common_expression::BLOCK_NAME_COL_NAME; -use databend_common_expression::FILENAME_COLUMN_NAME; use databend_common_expression::FILE_ROW_NUMBER_COLUMN_NAME; +use databend_common_expression::FILENAME_COLUMN_NAME; use databend_common_expression::ROW_ID_COL_NAME; use databend_common_expression::SEARCH_MATCHED_COL_NAME; use databend_common_expression::SEARCH_SCORE_COL_NAME; diff --git a/src/query/sql/src/planner/binder/location.rs b/src/query/sql/src/planner/binder/location.rs index db0bb4a3c6f90..13b82aaaf04d4 100644 --- a/src/query/sql/src/planner/binder/location.rs +++ b/src/query/sql/src/planner/binder/location.rs @@ -28,6 +28,9 @@ use databend_common_catalog::table_context::TableContext; use databend_common_config::GlobalConfig; use databend_common_exception::ErrorCode; use databend_common_meta_app::storage::S3StorageClass; +use databend_common_meta_app::storage::STORAGE_GCS_DEFAULT_ENDPOINT; +use databend_common_meta_app::storage::STORAGE_IPFS_DEFAULT_ENDPOINT; +use databend_common_meta_app::storage::STORAGE_S3_DEFAULT_ENDPOINT; use databend_common_meta_app::storage::StorageAzblobConfig; use databend_common_meta_app::storage::StorageCosConfig; use databend_common_meta_app::storage::StorageFsConfig; @@ -40,15 +43,12 @@ use databend_common_meta_app::storage::StorageOssConfig; use databend_common_meta_app::storage::StorageParams; use databend_common_meta_app::storage::StorageS3Config; use databend_common_meta_app::storage::StorageWebhdfsConfig; -use databend_common_meta_app::storage::STORAGE_GCS_DEFAULT_ENDPOINT; -use databend_common_meta_app::storage::STORAGE_IPFS_DEFAULT_ENDPOINT; -use databend_common_meta_app::storage::STORAGE_S3_DEFAULT_ENDPOINT; use databend_common_storage::STDIN_FD; -use log::info; +use databend_common_storage::Scheme; use log::LevelFilter; +use log::info; use opendal::raw::normalize_path; use opendal::raw::normalize_root; -use opendal::Scheme; /// secure_omission will fix omitted endpoint url schemes into 'https://' #[inline] diff --git a/src/query/sql/src/planner/binder/mod.rs b/src/query/sql/src/planner/binder/mod.rs index 989571eab718c..fbc473b70241d 100644 --- a/src/query/sql/src/planner/binder/mod.rs +++ b/src/query/sql/src/planner/binder/mod.rs @@ -64,9 +64,9 @@ mod window; pub use aggregate::AggregateInfo; pub use async_function_desc::AsyncFunctionDesc; pub use bind_context::*; -pub use bind_mutation::target_probe; pub use bind_mutation::MutationStrategy; pub use bind_mutation::MutationType; +pub use bind_mutation::target_probe; pub use bind_query::bind_values; pub use bind_table_reference::is_range_join_condition; pub use bind_table_reference::parse_result_scan_args; @@ -88,6 +88,6 @@ pub use location::parse_uri_location; pub use scalar::ScalarBinder; pub use scalar_common::*; pub use stream_column_factory::STREAM_COLUMN_FACTORY; -pub use window::bind_window_function_info; pub use window::WindowFunctionInfo; pub use window::WindowOrderByInfo; +pub use window::bind_window_function_info; diff --git a/src/query/sql/src/planner/binder/presign.rs b/src/query/sql/src/planner/binder/presign.rs index 73e9d954a4fe6..0555a838b5684 100644 --- a/src/query/sql/src/planner/binder/presign.rs +++ b/src/query/sql/src/planner/binder/presign.rs @@ -17,12 +17,12 @@ use databend_common_ast::ast::PresignLocation; use databend_common_ast::ast::PresignStmt; use databend_common_exception::Result; -use crate::binder::resolve_stage_location; +use crate::BindContext; use crate::binder::Binder; +use crate::binder::resolve_stage_location; use crate::plans::Plan; use crate::plans::PresignAction; use crate::plans::PresignPlan; -use crate::BindContext; impl Binder { #[async_backtrace::framed] diff --git a/src/query/sql/src/planner/binder/project.rs b/src/query/sql/src/planner/binder/project.rs index e65de9b97c342..001ac2bbce1c8 100644 --- a/src/query/sql/src/planner/binder/project.rs +++ b/src/query/sql/src/planner/binder/project.rs @@ -16,6 +16,7 @@ use std::collections::HashMap; use std::collections::HashSet; use std::sync::Arc; +use databend_common_ast::Span; use databend_common_ast::ast::ColumnFilter; use databend_common_ast::ast::ColumnID; use databend_common_ast::ast::ColumnRef; @@ -27,7 +28,6 @@ use databend_common_ast::ast::Literal; use databend_common_ast::ast::SelectTarget; use databend_common_ast::parser::parse_expr; use databend_common_ast::parser::tokenize_sql; -use databend_common_ast::Span; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::Column; @@ -44,21 +44,25 @@ use derive_visitor::VisitorMut; use itertools::Itertools; use super::AggregateInfo; +use crate::IndexType; +use crate::NameResolutionContext; +use crate::TypeChecker; +use crate::WindowChecker; +use crate::binder::ExprContext; +use crate::binder::Visibility; use crate::binder::aggregate::find_replaced_aggregate_function; use crate::binder::select::SelectItem; use crate::binder::select::SelectList; -use crate::binder::window::find_replaced_window_function; use crate::binder::window::WindowInfo; -use crate::binder::ExprContext; -use crate::binder::Visibility; +use crate::binder::window::find_replaced_window_function; use crate::optimizer::ir::SExpr; -use crate::planner::binder::scalar::ScalarBinder; use crate::planner::binder::BindContext; use crate::planner::binder::Binder; use crate::planner::binder::ColumnBinding; +use crate::planner::binder::scalar::ScalarBinder; +use crate::planner::semantic::GroupingChecker; use crate::planner::semantic::compare_table_name; use crate::planner::semantic::normalize_identifier; -use crate::planner::semantic::GroupingChecker; use crate::plans::BoundColumnRef; use crate::plans::EvalScalar; use crate::plans::ScalarExpr; @@ -66,10 +70,6 @@ use crate::plans::ScalarItem; use crate::plans::SubqueryExpr; use crate::plans::SubqueryType; use crate::plans::VisitorMut as _; -use crate::IndexType; -use crate::NameResolutionContext; -use crate::TypeChecker; -use crate::WindowChecker; #[derive(VisitorMut)] #[visitor(Identifier(enter))] diff --git a/src/query/sql/src/planner/binder/project_set.rs b/src/query/sql/src/planner/binder/project_set.rs index 3d8e757745c66..cf1f50d7f6ea2 100644 --- a/src/query/sql/src/planner/binder/project_set.rs +++ b/src/query/sql/src/planner/binder/project_set.rs @@ -21,23 +21,23 @@ use databend_common_exception::Result; use databend_common_expression::FunctionKind; use databend_common_functions::BUILTIN_FUNCTIONS; +use crate::BindContext; +use crate::Binder; +use crate::ColumnBinding; +use crate::MetadataRef; +use crate::ScalarExpr; +use crate::Visibility; +use crate::binder::ColumnBindingBuilder; use crate::binder::aggregate::AggregateRewriter; use crate::binder::select::SelectList; -use crate::binder::ColumnBindingBuilder; use crate::format_scalar; use crate::optimizer::ir::SExpr; -use crate::plans::walk_expr_mut; use crate::plans::BoundColumnRef; use crate::plans::FunctionCall; use crate::plans::ProjectSet; use crate::plans::ScalarItem; use crate::plans::VisitorMut; -use crate::BindContext; -use crate::Binder; -use crate::ColumnBinding; -use crate::MetadataRef; -use crate::ScalarExpr; -use crate::Visibility; +use crate::plans::walk_expr_mut; #[derive(Default, Clone, PartialEq, Eq, Debug)] pub struct SetReturningInfo { @@ -66,7 +66,7 @@ impl<'a> SetReturningAnalyzer<'a> { } } - fn as_aggregate_rewriter(&mut self) -> AggregateRewriter { + fn as_aggregate_rewriter(&mut self) -> AggregateRewriter<'_> { AggregateRewriter::new(self.bind_context, self.metadata.clone()) } } diff --git a/src/query/sql/src/planner/binder/qualify.rs b/src/query/sql/src/planner/binder/qualify.rs index d72269f2aedb2..f4e81dceb58e6 100644 --- a/src/query/sql/src/planner/binder/qualify.rs +++ b/src/query/sql/src/planner/binder/qualify.rs @@ -19,23 +19,23 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use super::Finder; -use crate::binder::split_conjunctions; -use crate::binder::window::WindowRewriter; +use crate::BindContext; +use crate::Binder; use crate::binder::ColumnBindingBuilder; use crate::binder::ExprContext; use crate::binder::ScalarBinder; use crate::binder::Visibility; +use crate::binder::split_conjunctions; +use crate::binder::window::WindowRewriter; use crate::optimizer::ir::SExpr; use crate::planner::semantic::GroupingChecker; -use crate::plans::walk_expr_mut; use crate::plans::BoundColumnRef; use crate::plans::Filter; use crate::plans::ScalarExpr; use crate::plans::SubqueryExpr; use crate::plans::Visitor; use crate::plans::VisitorMut; -use crate::BindContext; -use crate::Binder; +use crate::plans::walk_expr_mut; impl Binder { /// Analyze window in qualify clause, this will rewrite window functions. diff --git a/src/query/sql/src/planner/binder/replace.rs b/src/query/sql/src/planner/binder/replace.rs index 60ec4c45396b8..b91f58ac98276 100644 --- a/src/query/sql/src/planner/binder/replace.rs +++ b/src/query/sql/src/planner/binder/replace.rs @@ -21,6 +21,7 @@ use databend_common_catalog::lock::LockTableOption; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use crate::BindContext; use crate::binder::Binder; use crate::normalize_identifier; use crate::plans::CopyIntoTableMode; @@ -28,7 +29,6 @@ use crate::plans::InsertInputSource; use crate::plans::InsertValue; use crate::plans::Plan; use crate::plans::Replace; -use crate::BindContext; impl Binder { #[async_backtrace::framed] pub(in crate::planner::binder) async fn bind_replace( diff --git a/src/query/sql/src/planner/binder/report.rs b/src/query/sql/src/planner/binder/report.rs index d1e7f47f739ea..18532c4a18c94 100644 --- a/src/query/sql/src/planner/binder/report.rs +++ b/src/query/sql/src/planner/binder/report.rs @@ -14,8 +14,8 @@ use databend_common_exception::Result; -use crate::plans::Plan; use crate::Binder; +use crate::plans::Plan; impl Binder { pub async fn bind_report_issue(&mut self, sql: &str) -> Result { diff --git a/src/query/sql/src/planner/binder/scalar.rs b/src/query/sql/src/planner/binder/scalar.rs index 8230521138822..b2c28fc7d332c 100644 --- a/src/query/sql/src/planner/binder/scalar.rs +++ b/src/query/sql/src/planner/binder/scalar.rs @@ -17,14 +17,14 @@ use std::sync::Arc; use databend_common_ast::ast::Expr; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::FunctionContext; +use databend_common_expression::types::DataType; +use crate::MetadataRef; use crate::planner::binder::BindContext; use crate::planner::semantic::NameResolutionContext; use crate::planner::semantic::TypeChecker; use crate::plans::ScalarExpr; -use crate::MetadataRef; /// Helper for binding scalar expression with `BindContext`. pub struct ScalarBinder<'a> { diff --git a/src/query/sql/src/planner/binder/scalar_common.rs b/src/query/sql/src/planner/binder/scalar_common.rs index ca35096da6836..b90fd579107bf 100644 --- a/src/query/sql/src/planner/binder/scalar_common.rs +++ b/src/query/sql/src/planner/binder/scalar_common.rs @@ -16,17 +16,17 @@ use std::borrow::Cow; use std::collections::HashSet; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::Scalar; +use databend_common_expression::types::DataType; use crate::optimizer::ir::RelationalProperty; -use crate::plans::walk_expr; use crate::plans::BoundColumnRef; use crate::plans::CastExpr; use crate::plans::ConstantExpr; use crate::plans::FunctionCall; use crate::plans::ScalarExpr; use crate::plans::Visitor; +use crate::plans::walk_expr; // Visitor that find Expressions that match a particular predicate pub struct Finder<'a, F> diff --git a/src/query/sql/src/planner/binder/select.rs b/src/query/sql/src/planner/binder/select.rs index 4e97aba60ae2a..2427a3ff67355 100644 --- a/src/query/sql/src/planner/binder/select.rs +++ b/src/query/sql/src/planner/binder/select.rs @@ -16,31 +16,35 @@ use std::collections::HashMap; use std::collections::HashSet; use std::sync::Arc; +use databend_common_ast::Span; use databend_common_ast::ast::Expr; use databend_common_ast::ast::SelectStmt; use databend_common_ast::ast::SelectTarget; use databend_common_ast::ast::SetExpr; use databend_common_ast::ast::SetOperator; -use databend_common_ast::Span; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::ROW_ID_COL_NAME; +use databend_common_expression::ROW_ID_COLUMN_ID; use databend_common_expression::type_check::common_super_type; use databend_common_expression::types::DataType; -use databend_common_expression::ROW_ID_COLUMN_ID; -use databend_common_expression::ROW_ID_COL_NAME; use databend_common_functions::BUILTIN_FUNCTIONS; -use super::sort::OrderItem; use super::Finder; -use crate::binder::bind_table_reference::JoinConditions; -use crate::binder::scalar_common::split_conjunctions; +use super::sort::OrderItem; +use crate::ColumnEntry; +use crate::ColumnSet; +use crate::IndexType; +use crate::Visibility; use crate::binder::ColumnBindingBuilder; use crate::binder::ExprContext; use crate::binder::INTERNAL_COLUMN_FACTORY; +use crate::binder::bind_table_reference::JoinConditions; +use crate::binder::scalar_common::split_conjunctions; use crate::optimizer::ir::SExpr; -use crate::planner::binder::scalar::ScalarBinder; use crate::planner::binder::BindContext; use crate::planner::binder::Binder; +use crate::planner::binder::scalar::ScalarBinder; use crate::plans::BoundColumnRef; use crate::plans::CastExpr; use crate::plans::Filter; @@ -49,10 +53,6 @@ use crate::plans::ScalarExpr; use crate::plans::ScalarItem; use crate::plans::UnionAll; use crate::plans::Visitor as _; -use crate::ColumnEntry; -use crate::ColumnSet; -use crate::IndexType; -use crate::Visibility; // A normalized IR for `SELECT` clause. #[derive(Debug, Default)] @@ -161,15 +161,15 @@ impl Binder { SetOperator::Intersect if !all => { // Transfer Intersect to Semi join self.bind_intersect( - (left.span(),left_expr,left_bind_context), - (right.span(),right_expr,right_bind_context), + (left.span(), left_expr, left_bind_context), + (right.span(), right_expr, right_bind_context), ) } SetOperator::Except if !all => { // Transfer Except to Anti join self.bind_except( - (left.span(),left_expr,left_bind_context), - (right.span(),right_expr,right_bind_context), + (left.span(), left_expr, left_bind_context), + (right.span(), right_expr, right_bind_context), ) } SetOperator::Union => self.bind_union( diff --git a/src/query/sql/src/planner/binder/show.rs b/src/query/sql/src/planner/binder/show.rs index 139fda462b82d..cf8c8a30978c3 100644 --- a/src/query/sql/src/planner/binder/show.rs +++ b/src/query/sql/src/planner/binder/show.rs @@ -18,11 +18,11 @@ use databend_common_ast::ast::ShowOptions; use databend_common_exception::Result; use log::debug; -use crate::plans::Plan; -use crate::plans::RewriteKind; use crate::BindContext; use crate::Binder; use crate::SelectBuilder; +use crate::plans::Plan; +use crate::plans::RewriteKind; impl Binder { #[async_backtrace::framed] diff --git a/src/query/sql/src/planner/binder/sort.rs b/src/query/sql/src/planner/binder/sort.rs index 22dbd6645b133..03a4700d73ff4 100644 --- a/src/query/sql/src/planner/binder/sort.rs +++ b/src/query/sql/src/planner/binder/sort.rs @@ -22,12 +22,14 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use super::ExprContext; +use crate::BindContext; +use crate::IndexType; +use crate::binder::Binder; +use crate::binder::ColumnBinding; use crate::binder::aggregate::AggregateRewriter; use crate::binder::scalar::ScalarBinder; use crate::binder::select::SelectList; use crate::binder::window::WindowRewriter; -use crate::binder::Binder; -use crate::binder::ColumnBinding; use crate::optimizer::ir::SExpr; use crate::planner::semantic::GroupingChecker; use crate::plans::BoundColumnRef; @@ -40,8 +42,6 @@ use crate::plans::Sort; use crate::plans::SortItem; use crate::plans::UDFCall; use crate::plans::VisitorMut as _; -use crate::BindContext; -use crate::IndexType; #[derive(Debug)] pub struct OrderItems { @@ -67,7 +67,8 @@ impl Binder { distinct: bool, ) -> Result { bind_context.set_expr_context(ExprContext::OrderByClause); - let default_nulls_first = self.ctx.get_settings().get_nulls_first(); + let settings = self.ctx.get_settings(); + let default_nulls_first = settings.get_nulls_first(); let mut order_items = Vec::with_capacity(order_by.len()); for order in order_by { diff --git a/src/query/sql/src/planner/binder/statement_settings.rs b/src/query/sql/src/planner/binder/statement_settings.rs index 1becd783a062c..c9fb37247ee23 100644 --- a/src/query/sql/src/planner/binder/statement_settings.rs +++ b/src/query/sql/src/planner/binder/statement_settings.rs @@ -21,17 +21,17 @@ use databend_common_ast::ast::Settings; use databend_common_ast::ast::Statement; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::cast_scalar; -use databend_common_expression::types::DataType; use databend_common_expression::Constant; use databend_common_expression::ConstantFolder; use databend_common_expression::Expr; +use databend_common_expression::cast_scalar; +use databend_common_expression::types::DataType; use databend_common_functions::BUILTIN_FUNCTIONS; -use crate::plans::Plan; use crate::BindContext; use crate::Binder; use crate::TypeChecker; +use crate::plans::Plan; impl Binder { pub async fn bind_statement_settings( diff --git a/src/query/sql/src/planner/binder/table.rs b/src/query/sql/src/planner/binder/table.rs index 037f70d0af34c..5c8bdeefb3793 100644 --- a/src/query/sql/src/planner/binder/table.rs +++ b/src/query/sql/src/planner/binder/table.rs @@ -20,6 +20,7 @@ use std::sync::Arc; use chrono::TimeZone; use chrono::Utc; use dashmap::DashMap; +use databend_common_ast::Span; use databend_common_ast::ast; use databend_common_ast::ast::ColumnRef; use databend_common_ast::ast::Expr; @@ -34,7 +35,6 @@ use databend_common_ast::ast::TemporalClause; use databend_common_ast::ast::TimeTravelPoint; use databend_common_ast::parser::parse_expr; use databend_common_ast::parser::tokenize_sql; -use databend_common_ast::Span; use databend_common_catalog::catalog_kind::CATALOG_DEFAULT; use databend_common_catalog::table::NavigationPoint; use databend_common_catalog::table::Table; @@ -42,15 +42,15 @@ use databend_common_catalog::table::TimeNavigation; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::is_stream_column; -use databend_common_expression::type_check::check_number; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; use databend_common_expression::Constant; use databend_common_expression::ConstantFolder; use databend_common_expression::DataField; use databend_common_expression::FunctionContext; use databend_common_expression::TableField; +use databend_common_expression::is_stream_column; +use databend_common_expression::type_check::check_number; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_license::license::Feature; use databend_common_license::license_manager::LicenseManagerSwitch; @@ -67,28 +67,28 @@ use databend_enterprise_row_access_policy_feature::get_row_access_policy_handler use databend_storages_common_table_meta::table::ChangeType; use log::info; -use crate::binder::split_conjunctions; +use crate::BaseTableColumn; +use crate::BindContext; +use crate::ColumnEntry; +use crate::IndexType; +use crate::ScalarBinder; +use crate::ScalarExpr; +use crate::VirtualColumn; use crate::binder::Binder; use crate::binder::ColumnBindingBuilder; use crate::binder::CteInfo; use crate::binder::ExprContext; use crate::binder::Visibility; +use crate::binder::split_conjunctions; use crate::optimizer::ir::SExpr; -use crate::planner::semantic::normalize_identifier; use crate::planner::semantic::TypeChecker; +use crate::planner::semantic::normalize_identifier; use crate::plans::DummyTableScan; use crate::plans::RecursiveCteScan; use crate::plans::RelOperator; use crate::plans::Scan; use crate::plans::SecureFilter; use crate::plans::Statistics; -use crate::BaseTableColumn; -use crate::BindContext; -use crate::ColumnEntry; -use crate::IndexType; -use crate::ScalarBinder; -use crate::ScalarExpr; -use crate::VirtualColumn; impl Binder { pub fn bind_dummy_table( diff --git a/src/query/sql/src/planner/binder/table_args.rs b/src/query/sql/src/planner/binder/table_args.rs index fd542358efca6..7ec5873a294db 100644 --- a/src/query/sql/src/planner/binder/table_args.rs +++ b/src/query/sql/src/planner/binder/table_args.rs @@ -24,9 +24,9 @@ use databend_common_expression::ConstantFolder; use databend_common_expression::Scalar; use databend_common_functions::BUILTIN_FUNCTIONS; -use crate::plans::ConstantExpr; use crate::ScalarBinder; use crate::ScalarExpr; +use crate::plans::ConstantExpr; pub fn bind_table_args( scalar_binder: &mut ScalarBinder<'_>, diff --git a/src/query/sql/src/planner/binder/udf.rs b/src/query/sql/src/planner/binder/udf.rs index fa0974d92c885..af0380a3bd50d 100644 --- a/src/query/sql/src/planner/binder/udf.rs +++ b/src/query/sql/src/planner/binder/udf.rs @@ -24,9 +24,9 @@ use databend_common_ast::ast::UDFArgs; use databend_common_ast::ast::UDFDefinition; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::DataField; use databend_common_expression::types::DataType; use databend_common_expression::udf_client::UDFFlightClient; -use databend_common_expression::DataField; use databend_common_functions::is_builtin_function; use databend_common_meta_app::principal::LambdaUDF; use databend_common_meta_app::principal::ScalarUDF; @@ -34,11 +34,14 @@ use databend_common_meta_app::principal::UDAFScript; use databend_common_meta_app::principal::UDFDefinition as PlanUDFDefinition; use databend_common_meta_app::principal::UDFScript; use databend_common_meta_app::principal::UDFServer; +use databend_common_meta_app::principal::UDTF; use databend_common_meta_app::principal::UDTFServer; use databend_common_meta_app::principal::UserDefinedFunction; -use databend_common_meta_app::principal::UDTF; use databend_common_users::UserApiProvider; +use crate::BindContext; +use crate::Binder; +use crate::UdfRewriter; use crate::normalize_identifier; use crate::optimizer::ir::SExpr; use crate::planner::expression::UDFValidator; @@ -48,9 +51,6 @@ use crate::plans::CreateUDFPlan; use crate::plans::DropUDFPlan; use crate::plans::Plan; use crate::plans::UDFLanguage; -use crate::BindContext; -use crate::Binder; -use crate::UdfRewriter; impl Binder { pub(in crate::planner::binder) async fn bind_udf_definition( diff --git a/src/query/sql/src/planner/binder/util.rs b/src/query/sql/src/planner/binder/util.rs index 6efdc80f99514..173558e38294c 100644 --- a/src/query/sql/src/planner/binder/util.rs +++ b/src/query/sql/src/planner/binder/util.rs @@ -12,23 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_ast::ast::quote::QuotedIdent; use databend_common_ast::ast::Identifier; use databend_common_ast::ast::IdentifierType; use databend_common_ast::ast::TableAlias; +use databend_common_ast::ast::quote::QuotedIdent; use databend_common_ast::parser::Dialect; use databend_common_ast::span::merge_span; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::types::DataType; +use crate::Binder; +use crate::NameResolutionContext; +use crate::NameResolutionSuggest; use crate::normalize_identifier; use crate::optimizer::ir::SExpr; use crate::plans::Operator; use crate::plans::RelOperator; -use crate::Binder; -use crate::NameResolutionContext; -use crate::NameResolutionSuggest; /// Ident name can not contain ' or " /// Forbidden ' or " in UserName and RoleName, to prevent Meta injection problem diff --git a/src/query/sql/src/planner/binder/virtual_column.rs b/src/query/sql/src/planner/binder/virtual_column.rs index 9025e779abd8e..642ac78f81704 100644 --- a/src/query/sql/src/planner/binder/virtual_column.rs +++ b/src/query/sql/src/planner/binder/virtual_column.rs @@ -24,14 +24,14 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use log::debug; +use crate::BindContext; +use crate::SelectBuilder; use crate::binder::Binder; use crate::normalize_identifier; use crate::plans::Plan; use crate::plans::RefreshSelection; use crate::plans::RefreshVirtualColumnPlan; use crate::plans::RewriteKind; -use crate::BindContext; -use crate::SelectBuilder; impl Binder { #[async_backtrace::framed] diff --git a/src/query/sql/src/planner/binder/window.rs b/src/query/sql/src/planner/binder/window.rs index e3b9b6bc1248d..14694df7efac7 100644 --- a/src/query/sql/src/planner/binder/window.rs +++ b/src/query/sql/src/planner/binder/window.rs @@ -15,19 +15,23 @@ use std::collections::HashMap; use std::sync::Arc; +use databend_common_ast::Span; use databend_common_ast::ast::WindowDefinition; use databend_common_ast::ast::WindowSpec; -use databend_common_ast::Span; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use itertools::Itertools; use super::select::SelectList; +use crate::BindContext; +use crate::Binder; +use crate::IndexType; +use crate::MetadataRef; +use crate::Visibility; use crate::binder::ColumnBinding; use crate::binder::ColumnBindingBuilder; use crate::optimizer::ir::SExpr; -use crate::plans::walk_expr_mut; use crate::plans::AggregateFunction; use crate::plans::AggregateFunctionScalarSortDesc; use crate::plans::BoundColumnRef; @@ -46,11 +50,7 @@ use crate::plans::WindowFuncFrame; use crate::plans::WindowFuncType; use crate::plans::WindowOrderBy; use crate::plans::WindowPartition; -use crate::BindContext; -use crate::Binder; -use crate::IndexType; -use crate::MetadataRef; -use crate::Visibility; +use crate::plans::walk_expr_mut; impl Binder { pub(super) fn bind_window_function( @@ -490,7 +490,7 @@ impl<'a> WindowRewriter<'a> { } } - pub fn as_window_aggregate_rewriter(&self) -> WindowAggregateRewriter { + pub fn as_window_aggregate_rewriter(&self) -> WindowAggregateRewriter<'_> { WindowAggregateRewriter { bind_context: self.bind_context, } @@ -631,7 +631,8 @@ pub fn bind_window_function_info( child }; - let default_nulls_first = ctx.get_settings().get_nulls_first(); + let settings = ctx.get_settings(); + let default_nulls_first = settings.get_nulls_first(); let mut sort_items: Vec = vec![]; if !window_plan.partition_by.is_empty() { diff --git a/src/query/sql/src/planner/dataframe.rs b/src/query/sql/src/planner/dataframe.rs index 1fd54cbef93e2..0abab05119c88 100644 --- a/src/query/sql/src/planner/dataframe.rs +++ b/src/query/sql/src/planner/dataframe.rs @@ -26,8 +26,8 @@ use databend_common_ast::ast::JoinOperator; use databend_common_ast::ast::OrderByExpr; use databend_common_ast::ast::SelectTarget; use databend_common_ast::ast::TableReference; -use databend_common_catalog::catalog::CatalogManager; use databend_common_catalog::catalog::CATALOG_DEFAULT; +use databend_common_catalog::catalog::CatalogManager; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; @@ -35,12 +35,12 @@ use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use parking_lot::RwLock; -use crate::optimizer::ir::SExpr; -use crate::plans::Limit; use crate::BindContext; use crate::Binder; use crate::Metadata; use crate::NameResolutionContext; +use crate::optimizer::ir::SExpr; +use crate::plans::Limit; pub struct Dataframe { query_ctx: Arc, diff --git a/src/query/sql/src/planner/execution/mod.rs b/src/query/sql/src/planner/execution/mod.rs index 096821bcb839e..06ef4dcd46c44 100644 --- a/src/query/sql/src/planner/execution/mod.rs +++ b/src/query/sql/src/planner/execution/mod.rs @@ -16,6 +16,6 @@ mod query_executor; mod stream_column; pub use query_executor::QueryExecutor; -pub use stream_column::StreamContext; pub use stream_column::CURRENT_BLOCK_ID_COL_NAME; pub use stream_column::CURRENT_BLOCK_ROW_NUM_COL_NAME; +pub use stream_column::StreamContext; diff --git a/src/query/sql/src/planner/execution/stream_column.rs b/src/query/sql/src/planner/execution/stream_column.rs index cadb6e976155b..4a00e36f71bd2 100644 --- a/src/query/sql/src/planner/execution/stream_column.rs +++ b/src/query/sql/src/planner/execution/stream_column.rs @@ -21,18 +21,18 @@ use databend_common_catalog::plan::StreamColumnType; use databend_common_exception::Result; use databend_common_expression::DataBlock; use databend_common_expression::FunctionContext; -use databend_common_expression::TableSchema; use databend_common_expression::ORIGIN_BLOCK_ID_COL_NAME; use databend_common_expression::ORIGIN_BLOCK_ROW_NUM_COL_NAME; use databend_common_expression::ORIGIN_VERSION_COL_NAME; +use databend_common_expression::TableSchema; +use crate::ColumnBindingBuilder; +use crate::ScalarExpr; +use crate::Visibility; use crate::evaluator::BlockOperator; use crate::plans::BoundColumnRef; use crate::plans::ConstantExpr; use crate::plans::FunctionCall; -use crate::ColumnBindingBuilder; -use crate::ScalarExpr; -use crate::Visibility; pub const CURRENT_BLOCK_ID_COL_NAME: &str = "_current_block_id"; pub const CURRENT_BLOCK_ROW_NUM_COL_NAME: &str = "_current_block_row_num"; diff --git a/src/query/sql/src/planner/expression/expression_parser.rs b/src/query/sql/src/planner/expression/expression_parser.rs index 84952a4283b82..1425a72c14e61 100644 --- a/src/query/sql/src/planner/expression/expression_parser.rs +++ b/src/query/sql/src/planner/expression/expression_parser.rs @@ -23,10 +23,6 @@ use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::infer_table_schema; -use databend_common_expression::type_check::check_function; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; use databend_common_expression::Constant; use databend_common_expression::DataSchemaRef; use databend_common_expression::Expr; @@ -35,15 +31,14 @@ use databend_common_expression::RemoteExpr; use databend_common_expression::Scalar; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; +use databend_common_expression::infer_table_schema; +use databend_common_expression::type_check::check_function; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; use databend_common_functions::BUILTIN_FUNCTIONS; use derive_visitor::DriveMut; use parking_lot::RwLock; -use crate::binder::ColumnBindingBuilder; -use crate::binder::ExprContext; -use crate::planner::binder::BindContext; -use crate::planner::semantic::NameResolutionContext; -use crate::planner::semantic::TypeChecker; use crate::BaseTableColumn; use crate::ColumnEntry; use crate::IdentifierNormalizer; @@ -51,6 +46,11 @@ use crate::Metadata; use crate::MetadataRef; use crate::ScalarExpr; use crate::Visibility; +use crate::binder::ColumnBindingBuilder; +use crate::binder::ExprContext; +use crate::planner::binder::BindContext; +use crate::planner::semantic::NameResolutionContext; +use crate::planner::semantic::TypeChecker; pub fn bind_table(table_meta: Arc) -> Result<(BindContext, MetadataRef)> { let mut bind_context = BindContext::new(); diff --git a/src/query/sql/src/planner/format/display.rs b/src/query/sql/src/planner/format/display.rs index f1f6a86251b9c..2fa123d0e4e70 100644 --- a/src/query/sql/src/planner/format/display.rs +++ b/src/query/sql/src/planner/format/display.rs @@ -16,15 +16,15 @@ use databend_common_ast::ast::FormatTreeNode; use databend_common_exception::Result; use itertools::Itertools; +use crate::ColumnEntry; +use crate::IndexType; +use crate::Metadata; +use crate::ScalarExpr; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::RelationalProperty; use crate::optimizer::ir::SExpr; use crate::optimizer::ir::StatInfo; use crate::plans::RelOperator; -use crate::ColumnEntry; -use crate::IndexType; -use crate::Metadata; -use crate::ScalarExpr; /// A trait for humanizing IDs. pub trait IdHumanizer { diff --git a/src/query/sql/src/planner/format/mod.rs b/src/query/sql/src/planner/format/mod.rs index c2e2ee970bf61..ba66446ce1644 100644 --- a/src/query/sql/src/planner/format/mod.rs +++ b/src/query/sql/src/planner/format/mod.rs @@ -16,6 +16,6 @@ mod display; mod display_plan; mod display_rel_operator; -pub use display::format_scalar; pub use display::FormatOptions; pub use display::MetadataIdHumanizer; +pub use display::format_scalar; diff --git a/src/query/sql/src/planner/metadata/bloom_index.rs b/src/query/sql/src/planner/metadata/bloom_index.rs index 3f02519e9f466..a370a018bfe95 100644 --- a/src/query/sql/src/planner/metadata/bloom_index.rs +++ b/src/query/sql/src/planner/metadata/bloom_index.rs @@ -15,17 +15,17 @@ use std::collections::BTreeMap; use std::str::FromStr; +use databend_common_ast::parser::Dialect; use databend_common_ast::parser::parse_comma_separated_idents; use databend_common_ast::parser::tokenize_sql; -use databend_common_ast::parser::Dialect; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::is_stream_column_id; use databend_common_expression::ComputedExpr; use databend_common_expression::FieldIndex; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; +use databend_common_expression::is_stream_column_id; use databend_common_meta_app::tenant::Tenant; use databend_common_settings::Settings; diff --git a/src/query/sql/src/planner/metadata/hll_columns.rs b/src/query/sql/src/planner/metadata/hll_columns.rs index ec708a5744153..7ac84e4aa5ef9 100644 --- a/src/query/sql/src/planner/metadata/hll_columns.rs +++ b/src/query/sql/src/planner/metadata/hll_columns.rs @@ -15,9 +15,9 @@ use std::collections::BTreeMap; use std::str::FromStr; +use databend_common_ast::parser::Dialect; use databend_common_ast::parser::parse_comma_separated_idents; use databend_common_ast::parser::tokenize_sql; -use databend_common_ast::parser::Dialect; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::ComputedExpr; @@ -28,8 +28,8 @@ use databend_common_expression::TableSchemaRef; use databend_common_meta_app::tenant::Tenant; use databend_common_settings::Settings; -use crate::normalize_identifier; use crate::NameResolutionContext; +use crate::normalize_identifier; #[derive(Clone)] pub enum ApproxDistinctColumns { diff --git a/src/query/sql/src/planner/metadata/metadata.rs b/src/query/sql/src/planner/metadata/metadata.rs index 946e376180dc1..37be5574b551e 100644 --- a/src/query/sql/src/planner/metadata/metadata.rs +++ b/src/query/sql/src/planner/metadata/metadata.rs @@ -25,12 +25,12 @@ use databend_common_ast::ast::Literal; use databend_common_catalog::plan::DataSourcePlan; use databend_common_catalog::plan::InternalColumn; use databend_common_catalog::table::Table; -use databend_common_expression::display::display_tuple_field_name; -use databend_common_expression::is_stream_column_id; -use databend_common_expression::types::DataType; use databend_common_expression::ComputedExpr; use databend_common_expression::TableDataType; use databend_common_expression::TableField; +use databend_common_expression::display::display_tuple_field_name; +use databend_common_expression::is_stream_column_id; +use databend_common_expression::types::DataType; use parking_lot::RwLock; use crate::optimizer::ir::SExpr; diff --git a/src/query/sql/src/planner/mod.rs b/src/query/sql/src/planner/mod.rs index 1a84b3c62f98e..ddff0f75a140a 100644 --- a/src/query/sql/src/planner/mod.rs +++ b/src/query/sql/src/planner/mod.rs @@ -26,7 +26,6 @@ pub mod optimizer; mod planner_cache; pub mod plans; -pub use binder::parse_result_scan_args; pub use binder::BindContext; pub use binder::Binder; pub use binder::ColumnBinding; @@ -35,17 +34,18 @@ pub use binder::DefaultExprBinder; pub use binder::ScalarBinder; pub use binder::SelectBuilder; pub use binder::Visibility; +pub use binder::parse_result_scan_args; pub use execution::*; pub use expression::*; pub use format::*; pub use metadata::*; pub use optimizer::optimize; -pub use planner::get_query_kind; pub use planner::PlanExtras; pub use planner::Planner; -pub use plans::InsertInputSource; -pub use plans::ScalarExpr; +pub use planner::get_query_kind; pub use plans::DELETE_NAME; pub use plans::INSERT_NAME; +pub use plans::InsertInputSource; +pub use plans::ScalarExpr; pub use plans::UPDATE_NAME; pub use semantic::*; diff --git a/src/query/sql/src/planner/optimizer/cost/cost.rs b/src/query/sql/src/planner/optimizer/cost/cost.rs index 1bd662632c8ad..e426148c897ff 100644 --- a/src/query/sql/src/planner/optimizer/cost/cost.rs +++ b/src/query/sql/src/planner/optimizer/cost/cost.rs @@ -18,11 +18,11 @@ use std::ops::AddAssign; use databend_common_exception::Result; +use crate::IndexType; use crate::optimizer::ir::MExpr; use crate::optimizer::ir::Memo; use crate::optimizer::ir::PhysicalProperty; use crate::optimizer::ir::RequiredProperty; -use crate::IndexType; #[derive(Debug, Clone, Copy, PartialEq, PartialOrd)] pub struct Cost(pub f64); diff --git a/src/query/sql/src/planner/optimizer/ir/expr/extract.rs b/src/query/sql/src/planner/optimizer/ir/expr/extract.rs index 90d73478be83c..c3872b9b22374 100644 --- a/src/query/sql/src/planner/optimizer/ir/expr/extract.rs +++ b/src/query/sql/src/planner/optimizer/ir/expr/extract.rs @@ -17,10 +17,10 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use crate::optimizer::ir::expr::MExpr; -use crate::optimizer::ir::expr::SExpr; use crate::optimizer::ir::Group; use crate::optimizer::ir::Memo; +use crate::optimizer::ir::expr::MExpr; +use crate::optimizer::ir::expr::SExpr; use crate::plans::Operator; use crate::plans::RelOp; use crate::plans::RelOperator; diff --git a/src/query/sql/src/planner/optimizer/ir/expr/m_expr.rs b/src/query/sql/src/planner/optimizer/ir/expr/m_expr.rs index 5a989ad92d6b6..b35fbb412a046 100644 --- a/src/query/sql/src/planner/optimizer/ir/expr/m_expr.rs +++ b/src/query/sql/src/planner/optimizer/ir/expr/m_expr.rs @@ -17,14 +17,14 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use crate::IndexType; +use crate::optimizer::ir::Memo; use crate::optimizer::ir::expr::PatternExtractor; use crate::optimizer::ir::group::Group; -use crate::optimizer::ir::Memo; use crate::optimizer::optimizers::rule::AppliedRules; use crate::optimizer::optimizers::rule::RulePtr; use crate::optimizer::optimizers::rule::TransformResult; use crate::plans::RelOperator; -use crate::IndexType; /// `MExpr` is abbreviation of multiple expression, which is the representation of relational /// expressions inside `Memo`. diff --git a/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs b/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs index c3fe829c7e81f..045e8f5fc78ad 100644 --- a/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs +++ b/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs @@ -23,17 +23,17 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use educe::Educe; -use crate::optimizer::ir::property::RelExpr; -use crate::optimizer::ir::property::RelationalProperty; +use crate::IndexType; use crate::optimizer::ir::SExprVisitor; use crate::optimizer::ir::StatInfo; use crate::optimizer::ir::VisitAction; +use crate::optimizer::ir::property::RelExpr; +use crate::optimizer::ir::property::RelationalProperty; use crate::optimizer::optimizers::rule::AppliedRules; use crate::optimizer::optimizers::rule::RuleID; use crate::plans::Exchange; use crate::plans::Operator; use crate::plans::RelOperator; -use crate::IndexType; /// `SExpr` is abbreviation of single expression, which is a tree of relational operators. #[derive(Educe)] diff --git a/src/query/sql/src/planner/optimizer/ir/expr/visitor.rs b/src/query/sql/src/planner/optimizer/ir/expr/visitor.rs index d6eee9e030504..8261213c1467e 100644 --- a/src/query/sql/src/planner/optimizer/ir/expr/visitor.rs +++ b/src/query/sql/src/planner/optimizer/ir/expr/visitor.rs @@ -129,7 +129,7 @@ pub fn visit_sexpr(visitor: &mut V, expr: &SExpr) -> Result Some(new_expr), _ => None, - }) + }); } VisitAction::Stop => return Ok(None), VisitAction::Replace(new_expr) => return Ok(Some(new_expr)), @@ -197,7 +197,7 @@ pub async fn visit_sexpr_async( return visitor.post_visit(expr).await.map(|action| match action { VisitAction::Replace(new_expr) => Some(new_expr), _ => None, - }) + }); } VisitAction::Stop => return Ok(None), VisitAction::Replace(new_expr) => return Ok(Some(new_expr)), diff --git a/src/query/sql/src/planner/optimizer/ir/group.rs b/src/query/sql/src/planner/optimizer/ir/group.rs index 5738435368f28..24618a81e4829 100644 --- a/src/query/sql/src/planner/optimizer/ir/group.rs +++ b/src/query/sql/src/planner/optimizer/ir/group.rs @@ -18,12 +18,12 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use crate::IndexType; use crate::optimizer::cost::CostContext; +use crate::optimizer::ir::StatInfo; use crate::optimizer::ir::expr::MExpr; use crate::optimizer::ir::property::RelationalProperty; use crate::optimizer::ir::property::RequiredProperty; -use crate::optimizer::ir::StatInfo; -use crate::IndexType; /// State of a `Group` #[derive(Copy, Clone, Debug, PartialEq, Eq)] diff --git a/src/query/sql/src/planner/optimizer/ir/memo.rs b/src/query/sql/src/planner/optimizer/ir/memo.rs index d31c9801996a5..7b4a81605da10 100644 --- a/src/query/sql/src/planner/optimizer/ir/memo.rs +++ b/src/query/sql/src/planner/optimizer/ir/memo.rs @@ -18,15 +18,15 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use crate::IndexType; +use crate::optimizer::ir::Group; +use crate::optimizer::ir::GroupState; use crate::optimizer::ir::expr::MExpr; use crate::optimizer::ir::expr::SExpr; use crate::optimizer::ir::property::RelExpr; use crate::optimizer::ir::property::RelationalProperty; use crate::optimizer::ir::property::StatInfo; -use crate::optimizer::ir::Group; -use crate::optimizer::ir::GroupState; use crate::plans::RelOperator; -use crate::IndexType; /// `Memo` is a search space which memoize possible plans of a query. /// The plans inside `Memo` are organized with `Group`s. diff --git a/src/query/sql/src/planner/optimizer/ir/mod.rs b/src/query/sql/src/planner/optimizer/ir/mod.rs index 638f7067a7024..76482ede84989 100644 --- a/src/query/sql/src/planner/optimizer/ir/mod.rs +++ b/src/query/sql/src/planner/optimizer/ir/mod.rs @@ -43,7 +43,7 @@ pub use property::Statistics; pub use stats::ColumnStat; pub use stats::ColumnStatSet; pub use stats::HistogramBuilder; +pub use stats::MAX_SELECTIVITY; pub use stats::NewStatistic; pub use stats::SelectivityEstimator; pub use stats::UniformSampleSet; -pub use stats::MAX_SELECTIVITY; diff --git a/src/query/sql/src/planner/optimizer/ir/property/builder.rs b/src/query/sql/src/planner/optimizer/ir/property/builder.rs index d06ba0fcfc1b0..01af90a4c2551 100644 --- a/src/query/sql/src/planner/optimizer/ir/property/builder.rs +++ b/src/query/sql/src/planner/optimizer/ir/property/builder.rs @@ -18,15 +18,15 @@ use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use crate::IndexType; +use crate::optimizer::ir::StatInfo; use crate::optimizer::ir::expr::MExpr; use crate::optimizer::ir::expr::SExpr; use crate::optimizer::ir::memo::Memo; use crate::optimizer::ir::property::PhysicalProperty; use crate::optimizer::ir::property::RelationalProperty; use crate::optimizer::ir::property::RequiredProperty; -use crate::optimizer::ir::StatInfo; use crate::plans::Operator; -use crate::IndexType; /// A helper to access children of `SExpr` and `MExpr` in /// a unified view. diff --git a/src/query/sql/src/planner/optimizer/ir/property/property.rs b/src/query/sql/src/planner/optimizer/ir/property/property.rs index 366305ab3cf41..ec5b836d5f252 100644 --- a/src/query/sql/src/planner/optimizer/ir/property/property.rs +++ b/src/query/sql/src/planner/optimizer/ir/property/property.rs @@ -15,11 +15,11 @@ use std::fmt::Display; use std::fmt::Formatter; +use crate::ColumnSet; use crate::optimizer::ir::ColumnStatSet; use crate::plans::ScalarExpr; use crate::plans::ScalarItem; use crate::plans::SortItem; -use crate::ColumnSet; #[derive(Default, Clone, Debug, PartialEq, Eq, Hash)] pub struct RequiredProperty { @@ -115,10 +115,9 @@ impl Distribution { | (Distribution::Broadcast, Distribution::Broadcast) | (Distribution::NodeToNodeHash(_), Distribution::Broadcast) => true, - ( - Distribution::NodeToNodeHash(ref keys), - Distribution::NodeToNodeHash(ref other_keys), - ) => keys == other_keys, + (Distribution::NodeToNodeHash(keys), Distribution::NodeToNodeHash(other_keys)) => { + keys == other_keys + } _ => false, } } @@ -131,7 +130,7 @@ impl Display for Distribution { Distribution::Random => write!(f, "Random"), Distribution::Serial => write!(f, "Serial"), Distribution::Broadcast => write!(f, "Broadcast"), - Distribution::NodeToNodeHash(ref keys) => write!( + Distribution::NodeToNodeHash(keys) => write!( f, "Hash({})", keys.iter() diff --git a/src/query/sql/src/planner/optimizer/ir/stats/histogram.rs b/src/query/sql/src/planner/optimizer/ir/stats/histogram.rs index 2c57e8868a451..b00088ee2fde0 100644 --- a/src/query/sql/src/planner/optimizer/ir/stats/histogram.rs +++ b/src/query/sql/src/planner/optimizer/ir/stats/histogram.rs @@ -277,8 +277,11 @@ impl SampleSet for UniformSampleSet { _ => Err(format!( "Unsupported datum type for histogram calculation: {} (type: {}), {} (type: {}). Only numeric types are supported.", - self.min, self.min.type_name(), self.max, self.max.type_name() - )) + self.min, + self.min.type_name(), + self.max, + self.max.type_name() + )), } } } diff --git a/src/query/sql/src/planner/optimizer/ir/stats/mod.rs b/src/query/sql/src/planner/optimizer/ir/stats/mod.rs index f2b9d700a8cca..c0a96b9f0c308 100644 --- a/src/query/sql/src/planner/optimizer/ir/stats/mod.rs +++ b/src/query/sql/src/planner/optimizer/ir/stats/mod.rs @@ -21,5 +21,5 @@ pub use column_stat::ColumnStatSet; pub use column_stat::NewStatistic; pub use histogram::HistogramBuilder; pub use histogram::UniformSampleSet; -pub use selectivity::SelectivityEstimator; pub use selectivity::MAX_SELECTIVITY; +pub use selectivity::SelectivityEstimator; diff --git a/src/query/sql/src/planner/optimizer/ir/stats/selectivity.rs b/src/query/sql/src/planner/optimizer/ir/stats/selectivity.rs index b6278a67aa05e..bd66ccd53fd74 100644 --- a/src/query/sql/src/planner/optimizer/ir/stats/selectivity.rs +++ b/src/query/sql/src/planner/optimizer/ir/stats/selectivity.rs @@ -12,26 +12,27 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::cmp::max; use std::cmp::Ordering; +use std::cmp::max; use std::collections::HashSet; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::type_check; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::NumberScalar; use databend_common_expression::Constant; use databend_common_expression::ConstantFolder; use databend_common_expression::Expr; use databend_common_expression::FunctionContext; use databend_common_expression::Scalar; +use databend_common_expression::type_check; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::NumberScalar; use databend_common_functions::BUILTIN_FUNCTIONS; -use databend_common_storage::Datum; use databend_common_storage::DEFAULT_HISTOGRAM_BUCKETS; +use databend_common_storage::Datum; use databend_common_storage::F64; +use crate::IndexType; use crate::optimizer::ir::ColumnStat; use crate::optimizer::ir::HistogramBuilder; use crate::optimizer::ir::Statistics; @@ -40,7 +41,6 @@ use crate::plans::ComparisonOp; use crate::plans::ConstantExpr; use crate::plans::FunctionCall; use crate::plans::ScalarExpr; -use crate::IndexType; /// A default selectivity factor for a predicate /// that we cannot estimate the selectivity for it. diff --git a/src/query/sql/src/planner/optimizer/optimizer.rs b/src/query/sql/src/planner/optimizer/optimizer.rs index 04200e46d504c..2b9b8da721b62 100644 --- a/src/query/sql/src/planner/optimizer/optimizer.rs +++ b/src/query/sql/src/planner/optimizer/optimizer.rs @@ -20,11 +20,16 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use log::info; -use crate::binder::target_probe; +use crate::InsertInputSource; use crate::binder::MutationStrategy; use crate::binder::MutationType; +use crate::binder::target_probe; +use crate::optimizer::OptimizerContext; use crate::optimizer::ir::Memo; use crate::optimizer::ir::SExpr; +use crate::optimizer::optimizers::CTEFilterPushdownOptimizer; +use crate::optimizer::optimizers::CascadesOptimizer; +use crate::optimizer::optimizers::DPhpyOptimizer; use crate::optimizer::optimizers::distributed::BroadcastToShuffleOptimizer; use crate::optimizer::optimizers::operator::CleanupUnusedCTEOptimizer; use crate::optimizer::optimizers::operator::DeduplicateJoinConditionOptimizer; @@ -34,15 +39,11 @@ use crate::optimizer::optimizers::operator::RuleStatsAggregateOptimizer; use crate::optimizer::optimizers::operator::SingleToInnerOptimizer; use crate::optimizer::optimizers::operator::SubqueryDecorrelatorOptimizer; use crate::optimizer::optimizers::recursive::RecursiveRuleOptimizer; +use crate::optimizer::optimizers::rule::DEFAULT_REWRITE_RULES; use crate::optimizer::optimizers::rule::RuleEagerAggregation; use crate::optimizer::optimizers::rule::RuleID; -use crate::optimizer::optimizers::rule::DEFAULT_REWRITE_RULES; -use crate::optimizer::optimizers::CTEFilterPushdownOptimizer; -use crate::optimizer::optimizers::CascadesOptimizer; -use crate::optimizer::optimizers::DPhpyOptimizer; use crate::optimizer::pipeline::OptimizerPipeline; use crate::optimizer::statistics::CollectStatisticsOptimizer; -use crate::optimizer::OptimizerContext; use crate::plans::ConstantTableScan; use crate::plans::CopyIntoLocationPlan; use crate::plans::Join; @@ -55,7 +56,6 @@ use crate::plans::Plan; use crate::plans::RelOp; use crate::plans::RelOperator; use crate::plans::SetScalarsOrQuery; -use crate::InsertInputSource; #[fastrace::trace] #[async_recursion(# [recursive::recursive])] diff --git a/src/query/sql/src/planner/optimizer/optimizer_context.rs b/src/query/sql/src/planner/optimizer/optimizer_context.rs index 695bd266b72a2..c6ff259edd0f9 100644 --- a/src/query/sql/src/planner/optimizer/optimizer_context.rs +++ b/src/query/sql/src/planner/optimizer/optimizer_context.rs @@ -22,9 +22,9 @@ use databend_common_settings::Settings; use educe::Educe; use parking_lot::RwLock; +use crate::MetadataRef; use crate::optimizer::optimizers::rule::RuleID; use crate::planner::QueryExecutor; -use crate::MetadataRef; #[derive(Educe)] #[educe(Debug)] diff --git a/src/query/sql/src/planner/optimizer/optimizers/cascades/cascade.rs b/src/query/sql/src/planner/optimizer/optimizers/cascades/cascade.rs index 5667a667ff565..20c8eae11dde3 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/cascades/cascade.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/cascades/cascade.rs @@ -19,6 +19,9 @@ use databend_common_exception::Result; use log::debug; use log::info; +use crate::IndexType; +use crate::optimizer::Optimizer; +use crate::optimizer::OptimizerContext; use crate::optimizer::cost::CostModel; use crate::optimizer::ir::Distribution; use crate::optimizer::ir::Memo; @@ -27,18 +30,15 @@ use crate::optimizer::ir::RequiredProperty; use crate::optimizer::ir::SExpr; use crate::optimizer::optimizers::cascades::cost::DefaultCostModel; use crate::optimizer::optimizers::cascades::rule::StrategyFactory; +use crate::optimizer::optimizers::cascades::tasks::DEFAULT_TASK_LIMIT; use crate::optimizer::optimizers::cascades::tasks::OptimizeGroupTask; use crate::optimizer::optimizers::cascades::tasks::Task; use crate::optimizer::optimizers::cascades::tasks::TaskManager; -use crate::optimizer::optimizers::cascades::tasks::DEFAULT_TASK_LIMIT; use crate::optimizer::optimizers::distributed::DistributedOptimizer; use crate::optimizer::optimizers::distributed::SortAndLimitPushDownOptimizer; use crate::optimizer::optimizers::rule::RuleSet; use crate::optimizer::optimizers::rule::TransformResult; -use crate::optimizer::Optimizer; -use crate::optimizer::OptimizerContext; use crate::plans::RelOperator; -use crate::IndexType; /// A cascades-style search engine to enumerate possible alternations of a relational expression and /// find the optimal one. diff --git a/src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/apply_rule.rs b/src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/apply_rule.rs index 43646e5198791..3747c83a7cf36 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/apply_rule.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/apply_rule.rs @@ -15,12 +15,12 @@ use databend_common_exception::Result; use educe::Educe; -use crate::optimizer::optimizers::cascades::tasks::SharedCounter; +use crate::IndexType; use crate::optimizer::optimizers::cascades::CascadesOptimizer; +use crate::optimizer::optimizers::cascades::tasks::SharedCounter; use crate::optimizer::optimizers::rule::RuleFactory; use crate::optimizer::optimizers::rule::RuleID; use crate::optimizer::optimizers::rule::TransformResult; -use crate::IndexType; #[derive(Educe)] #[educe(Debug)] diff --git a/src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/explore_expr.rs b/src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/explore_expr.rs index 13bc871c3cae4..cbf3a05f36824 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/explore_expr.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/explore_expr.rs @@ -19,13 +19,13 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use educe::Educe; +use super::Task; use super::apply_rule::ApplyRuleTask; use super::explore_group::ExploreGroupTask; -use super::Task; +use crate::IndexType; +use crate::optimizer::optimizers::cascades::CascadesOptimizer; use crate::optimizer::optimizers::cascades::tasks::SharedCounter; use crate::optimizer::optimizers::cascades::tasks::TaskManager; -use crate::optimizer::optimizers::cascades::CascadesOptimizer; -use crate::IndexType; #[derive(Clone, Copy, Debug)] pub enum ExploreExprState { diff --git a/src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/explore_group.rs b/src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/explore_group.rs index 377fbd3066297..5c8df899271f9 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/explore_group.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/explore_group.rs @@ -19,13 +19,13 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use educe::Educe; -use super::explore_expr::ExploreExprTask; use super::Task; +use super::explore_expr::ExploreExprTask; +use crate::IndexType; use crate::optimizer::ir::GroupState; +use crate::optimizer::optimizers::cascades::CascadesOptimizer; use crate::optimizer::optimizers::cascades::tasks::SharedCounter; use crate::optimizer::optimizers::cascades::tasks::TaskManager; -use crate::optimizer::optimizers::cascades::CascadesOptimizer; -use crate::IndexType; #[derive(Clone, Copy, Debug)] pub enum ExploreGroupState { diff --git a/src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/mod.rs b/src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/mod.rs index 322141dedcda4..e295cfa2b471b 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/mod.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/mod.rs @@ -27,5 +27,5 @@ pub use optimize_expr::OptimizeExprTask; pub use optimize_group::OptimizeGroupTask; pub use task::SharedCounter; pub use task::Task; -pub use task_manager::TaskManager; pub use task_manager::DEFAULT_TASK_LIMIT; +pub use task_manager::TaskManager; diff --git a/src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/optimize_expr.rs b/src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/optimize_expr.rs index 48432822b2e0f..f76c25248b4df 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/optimize_expr.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/optimize_expr.rs @@ -19,8 +19,9 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use educe::Educe; -use super::optimize_group::OptimizeGroupTask; use super::Task; +use super::optimize_group::OptimizeGroupTask; +use crate::IndexType; use crate::optimizer::cost::Cost; use crate::optimizer::cost::CostContext; use crate::optimizer::ir::Distribution; @@ -31,11 +32,10 @@ use crate::optimizer::ir::PatternExtractor; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::RequiredProperty; use crate::optimizer::ir::SExpr; +use crate::optimizer::optimizers::cascades::CascadesOptimizer; use crate::optimizer::optimizers::cascades::tasks::SharedCounter; use crate::optimizer::optimizers::cascades::tasks::TaskManager; -use crate::optimizer::optimizers::cascades::CascadesOptimizer; use crate::plans::RelOperator; -use crate::IndexType; #[derive(Clone, Copy, Debug)] pub enum OptimizeExprState { diff --git a/src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/optimize_group.rs b/src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/optimize_group.rs index db6a766b855ef..6349f69f8cc09 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/optimize_group.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/optimize_group.rs @@ -19,16 +19,16 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use educe::Educe; -use super::optimize_expr::OptimizeExprTask; use super::Task; +use super::optimize_expr::OptimizeExprTask; +use crate::IndexType; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::RequiredProperty; +use crate::optimizer::optimizers::cascades::CascadesOptimizer; use crate::optimizer::optimizers::cascades::tasks::ExploreGroupTask; use crate::optimizer::optimizers::cascades::tasks::SharedCounter; use crate::optimizer::optimizers::cascades::tasks::TaskManager; -use crate::optimizer::optimizers::cascades::CascadesOptimizer; use crate::plans::Operator; -use crate::IndexType; #[derive(Clone, Copy, Debug)] pub enum OptimizeGroupState { diff --git a/src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/task.rs b/src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/task.rs index 8aab31de86874..ae187c0fa0424 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/task.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/task.rs @@ -18,13 +18,13 @@ use std::rc::Rc; use databend_common_base::runtime::drop_guard; use databend_common_exception::Result; +use crate::optimizer::optimizers::CascadesOptimizer; use crate::optimizer::optimizers::cascades::tasks::ApplyRuleTask; use crate::optimizer::optimizers::cascades::tasks::ExploreExprTask; use crate::optimizer::optimizers::cascades::tasks::ExploreGroupTask; use crate::optimizer::optimizers::cascades::tasks::OptimizeExprTask; use crate::optimizer::optimizers::cascades::tasks::OptimizeGroupTask; use crate::optimizer::optimizers::cascades::tasks::TaskManager; -use crate::optimizer::optimizers::CascadesOptimizer; #[derive(Debug)] pub struct SharedCounter { diff --git a/src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/task_manager.rs b/src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/task_manager.rs index 5a0cec5e26e72..ec4c69a0994cd 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/task_manager.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/task_manager.rs @@ -18,8 +18,8 @@ use std::time::Duration; use databend_common_exception::Result; use log::info; -use crate::optimizer::optimizers::cascades::tasks::Task; use crate::optimizer::optimizers::CascadesOptimizer; +use crate::optimizer::optimizers::cascades::tasks::Task; /// It will cost about 4000ns to execute a task in average, /// so the default task limit is 1,250,000 which means the diff --git a/src/query/sql/src/planner/optimizer/optimizers/cte_filter_pushdown.rs b/src/query/sql/src/planner/optimizer/optimizers/cte_filter_pushdown.rs index da3e34dc15c9d..3a148253b9b71 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/cte_filter_pushdown.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/cte_filter_pushdown.rs @@ -19,12 +19,12 @@ use async_trait::async_trait; use databend_common_ast::Span; use databend_common_exception::Result; +use crate::optimizer::Optimizer; +use crate::optimizer::OptimizerContext; use crate::optimizer::ir::SExpr; use crate::optimizer::optimizers::operator::PullUpFilterOptimizer; use crate::optimizer::optimizers::recursive::RecursiveRuleOptimizer; use crate::optimizer::optimizers::rule::DEFAULT_REWRITE_RULES; -use crate::optimizer::Optimizer; -use crate::optimizer::OptimizerContext; use crate::plans::BoundColumnRef; use crate::plans::Filter; use crate::plans::FunctionCall; diff --git a/src/query/sql/src/planner/optimizer/optimizers/distributed/distributed.rs b/src/query/sql/src/planner/optimizer/optimizers/distributed/distributed.rs index c317c56ed5ed0..da6060d7c1e3c 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/distributed/distributed.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/distributed/distributed.rs @@ -18,12 +18,12 @@ use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; use super::sort_and_limit::SortAndLimitPushDownOptimizer; +use crate::optimizer::OptimizerContext; use crate::optimizer::ir::Distribution; use crate::optimizer::ir::PropertyEnforcer; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::RequiredProperty; use crate::optimizer::ir::SExpr; -use crate::optimizer::OptimizerContext; use crate::plans::Exchange; /// DistributedOptimizer optimizes a query plan for distributed execution. diff --git a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs index 97cd27bb085b2..5c479b2def710 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs @@ -21,24 +21,24 @@ use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use crate::IndexType; +use crate::MetadataRef; +use crate::ScalarExpr; +use crate::optimizer::Optimizer; +use crate::optimizer::OptimizerContext; use crate::optimizer::ir::SExpr; -use crate::optimizer::optimizers::hyper_dp::util::intersect; -use crate::optimizer::optimizers::hyper_dp::util::union; use crate::optimizer::optimizers::hyper_dp::JoinNode; use crate::optimizer::optimizers::hyper_dp::JoinRelation; use crate::optimizer::optimizers::hyper_dp::QueryGraph; use crate::optimizer::optimizers::hyper_dp::RelationSetTree; +use crate::optimizer::optimizers::hyper_dp::util::intersect; +use crate::optimizer::optimizers::hyper_dp::util::union; use crate::optimizer::optimizers::rule::RuleFactory; use crate::optimizer::optimizers::rule::RuleID; use crate::optimizer::optimizers::rule::TransformResult; -use crate::optimizer::Optimizer; -use crate::optimizer::OptimizerContext; use crate::plans::Filter; use crate::plans::JoinType; use crate::plans::RelOperator; -use crate::IndexType; -use crate::MetadataRef; -use crate::ScalarExpr; const EMIT_THRESHOLD: usize = 10000; const RELATION_THRESHOLD: usize = 10; diff --git a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/join_node.rs b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/join_node.rs index 0f08abd205ff7..4e179c838a7ef 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/join_node.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/join_node.rs @@ -16,6 +16,8 @@ use std::sync::Arc; use databend_common_exception::Result; +use crate::IndexType; +use crate::ScalarExpr; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::SExpr; use crate::optimizer::optimizers::hyper_dp::JoinRelation; @@ -23,8 +25,6 @@ use crate::plans::Join; use crate::plans::JoinEquiCondition; use crate::plans::JoinType; use crate::plans::RelOperator; -use crate::IndexType; -use crate::ScalarExpr; #[derive(Clone)] pub struct JoinNode { diff --git a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/join_relation.rs b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/join_relation.rs index c98bc96baa17b..b17a14ea6f301 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/join_relation.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/join_relation.rs @@ -19,10 +19,10 @@ use ahash::HashMap; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use crate::optimizer::ir::RelExpr; -use crate::optimizer::ir::SExpr; use crate::IndexType; use crate::MetadataRef; +use crate::optimizer::ir::RelExpr; +use crate::optimizer::ir::SExpr; pub struct JoinRelation { s_expr: SExpr, diff --git a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/query_graph.rs b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/query_graph.rs index 0b94522598b0b..217b027ed9e5b 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/query_graph.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/query_graph.rs @@ -17,9 +17,9 @@ use std::collections::HashSet; use databend_common_exception::Result; -use crate::optimizer::optimizers::hyper_dp::util::is_subset; use crate::IndexType; use crate::ScalarExpr; +use crate::optimizer::optimizers::hyper_dp::util::is_subset; #[derive(Clone, Debug)] struct NeighborInfo { diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/aggregate/normalize_aggregate.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/aggregate/normalize_aggregate.rs index 86526124f5167..7dd092f646e74 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/operator/aggregate/normalize_aggregate.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/aggregate/normalize_aggregate.rs @@ -16,16 +16,16 @@ use std::sync::Arc; use databend_common_exception::Result; -use crate::optimizer::ir::SExpr; +use crate::ColumnBinding; +use crate::Visibility; use crate::optimizer::Optimizer; +use crate::optimizer::ir::SExpr; use crate::plans::Aggregate; use crate::plans::BoundColumnRef; use crate::plans::EvalScalar; use crate::plans::RelOperator; use crate::plans::ScalarExpr; use crate::plans::ScalarItem; -use crate::ColumnBinding; -use crate::Visibility; pub struct RuleNormalizeAggregateOptimizer {} diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/aggregate/stats_aggregate.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/aggregate/stats_aggregate.rs index 9f8eb81c06b31..2c168d41d0dfb 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/operator/aggregate/stats_aggregate.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/aggregate/stats_aggregate.rs @@ -18,9 +18,12 @@ use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; use databend_common_expression::types::DataType; -use crate::optimizer::ir::SExpr; +use crate::ColumnBindingBuilder; +use crate::MetadataRef; +use crate::Visibility; use crate::optimizer::Optimizer; use crate::optimizer::OptimizerContext; +use crate::optimizer::ir::SExpr; use crate::plans::Aggregate; use crate::plans::AggregateFunction; use crate::plans::BoundColumnRef; @@ -32,9 +35,6 @@ use crate::plans::RelOp; use crate::plans::RelOperator; use crate::plans::ScalarExpr; use crate::plans::ScalarItem; -use crate::ColumnBindingBuilder; -use crate::MetadataRef; -use crate::Visibility; // Replace aggregate function with scalar from table's accurate stats function pub struct RuleStatsAggregateOptimizer { diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs index 7f6274efc445f..a2e23c8dee34a 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs @@ -17,8 +17,8 @@ use std::sync::Arc; use databend_common_exception::Result; -use crate::optimizer::ir::SExpr; use crate::optimizer::Optimizer; +use crate::optimizer::ir::SExpr; use crate::plans::RelOperator; /// Optimizer that removes unused CTEs from the query plan. diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/decorrelate.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/decorrelate.rs index f5115251e9efc..3a6c835e2e3a2 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/decorrelate.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/decorrelate.rs @@ -18,14 +18,15 @@ use std::sync::Arc; use databend_common_ast::Span; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::type_check::common_super_type; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberScalar; use databend_common_expression::ColumnBuilder; use databend_common_expression::Scalar; use databend_common_expression::ScalarRef; +use databend_common_expression::type_check::common_super_type; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberScalar; use databend_common_functions::BUILTIN_FUNCTIONS; +use crate::ColumnSet; use crate::binder::ColumnBindingBuilder; use crate::binder::JoinPredicate; use crate::binder::Visibility; @@ -49,7 +50,6 @@ use crate::plans::ScalarExpr; use crate::plans::SubqueryComparisonOp; use crate::plans::SubqueryExpr; use crate::plans::SubqueryType; -use crate::ColumnSet; impl SubqueryDecorrelatorOptimizer { // Try to decorrelate a `CrossApply` into `SemiJoin` or `AntiJoin`. diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/flatten_plan.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/flatten_plan.rs index 641858f7c924e..6b7259cd2967e 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/flatten_plan.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/flatten_plan.rs @@ -17,11 +17,15 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; use databend_common_expression::DataField; use databend_common_expression::DataSchema; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; +use crate::ColumnEntry; +use crate::ColumnSet; +use crate::IndexType; +use crate::Metadata; use crate::binder::ColumnBindingBuilder; use crate::binder::Visibility; use crate::optimizer::ir::RelExpr; @@ -48,10 +52,6 @@ use crate::plans::Scan; use crate::plans::Sort; use crate::plans::UnionAll; use crate::plans::Window; -use crate::ColumnEntry; -use crate::ColumnSet; -use crate::IndexType; -use crate::Metadata; impl SubqueryDecorrelatorOptimizer { #[recursive::recursive] @@ -860,7 +860,7 @@ impl SubqueryDecorrelatorOptimizer { if aggregate.grouping_sets.is_some() { return Err(ErrorCode::Unimplemented( "join left plan can't contain aggregate with GROUPING SETS to dcorrelated join right plan", - )); + )); } aggregate.into() } @@ -868,7 +868,7 @@ impl SubqueryDecorrelatorOptimizer { return Err(ErrorCode::Unimplemented(format!( "join left plan can't contain {:?} to dcorrelated join right plan", plan.rel_op() - ))) + ))); } }; Ok(op) diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/flatten_scalar.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/flatten_scalar.rs index 603b892c6908a..be1f09fae49c4 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/flatten_scalar.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/flatten_scalar.rs @@ -15,6 +15,7 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use crate::ColumnSet; use crate::binder::ColumnBindingBuilder; use crate::optimizer::optimizers::operator::SubqueryDecorrelatorOptimizer; use crate::plans::AggregateFunction; @@ -24,7 +25,6 @@ use crate::plans::CastExpr; use crate::plans::FunctionCall; use crate::plans::ScalarExpr; use crate::plans::UDFCall; -use crate::ColumnSet; impl SubqueryDecorrelatorOptimizer { #[recursive::recursive] diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/subquery_decorrelator.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/subquery_decorrelator.rs index 50e9498237c0a..8cc561f33a1b8 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/subquery_decorrelator.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/subquery_decorrelator.rs @@ -19,18 +19,21 @@ use std::vec; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::Scalar; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::NumberScalar; -use databend_common_expression::Scalar; use databend_common_functions::aggregates::AggregateCountFunction; -use crate::binder::wrap_cast; +use crate::Binder; +use crate::IndexType; +use crate::MetadataRef; use crate::binder::ColumnBindingBuilder; use crate::binder::Visibility; -use crate::optimizer::ir::SExpr; +use crate::binder::wrap_cast; use crate::optimizer::Optimizer; use crate::optimizer::OptimizerContext; +use crate::optimizer::ir::SExpr; use crate::plans::Aggregate; use crate::plans::AggregateFunction; use crate::plans::BoundColumnRef; @@ -54,9 +57,6 @@ use crate::plans::UDAFCall; use crate::plans::UDFCall; use crate::plans::UDFLambdaCall; use crate::plans::WindowFuncType; -use crate::Binder; -use crate::IndexType; -use crate::MetadataRef; #[allow(clippy::enum_variant_names)] #[derive(Debug)] diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/filter/common.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/filter/common.rs index a4333ac9b389f..7f841be1dd201 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/operator/filter/common.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/filter/common.rs @@ -13,15 +13,15 @@ // limitations under the License. use databend_common_base::base::OrderedFloat; +use databend_common_expression::Scalar; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::NumberScalar; -use databend_common_expression::Scalar; +use crate::ScalarExpr; use crate::plans::BoundColumnRef; use crate::plans::CastExpr; use crate::plans::ConstantExpr; -use crate::ScalarExpr; pub fn check_uint_range(max: u64, value: &Scalar) -> (bool, u64) { let value = match *value { diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/filter/deduplicate_join_condition.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/filter/deduplicate_join_condition.rs index ae2181708fee8..6415b8d83f35a 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/operator/filter/deduplicate_join_condition.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/filter/deduplicate_join_condition.rs @@ -17,12 +17,12 @@ use std::sync::Arc; use databend_common_exception::Result; -use crate::optimizer::ir::SExpr; +use crate::ScalarExpr; use crate::optimizer::Optimizer; +use crate::optimizer::ir::SExpr; use crate::plans::Join; use crate::plans::JoinType; use crate::plans::RelOperator; -use crate::ScalarExpr; // The DeduplicateJoinConditionOptimizer uses the Union-Find algorithm to remove duplicate join conditions. // For example: SELECT * FROM t1, t2, t3 WHERE t1.id = t2.id AND t2.id = t3.id AND t3.id = t1.id diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/filter/equivalent_constants_visitor.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/filter/equivalent_constants_visitor.rs index 9134a1006e5ba..ff89a8fd50325 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/operator/filter/equivalent_constants_visitor.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/filter/equivalent_constants_visitor.rs @@ -16,13 +16,13 @@ use std::collections::HashMap; use databend_common_exception::Result; +use crate::ScalarExpr; use crate::optimizer::optimizers::operator::filter::remove_trivial_type_cast; -use crate::plans::walk_expr_mut; use crate::plans::BoundColumnRef; use crate::plans::ComparisonOp; use crate::plans::FunctionCall; use crate::plans::VisitorMut; -use crate::ScalarExpr; +use crate::plans::walk_expr_mut; // EquivalentConstantsVisitor is used to collect the equivalent relationship between Column and Scalar from the bottom to the bottom, // replacing the Column in the upper expression with Scalar to make the Predicate easier to push down diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/filter/infer_filter.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/filter/infer_filter.rs index 27b5393399ace..1445c68387fe6 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/operator/filter/infer_filter.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/filter/infer_filter.rs @@ -17,13 +17,14 @@ use std::collections::HashSet; use databend_common_base::base::OrderedFloat; use databend_common_exception::Result; +use databend_common_expression::Scalar; use databend_common_expression::type_check::common_super_type; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::NumberScalar; -use databend_common_expression::Scalar; use databend_common_functions::BUILTIN_FUNCTIONS; +use crate::ColumnSet; use crate::optimizer::optimizers::operator::filter::check_float_range; use crate::optimizer::optimizers::operator::filter::check_int_range; use crate::optimizer::optimizers::operator::filter::check_uint_range; @@ -33,7 +34,6 @@ use crate::plans::ConstantExpr; use crate::plans::FunctionCall; use crate::plans::ScalarExpr; use crate::plans::VisitorMut; -use crate::ColumnSet; // The InferFilterOptimizer tries to infer new predicates from existing predicates, for example: // 1. [A > 1 and A > 5] => [A > 5], [A > 1 and A <= 1 => false], [A = 1 and A < 10] => [A = 1] @@ -154,11 +154,13 @@ impl<'a> InferFilterOptimizer<'a> { } if self.is_falsy { - new_predicates = vec![ConstantExpr { - span: None, - value: Scalar::Boolean(false), - } - .into()]; + new_predicates = vec![ + ConstantExpr { + span: None, + value: Scalar::Boolean(false), + } + .into(), + ]; } else { // Derive new predicates from remaining predicates. new_predicates.extend(self.derive_remaining_predicates(remaining_predicates)); diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/filter/pull_up_filter.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/filter/pull_up_filter.rs index ef30a55c6571d..73b6a89775e7c 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/operator/filter/pull_up_filter.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/filter/pull_up_filter.rs @@ -16,12 +16,14 @@ use std::sync::Arc; use databend_common_exception::Result; +use crate::MetadataRef; +use crate::ScalarExpr; use crate::binder::split_conjunctions; +use crate::optimizer::Optimizer; +use crate::optimizer::OptimizerContext; use crate::optimizer::ir::SExpr; use crate::optimizer::optimizers::operator::InferFilterOptimizer; use crate::optimizer::optimizers::operator::NormalizeDisjunctiveFilterOptimizer; -use crate::optimizer::Optimizer; -use crate::optimizer::OptimizerContext; use crate::plans::EvalScalar; use crate::plans::Filter; use crate::plans::FunctionCall; @@ -30,8 +32,6 @@ use crate::plans::JoinType; use crate::plans::RelOperator; use crate::plans::ScalarItem; use crate::plans::WindowFuncType; -use crate::MetadataRef; -use crate::ScalarExpr; // The PullUpFilterOptimizer will pull up filters to the top of the plan tree and infer new filters. pub struct PullUpFilterOptimizer { diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/join/single_to_inner.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/join/single_to_inner.rs index 94b7c863fbd4a..43988f17b2099 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/operator/join/single_to_inner.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/join/single_to_inner.rs @@ -16,8 +16,8 @@ use std::sync::Arc; use databend_common_exception::Result; -use crate::optimizer::ir::SExpr; use crate::optimizer::Optimizer; +use crate::optimizer::ir::SExpr; use crate::plans::JoinType; use crate::plans::RelOperator; diff --git a/src/query/sql/src/planner/optimizer/optimizers/recursive/recursive.rs b/src/query/sql/src/planner/optimizer/optimizers/recursive/recursive.rs index 4525e39889906..34409cacd1180 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/recursive/recursive.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/recursive/recursive.rs @@ -18,13 +18,13 @@ use std::time::Instant; use databend_common_exception::Result; +use crate::optimizer::Optimizer; +use crate::optimizer::OptimizerContext; use crate::optimizer::ir::SExpr; use crate::optimizer::optimizers::rule::RuleFactory; use crate::optimizer::optimizers::rule::RuleID; use crate::optimizer::optimizers::rule::TransformResult; use crate::optimizer::pipeline::OptimizerTraceCollector; -use crate::optimizer::Optimizer; -use crate::optimizer::OptimizerContext; /// Optimizer that recursively applies a set of transformation rules #[derive(Clone)] diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/agg_index/query_rewrite.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/agg_index/query_rewrite.rs index 09e75a0a9f594..4527313e97145 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/agg_index/query_rewrite.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/agg_index/query_rewrite.rs @@ -20,15 +20,20 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::infer_schema_type; -use databend_common_expression::types::DataType; use databend_common_expression::Scalar; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::infer_schema_type; +use databend_common_expression::types::DataType; use databend_common_functions::aggregates::AggregateFunctionFactory; use itertools::Itertools; use log::info; +use crate::ColumnBinding; +use crate::ColumnEntry; +use crate::IndexType; +use crate::ScalarExpr; +use crate::Visibility; use crate::binder::ColumnBindingBuilder; use crate::optimizer::ir::SExpr; use crate::plans::AggIndexInfo; @@ -40,11 +45,6 @@ use crate::plans::FunctionCall; use crate::plans::RelOperator; use crate::plans::ScalarItem; use crate::plans::SortItem; -use crate::ColumnBinding; -use crate::ColumnEntry; -use crate::IndexType; -use crate::ScalarExpr; -use crate::Visibility; pub fn try_rewrite( table_index: IndexType, diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_eager_aggregation.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_eager_aggregation.rs index ec3e74b214caa..198310ddef8d1 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_eager_aggregation.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_eager_aggregation.rs @@ -16,14 +16,20 @@ use std::collections::HashMap; use std::sync::Arc; use databend_common_exception::ErrorCode; -use databend_common_expression::types::number::NumberDataType; use databend_common_expression::types::ArgType; use databend_common_expression::types::DataType; use databend_common_expression::types::UInt64Type; +use databend_common_expression::types::number::NumberDataType; use databend_common_functions::aggregates::AggregateFunctionFactory; -use crate::binder::wrap_cast; +use crate::ColumnSet; +use crate::IndexType; +use crate::MetadataRef; +use crate::ScalarExpr; +use crate::Visibility; use crate::binder::ColumnBindingBuilder; +use crate::binder::wrap_cast; +use crate::optimizer::Optimizer; use crate::optimizer::ir::Matcher; use crate::optimizer::ir::SExpr; use crate::optimizer::ir::SExprVisitor; @@ -32,7 +38,6 @@ use crate::optimizer::ir::VisitAction; use crate::optimizer::optimizers::rule::Rule; use crate::optimizer::optimizers::rule::RuleID; use crate::optimizer::optimizers::rule::TransformResult; -use crate::optimizer::Optimizer; use crate::plans::Aggregate; use crate::plans::AggregateFunction; use crate::plans::AggregateMode; @@ -42,11 +47,6 @@ use crate::plans::FunctionCall; use crate::plans::JoinType; use crate::plans::RelOp; use crate::plans::ScalarItem; -use crate::ColumnSet; -use crate::IndexType; -use crate::MetadataRef; -use crate::ScalarExpr; -use crate::Visibility; /// Rule to push aggregation past a join to reduces the number of input rows to the join. /// Read the paper "Eager aggregation and lazy aggregation" for more details. diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_fold_count_aggregate.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_fold_count_aggregate.rs index e62f2c65b0ac6..cb923f270f941 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_fold_count_aggregate.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_fold_count_aggregate.rs @@ -15,8 +15,8 @@ use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::types::NumberScalar; use databend_common_expression::Scalar; +use databend_common_expression::types::NumberScalar; use crate::optimizer::ir::Matcher; use crate::optimizer::ir::RelExpr; diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs index 7d6c82ba27e76..0fb568a7f29f6 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs @@ -18,17 +18,18 @@ use std::hash::Hasher; use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::types::NumberScalar; use databend_common_expression::Scalar; +use databend_common_expression::types::NumberScalar; +use crate::IndexType; +use crate::ScalarExpr; +use crate::optimizer::OptimizerContext; use crate::optimizer::ir::Matcher; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::SExpr; use crate::optimizer::optimizers::rule::Rule; use crate::optimizer::optimizers::rule::RuleID; use crate::optimizer::optimizers::rule::TransformResult; -use crate::optimizer::OptimizerContext; -use crate::plans::walk_expr_mut; use crate::plans::Aggregate; use crate::plans::AggregateMode; use crate::plans::CastExpr; @@ -40,8 +41,7 @@ use crate::plans::RelOp; use crate::plans::Sequence; use crate::plans::UnionAll; use crate::plans::VisitorMut; -use crate::IndexType; -use crate::ScalarExpr; +use crate::plans::walk_expr_mut; const ID: RuleID = RuleID::GroupingSetsToUnion; // Split `Grouping Sets` into `Union All` of `Group by` diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_hierarchical_grouping_sets.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_hierarchical_grouping_sets.rs index aad80a90fd576..acca613e2c418 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_hierarchical_grouping_sets.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_hierarchical_grouping_sets.rs @@ -19,19 +19,21 @@ use std::hash::Hasher; use std::sync::Arc; use databend_common_exception::Result; +use databend_common_expression::Scalar; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::NumberScalar; -use databend_common_expression::Scalar; +use crate::ColumnBindingBuilder; +use crate::IndexType; +use crate::Visibility; +use crate::optimizer::OptimizerContext; use crate::optimizer::ir::Matcher; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::SExpr; use crate::optimizer::optimizers::rule::Rule; use crate::optimizer::optimizers::rule::RuleID; use crate::optimizer::optimizers::rule::TransformResult; -use crate::optimizer::OptimizerContext; -use crate::plans::walk_expr_mut; use crate::plans::Aggregate; use crate::plans::AggregateMode; use crate::plans::BoundColumnRef; @@ -46,9 +48,7 @@ use crate::plans::ScalarItem; use crate::plans::Sequence; use crate::plans::UnionAll; use crate::plans::VisitorMut; -use crate::ColumnBindingBuilder; -use crate::IndexType; -use crate::Visibility; +use crate::plans::walk_expr_mut; const ID: RuleID = RuleID::HierarchicalGroupingSetsToUnion; @@ -609,7 +609,7 @@ impl RuleHierarchicalGroupingSetsToUnion { for agg_func in hierarchical_agg.aggregate_functions.iter_mut() { // Get the original data type before modifying the function let original_data_type = agg_func.scalar.data_type()?; - if let ScalarExpr::AggregateFunction(ref mut func) = &mut agg_func.scalar { + if let ScalarExpr::AggregateFunction(func) = &mut agg_func.scalar { match func.func_name.as_str() { "count" => { // COUNT(*) from pre-aggregated -> sum0(pre_computed_count) diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_try_apply_agg_index.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_try_apply_agg_index.rs index 34deb672d077d..c980fc56eb1c1 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_try_apply_agg_index.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_try_apply_agg_index.rs @@ -15,14 +15,14 @@ use databend_common_exception::Result; use super::agg_index; +use crate::IndexType; +use crate::MetadataRef; use crate::optimizer::ir::Matcher; use crate::optimizer::ir::SExpr; use crate::optimizer::optimizers::rule::Rule; use crate::optimizer::optimizers::rule::RuleID; use crate::plans::RelOp; use crate::plans::RelOperator; -use crate::IndexType; -use crate::MetadataRef; pub struct RuleTryApplyAggIndex { id: RuleID, diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/constant.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/constant.rs index 4d84da710b462..a98c4efb8f513 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/constant.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/constant.rs @@ -14,8 +14,8 @@ use databend_common_expression::Scalar; -use crate::plans::ConstantExpr; use crate::ScalarExpr; +use crate::plans::ConstantExpr; pub fn is_true(predicate: &ScalarExpr) -> bool { matches!( diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/factory.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/factory.rs index 5791f68e3a279..d1b700f98fc18 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/factory.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/factory.rs @@ -16,6 +16,7 @@ use std::sync::Arc; use databend_common_exception::Result; +use crate::optimizer::OptimizerContext; use crate::optimizer::optimizers::rule::RuleCommuteJoin; use crate::optimizer::optimizers::rule::RuleCommuteJoinBaseTable; use crate::optimizer::optimizers::rule::RuleDeduplicateSort; @@ -62,7 +63,6 @@ use crate::optimizer::optimizers::rule::RulePushDownSortScan; use crate::optimizer::optimizers::rule::RuleSemiToInnerJoin; use crate::optimizer::optimizers::rule::RuleSplitAggregate; use crate::optimizer::optimizers::rule::RuleTryApplyAggIndex; -use crate::optimizer::OptimizerContext; pub struct RuleFactory; diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_eliminate_filter.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_eliminate_filter.rs index e6f434dacca54..40ca36677edf5 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_eliminate_filter.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_eliminate_filter.rs @@ -19,20 +19,20 @@ use databend_common_expression::DataField; use databend_common_expression::DataSchemaRefExt; use itertools::Itertools; +use crate::MetadataRef; use crate::optimizer::ir::Matcher; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::SExpr; -use crate::optimizer::optimizers::rule::constant::is_falsy; use crate::optimizer::optimizers::rule::Rule; use crate::optimizer::optimizers::rule::RuleID; use crate::optimizer::optimizers::rule::TransformResult; +use crate::optimizer::optimizers::rule::constant::is_falsy; use crate::plans::ConstantTableScan; use crate::plans::Filter; use crate::plans::Operator; use crate::plans::RelOp; use crate::plans::RelOperator; use crate::plans::ScalarExpr; -use crate::MetadataRef; pub struct RuleEliminateFilter { id: RuleID, diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_filter_nulls.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_filter_nulls.rs index 775e28b6326ea..70ab719f151e9 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_filter_nulls.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_filter_nulls.rs @@ -16,6 +16,7 @@ use std::sync::Arc; use databend_common_exception::Result; +use crate::ScalarExpr; use crate::optimizer::ir::Matcher; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::SExpr; @@ -29,7 +30,6 @@ use crate::plans::Join; use crate::plans::JoinType; use crate::plans::RelOp; use crate::plans::RelOperator; -use crate::ScalarExpr; const NULL_THRESHOLD_RATIO: f64 = 0.2; diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_merge_filter_into_mutation.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_merge_filter_into_mutation.rs index 7e0190073cc60..acff830b891d2 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_merge_filter_into_mutation.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_merge_filter_into_mutation.rs @@ -17,6 +17,7 @@ use std::sync::Arc; use databend_common_exception::Result; use databend_common_expression::types::DataType; +use crate::MetadataRef; use crate::binder::MutationType; use crate::optimizer::ir::Matcher; use crate::optimizer::ir::SExpr; @@ -27,7 +28,6 @@ use crate::plans::Filter; use crate::plans::MutationSource; use crate::plans::RelOp; use crate::plans::RelOperator; -use crate::MetadataRef; pub struct RuleMergeFilterIntoMutation { id: RuleID, diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_push_down_filter_eval_scalar.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_push_down_filter_eval_scalar.rs index 8b79fb74569f1..8ff147b24fed9 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_push_down_filter_eval_scalar.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_push_down_filter_eval_scalar.rs @@ -22,13 +22,13 @@ use crate::optimizer::ir::SExpr; use crate::optimizer::optimizers::rule::Rule; use crate::optimizer::optimizers::rule::RuleID; use crate::optimizer::optimizers::rule::TransformResult; -use crate::plans::walk_expr_mut; use crate::plans::EvalScalar; use crate::plans::Filter; use crate::plans::RelOp; use crate::plans::ScalarExpr; use crate::plans::ScalarItem; use crate::plans::VisitorMut; +use crate::plans::walk_expr_mut; pub struct RulePushDownFilterEvalScalar { id: RuleID, diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_push_down_filter_join.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_push_down_filter_join.rs index 349f2e2700c71..986d2e157e23e 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_push_down_filter_join.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_push_down_filter_join.rs @@ -16,6 +16,7 @@ use std::sync::Arc; use databend_common_exception::Result; +use crate::MetadataRef; use crate::binder::JoinPredicate; use crate::optimizer::ir::Matcher; use crate::optimizer::ir::RelExpr; @@ -23,15 +24,15 @@ use crate::optimizer::ir::SExpr; use crate::optimizer::optimizers::operator::EquivalentConstantsVisitor; use crate::optimizer::optimizers::operator::InferFilterOptimizer; use crate::optimizer::optimizers::operator::JoinProperty; +use crate::optimizer::optimizers::rule::Rule; +use crate::optimizer::optimizers::rule::RuleID; +use crate::optimizer::optimizers::rule::TransformResult; use crate::optimizer::optimizers::rule::can_filter_null; use crate::optimizer::optimizers::rule::constant::false_constant; use crate::optimizer::optimizers::rule::constant::is_falsy; use crate::optimizer::optimizers::rule::convert_mark_to_semi_join; use crate::optimizer::optimizers::rule::outer_join_to_inner_join; use crate::optimizer::optimizers::rule::rewrite_predicates; -use crate::optimizer::optimizers::rule::Rule; -use crate::optimizer::optimizers::rule::RuleID; -use crate::optimizer::optimizers::rule::TransformResult; use crate::plans::ComparisonOp; use crate::plans::Filter; use crate::plans::FunctionCall; @@ -42,7 +43,6 @@ use crate::plans::Operator; use crate::plans::RelOp; use crate::plans::ScalarExpr; use crate::plans::VisitorMut; -use crate::MetadataRef; pub struct RulePushDownFilterJoin { id: RuleID, diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_push_down_filter_scan.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_push_down_filter_scan.rs index 62b9913fb0ecf..38be3ad99276d 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_push_down_filter_scan.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_push_down_filter_scan.rs @@ -16,6 +16,10 @@ use std::sync::Arc; use databend_common_exception::Result; +use crate::ColumnEntry; +use crate::MetadataRef; +use crate::ScalarExpr; +use crate::TableEntry; use crate::binder::ColumnBindingBuilder; use crate::optimizer::ir::Matcher; use crate::optimizer::ir::SExpr; @@ -29,10 +33,6 @@ use crate::plans::RelOp; use crate::plans::Scan; use crate::plans::SubqueryExpr; use crate::plans::VisitorMut; -use crate::ColumnEntry; -use crate::MetadataRef; -use crate::ScalarExpr; -use crate::TableEntry; pub struct RulePushDownFilterScan { id: RuleID, diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_push_down_filter_union.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_push_down_filter_union.rs index 448696cdc992d..be8c07c12fdd1 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_push_down_filter_union.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_push_down_filter_union.rs @@ -17,6 +17,8 @@ use std::sync::Arc; use ahash::HashMap; use databend_common_exception::Result; +use crate::IndexType; +use crate::Visibility; use crate::binder::ColumnBindingBuilder; use crate::optimizer::ir::Matcher; use crate::optimizer::ir::SExpr; @@ -29,8 +31,6 @@ use crate::plans::RelOp; use crate::plans::ScalarExpr; use crate::plans::UnionAll; use crate::plans::VisitorMut; -use crate::IndexType; -use crate::Visibility; // For a union query, it's not allowed to add `filter` after union // Such as: `(select * from t1 union all select * from t2) where a > 1`, it's invalid. diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_push_down_filter_window.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_push_down_filter_window.rs index 382afb854e2fc..85355c4baead9 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_push_down_filter_window.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_push_down_filter_window.rs @@ -14,6 +14,7 @@ use std::sync::Arc; +use crate::ColumnSet; use crate::optimizer::ir::Matcher; use crate::optimizer::ir::SExpr; use crate::optimizer::optimizers::rule::Rule; @@ -22,7 +23,6 @@ use crate::optimizer::optimizers::rule::TransformResult; use crate::plans::Filter; use crate::plans::RelOp; use crate::plans::Window; -use crate::ColumnSet; /// Input: Filter /// \ diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_push_down_filter_window_top_n.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_push_down_filter_window_top_n.rs index 42347e145b96a..2958b30807a04 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_push_down_filter_window_top_n.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_push_down_filter_window_top_n.rs @@ -15,12 +15,13 @@ use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::type_check::check_number; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRefExt; use databend_common_expression::FunctionContext; +use databend_common_expression::type_check::check_number; use databend_common_functions::BUILTIN_FUNCTIONS; +use crate::MetadataRef; use crate::optimizer::ir::Matcher; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::SExpr; @@ -37,7 +38,6 @@ use crate::plans::ScalarExpr; use crate::plans::Sort; use crate::plans::Window; use crate::plans::WindowFuncType; -use crate::MetadataRef; /// Input: Filter /// \ diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_push_down_prewhere.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_push_down_prewhere.rs index eba60ad5b6b10..8612594dd2d3c 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_push_down_prewhere.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_push_down_prewhere.rs @@ -16,11 +16,15 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::is_internal_column; -use databend_common_expression::TableSchemaRef; use databend_common_expression::SEARCH_MATCHED_COL_NAME; use databend_common_expression::SEARCH_SCORE_COL_NAME; +use databend_common_expression::TableSchemaRef; +use databend_common_expression::is_internal_column; +use crate::ColumnSet; +use crate::IndexType; +use crate::MetadataRef; +use crate::Visibility; use crate::optimizer::ir::Matcher; use crate::optimizer::ir::SExpr; use crate::optimizer::optimizers::rule::Rule; @@ -33,10 +37,6 @@ use crate::plans::ScalarExpr; use crate::plans::Scan; use crate::plans::SubqueryExpr; use crate::plans::Visitor; -use crate::ColumnSet; -use crate::IndexType; -use crate::MetadataRef; -use crate::Visibility; /// Input: Filter /// \ diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_push_down_sort_expression.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_push_down_sort_expression.rs index 7b0f3ecc62a23..fa84672e81178 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_push_down_sort_expression.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/filter_rules/rule_push_down_sort_expression.rs @@ -14,6 +14,7 @@ use std::sync::Arc; +use crate::MetadataRef; use crate::optimizer::ir::Matcher; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::SExpr; @@ -24,7 +25,6 @@ use crate::plans::EvalScalar; use crate::plans::RelOp; use crate::plans::RelOperator; use crate::plans::Sort; -use crate::MetadataRef; /// Input: Order by /// \ diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/join_rules/push_down_filter_join/extract_or_predicates.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/join_rules/push_down_filter_join/extract_or_predicates.rs index 87be66b639186..332852f89b551 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/join_rules/push_down_filter_join/extract_or_predicates.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/join_rules/push_down_filter_join/extract_or_predicates.rs @@ -15,12 +15,12 @@ use databend_common_exception::Result; use itertools::Itertools; +use crate::ColumnSet; +use crate::ScalarExpr; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::SExpr; use crate::plans::Filter; use crate::plans::FunctionCall; -use crate::ColumnSet; -use crate::ScalarExpr; pub fn rewrite_predicates(s_expr: &SExpr) -> Result> { let filter: Filter = s_expr.plan().clone().try_into()?; diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/join_rules/push_down_filter_join/mark_join_to_semi_join.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/join_rules/push_down_filter_join/mark_join_to_semi_join.rs index b472f7d6495d5..233f7bd71d6e8 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/join_rules/push_down_filter_join/mark_join_to_semi_join.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/join_rules/push_down_filter_join/mark_join_to_semi_join.rs @@ -16,11 +16,11 @@ use std::sync::Arc; use databend_common_exception::Result; +use crate::ScalarExpr; use crate::optimizer::ir::SExpr; use crate::plans::Filter; use crate::plans::Join; use crate::plans::JoinType; -use crate::ScalarExpr; pub fn convert_mark_to_semi_join(s_expr: &SExpr) -> Result<(SExpr, bool)> { let mut filter: Filter = s_expr.plan().clone().try_into()?; diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/join_rules/push_down_filter_join/outer_join_to_inner_join.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/join_rules/push_down_filter_join/outer_join_to_inner_join.rs index c955c3aa309d7..1c33fdc13bd6c 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/join_rules/push_down_filter_join/outer_join_to_inner_join.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/join_rules/push_down_filter_join/outer_join_to_inner_join.rs @@ -16,7 +16,6 @@ use std::collections::HashMap; use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::ConstantFolder; use databend_common_expression::DataBlock; use databend_common_expression::DataField; @@ -26,8 +25,13 @@ use databend_common_expression::Expr; use databend_common_expression::FunctionContext; use databend_common_expression::Scalar; use databend_common_expression::Value; +use databend_common_expression::types::DataType; use databend_common_functions::BUILTIN_FUNCTIONS; +use crate::ColumnSet; +use crate::MetadataRef; +use crate::ScalarExpr; +use crate::TypeCheck; use crate::binder::JoinPredicate; use crate::executor::cast_expr_to_non_null_boolean; use crate::optimizer::ir::RelExpr; @@ -36,10 +40,6 @@ use crate::plans::ConstantExpr; use crate::plans::Filter; use crate::plans::Join; use crate::plans::JoinType; -use crate::ColumnSet; -use crate::MetadataRef; -use crate::ScalarExpr; -use crate::TypeCheck; pub fn outer_join_to_inner_join(s_expr: &SExpr, metadata: MetadataRef) -> Result<(SExpr, bool)> { let mut join: Join = s_expr.child(0)?.plan().clone().try_into()?; diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/join_rules/rule_semi_to_inner_join.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/join_rules/rule_semi_to_inner_join.rs index 3e5551345c31e..4daa70bc9b305 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/join_rules/rule_semi_to_inner_join.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/join_rules/rule_semi_to_inner_join.rs @@ -19,6 +19,8 @@ use std::sync::Arc; use databend_common_exception::Result; use databend_common_expression::types::DataType; +use crate::IndexType; +use crate::ScalarExpr; use crate::optimizer::ir::Matcher; use crate::optimizer::ir::SExpr; use crate::optimizer::optimizers::rule::Rule; @@ -28,8 +30,6 @@ use crate::plans::Join; use crate::plans::JoinType; use crate::plans::RelOp; use crate::plans::RelOperator; -use crate::IndexType; -use crate::ScalarExpr; pub struct RuleSemiToInnerJoin { id: RuleID, diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/limit_rules/rule_push_down_limit.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/limit_rules/rule_push_down_limit.rs index 050b813d79e3c..0f41bffa77d53 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/limit_rules/rule_push_down_limit.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/limit_rules/rule_push_down_limit.rs @@ -18,6 +18,7 @@ use databend_common_exception::Result; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRefExt; +use crate::MetadataRef; use crate::optimizer::ir::Matcher; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::SExpr; @@ -29,7 +30,6 @@ use crate::plans::Limit; use crate::plans::Operator; use crate::plans::RelOp; use crate::plans::RelOperator; -use crate::MetadataRef; pub struct RulePushDownLimit { id: RuleID, diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/mod.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/mod.rs index 8c6c6ed70c3d1..a945bf5e80c06 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/mod.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/mod.rs @@ -32,10 +32,10 @@ pub use factory::RuleFactory; pub use filter_rules::*; pub use join_rules::*; pub use limit_rules::*; +pub use rule::DEFAULT_REWRITE_RULES; pub use rule::Rule; pub use rule::RuleID; pub use rule::RulePtr; -pub use rule::DEFAULT_REWRITE_RULES; pub use rule_set::AppliedRules; pub use rule_set::RuleSet; pub use scalar_rules::*; diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/scalar_rules/rule_eliminate_eval_scalar.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/scalar_rules/rule_eliminate_eval_scalar.rs index 1acb7d127ccb2..62c529ff52245 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/scalar_rules/rule_eliminate_eval_scalar.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/scalar_rules/rule_eliminate_eval_scalar.rs @@ -14,6 +14,9 @@ use databend_common_exception::Result; +use crate::ColumnSet; +use crate::MetadataRef; +use crate::ScalarExpr; use crate::optimizer::ir::Matcher; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::SExpr; @@ -23,9 +26,6 @@ use crate::optimizer::optimizers::rule::TransformResult; use crate::plans::EvalScalar; use crate::plans::Operator; use crate::plans::RelOp; -use crate::ColumnSet; -use crate::MetadataRef; -use crate::ScalarExpr; pub struct RuleEliminateEvalScalar { id: RuleID, diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/scalar_rules/rule_merge_eval_scalar.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/scalar_rules/rule_merge_eval_scalar.rs index ce3589089b7e2..77ba1a84ed048 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/scalar_rules/rule_merge_eval_scalar.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/scalar_rules/rule_merge_eval_scalar.rs @@ -16,6 +16,7 @@ use std::sync::Arc; use databend_common_exception::Result; +use crate::ColumnSet; use crate::optimizer::ir::Matcher; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::SExpr; @@ -24,7 +25,6 @@ use crate::optimizer::optimizers::rule::RuleID; use crate::optimizer::optimizers::rule::TransformResult; use crate::plans::EvalScalar; use crate::plans::RelOp; -use crate::ColumnSet; // Merge two adjacent `EvalScalar`s into one pub struct RuleMergeEvalScalar { diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/scalar_rules/rule_normalize_scalar.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/scalar_rules/rule_normalize_scalar.rs index 1ad702d401fb4..6f4a4cdfff9eb 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/scalar_rules/rule_normalize_scalar.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/scalar_rules/rule_normalize_scalar.rs @@ -19,10 +19,10 @@ use databend_common_expression::Scalar; use crate::optimizer::ir::Matcher; use crate::optimizer::ir::SExpr; -use crate::optimizer::optimizers::rule::constant::is_falsy; -use crate::optimizer::optimizers::rule::constant::is_true; use crate::optimizer::optimizers::rule::Rule; use crate::optimizer::optimizers::rule::RuleID; +use crate::optimizer::optimizers::rule::constant::is_falsy; +use crate::optimizer::optimizers::rule::constant::is_true; use crate::plans::ConstantExpr; use crate::plans::Filter; use crate::plans::RelOp; @@ -40,11 +40,13 @@ fn remove_true_predicate(predicates: Vec) -> Vec { fn normalize_falsy_predicate(predicates: Vec) -> Vec { if predicates.iter().any(is_falsy) { - vec![ConstantExpr { - span: None, - value: Scalar::Boolean(false), - } - .into()] + vec![ + ConstantExpr { + span: None, + value: Scalar::Boolean(false), + } + .into(), + ] } else { predicates } diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/union_rules/rule_eliminate_union.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/union_rules/rule_eliminate_union.rs index 46867867acde7..7b11186a5edd2 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/union_rules/rule_eliminate_union.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/union_rules/rule_eliminate_union.rs @@ -18,6 +18,7 @@ use databend_common_exception::Result; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRefExt; +use crate::MetadataRef; use crate::optimizer::ir::Matcher; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::SExpr; @@ -29,7 +30,6 @@ use crate::plans::Operator; use crate::plans::RelOp; use crate::plans::RelOperator; use crate::plans::UnionAll; -use crate::MetadataRef; pub struct RuleEliminateUnion { id: RuleID, diff --git a/src/query/sql/src/planner/optimizer/pipeline/common.rs b/src/query/sql/src/planner/optimizer/pipeline/common.rs index 26aa6e6556415..45e70918db8e3 100644 --- a/src/query/sql/src/planner/optimizer/pipeline/common.rs +++ b/src/query/sql/src/planner/optimizer/pipeline/common.rs @@ -14,9 +14,9 @@ use databend_common_catalog::table::DistributionLevel; +use crate::MetadataRef; use crate::optimizer::ir::SExpr; use crate::plans::RelOperator; -use crate::MetadataRef; /// Check if a query will read data from local tables(e.g. system tables). pub fn contains_local_table_scan(s_expr: &SExpr, metadata: &MetadataRef) -> bool { diff --git a/src/query/sql/src/planner/optimizer/pipeline/pipeline.rs b/src/query/sql/src/planner/optimizer/pipeline/pipeline.rs index f90580450f416..3552c37838069 100644 --- a/src/query/sql/src/planner/optimizer/pipeline/pipeline.rs +++ b/src/query/sql/src/planner/optimizer/pipeline/pipeline.rs @@ -20,11 +20,11 @@ use log::info; use super::common::contains_local_table_scan; use super::common::contains_warehouse_table_scan; +use crate::optimizer::Optimizer; +use crate::optimizer::OptimizerContext; use crate::optimizer::ir::Memo; use crate::optimizer::ir::SExpr; use crate::optimizer::pipeline::OptimizerTraceCollector; -use crate::optimizer::Optimizer; -use crate::optimizer::OptimizerContext; /// A pipeline of optimizers that are executed in sequence. pub struct OptimizerPipeline { @@ -75,11 +75,7 @@ impl OptimizerPipeline { /// Add an optimizer to the pipeline conditionally pub fn add_if(self, condition: bool, optimizer: T) -> Self { - if condition { - self.add(optimizer) - } else { - self - } + if condition { self.add(optimizer) } else { self } } /// Configure distributed optimization based on table types diff --git a/src/query/sql/src/planner/optimizer/pipeline/trace/expr_diff.rs b/src/query/sql/src/planner/optimizer/pipeline/trace/expr_diff.rs index 80e742e3b2d3c..22bd5e0d3bb1e 100644 --- a/src/query/sql/src/planner/optimizer/pipeline/trace/expr_diff.rs +++ b/src/query/sql/src/planner/optimizer/pipeline/trace/expr_diff.rs @@ -16,10 +16,10 @@ use databend_common_exception::Result; use similar::ChangeTag; use similar::TextDiff; +use crate::Metadata; use crate::optimizer::ir::SExpr; use crate::planner::format::FormatOptions; use crate::planner::format::MetadataIdHumanizer; -use crate::Metadata; impl SExpr { /// Compares this SExpr with another SExpr and returns a diff of their string representations. diff --git a/src/query/sql/src/planner/optimizer/pipeline/trace/trace.rs b/src/query/sql/src/planner/optimizer/pipeline/trace/trace.rs index 238e88147d7db..9d8e80ade3eed 100644 --- a/src/query/sql/src/planner/optimizer/pipeline/trace/trace.rs +++ b/src/query/sql/src/planner/optimizer/pipeline/trace/trace.rs @@ -19,8 +19,8 @@ use databend_common_exception::Result; use log::info; use parking_lot::Mutex; -use crate::optimizer::ir::SExpr; use crate::Metadata; +use crate::optimizer::ir::SExpr; /// Represents a trace entry for a rule execution #[derive(Clone)] diff --git a/src/query/sql/src/planner/optimizer/statistics/collect_statistics.rs b/src/query/sql/src/planner/optimizer/statistics/collect_statistics.rs index f992ebadaeeef..f00cffbba50c0 100644 --- a/src/query/sql/src/planner/optimizer/statistics/collect_statistics.rs +++ b/src/query/sql/src/planner/optimizer/statistics/collect_statistics.rs @@ -17,23 +17,23 @@ use std::sync::Arc; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::NumberScalar; -use databend_common_expression::types::F64; use databend_common_expression::ColumnId; use databend_common_expression::Scalar; +use databend_common_expression::types::F64; +use databend_common_expression::types::NumberScalar; -use crate::optimizer::ir::SExpr; +use crate::BaseTableColumn; +use crate::ColumnEntry; +use crate::MetadataRef; +use crate::ScalarExpr; use crate::optimizer::Optimizer; use crate::optimizer::OptimizerContext; +use crate::optimizer::ir::SExpr; use crate::plans::ConstantExpr; use crate::plans::Filter; use crate::plans::FunctionCall; use crate::plans::RelOperator; use crate::plans::Statistics; -use crate::BaseTableColumn; -use crate::ColumnEntry; -use crate::MetadataRef; -use crate::ScalarExpr; // The CollectStatisticsOptimizer will collect statistics for each leaf node in SExpr. pub struct CollectStatisticsOptimizer { diff --git a/src/query/sql/src/planner/planner.rs b/src/query/sql/src/planner/planner.rs index eee8009f51f99..6d72d8b866143 100644 --- a/src/query/sql/src/planner/planner.rs +++ b/src/query/sql/src/planner/planner.rs @@ -24,13 +24,13 @@ use databend_common_ast::ast::InsertStmt; use databend_common_ast::ast::Literal; use databend_common_ast::ast::ReplaceStmt; use databend_common_ast::ast::Statement; +use databend_common_ast::parser::Dialect; use databend_common_ast::parser::parse_raw_insert_stmt; use databend_common_ast::parser::parse_raw_replace_stmt; use databend_common_ast::parser::parse_sql; use databend_common_ast::parser::token::Token; use databend_common_ast::parser::token::TokenKind; use databend_common_ast::parser::token::Tokenizer; -use databend_common_ast::parser::Dialect; use databend_common_catalog::catalog::CatalogManager; use databend_common_catalog::query_kind::QueryKind; use databend_common_catalog::session_type::SessionType; @@ -44,15 +44,15 @@ use parking_lot::RwLock; use super::semantic::AggregateRewriter; use super::semantic::DistinctToGroupBy; -use crate::optimizer::optimize; -use crate::optimizer::OptimizerContext; -use crate::planner::QueryExecutor; -use crate::plans::Plan; use crate::Binder; use crate::CountSetOps; use crate::Metadata; use crate::NameResolutionContext; use crate::VariableNormalizer; +use crate::optimizer::OptimizerContext; +use crate::optimizer::optimize; +use crate::planner::QueryExecutor; +use crate::plans::Plan; const PROBE_INSERT_INITIAL_TOKENS: usize = 128; @@ -146,7 +146,9 @@ impl Planner { .collect::>() .unwrap() } else { - (&mut tokenizer).collect::>()? + (&mut tokenizer) + .collect::>() + .map_err(ErrorCode::from)? }; let session_type = self.ctx.get_session_type(); let in_streaming_load = session_type == SessionType::HTTPStreamingLoad; @@ -156,13 +158,17 @@ impl Planner { // Step 2: Parse the SQL. let (mut stmt, format) = if is_insert_stmt { ( - parse_raw_insert_stmt(&tokens, sql_dialect, in_streaming_load)?, + parse_raw_insert_stmt(&tokens, sql_dialect, in_streaming_load) + .map_err(ErrorCode::from)?, None, ) } else if is_replace_stmt { - (parse_raw_replace_stmt(&tokens, sql_dialect)?, None) + ( + parse_raw_replace_stmt(&tokens, sql_dialect).map_err(ErrorCode::from)?, + None, + ) } else { - parse_sql(&tokens, sql_dialect)? + parse_sql(&tokens, sql_dialect).map_err(ErrorCode::from)? }; if !matches!(stmt, Statement::SetStmt { .. }) && sql_dialect == Dialect::PRQL diff --git a/src/query/sql/src/planner/planner_cache.rs b/src/query/sql/src/planner/planner_cache.rs index cc3453cb432cd..7e92a6af5bd3c 100644 --- a/src/query/sql/src/planner/planner_cache.rs +++ b/src/query/sql/src/planner/planner_cache.rs @@ -36,10 +36,10 @@ use itertools::Itertools; use sha2::Digest; use sha2::Sha256; -use crate::normalize_identifier; -use crate::plans::Plan; use crate::NameResolutionContext; use crate::Planner; +use crate::normalize_identifier; +use crate::plans::Plan; #[derive(Clone)] pub struct PlanCacheItem { diff --git a/src/query/sql/src/planner/plans/aggregate.rs b/src/query/sql/src/planner/plans/aggregate.rs index 76aa60b62f40d..1c3dd73d9c035 100644 --- a/src/query/sql/src/planner/plans/aggregate.rs +++ b/src/query/sql/src/planner/plans/aggregate.rs @@ -19,6 +19,9 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::types::DataType; +use crate::ColumnSet; +use crate::IndexType; +use crate::ScalarExpr; use crate::optimizer::ir::Distribution; use crate::optimizer::ir::PhysicalProperty; use crate::optimizer::ir::RelExpr; @@ -26,13 +29,10 @@ use crate::optimizer::ir::RelationalProperty; use crate::optimizer::ir::RequiredProperty; use crate::optimizer::ir::StatInfo; use crate::optimizer::ir::Statistics; -use crate::plans::sort::SortItem; use crate::plans::Operator; use crate::plans::RelOp; use crate::plans::ScalarItem; -use crate::ColumnSet; -use crate::IndexType; -use crate::ScalarExpr; +use crate::plans::sort::SortItem; const DEFAULT_AGGREGATE_RATIO: f64 = 1f64 / 3f64; const AGGREGATE_COLUMN_CORRELATION_COEFFICIENT: f64 = 0.75_f64; @@ -96,7 +96,7 @@ impl Aggregate { .collect() } else { Ok(vec![ - self.group_items[0].bound_column_expr("_group_item_0".to_string())? + self.group_items[0].bound_column_expr("_group_item_0".to_string())?, ]) } } diff --git a/src/query/sql/src/planner/plans/async_function.rs b/src/query/sql/src/planner/plans/async_function.rs index 1c8edaa0b66fb..a316641f7c657 100644 --- a/src/query/sql/src/planner/plans/async_function.rs +++ b/src/query/sql/src/planner/plans/async_function.rs @@ -17,6 +17,8 @@ use std::sync::Arc; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; +use crate::ColumnSet; +use crate::ScalarExpr; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::RelationalProperty; use crate::optimizer::ir::RequiredProperty; @@ -24,8 +26,6 @@ use crate::optimizer::ir::StatInfo; use crate::plans::Operator; use crate::plans::RelOp; use crate::plans::ScalarItem; -use crate::ColumnSet; -use crate::ScalarExpr; /// `AsyncFunction` is a plan that evaluate a series of async functions. #[derive(Debug, Clone, PartialEq, Eq, Hash)] diff --git a/src/query/sql/src/planner/plans/cache_scan.rs b/src/query/sql/src/planner/plans/cache_scan.rs index b48e6ae20ef98..3df5fc473537c 100644 --- a/src/query/sql/src/planner/plans/cache_scan.rs +++ b/src/query/sql/src/planner/plans/cache_scan.rs @@ -18,6 +18,7 @@ use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; +use crate::ColumnSet; use crate::optimizer::ir::Distribution; use crate::optimizer::ir::PhysicalProperty; use crate::optimizer::ir::RelExpr; @@ -25,7 +26,6 @@ use crate::optimizer::ir::RelationalProperty; use crate::optimizer::ir::RequiredProperty; use crate::plans::Operator; use crate::plans::RelOp; -use crate::ColumnSet; #[derive(Clone, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] pub enum CacheSource { diff --git a/src/query/sql/src/planner/plans/constant_table_scan.rs b/src/query/sql/src/planner/plans/constant_table_scan.rs index 3dd153b4fc926..e3b29ca1c530d 100644 --- a/src/query/sql/src/planner/plans/constant_table_scan.rs +++ b/src/query/sql/src/planner/plans/constant_table_scan.rs @@ -17,16 +17,18 @@ use std::sync::Arc; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::AccessType; -use databend_common_expression::types::NumberType; use databend_common_expression::Column; use databend_common_expression::ColumnBuilder; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRef; +use databend_common_expression::types::AccessType; +use databend_common_expression::types::NumberType; use databend_common_functions::aggregates::eval_aggr; -use databend_common_storage::Datum; use databend_common_storage::DEFAULT_HISTOGRAM_BUCKETS; +use databend_common_storage::Datum; +use crate::ColumnSet; +use crate::IndexType; use crate::optimizer::ir::ColumnStat; use crate::optimizer::ir::ColumnStatSet; use crate::optimizer::ir::Distribution; @@ -39,8 +41,6 @@ use crate::optimizer::ir::StatInfo; use crate::optimizer::ir::Statistics; use crate::plans::Operator; use crate::plans::RelOp; -use crate::ColumnSet; -use crate::IndexType; // Constant table is a table with constant values. #[derive(Clone, Debug)] diff --git a/src/query/sql/src/planner/plans/copy_into_location.rs b/src/query/sql/src/planner/plans/copy_into_location.rs index 22536f3d7a861..b2b32137a5f02 100644 --- a/src/query/sql/src/planner/plans/copy_into_location.rs +++ b/src/query/sql/src/planner/plans/copy_into_location.rs @@ -15,11 +15,11 @@ use std::fmt::Debug; use std::fmt::Formatter; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; use databend_storages_common_stage::CopyIntoLocationInfo; use crate::plans::Plan; diff --git a/src/query/sql/src/planner/plans/copy_into_table.rs b/src/query/sql/src/planner/plans/copy_into_table.rs index 6ee3b0b805bb3..684d08eb7a0d0 100644 --- a/src/query/sql/src/planner/plans/copy_into_table.rs +++ b/src/query/sql/src/planner/plans/copy_into_table.rs @@ -24,13 +24,13 @@ use databend_common_catalog::table_context::FilteredCopyFiles; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; use databend_common_expression::DataField; use databend_common_expression::DataSchema; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use databend_common_expression::Scalar; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; use databend_common_meta_app::principal::COPY_MAX_FILES_COMMIT_MSG; use databend_common_meta_app::principal::COPY_MAX_FILES_PER_COMMIT; use databend_common_meta_app::schema::CatalogInfo; diff --git a/src/query/sql/src/planner/plans/data_mask.rs b/src/query/sql/src/planner/plans/data_mask.rs index 845fdc737a17d..5c1d3890cf96d 100644 --- a/src/query/sql/src/planner/plans/data_mask.rs +++ b/src/query/sql/src/planner/plans/data_mask.rs @@ -16,10 +16,10 @@ use std::sync::Arc; use chrono::Utc; use databend_common_ast::ast::DataMaskPolicy; -use databend_common_expression::types::DataType; use databend_common_expression::DataField; use databend_common_expression::DataSchema; use databend_common_expression::DataSchemaRef; +use databend_common_expression::types::DataType; use databend_common_meta_app::data_mask::CreateDatamaskReq; use databend_common_meta_app::data_mask::DataMaskNameIdent; use databend_common_meta_app::data_mask::DatamaskMeta; diff --git a/src/query/sql/src/planner/plans/ddl/account.rs b/src/query/sql/src/planner/plans/ddl/account.rs index eca17ffb80203..9be6736f48cec 100644 --- a/src/query/sql/src/planner/plans/ddl/account.rs +++ b/src/query/sql/src/planner/plans/ddl/account.rs @@ -16,10 +16,10 @@ use chrono::DateTime; use chrono::Utc; use databend_common_ast::ast::AlterPasswordAction; use databend_common_ast::ast::PasswordSetOptions; -use databend_common_expression::types::DataType; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; +use databend_common_expression::types::DataType; use databend_common_meta_app::principal::AuthInfo; use databend_common_meta_app::principal::GrantObject; use databend_common_meta_app::principal::PrincipalIdentity; diff --git a/src/query/sql/src/planner/plans/ddl/connection.rs b/src/query/sql/src/planner/plans/ddl/connection.rs index 03429f4bcb7e2..65db1671e1c07 100644 --- a/src/query/sql/src/planner/plans/ddl/connection.rs +++ b/src/query/sql/src/planner/plans/ddl/connection.rs @@ -15,10 +15,10 @@ use std::collections::BTreeMap; use std::fmt::Debug; -use databend_common_expression::types::DataType; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; +use databend_common_expression::types::DataType; use databend_common_meta_app::schema::CreateOption; #[derive(Clone, Debug, PartialEq, Eq)] diff --git a/src/query/sql/src/planner/plans/ddl/database.rs b/src/query/sql/src/planner/plans/ddl/database.rs index eae2df5e4b8fa..1616fccafa614 100644 --- a/src/query/sql/src/planner/plans/ddl/database.rs +++ b/src/query/sql/src/planner/plans/ddl/database.rs @@ -16,12 +16,12 @@ use std::sync::Arc; use databend_common_expression::DataSchema; use databend_common_expression::DataSchemaRef; -use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_app::schema::CreateDatabaseReq; use databend_common_meta_app::schema::CreateOption; use databend_common_meta_app::schema::DatabaseMeta; use databend_common_meta_app::schema::DropDatabaseReq; use databend_common_meta_app::schema::UndropDatabaseReq; +use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_app::tenant::Tenant; /// Create. diff --git a/src/query/sql/src/planner/plans/ddl/file_format.rs b/src/query/sql/src/planner/plans/ddl/file_format.rs index e2fe06bb6abc1..1f703fb5fa44c 100644 --- a/src/query/sql/src/planner/plans/ddl/file_format.rs +++ b/src/query/sql/src/planner/plans/ddl/file_format.rs @@ -14,10 +14,10 @@ use std::fmt::Debug; -use databend_common_expression::types::DataType; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; +use databend_common_expression::types::DataType; use databend_common_meta_app::principal::FileFormatParams; use databend_common_meta_app::schema::CreateOption; diff --git a/src/query/sql/src/planner/plans/ddl/notification.rs b/src/query/sql/src/planner/plans/ddl/notification.rs index 2e2f0ac58c6fb..23493984f1a15 100644 --- a/src/query/sql/src/planner/plans/ddl/notification.rs +++ b/src/query/sql/src/planner/plans/ddl/notification.rs @@ -19,12 +19,12 @@ use std::sync::Arc; use databend_common_ast::ast::AlterNotificationOptions; use databend_common_ast::ast::NotificationWebhookOptions; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType::UInt64; use databend_common_expression::DataField; use databend_common_expression::DataSchema; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType::UInt64; use databend_common_meta_app::tenant::Tenant; pub fn notification_schema() -> DataSchemaRef { diff --git a/src/query/sql/src/planner/plans/ddl/procedure.rs b/src/query/sql/src/planner/plans/ddl/procedure.rs index 9bdf2b48b5f5a..12b0f3a6fcbd4 100644 --- a/src/query/sql/src/planner/plans/ddl/procedure.rs +++ b/src/query/sql/src/planner/plans/ddl/procedure.rs @@ -14,10 +14,10 @@ use databend_common_ast::ast::Expr; use databend_common_ast::ast::ScriptBlock; -use databend_common_expression::types::DataType; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; +use databend_common_expression::types::DataType; use databend_common_meta_app::principal::CreateProcedureReq; use databend_common_meta_app::principal::DropProcedureReq; use databend_common_meta_app::principal::GetProcedureReq; diff --git a/src/query/sql/src/planner/plans/ddl/sequence.rs b/src/query/sql/src/planner/plans/ddl/sequence.rs index 4f304a0b7928f..1f41457149234 100644 --- a/src/query/sql/src/planner/plans/ddl/sequence.rs +++ b/src/query/sql/src/planner/plans/ddl/sequence.rs @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; use databend_common_meta_app::schema::CreateOption; use databend_common_meta_app::schema::SequenceIdent; diff --git a/src/query/sql/src/planner/plans/ddl/table.rs b/src/query/sql/src/planner/plans/ddl/table.rs index 1086d67b98d28..2f4f93e355202 100644 --- a/src/query/sql/src/planner/plans/ddl/table.rs +++ b/src/query/sql/src/planner/plans/ddl/table.rs @@ -18,8 +18,6 @@ use std::time::Duration; use databend_common_ast::ast::Engine; use databend_common_ast::ast::Identifier; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; use databend_common_expression::DataField; use databend_common_expression::DataSchema; use databend_common_expression::DataSchemaRef; @@ -27,6 +25,8 @@ use databend_common_expression::DataSchemaRefExt; use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRef; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; use databend_common_meta_app::schema::Constraint; use databend_common_meta_app::schema::CreateOption; use databend_common_meta_app::schema::TableIndex; diff --git a/src/query/sql/src/planner/plans/ddl/task.rs b/src/query/sql/src/planner/plans/ddl/task.rs index cb8b8ab994bb7..8251f3dbf67d4 100644 --- a/src/query/sql/src/planner/plans/ddl/task.rs +++ b/src/query/sql/src/planner/plans/ddl/task.rs @@ -19,14 +19,14 @@ use databend_common_ast::ast::AlterTaskOptions; use databend_common_ast::ast::ScheduleOptions; use databend_common_ast::ast::ShowLimit; use databend_common_ast::ast::TaskSql; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType::Int32; -use databend_common_expression::types::NumberDataType::Int64; -use databend_common_expression::types::NumberDataType::UInt64; use databend_common_expression::DataField; use databend_common_expression::DataSchema; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType::Int32; +use databend_common_expression::types::NumberDataType::Int64; +use databend_common_expression::types::NumberDataType::UInt64; use databend_common_meta_app::schema::CreateOption; use databend_common_meta_app::tenant::Tenant; diff --git a/src/query/sql/src/planner/plans/ddl/warehouse.rs b/src/query/sql/src/planner/plans/ddl/warehouse.rs index 261811d176117..7235c5db744eb 100644 --- a/src/query/sql/src/planner/plans/ddl/warehouse.rs +++ b/src/query/sql/src/planner/plans/ddl/warehouse.rs @@ -15,10 +15,10 @@ use std::collections::BTreeMap; use std::collections::HashMap; -use databend_common_expression::types::DataType; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; +use databend_common_expression::types::DataType; #[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)] pub struct UseWarehousePlan { diff --git a/src/query/sql/src/planner/plans/dummy_table_scan.rs b/src/query/sql/src/planner/plans/dummy_table_scan.rs index ba3b09e980a80..3783962ebd2db 100644 --- a/src/query/sql/src/planner/plans/dummy_table_scan.rs +++ b/src/query/sql/src/planner/plans/dummy_table_scan.rs @@ -18,6 +18,7 @@ use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use crate::ColumnSet; use crate::optimizer::ir::Distribution; use crate::optimizer::ir::PhysicalProperty; use crate::optimizer::ir::RelExpr; @@ -27,7 +28,6 @@ use crate::optimizer::ir::StatInfo; use crate::optimizer::ir::Statistics; use crate::plans::Operator; use crate::plans::RelOp; -use crate::ColumnSet; #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct DummyTableScan; diff --git a/src/query/sql/src/planner/plans/eval_scalar.rs b/src/query/sql/src/planner/plans/eval_scalar.rs index d7c1d9354ae0e..9c547ba534e09 100644 --- a/src/query/sql/src/planner/plans/eval_scalar.rs +++ b/src/query/sql/src/planner/plans/eval_scalar.rs @@ -16,6 +16,11 @@ use std::sync::Arc; use databend_common_exception::Result; +use crate::ColumnBinding; +use crate::ColumnBindingBuilder; +use crate::ColumnSet; +use crate::IndexType; +use crate::Visibility; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::RelationalProperty; use crate::optimizer::ir::StatInfo; @@ -23,11 +28,6 @@ use crate::plans::BoundColumnRef; use crate::plans::Operator; use crate::plans::RelOp; use crate::plans::ScalarExpr; -use crate::ColumnBinding; -use crate::ColumnBindingBuilder; -use crate::ColumnSet; -use crate::IndexType; -use crate::Visibility; /// Evaluate scalar expression #[derive(Clone, Debug, PartialEq, Eq, Hash)] diff --git a/src/query/sql/src/planner/plans/expression_scan.rs b/src/query/sql/src/planner/plans/expression_scan.rs index f439a6ebbfc55..75fda1b223b77 100644 --- a/src/query/sql/src/planner/plans/expression_scan.rs +++ b/src/query/sql/src/planner/plans/expression_scan.rs @@ -15,17 +15,17 @@ use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::DataSchemaRef; +use databend_common_expression::types::DataType; +use crate::ColumnSet; +use crate::ScalarExpr; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::RelationalProperty; use crate::optimizer::ir::StatInfo; use crate::optimizer::ir::Statistics; use crate::plans::Operator; use crate::plans::RelOp; -use crate::ColumnSet; -use crate::ScalarExpr; // Constant table is a table with constant values. #[derive(Clone, Debug, PartialEq, Eq)] diff --git a/src/query/sql/src/planner/plans/filter.rs b/src/query/sql/src/planner/plans/filter.rs index 8242df09418cf..fc02de99c0b93 100644 --- a/src/query/sql/src/planner/plans/filter.rs +++ b/src/query/sql/src/planner/plans/filter.rs @@ -18,16 +18,16 @@ use std::sync::Arc; use databend_common_exception::Result; +use crate::ColumnSet; +use crate::optimizer::ir::MAX_SELECTIVITY; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::RelationalProperty; use crate::optimizer::ir::SelectivityEstimator; use crate::optimizer::ir::StatInfo; use crate::optimizer::ir::Statistics; -use crate::optimizer::ir::MAX_SELECTIVITY; use crate::plans::Operator; use crate::plans::RelOp; use crate::plans::ScalarExpr; -use crate::ColumnSet; #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct Filter { diff --git a/src/query/sql/src/planner/plans/insert.rs b/src/query/sql/src/planner/plans/insert.rs index d9010a839e71c..0586c91cfde2f 100644 --- a/src/query/sql/src/planner/plans/insert.rs +++ b/src/query/sql/src/planner/plans/insert.rs @@ -17,9 +17,6 @@ use std::sync::Arc; use databend_common_ast::ast::FormatTreeNode; use databend_common_base::base::tokio::sync::mpsc::Receiver; use databend_common_exception::Result; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; use databend_common_expression::BlockThresholds; use databend_common_expression::DataBlock; use databend_common_expression::DataField; @@ -29,6 +26,9 @@ use databend_common_expression::FromData; use databend_common_expression::RemoteDefaultExpr; use databend_common_expression::Scalar; use databend_common_expression::TableSchemaRef; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; use databend_common_meta_app::principal::FileFormatParams; use databend_common_meta_app::schema::TableInfo; use enum_as_inner::EnumAsInner; @@ -37,10 +37,10 @@ use serde::Deserialize; use serde::Serialize; use super::Plan; +use crate::INSERT_NAME; use crate::planner::format::FormatOptions; use crate::planner::format::MetadataIdHumanizer; use crate::plans::CopyIntoTablePlan; -use crate::INSERT_NAME; #[derive(Clone, Debug, EnumAsInner)] pub enum InsertInputSource { diff --git a/src/query/sql/src/planner/plans/insert_multi_table.rs b/src/query/sql/src/planner/plans/insert_multi_table.rs index 95e3a7b6b4887..b52d303d61db7 100644 --- a/src/query/sql/src/planner/plans/insert_multi_table.rs +++ b/src/query/sql/src/planner/plans/insert_multi_table.rs @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; use super::Plan; use crate::MetadataRef; diff --git a/src/query/sql/src/planner/plans/join.rs b/src/query/sql/src/planner/plans/join.rs index 83d65f67d4a61..1d26198edb9b6 100644 --- a/src/query/sql/src/planner/plans/join.rs +++ b/src/query/sql/src/planner/plans/join.rs @@ -21,10 +21,12 @@ use std::sync::Arc; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; use databend_common_expression::types::F64; +use databend_common_storage::DEFAULT_HISTOGRAM_BUCKETS; use databend_common_storage::Datum; use databend_common_storage::Histogram; -use databend_common_storage::DEFAULT_HISTOGRAM_BUCKETS; +use crate::ColumnSet; +use crate::IndexType; use crate::optimizer::ir::ColumnStat; use crate::optimizer::ir::Distribution; use crate::optimizer::ir::HistogramBuilder; @@ -40,8 +42,6 @@ use crate::optimizer::ir::UniformSampleSet; use crate::plans::Operator; use crate::plans::RelOp; use crate::plans::ScalarExpr; -use crate::ColumnSet; -use crate::IndexType; #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, serde::Serialize, serde::Deserialize)] pub enum JoinType { diff --git a/src/query/sql/src/planner/plans/limit.rs b/src/query/sql/src/planner/plans/limit.rs index 67f5c5fc5a574..92a00b1d34738 100644 --- a/src/query/sql/src/planner/plans/limit.rs +++ b/src/query/sql/src/planner/plans/limit.rs @@ -17,6 +17,7 @@ use std::sync::Arc; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; +use crate::ColumnSet; use crate::optimizer::ir::Distribution; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::RelationalProperty; @@ -25,7 +26,6 @@ use crate::optimizer::ir::StatInfo; use crate::optimizer::ir::Statistics; use crate::plans::Operator; use crate::plans::RelOp; -use crate::ColumnSet; #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct Limit { diff --git a/src/query/sql/src/planner/plans/materialized_cte.rs b/src/query/sql/src/planner/plans/materialized_cte.rs index a1cec441397a5..774a68e6502c1 100644 --- a/src/query/sql/src/planner/plans/materialized_cte.rs +++ b/src/query/sql/src/planner/plans/materialized_cte.rs @@ -18,6 +18,7 @@ use std::sync::Arc; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; +use crate::ColumnBinding; use crate::optimizer::ir::PhysicalProperty; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::RelationalProperty; @@ -25,7 +26,6 @@ use crate::optimizer::ir::RequiredProperty; use crate::optimizer::ir::StatInfo; use crate::plans::Operator; use crate::plans::RelOp; -use crate::ColumnBinding; #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct MaterializedCTE { diff --git a/src/query/sql/src/planner/plans/mod.rs b/src/query/sql/src/planner/plans/mod.rs index 5bb94c890320f..8582b282d69b9 100644 --- a/src/query/sql/src/planner/plans/mod.rs +++ b/src/query/sql/src/planner/plans/mod.rs @@ -81,12 +81,12 @@ pub use join::*; pub use kill::KillPlan; pub use limit::*; pub use materialized_cte::*; -pub use mutation::MatchedEvaluator; -pub use mutation::Mutation; -pub use mutation::UnmatchedEvaluator; pub use mutation::DELETE_NAME; pub use mutation::INSERT_NAME; +pub use mutation::MatchedEvaluator; +pub use mutation::Mutation; pub use mutation::UPDATE_NAME; +pub use mutation::UnmatchedEvaluator; pub use mutation_source::MutationSource; pub use operator::*; pub use optimize::*; diff --git a/src/query/sql/src/planner/plans/mutation.rs b/src/query/sql/src/planner/plans/mutation.rs index 757fcf060d742..62560a6a4d7bb 100644 --- a/src/query/sql/src/planner/plans/mutation.rs +++ b/src/query/sql/src/planner/plans/mutation.rs @@ -17,23 +17,23 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; use databend_common_expression::FieldIndex; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; use databend_common_pipeline::core::LockGuard; -use crate::binder::MutationStrategy; -use crate::binder::MutationType; -use crate::plans::Operator; -use crate::plans::RelOp; use crate::BindContext; use crate::ColumnSet; use crate::IndexType; use crate::MetadataRef; use crate::ScalarExpr; +use crate::binder::MutationStrategy; +use crate::binder::MutationType; +use crate::plans::Operator; +use crate::plans::RelOp; #[derive(Clone, Debug, PartialEq)] pub struct UnmatchedEvaluator { @@ -151,11 +151,7 @@ impl Mutation { .iter() .filter_map( |(field, include)| { - if *include { - Some(field.clone()) - } else { - None - } + if *include { Some(field.clone()) } else { None } }, ) .collect(); diff --git a/src/query/sql/src/planner/plans/mutation_source.rs b/src/query/sql/src/planner/plans/mutation_source.rs index 8d1811d503bb4..905d950735ea0 100644 --- a/src/query/sql/src/planner/plans/mutation_source.rs +++ b/src/query/sql/src/planner/plans/mutation_source.rs @@ -20,6 +20,8 @@ use databend_common_exception::Result; use databend_common_expression::TableSchema; use super::ScalarExpr; +use crate::ColumnSet; +use crate::IndexType; use crate::binder::MutationType; use crate::optimizer::ir::Distribution; use crate::optimizer::ir::PhysicalProperty; @@ -30,8 +32,6 @@ use crate::optimizer::ir::StatInfo; use crate::optimizer::ir::Statistics as OpStatistics; use crate::plans::Operator; use crate::plans::RelOp; -use crate::ColumnSet; -use crate::IndexType; #[derive(Clone, Debug, Default)] pub struct MutationSource { diff --git a/src/query/sql/src/planner/plans/operator.rs b/src/query/sql/src/planner/plans/operator.rs index 0fcd423233293..e2db3ed60eb05 100644 --- a/src/query/sql/src/planner/plans/operator.rs +++ b/src/query/sql/src/planner/plans/operator.rs @@ -22,6 +22,7 @@ use enum_as_inner::EnumAsInner; use super::MutationSource; use super::SubqueryExpr; +use crate::ScalarExpr; use crate::impl_match_rel_op; use crate::impl_try_from_rel_operator; use crate::match_rel_op; @@ -30,8 +31,6 @@ use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::RelationalProperty; use crate::optimizer::ir::RequiredProperty; use crate::optimizer::ir::StatInfo; -use crate::plans::r_cte_scan::RecursiveCteScan; -use crate::plans::sequence::Sequence; use crate::plans::Aggregate; use crate::plans::AsyncFunction; use crate::plans::CacheScan; @@ -54,7 +53,8 @@ use crate::plans::Sort; use crate::plans::Udf; use crate::plans::UnionAll; use crate::plans::Window; -use crate::ScalarExpr; +use crate::plans::r_cte_scan::RecursiveCteScan; +use crate::plans::sequence::Sequence; pub trait Operator { /// Get relational operator kind diff --git a/src/query/sql/src/planner/plans/plan.rs b/src/query/sql/src/planner/plans/plan.rs index 6df6c68d6715c..53dff012e5495 100644 --- a/src/query/sql/src/planner/plans/plan.rs +++ b/src/query/sql/src/planner/plans/plan.rs @@ -18,11 +18,11 @@ use std::sync::Arc; use databend_common_ast::ast::ExplainKind; use databend_common_catalog::query_kind::QueryKind; -use databend_common_expression::types::DataType; use databend_common_expression::DataField; use databend_common_expression::DataSchema; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; +use databend_common_expression::types::DataType; use educe::Educe; use super::CreateDictionaryPlan; @@ -30,10 +30,10 @@ use super::DropDictionaryPlan; use super::ModifyTableConnectionPlan; use super::RenameDictionaryPlan; use super::ShowCreateDictionaryPlan; +use crate::BindContext; +use crate::MetadataRef; use crate::binder::ExplainConfig; use crate::optimizer::ir::SExpr; -use crate::plans::copy_into_location::CopyIntoLocationPlan; -use crate::plans::row_access_policy::CreateRowAccessPolicyPlan; use crate::plans::AddTableColumnPlan; use crate::plans::AddTableConstraintPlan; use crate::plans::AddTableRowAccessPolicyPlan; @@ -179,8 +179,8 @@ use crate::plans::UseWarehousePlan; use crate::plans::VacuumDropTablePlan; use crate::plans::VacuumTablePlan; use crate::plans::VacuumTemporaryFilesPlan; -use crate::BindContext; -use crate::MetadataRef; +use crate::plans::copy_into_location::CopyIntoLocationPlan; +use crate::plans::row_access_policy::CreateRowAccessPolicyPlan; #[derive(Educe)] #[educe( diff --git a/src/query/sql/src/planner/plans/presign.rs b/src/query/sql/src/planner/plans/presign.rs index a147e5e9239c6..3de7d66a94c7d 100644 --- a/src/query/sql/src/planner/plans/presign.rs +++ b/src/query/sql/src/planner/plans/presign.rs @@ -14,10 +14,10 @@ use std::time::Duration; -use databend_common_expression::types::DataType; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; +use databend_common_expression::types::DataType; use databend_common_meta_app::principal::StageInfo; #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/src/query/sql/src/planner/plans/project_set.rs b/src/query/sql/src/planner/plans/project_set.rs index 1876b9e213001..8d1b2ebe17c54 100644 --- a/src/query/sql/src/planner/plans/project_set.rs +++ b/src/query/sql/src/planner/plans/project_set.rs @@ -17,13 +17,13 @@ use std::sync::Arc; use databend_common_exception::Result; +use crate::ScalarExpr; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::RelationalProperty; use crate::optimizer::ir::StatInfo; use crate::plans::Operator; use crate::plans::RelOp; use crate::plans::ScalarItem; -use crate::ScalarExpr; /// `ProjectSet` is a plan that evaluate a series of /// set-returning functions, zip the result together, diff --git a/src/query/sql/src/planner/plans/r_cte_scan.rs b/src/query/sql/src/planner/plans/r_cte_scan.rs index a39bb2fc9e497..2259ec82a2875 100644 --- a/src/query/sql/src/planner/plans/r_cte_scan.rs +++ b/src/query/sql/src/planner/plans/r_cte_scan.rs @@ -21,6 +21,7 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::DataField; +use crate::ColumnSet; use crate::optimizer::ir::Distribution; use crate::optimizer::ir::PhysicalProperty; use crate::optimizer::ir::RelExpr; @@ -28,7 +29,6 @@ use crate::optimizer::ir::RelationalProperty; use crate::optimizer::ir::RequiredProperty; use crate::plans::Operator; use crate::plans::RelOp; -use crate::ColumnSet; #[derive(Clone, Debug, PartialEq, Eq)] pub struct RecursiveCteScan { diff --git a/src/query/sql/src/planner/plans/recluster.rs b/src/query/sql/src/planner/plans/recluster.rs index 5df70ca198475..796520f0273bc 100644 --- a/src/query/sql/src/planner/plans/recluster.rs +++ b/src/query/sql/src/planner/plans/recluster.rs @@ -21,19 +21,19 @@ use databend_common_ast::parser::tokenize_sql; use databend_common_catalog::catalog::CatalogManager; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::NumberScalar; use databend_common_expression::Scalar; +use databend_common_expression::types::NumberScalar; +use crate::Binder; +use crate::MetadataRef; +use crate::NameResolutionContext; +use crate::ScalarExpr; +use crate::optimizer::OptimizerContext; use crate::optimizer::ir::SExpr; use crate::optimizer::optimize; -use crate::optimizer::OptimizerContext; use crate::plans::ConstantExpr; use crate::plans::Plan; use crate::plans::RelOperator; -use crate::Binder; -use crate::MetadataRef; -use crate::NameResolutionContext; -use crate::ScalarExpr; #[derive(Debug, Clone)] pub struct ReclusterPlan { diff --git a/src/query/sql/src/planner/plans/replace.rs b/src/query/sql/src/planner/plans/replace.rs index 9d4d5e7368718..cb05631cd31de 100644 --- a/src/query/sql/src/planner/plans/replace.rs +++ b/src/query/sql/src/planner/plans/replace.rs @@ -16,18 +16,18 @@ use std::sync::Arc; use databend_common_ast::ast::Expr; use databend_common_ast::ast::FormatTreeNode; -use databend_common_expression::types::StringType; use databend_common_expression::DataBlock; use databend_common_expression::DataSchemaRef; use databend_common_expression::FromData; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; +use databend_common_expression::types::StringType; use databend_common_meta_types::MetaId; use databend_common_pipeline::core::LockGuard; use super::insert::format_insert_source; -use crate::plans::InsertInputSource; use crate::FormatOptions; +use crate::plans::InsertInputSource; #[derive(Clone)] pub struct Replace { diff --git a/src/query/sql/src/planner/plans/row_access_policy.rs b/src/query/sql/src/planner/plans/row_access_policy.rs index ade7f53f63692..6a902de59643b 100644 --- a/src/query/sql/src/planner/plans/row_access_policy.rs +++ b/src/query/sql/src/planner/plans/row_access_policy.rs @@ -16,10 +16,10 @@ use std::sync::Arc; use chrono::Utc; use databend_common_ast::ast::RowAccessPolicyDefinition; -use databend_common_expression::types::DataType; use databend_common_expression::DataField; use databend_common_expression::DataSchema; use databend_common_expression::DataSchemaRef; +use databend_common_expression::types::DataType; use databend_common_meta_app::row_access_policy::CreateRowAccessPolicyReq; use databend_common_meta_app::row_access_policy::DropRowAccessPolicyReq; use databend_common_meta_app::row_access_policy::RowAccessPolicyMeta; diff --git a/src/query/sql/src/planner/plans/scalar_expr.rs b/src/query/sql/src/planner/plans/scalar_expr.rs index a97decf332b08..edccf83755409 100644 --- a/src/query/sql/src/planner/plans/scalar_expr.rs +++ b/src/query/sql/src/planner/plans/scalar_expr.rs @@ -22,23 +22,23 @@ use std::hash::Hasher; use std::str::FromStr; use std::sync::Arc; -use databend_common_ast::ast::BinaryOperator; use databend_common_ast::Range; use databend_common_ast::Span; +use databend_common_ast::ast::BinaryOperator; use databend_common_catalog::catalog::Catalog; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberScalar; use databend_common_expression::AutoIncrementExpr; use databend_common_expression::FunctionKind; use databend_common_expression::RemoteExpr; -use databend_common_expression::Scalar; use databend_common_expression::SEARCH_MATCHED_COL_NAME; use databend_common_expression::SEARCH_SCORE_COL_NAME; +use databend_common_expression::Scalar; use databend_common_expression::VECTOR_SCORE_COL_NAME; -use databend_common_functions::aggregates::AggregateFunctionSortDesc; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberScalar; use databend_common_functions::BUILTIN_FUNCTIONS; +use databend_common_functions::aggregates::AggregateFunctionSortDesc; use databend_common_meta_app::principal::AutoIncrementKey; use databend_common_meta_app::principal::StageInfo; use databend_common_meta_app::schema::GetAutoIncrementNextValueReq; @@ -52,11 +52,11 @@ use itertools::Itertools; use super::WindowFuncFrame; use super::WindowFuncType; -use crate::binder::ColumnBinding; -use crate::optimizer::ir::SExpr; use crate::ColumnSet; use crate::IndexType; use crate::MetadataRef; +use crate::binder::ColumnBinding; +use crate::optimizer::ir::SExpr; #[derive(Debug)] pub enum ScalarExpr { @@ -1260,7 +1260,10 @@ impl AsyncFunctionCall { AsyncFunctionArgument::SequenceFunction(sequence_name) => { if let Some(visibility_checker) = &visibility_checker { if !visibility_checker.check_seq_visibility(sequence_name) { - return Err(ErrorCode::PermissionDenied(format!("Permission denied: privilege ACCESS SEQUENCE is required on sequence {}", sequence_name))); + return Err(ErrorCode::PermissionDenied(format!( + "Permission denied: privilege ACCESS SEQUENCE is required on sequence {}", + sequence_name + ))); } } let req = GetSequenceNextValueReq { diff --git a/src/query/sql/src/planner/plans/scan.rs b/src/query/sql/src/planner/plans/scan.rs index d47b83b367be7..123f174e56d55 100644 --- a/src/query/sql/src/planner/plans/scan.rs +++ b/src/query/sql/src/planner/plans/scan.rs @@ -25,16 +25,19 @@ use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::TableSchemaRef; +use databend_common_storage::DEFAULT_HISTOGRAM_BUCKETS; use databend_common_storage::Datum; use databend_common_storage::Histogram; -use databend_common_storage::DEFAULT_HISTOGRAM_BUCKETS; use databend_storages_common_table_meta::table::ChangeType; use super::ScalarItem; +use crate::ColumnSet; +use crate::IndexType; use crate::optimizer::ir::ColumnStat; use crate::optimizer::ir::ColumnStatSet; use crate::optimizer::ir::Distribution; use crate::optimizer::ir::HistogramBuilder; +use crate::optimizer::ir::MAX_SELECTIVITY; use crate::optimizer::ir::PhysicalProperty; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::RelationalProperty; @@ -42,13 +45,10 @@ use crate::optimizer::ir::RequiredProperty; use crate::optimizer::ir::SelectivityEstimator; use crate::optimizer::ir::StatInfo; use crate::optimizer::ir::Statistics as OpStatistics; -use crate::optimizer::ir::MAX_SELECTIVITY; use crate::plans::Operator; use crate::plans::RelOp; use crate::plans::ScalarExpr; use crate::plans::SortItem; -use crate::ColumnSet; -use crate::IndexType; #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct Prewhere { @@ -329,7 +329,7 @@ impl Operator for Scan { .and_then(|stat| stat.num_rows); let cardinality = match (precise_cardinality, &self.prewhere) { - (Some(precise_cardinality), Some(ref prewhere)) => { + (Some(precise_cardinality), Some(prewhere)) => { let mut statistics = OpStatistics { precise_cardinality: Some(precise_cardinality), column_stats, diff --git a/src/query/sql/src/planner/plans/secure_filter.rs b/src/query/sql/src/planner/plans/secure_filter.rs index 171d88656eff8..df7a72e386726 100644 --- a/src/query/sql/src/planner/plans/secure_filter.rs +++ b/src/query/sql/src/planner/plans/secure_filter.rs @@ -18,17 +18,17 @@ use std::sync::Arc; use databend_common_exception::Result; +use crate::ColumnSet; +use crate::IndexType; +use crate::optimizer::ir::MAX_SELECTIVITY; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::RelationalProperty; use crate::optimizer::ir::SelectivityEstimator; use crate::optimizer::ir::StatInfo; use crate::optimizer::ir::Statistics; -use crate::optimizer::ir::MAX_SELECTIVITY; use crate::plans::Operator; use crate::plans::RelOp; use crate::plans::ScalarExpr; -use crate::ColumnSet; -use crate::IndexType; #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct SecureFilter { diff --git a/src/query/sql/src/planner/plans/sequence.rs b/src/query/sql/src/planner/plans/sequence.rs index b99c7b4d1338a..6f60b623bb9cf 100644 --- a/src/query/sql/src/planner/plans/sequence.rs +++ b/src/query/sql/src/planner/plans/sequence.rs @@ -17,6 +17,7 @@ use std::sync::Arc; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; +use crate::ScalarExpr; use crate::optimizer::ir::PhysicalProperty; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::RelationalProperty; @@ -24,7 +25,6 @@ use crate::optimizer::ir::RequiredProperty; use crate::optimizer::ir::StatInfo; use crate::plans::Operator; use crate::plans::RelOp; -use crate::ScalarExpr; #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct Sequence; diff --git a/src/query/sql/src/planner/plans/sort.rs b/src/query/sql/src/planner/plans/sort.rs index e396006ba7453..c3fdc03d382a8 100644 --- a/src/query/sql/src/planner/plans/sort.rs +++ b/src/query/sql/src/planner/plans/sort.rs @@ -18,6 +18,8 @@ use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; use super::WindowPartition; +use crate::ColumnSet; +use crate::IndexType; use crate::optimizer::ir::Distribution; use crate::optimizer::ir::PhysicalProperty; use crate::optimizer::ir::RelExpr; @@ -26,8 +28,6 @@ use crate::optimizer::ir::RequiredProperty; use crate::optimizer::ir::StatInfo; use crate::plans::Operator; use crate::plans::RelOp; -use crate::ColumnSet; -use crate::IndexType; #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct Sort { diff --git a/src/query/sql/src/planner/plans/udaf.rs b/src/query/sql/src/planner/plans/udaf.rs index 79553335a12a6..a8b08047f3d99 100644 --- a/src/query/sql/src/planner/plans/udaf.rs +++ b/src/query/sql/src/planner/plans/udaf.rs @@ -17,6 +17,7 @@ use std::sync::Arc; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; +use crate::ColumnSet; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::RelationalProperty; use crate::optimizer::ir::RequiredProperty; @@ -24,7 +25,6 @@ use crate::optimizer::ir::StatInfo; use crate::plans::Operator; use crate::plans::RelOp; use crate::plans::ScalarItem; -use crate::ColumnSet; /// `Udaf` is a plan that evaluate a series of udaf functions. #[derive(Debug, Clone, PartialEq, Eq, Hash)] diff --git a/src/query/sql/src/planner/plans/udf.rs b/src/query/sql/src/planner/plans/udf.rs index 63e59851f644c..2be572c692bf6 100644 --- a/src/query/sql/src/planner/plans/udf.rs +++ b/src/query/sql/src/planner/plans/udf.rs @@ -17,6 +17,8 @@ use std::sync::Arc; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; +use crate::ColumnSet; +use crate::ScalarExpr; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::RelationalProperty; use crate::optimizer::ir::RequiredProperty; @@ -24,8 +26,6 @@ use crate::optimizer::ir::StatInfo; use crate::plans::Operator; use crate::plans::RelOp; use crate::plans::ScalarItem; -use crate::ColumnSet; -use crate::ScalarExpr; /// `Udf` is a plan that evaluate a series of udf functions. #[derive(Debug, Clone, PartialEq, Eq, Hash)] diff --git a/src/query/sql/src/planner/plans/union_all.rs b/src/query/sql/src/planner/plans/union_all.rs index 0e64c71655c9a..d6a7606cffab5 100644 --- a/src/query/sql/src/planner/plans/union_all.rs +++ b/src/query/sql/src/planner/plans/union_all.rs @@ -17,6 +17,9 @@ use std::sync::Arc; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; +use crate::ColumnSet; +use crate::IndexType; +use crate::ScalarExpr; use crate::optimizer::ir::Distribution; use crate::optimizer::ir::PhysicalProperty; use crate::optimizer::ir::RelExpr; @@ -26,9 +29,6 @@ use crate::optimizer::ir::StatInfo; use crate::optimizer::ir::Statistics; use crate::plans::Operator; use crate::plans::RelOp; -use crate::ColumnSet; -use crate::IndexType; -use crate::ScalarExpr; #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct UnionAll { diff --git a/src/query/sql/src/planner/plans/window.rs b/src/query/sql/src/planner/plans/window.rs index 380c382573adf..258752c65ed6f 100644 --- a/src/query/sql/src/planner/plans/window.rs +++ b/src/query/sql/src/planner/plans/window.rs @@ -20,9 +20,9 @@ use databend_common_ast::Span; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::Scalar; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; -use databend_common_expression::Scalar; use educe::Educe; use enum_as_inner::EnumAsInner; use serde::Deserialize; @@ -30,6 +30,9 @@ use serde::Serialize; use super::AggregateFunction; use super::NthValueFunction; +use crate::ColumnSet; +use crate::IndexType; +use crate::ScalarExpr; use crate::binder::WindowOrderByInfo; use crate::optimizer::ir::Distribution; use crate::optimizer::ir::RelExpr; @@ -41,9 +44,6 @@ use crate::plans::NtileFunction; use crate::plans::Operator; use crate::plans::RelOp; use crate::plans::ScalarItem; -use crate::ColumnSet; -use crate::IndexType; -use crate::ScalarExpr; #[derive(Clone, Debug, Educe)] #[educe(PartialEq, Eq, Hash)] diff --git a/src/query/sql/src/planner/semantic/aggregating_index_visitor.rs b/src/query/sql/src/planner/semantic/aggregating_index_visitor.rs index 6b29dee7102fa..de3074b84f4ed 100644 --- a/src/query/sql/src/planner/semantic/aggregating_index_visitor.rs +++ b/src/query/sql/src/planner/semantic/aggregating_index_visitor.rs @@ -25,13 +25,13 @@ use databend_common_ast::ast::SelectStmt; use databend_common_ast::ast::SelectTarget; use databend_common_ast::ast::SetExpr; use databend_common_ast::ast::TableReference; +use databend_common_ast::parser::Dialect; use databend_common_ast::parser::parse_expr; use databend_common_ast::parser::tokenize_sql; -use databend_common_ast::parser::Dialect; -use databend_common_expression::FunctionKind; use databend_common_expression::BLOCK_NAME_COL_NAME; -use databend_common_functions::aggregates::AggregateFunctionFactory; +use databend_common_expression::FunctionKind; use databend_common_functions::BUILTIN_FUNCTIONS; +use databend_common_functions::aggregates::AggregateFunctionFactory; use derive_visitor::DriveMut; use derive_visitor::Visitor; use derive_visitor::VisitorMut; diff --git a/src/query/sql/src/planner/semantic/async_function_rewriter.rs b/src/query/sql/src/planner/semantic/async_function_rewriter.rs index 74dccc7174529..c8ae308128ced 100644 --- a/src/query/sql/src/planner/semantic/async_function_rewriter.rs +++ b/src/query/sql/src/planner/semantic/async_function_rewriter.rs @@ -19,8 +19,11 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use crate::ColumnBindingBuilder; +use crate::IndexType; +use crate::MetadataRef; +use crate::Visibility; use crate::optimizer::ir::SExpr; -use crate::plans::walk_expr_mut; use crate::plans::AsyncFunction; use crate::plans::AsyncFunctionCall; use crate::plans::BoundColumnRef; @@ -29,10 +32,7 @@ use crate::plans::RelOperator; use crate::plans::ScalarExpr; use crate::plans::ScalarItem; use crate::plans::VisitorMut; -use crate::ColumnBindingBuilder; -use crate::IndexType; -use crate::MetadataRef; -use crate::Visibility; +use crate::plans::walk_expr_mut; pub(crate) struct AsyncFunctionRewriter { metadata: MetadataRef, @@ -168,7 +168,7 @@ impl<'a> VisitorMut<'a> for AsyncFunctionRewriter { for (i, arg) in async_func.arguments.iter_mut().enumerate() { self.visit(arg)?; - let new_column_ref = if let ScalarExpr::BoundColumnRef(ref column_ref) = &arg { + let new_column_ref = if let ScalarExpr::BoundColumnRef(column_ref) = &arg { column_ref.clone() } else { let name = format!("{}_arg_{}", &async_func.display_name, i); diff --git a/src/query/sql/src/planner/semantic/grouping_check.rs b/src/query/sql/src/planner/semantic/grouping_check.rs index 1aaf88ea9af29..e90c364052bfe 100644 --- a/src/query/sql/src/planner/semantic/grouping_check.rs +++ b/src/query/sql/src/planner/semantic/grouping_check.rs @@ -15,13 +15,13 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use crate::BindContext; use crate::binder::ColumnBindingBuilder; use crate::binder::Visibility; -use crate::plans::walk_expr_mut; use crate::plans::BoundColumnRef; use crate::plans::ScalarExpr; use crate::plans::VisitorMut; -use crate::BindContext; +use crate::plans::walk_expr_mut; /// Check validity of scalar expression in a grouping context. /// The matched grouping item will be replaced with a BoundColumnRef diff --git a/src/query/sql/src/planner/semantic/lowering.rs b/src/query/sql/src/planner/semantic/lowering.rs index 810d031b33668..763e08678435f 100644 --- a/src/query/sql/src/planner/semantic/lowering.rs +++ b/src/query/sql/src/planner/semantic/lowering.rs @@ -16,20 +16,20 @@ use std::collections::HashMap; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::type_check; -use databend_common_expression::types::DataType; use databend_common_expression::ColumnIndex; use databend_common_expression::DataSchema; use databend_common_expression::Expr; use databend_common_expression::RawExpr; +use databend_common_expression::type_check; +use databend_common_expression::types::DataType; use databend_common_functions::BUILTIN_FUNCTIONS; -use crate::binder::DummyColumnType; -use crate::plans::ScalarExpr; use crate::ColumnBinding; use crate::ColumnEntry; use crate::IndexType; use crate::Metadata; +use crate::binder::DummyColumnType; +use crate::plans::ScalarExpr; pub trait TypeProvider { fn get_type(&self, column_id: &ColumnID) -> Result; diff --git a/src/query/sql/src/planner/semantic/mod.rs b/src/query/sql/src/planner/semantic/mod.rs index 0f22a8bc6c742..ed05a28a13f13 100644 --- a/src/query/sql/src/planner/semantic/mod.rs +++ b/src/query/sql/src/planner/semantic/mod.rs @@ -37,18 +37,18 @@ pub use count_set_ops::CountSetOps; pub use distinct_to_groupby::DistinctToGroupBy; pub use grouping_check::GroupingChecker; pub use lowering::*; -pub use name_resolution::compare_table_name; -pub use name_resolution::normalize_identifier; pub use name_resolution::IdentifierNormalizer; pub use name_resolution::NameResolutionContext; pub use name_resolution::NameResolutionSuggest; pub use name_resolution::VariableNormalizer; +pub use name_resolution::compare_table_name; +pub use name_resolution::normalize_identifier; +pub use type_check::StageLocationParam; +pub use type_check::TypeChecker; pub use type_check::resolve_type_name; pub use type_check::resolve_type_name_by_str; pub use type_check::resolve_type_name_udf; pub use type_check::validate_function_arg; -pub use type_check::StageLocationParam; -pub use type_check::TypeChecker; pub use udf_rewriter::UDFArgVisitor; pub(crate) use udf_rewriter::UdfRewriter; pub use view_rewriter::ViewRewriter; diff --git a/src/query/sql/src/planner/semantic/name_resolution.rs b/src/query/sql/src/planner/semantic/name_resolution.rs index 8360e20e753d1..697d6e107d79f 100644 --- a/src/query/sql/src/planner/semantic/name_resolution.rs +++ b/src/query/sql/src/planner/semantic/name_resolution.rs @@ -14,10 +14,10 @@ use std::sync::Arc; -use databend_common_ast::ast::quote::ident_needs_quote; use databend_common_ast::ast::Identifier; use databend_common_ast::ast::IdentifierType; use databend_common_ast::ast::MapAccessor; +use databend_common_ast::ast::quote::ident_needs_quote; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; diff --git a/src/query/sql/src/planner/semantic/tests/identifier_quote_test.rs b/src/query/sql/src/planner/semantic/tests/identifier_quote_test.rs index 0b12261f97e97..5eb9fefd526bf 100644 --- a/src/query/sql/src/planner/semantic/tests/identifier_quote_test.rs +++ b/src/query/sql/src/planner/semantic/tests/identifier_quote_test.rs @@ -15,13 +15,13 @@ mod tests { use databend_common_ast::ast::Identifier; + use databend_common_ast::parser::Dialect; use databend_common_ast::parser::parse_sql; use databend_common_ast::parser::tokenize_sql; - use databend_common_ast::parser::Dialect; use databend_common_exception::Result; - use crate::planner::semantic::name_resolution::normalize_identifier; use crate::planner::semantic::name_resolution::NameResolutionContext; + use crate::planner::semantic::name_resolution::normalize_identifier; #[test] fn test_sql_to_ast_to_sql_quoted_identifiers() -> Result<()> { diff --git a/src/query/sql/src/planner/semantic/type_check.rs b/src/query/sql/src/planner/semantic/type_check.rs index 03efeeb569327..15522eeb02852 100644 --- a/src/query/sql/src/planner/semantic/type_check.rs +++ b/src/query/sql/src/planner/semantic/type_check.rs @@ -22,6 +22,7 @@ use std::str::FromStr; use std::sync::Arc; use std::vec; +use databend_common_ast::Span; use databend_common_ast::ast::BinaryOperator; use databend_common_ast::ast::ColumnID; use databend_common_ast::ast::ColumnRef; @@ -47,10 +48,9 @@ use databend_common_ast::ast::Window; use databend_common_ast::ast::WindowFrame; use databend_common_ast::ast::WindowFrameBound; use databend_common_ast::ast::WindowFrameUnits; +use databend_common_ast::parser::Dialect; use databend_common_ast::parser::parse_expr; use databend_common_ast::parser::tokenize_sql; -use databend_common_ast::parser::Dialect; -use databend_common_ast::Span; use databend_common_base::runtime::GLOBAL_MEM_STAT; use databend_common_catalog::catalog::CatalogManager; use databend_common_catalog::plan::InternalColumn; @@ -63,25 +63,6 @@ use databend_common_compress::CompressAlgorithm; use databend_common_compress::DecompressDecoder; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::cast_scalar; -use databend_common_expression::display::display_tuple_field_name; -use databend_common_expression::expr; -use databend_common_expression::infer_schema_type; -use databend_common_expression::shrink_scalar; -use databend_common_expression::type_check; -use databend_common_expression::type_check::check_number; -use databend_common_expression::type_check::common_super_type; -use databend_common_expression::type_check::convert_escape_pattern; -use databend_common_expression::types::decimal::DecimalScalar; -use databend_common_expression::types::decimal::DecimalSize; -use databend_common_expression::types::i256; -use databend_common_expression::types::vector::VectorDataType; -use databend_common_expression::types::DataType; -use databend_common_expression::types::Decimal; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::NumberScalar; -use databend_common_expression::types::F32; -use databend_common_expression::udf_client::UDFFlightClient; use databend_common_expression::BlockEntry; use databend_common_expression::Column; use databend_common_expression::ColumnBuilder; @@ -94,13 +75,30 @@ use databend_common_expression::Expr as EExpr; use databend_common_expression::FunctionContext; use databend_common_expression::FunctionKind; use databend_common_expression::RawExpr; -use databend_common_expression::Scalar; -use databend_common_expression::TableDataType; use databend_common_expression::SEARCH_MATCHED_COL_NAME; use databend_common_expression::SEARCH_SCORE_COL_NAME; +use databend_common_expression::Scalar; +use databend_common_expression::TableDataType; use databend_common_expression::VECTOR_SCORE_COL_NAME; -use databend_common_functions::aggregates::AggregateFunctionFactory; -use databend_common_functions::is_builtin_function; +use databend_common_expression::cast_scalar; +use databend_common_expression::display::display_tuple_field_name; +use databend_common_expression::expr; +use databend_common_expression::infer_schema_type; +use databend_common_expression::shrink_scalar; +use databend_common_expression::type_check; +use databend_common_expression::type_check::check_number; +use databend_common_expression::type_check::common_super_type; +use databend_common_expression::type_check::convert_escape_pattern; +use databend_common_expression::types::DataType; +use databend_common_expression::types::Decimal; +use databend_common_expression::types::F32; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::NumberScalar; +use databend_common_expression::types::decimal::DecimalScalar; +use databend_common_expression::types::decimal::DecimalSize; +use databend_common_expression::types::i256; +use databend_common_expression::types::vector::VectorDataType; +use databend_common_expression::udf_client::UDFFlightClient; use databend_common_functions::ASYNC_FUNCTIONS; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_functions::GENERAL_LAMBDA_FUNCTIONS; @@ -108,6 +106,8 @@ use databend_common_functions::GENERAL_SEARCH_FUNCTIONS; use databend_common_functions::GENERAL_WINDOW_FUNCTIONS; use databend_common_functions::GENERAL_WITHIN_GROUP_FUNCTIONS; use databend_common_functions::RANK_WINDOW_FUNCTIONS; +use databend_common_functions::aggregates::AggregateFunctionFactory; +use databend_common_functions::is_builtin_function; use databend_common_license::license::Feature; use databend_common_license::license_manager::LicenseManagerSwitch; use databend_common_meta_app::principal::LambdaUDF; @@ -118,11 +118,11 @@ use databend_common_meta_app::principal::UDAFScript; use databend_common_meta_app::principal::UDFDefinition; use databend_common_meta_app::principal::UDFScript; use databend_common_meta_app::principal::UDFServer; -use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; use databend_common_meta_app::schema::DictionaryIdentity; use databend_common_meta_app::schema::GetSequenceReq; use databend_common_meta_app::schema::SequenceIdent; use databend_common_meta_app::schema::TableIndexType; +use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; use databend_common_meta_app::storage::StorageParams; use databend_common_storage::init_stage_operator; use databend_common_users::Object; @@ -132,28 +132,36 @@ use derive_visitor::DriveMut; use derive_visitor::Visitor; use derive_visitor::VisitorMut; use itertools::Itertools; -use jsonb::keypath::parse_key_paths; use jsonb::keypath::KeyPath; use jsonb::keypath::KeyPaths; +use jsonb::keypath::parse_key_paths; use serde_json::json; use serde_json::to_string; use simsearch::SimSearch; -use tantivy_query_grammar::parse_query_lenient; use tantivy_query_grammar::UserInputAst; use tantivy_query_grammar::UserInputLeaf; +use tantivy_query_grammar::parse_query_lenient; use unicase::Ascii; use super::name_resolution::NameResolutionContext; use super::normalize_identifier; +use crate::BaseTableColumn; +use crate::BindContext; +use crate::ColumnBinding; +use crate::ColumnEntry; +use crate::DefaultExprBinder; +use crate::IndexType; +use crate::MetadataRef; +use crate::UDFArgVisitor; +use crate::binder::Binder; +use crate::binder::ExprContext; +use crate::binder::InternalColumnBinding; +use crate::binder::NameResolutionResult; use crate::binder::bind_values; use crate::binder::resolve_file_location; use crate::binder::resolve_stage_location; use crate::binder::resolve_stage_locations; use crate::binder::wrap_cast; -use crate::binder::Binder; -use crate::binder::ExprContext; -use crate::binder::InternalColumnBinding; -use crate::binder::NameResolutionResult; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::SExpr; use crate::parse_lambda_expr; @@ -196,14 +204,6 @@ use crate::plans::WindowFuncFrameBound; use crate::plans::WindowFuncFrameUnits; use crate::plans::WindowFuncType; use crate::plans::WindowOrderBy; -use crate::BaseTableColumn; -use crate::BindContext; -use crate::ColumnBinding; -use crate::ColumnEntry; -use crate::DefaultExprBinder; -use crate::IndexType; -use crate::MetadataRef; -use crate::UDFArgVisitor; const DEFAULT_DECIMAL_PRECISION: i64 = 38; const DEFAULT_DECIMAL_SCALE: i64 = 0; @@ -1331,7 +1331,7 @@ impl<'a> TypeChecker<'a> { return Err(ErrorCode::SemanticError( "Hole or Placeholder expression is impossible in trivial query".to_string(), ) - .set_span(*span)) + .set_span(*span)); } Expr::StageLocation { span, location } => { self.resolve_stage_location(*span, location)? @@ -3663,7 +3663,7 @@ impl<'a> TypeChecker<'a> { _ => { return Err(ErrorCode::Internal( "Only support resolve datesub, date_sub, date_diff, date_add", - )) + )); } }; let mut args = vec![]; @@ -4512,8 +4512,8 @@ impl<'a> TypeChecker<'a> { } let nulls_first = nulls_first.unwrap_or_else(|| { - let default_nulls_first = self.ctx.get_settings().get_nulls_first(); - default_nulls_first(asc) + let settings = self.ctx.get_settings(); + settings.get_nulls_first()(asc) }); let func_name = match (asc, nulls_first) { @@ -4726,7 +4726,7 @@ impl<'a> TypeChecker<'a> { { return None; } - let ScalarExpr::BoundColumnRef(BoundColumnRef { ref column, .. }) = &args[0] else { + let ScalarExpr::BoundColumnRef(BoundColumnRef { column, .. }) = &args[0] else { return None; }; if column.index >= self.metadata.read().columns().len() { @@ -4791,38 +4791,44 @@ impl<'a> TypeChecker<'a> { let uni_case_func_name = Ascii::new(func_name); if Self::vector_functions().contains(&uni_case_func_name) { match args { - [ScalarExpr::BoundColumnRef(BoundColumnRef { - column: - ColumnBinding { - table_index, - database_name, - table_name, - column_name, - data_type, - .. - }, - .. - }), ScalarExpr::CastExpr(CastExpr { - argument, - target_type, - .. - })] - | [ScalarExpr::CastExpr(CastExpr { - argument, - target_type, - .. - }), ScalarExpr::BoundColumnRef(BoundColumnRef { - column: - ColumnBinding { - table_index, - database_name, - table_name, - column_name, - data_type, - .. - }, - .. - })] => { + [ + ScalarExpr::BoundColumnRef(BoundColumnRef { + column: + ColumnBinding { + table_index, + database_name, + table_name, + column_name, + data_type, + .. + }, + .. + }), + ScalarExpr::CastExpr(CastExpr { + argument, + target_type, + .. + }), + ] + | [ + ScalarExpr::CastExpr(CastExpr { + argument, + target_type, + .. + }), + ScalarExpr::BoundColumnRef(BoundColumnRef { + column: + ColumnBinding { + table_index, + database_name, + table_name, + column_name, + data_type, + .. + }, + .. + }), + ] => { let col_data_type = data_type.remove_nullable(); let target_type = target_type.remove_nullable(); if table_index.is_some() @@ -5243,9 +5249,9 @@ impl<'a> TypeChecker<'a> { return Ok(None); } - let udf = databend_common_base::runtime::block_on({ - UserApiProvider::instance().get_udf(&self.ctx.get_tenant(), udf_name) - })?; + let tenant = self.ctx.get_tenant(); + let provider = UserApiProvider::instance(); + let udf = databend_common_base::runtime::block_on(provider.get_udf(&tenant, udf_name))?; let Some(udf) = udf else { return Ok(None); @@ -6079,7 +6085,7 @@ impl<'a> TypeChecker<'a> { } // If the type of source column is a tuple, rewrite to json_object_keep_null function, // using the name of tuple inner fields as the object name. - if let ScalarExpr::BoundColumnRef(BoundColumnRef { ref column, .. }) = scalar { + if let ScalarExpr::BoundColumnRef(BoundColumnRef { column, .. }) = scalar { let column_entry = self.metadata.read().column(column.index).clone(); if let ColumnEntry::BaseTableColumn(BaseTableColumn { data_type, .. }) = column_entry { let new_scalar = Self::rewrite_cast_to_variant(span, scalar, &data_type, is_try); diff --git a/src/query/sql/src/planner/semantic/udf_rewriter.rs b/src/query/sql/src/planner/semantic/udf_rewriter.rs index 65f5766f48d19..5ac7116509390 100644 --- a/src/query/sql/src/planner/semantic/udf_rewriter.rs +++ b/src/query/sql/src/planner/semantic/udf_rewriter.rs @@ -19,13 +19,16 @@ use std::sync::Arc; use databend_common_ast::ast::Expr; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::convert_to_type_name; use databend_common_expression::types::DataType; +use databend_common_expression::types::convert_to_type_name; use derive_visitor::VisitorMut as StatementVisitorMut; use itertools::Itertools; +use crate::ColumnBindingBuilder; +use crate::IndexType; +use crate::MetadataRef; +use crate::Visibility; use crate::optimizer::ir::SExpr; -use crate::plans::walk_expr_mut; use crate::plans::BoundColumnRef; use crate::plans::EvalScalar; use crate::plans::RelOperator; @@ -34,10 +37,7 @@ use crate::plans::ScalarItem; use crate::plans::UDFCall; use crate::plans::Udf; use crate::plans::VisitorMut; -use crate::ColumnBindingBuilder; -use crate::IndexType; -use crate::MetadataRef; -use crate::Visibility; +use crate::plans::walk_expr_mut; pub(crate) struct UdfRewriter { metadata: MetadataRef, @@ -176,7 +176,7 @@ impl<'a> VisitorMut<'a> for UdfRewriter { for (i, arg) in udf.arguments.iter_mut().enumerate() { self.visit(arg)?; - let new_column_ref = if let ScalarExpr::BoundColumnRef(ref column_ref) = &arg { + let new_column_ref = if let ScalarExpr::BoundColumnRef(column_ref) = &arg { column_ref.clone() } else { let name = format!("{}_arg_{}", &udf.display_name, i); diff --git a/src/query/sql/src/planner/semantic/window_check.rs b/src/query/sql/src/planner/semantic/window_check.rs index c5558bedbd2b2..3f1c9dbe06d37 100644 --- a/src/query/sql/src/planner/semantic/window_check.rs +++ b/src/query/sql/src/planner/semantic/window_check.rs @@ -15,14 +15,14 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use crate::BindContext; +use crate::ScalarExpr; +use crate::Visibility; use crate::binder::ColumnBindingBuilder; -use crate::plans::walk_expr_mut; use crate::plans::BoundColumnRef; use crate::plans::SubqueryExpr; use crate::plans::VisitorMut; -use crate::BindContext; -use crate::ScalarExpr; -use crate::Visibility; +use crate::plans::walk_expr_mut; pub struct WindowChecker<'a> { bind_context: &'a BindContext, diff --git a/src/query/storages/basic/src/lib.rs b/src/query/storages/basic/src/lib.rs index 304939c788468..7d5538d9a1608 100644 --- a/src/query/storages/basic/src/lib.rs +++ b/src/query/storages/basic/src/lib.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(clippy::uninlined_format_args)] +#![allow(clippy::collapsible_if, clippy::uninlined_format_args)] #![feature(type_alias_impl_trait)] #![feature(impl_trait_in_assoc_type)] @@ -40,8 +40,8 @@ pub use null::NullTable; pub use random::{RandomPartInfo, RandomTable}; // Result cache storage exports pub use result_cache::{ - gen_result_cache_key, gen_result_cache_meta_key, gen_result_cache_prefix, ResultCacheMetaManager, ResultCacheReader, ResultScan, WriteResultCacheSink, + gen_result_cache_key, gen_result_cache_meta_key, gen_result_cache_prefix, }; // View storage exports pub use view::view_table; diff --git a/src/query/storages/basic/src/memory_table.rs b/src/query/storages/basic/src/memory_table.rs index 3aa8b8f67658e..8b998e80dc971 100644 --- a/src/query/storages/basic/src/memory_table.rs +++ b/src/query/storages/basic/src/memory_table.rs @@ -32,9 +32,9 @@ use databend_common_catalog::table::TableStatistics; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::BlockEntry; use databend_common_expression::DataBlock; +use databend_common_expression::types::DataType; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::UpdateStreamMetaReq; use databend_common_meta_app::schema::UpsertTableCopiedFileReq; @@ -48,8 +48,8 @@ use databend_common_pipeline::sinks::Sinker; use databend_common_pipeline::sources::SyncSource; use databend_common_pipeline::sources::SyncSourcer; use databend_common_storage::StorageMetrics; -use databend_storages_common_blocks::memory::InMemoryDataKey; use databend_storages_common_blocks::memory::IN_MEMORY_DATA; +use databend_storages_common_blocks::memory::InMemoryDataKey; use databend_storages_common_table_meta::meta::SnapshotId; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use databend_storages_common_table_meta::table::ChangeType; diff --git a/src/query/storages/basic/src/random/random_table.rs b/src/query/storages/basic/src/random/random_table.rs index 84c7ae65b2f74..3885c09fa0a4f 100644 --- a/src/query/storages/basic/src/random/random_table.rs +++ b/src/query/storages/basic/src/random/random_table.rs @@ -25,11 +25,11 @@ use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::Column; use databend_common_expression::DataBlock; use databend_common_expression::RandomOptions; use databend_common_expression::TableSchemaRef; +use databend_common_expression::types::DataType; use databend_common_meta_app::schema::TableInfo; use databend_common_pipeline::core::OutputPort; use databend_common_pipeline::core::Pipeline; diff --git a/src/query/storages/basic/src/result_cache/read/reader.rs b/src/query/storages/basic/src/result_cache/read/reader.rs index 30e84fb79758b..a6a45fc0c811d 100644 --- a/src/query/storages/basic/src/result_cache/read/reader.rs +++ b/src/query/storages/basic/src/result_cache/read/reader.rs @@ -23,8 +23,8 @@ use databend_common_storage::DataOperator; use opendal::Operator; use parquet::arrow::arrow_reader::ParquetRecordBatchReader; -use crate::result_cache::common::gen_result_cache_meta_key; use crate::result_cache::common::ResultCacheValue; +use crate::result_cache::common::gen_result_cache_meta_key; use crate::result_cache::meta_manager::ResultCacheMetaManager; pub struct ResultCacheReader { diff --git a/src/query/storages/basic/src/result_cache/table_function/table.rs b/src/query/storages/basic/src/result_cache/table_function/table.rs index 0bb418e74d927..c024d181dc55a 100644 --- a/src/query/storages/basic/src/result_cache/table_function/table.rs +++ b/src/query/storages/basic/src/result_cache/table_function/table.rs @@ -35,9 +35,9 @@ use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; use databend_common_pipeline::core::Pipeline; +use databend_common_storage::DataOperator; use databend_common_storage::parquet::infer_schema_with_extension; use databend_common_storage::read_metadata_async; -use databend_common_storage::DataOperator; use databend_common_storages_parquet::ParquetFilePart; use databend_common_storages_parquet::ParquetPart; use databend_common_storages_parquet::ParquetReaderBuilder; diff --git a/src/query/storages/basic/src/result_cache/write/sink.rs b/src/query/storages/basic/src/result_cache/write/sink.rs index 528feac2fa81c..de9c6dc77c152 100644 --- a/src/query/storages/basic/src/result_cache/write/sink.rs +++ b/src/query/storages/basic/src/result_cache/write/sink.rs @@ -30,9 +30,9 @@ use databend_common_storage::DataOperator; use tokio::time::Instant; use super::writer::ResultCacheWriter; +use crate::result_cache::common::ResultCacheValue; use crate::result_cache::common::gen_result_cache_dir; use crate::result_cache::common::gen_result_cache_meta_key; -use crate::result_cache::common::ResultCacheValue; use crate::result_cache::meta_manager::ResultCacheMetaManager; pub struct WriteResultCacheSink { diff --git a/src/query/storages/common/blocks/src/lib.rs b/src/query/storages/common/blocks/src/lib.rs index 0434a7671b1d0..8c9e296ba3a33 100644 --- a/src/query/storages/common/blocks/src/lib.rs +++ b/src/query/storages/common/blocks/src/lib.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(clippy::uninlined_format_args)] +#![allow(clippy::collapsible_if, clippy::uninlined_format_args)] mod parquet_rs; pub use parquet_rs::*; diff --git a/src/query/storages/common/blocks/src/parquet_rs.rs b/src/query/storages/common/blocks/src/parquet_rs.rs index 579a2e441dd18..0e03b24342c2a 100644 --- a/src/query/storages/common/blocks/src/parquet_rs.rs +++ b/src/query/storages/common/blocks/src/parquet_rs.rs @@ -15,10 +15,10 @@ use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::converts::arrow::table_schema_arrow_leaf_paths; use databend_common_expression::ColumnId; use databend_common_expression::DataBlock; use databend_common_expression::TableSchema; +use databend_common_expression::converts::arrow::table_schema_arrow_leaf_paths; use databend_storages_common_table_meta::meta::StatisticsOfColumns; use databend_storages_common_table_meta::table::TableCompression; use parquet::arrow::ArrowWriter; @@ -158,9 +158,9 @@ impl NdvProvider for &StatisticsOfColumns { mod tests { use std::collections::HashMap; - use databend_common_expression::types::number::NumberDataType; use databend_common_expression::TableDataType; use databend_common_expression::TableField; + use databend_common_expression::types::number::NumberDataType; use super::*; diff --git a/src/query/storages/common/cache/src/cache_items.rs b/src/query/storages/common/cache/src/cache_items.rs index 7245beb66c4f7..581757467e5bc 100644 --- a/src/query/storages/common/cache/src/cache_items.rs +++ b/src/query/storages/common/cache/src/cache_items.rs @@ -17,19 +17,19 @@ pub use databend_common_catalog::plan::PartStatistics; pub use databend_common_catalog::plan::Partitions; pub use databend_common_catalog::table::Table; use databend_common_exception::ErrorCode; -pub use databend_storages_common_index::filters::FilterImpl; pub use databend_storages_common_index::BloomIndexMeta; pub use databend_storages_common_index::InvertedIndexFile; pub use databend_storages_common_index::InvertedIndexMeta; pub use databend_storages_common_index::VectorIndexFile; pub use databend_storages_common_index::VectorIndexMeta; -pub use databend_storages_common_table_meta::meta::column_oriented_segment::ColumnOrientedSegment; +pub use databend_storages_common_index::filters::FilterImpl; pub use databend_storages_common_table_meta::meta::BlockMeta; pub use databend_storages_common_table_meta::meta::CompactSegmentInfo; pub use databend_storages_common_table_meta::meta::SegmentInfo; pub use databend_storages_common_table_meta::meta::SegmentStatistics; pub use databend_storages_common_table_meta::meta::TableSnapshot; pub use databend_storages_common_table_meta::meta::TableSnapshotStatistics; +pub use databend_storages_common_table_meta::meta::column_oriented_segment::ColumnOrientedSegment; pub use parquet::file::metadata::ParquetMetaData; use crate::HybridCache; diff --git a/src/query/storages/common/cache/src/caches.rs b/src/query/storages/common/cache/src/caches.rs index 4aa4f79d84978..35726131cd6db 100644 --- a/src/query/storages/common/cache/src/caches.rs +++ b/src/query/storages/common/cache/src/caches.rs @@ -12,18 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::sync::atomic::AtomicBool; use std::sync::Arc; +use std::sync::atomic::AtomicBool; use std::time::Instant; use arrow::array::ArrayRef; use databend_common_cache::MemSized; +use crate::CacheAccessor; +use crate::InMemoryLruCache; pub use crate::cache_items::*; use crate::manager::CacheManager; use crate::providers::HybridCache; -use crate::CacheAccessor; -use crate::InMemoryLruCache; /// In memory object cache of SegmentInfo pub type CompactSegmentInfoCache = InMemoryLruCache; diff --git a/src/query/storages/common/cache/src/lib.rs b/src/query/storages/common/cache/src/lib.rs index d978777843232..0e48088e44e83 100644 --- a/src/query/storages/common/cache/src/lib.rs +++ b/src/query/storages/common/cache/src/lib.rs @@ -16,6 +16,7 @@ #![feature(associated_type_defaults)] #![feature(assert_matches)] #![feature(io_const_error)] +#![allow(clippy::collapsible_if, clippy::let_and_return)] mod cache; mod cache_items; @@ -28,7 +29,6 @@ mod temp_dir; pub use cache::*; pub use caches::*; pub use manager::CacheManager; -pub use providers::disk_cache::disk_cache_lru::read_cache_content; pub use providers::DiskCacheAccessor; pub use providers::DiskCacheBuilder; pub use providers::DiskCacheError; @@ -40,5 +40,6 @@ pub use providers::LruDiskCache; pub use providers::LruDiskCacheBuilder; pub use providers::LruDiskCacheHolder; pub use providers::TableDataCacheKey; +pub use providers::disk_cache::disk_cache_lru::read_cache_content; pub use read::*; pub use temp_dir::*; diff --git a/src/query/storages/common/cache/src/manager.rs b/src/query/storages/common/cache/src/manager.rs index 886a9021b15f4..192a47ed27388 100644 --- a/src/query/storages/common/cache/src/manager.rs +++ b/src/query/storages/common/cache/src/manager.rs @@ -13,9 +13,9 @@ // limitations under the License. use std::path::PathBuf; +use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; -use std::sync::Arc; use databend_common_base::base::GlobalInstance; use databend_common_config::CacheConfig; @@ -26,6 +26,12 @@ use databend_common_exception::Result; use log::info; use parking_lot::RwLock; +use crate::CacheAccessor; +use crate::DiskCacheAccessor; +use crate::DiskCacheBuilder; +use crate::InMemoryLruCache; +use crate::SegmentStatisticsCache; +use crate::Unit; use crate::caches::BlockMetaCache; use crate::caches::BloomIndexFilterCache; use crate::caches::BloomIndexMetaCache; @@ -46,12 +52,6 @@ use crate::caches::VectorIndexFileCache; use crate::caches::VectorIndexMetaCache; use crate::providers::HybridCache; use crate::providers::HybridCacheExt; -use crate::CacheAccessor; -use crate::DiskCacheAccessor; -use crate::DiskCacheBuilder; -use crate::InMemoryLruCache; -use crate::SegmentStatisticsCache; -use crate::Unit; static DEFAULT_PARQUET_META_DATA_CACHE_ITEMS: usize = 3000; @@ -891,10 +891,10 @@ mod tests { use bytes::Bytes; use databend_common_config::CacheConfig; use databend_common_config::DiskCacheInnerConfig; + use databend_storages_common_index::BloomIndexMeta; use databend_storages_common_index::filters::FilterBuilder; use databend_storages_common_index::filters::FilterImpl; use databend_storages_common_index::filters::Xor8Builder; - use databend_storages_common_index::BloomIndexMeta; use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::CompactSegmentInfo; use databend_storages_common_table_meta::meta::Compression; @@ -1189,10 +1189,12 @@ mod tests { // ----- VERIFY INITIAL CACHE STATE ----- // Verify all caches are correctly populated assert!(!cache_manager.get_table_data_array_cache().is_empty()); - assert!(!cache_manager - .get_segment_block_metas_cache() - .unwrap() - .is_empty()); + assert!( + !cache_manager + .get_segment_block_metas_cache() + .unwrap() + .is_empty() + ); assert!(!cache_manager.get_column_data_cache().is_empty()); assert!(!cache_manager.get_block_meta_cache().is_empty()); assert!(!cache_manager.get_table_segment_cache().is_empty()); @@ -1207,22 +1209,26 @@ mod tests { // Verify basic caches are cleared assert!(in_memory_table_data_cache.is_empty()); - assert!(cache_manager - .get_segment_block_metas_cache() - .unwrap() - .is_empty()); + assert!( + cache_manager + .get_segment_block_metas_cache() + .unwrap() + .is_empty() + ); assert!(cache_manager.get_block_meta_cache().is_empty()); // Verify hybrid column data cache behavior: // - On-disk cache of table data should remain populated assert!(!cache_manager.get_column_data_cache().is_empty()); // - In-memory cache of table data should be cleared - assert!(cache_manager - .get_column_data_cache() - .unwrap() - .in_memory_cache() - .unwrap() - .is_empty()); + assert!( + cache_manager + .get_column_data_cache() + .unwrap() + .in_memory_cache() + .unwrap() + .is_empty() + ); // Verify extra caches remain intact after Basic clearance assert!(!cache_manager.get_table_segment_cache().is_empty()); @@ -1239,20 +1245,24 @@ mod tests { assert!(cache_manager.get_table_segment_cache().is_empty()); // Verify in-memory part of hybrid bloom index meta caches are cleared - assert!(cache_manager - .get_bloom_index_meta_cache() - .unwrap() - .in_memory_cache() - .unwrap() - .is_empty()); + assert!( + cache_manager + .get_bloom_index_meta_cache() + .unwrap() + .in_memory_cache() + .unwrap() + .is_empty() + ); // Verify in-memory part of hybrid bloom index filter caches are cleared - assert!(cache_manager - .get_bloom_index_filter_cache() - .unwrap() - .in_memory_cache() - .unwrap() - .is_empty()); + assert!( + cache_manager + .get_bloom_index_filter_cache() + .unwrap() + .in_memory_cache() + .unwrap() + .is_empty() + ); Ok(()) } diff --git a/src/query/storages/common/cache/src/providers/disk_cache/disk_cache.rs b/src/query/storages/common/cache/src/providers/disk_cache/disk_cache.rs index 3fb6f41610c2a..7de5180c92899 100644 --- a/src/query/storages/common/cache/src/providers/disk_cache/disk_cache.rs +++ b/src/query/storages/common/cache/src/providers/disk_cache/disk_cache.rs @@ -18,9 +18,9 @@ use std::io::IoSlice; use std::io::Write; use std::path::Path; use std::path::PathBuf; +use std::sync::Arc; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; -use std::sync::Arc; use std::time::Instant; use databend_common_cache::Cache; @@ -31,8 +31,8 @@ use log::error; use log::info; use log::warn; use parking_lot::RwLock; -use rayon::prelude::*; use rayon::ThreadPoolBuilder; +use rayon::prelude::*; use crate::CacheValue; use crate::DiskCacheKey; @@ -388,7 +388,7 @@ pub mod io_result { match self { Error::FileTooLarge => write!(f, "File too large"), Error::MalformedPath(p) => write!(f, "Malformed catch file path: {:?}", p), - Error::Io(ref e) => write!(f, "{e}"), + Error::Io(e) => write!(f, "{e}"), Error::Misc(msg) => write!(f, "{msg}"), } } @@ -397,7 +397,7 @@ pub mod io_result { impl StdError for Error { fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { match self { - Error::Io(ref e) => Some(e), + Error::Io(e) => Some(e), _ => None, } } diff --git a/src/query/storages/common/cache/src/providers/disk_cache/disk_cache_lru.rs b/src/query/storages/common/cache/src/providers/disk_cache/disk_cache_lru.rs index 5690a0a8423a9..86a8a6fd86825 100644 --- a/src/query/storages/common/cache/src/providers/disk_cache/disk_cache_lru.rs +++ b/src/query/storages/common/cache/src/providers/disk_cache/disk_cache_lru.rs @@ -24,8 +24,8 @@ use log::error; use log::warn; use parking_lot::RwLock; -use crate::providers::disk_cache::DiskCache; use crate::CacheAccessor; +use crate::providers::disk_cache::DiskCache; impl CacheAccessor for LruDiskCacheHolder { type V = Bytes; @@ -225,7 +225,7 @@ mod linux_read { buf: *mut libc::c_void, count: libc::size_t, ) -> libc::ssize_t { - libc::read(fd, buf, count) + unsafe { libc::read(fd, buf, count) } } } @@ -268,7 +268,7 @@ mod linux_read { "Invalid cache item, expects {} bytes, got {}, path {:?}", size, total_read, path ), - )) + )); } _ => { let err = std::io::Error::last_os_error(); @@ -292,8 +292,8 @@ mod linux_read { mod tests_read_content { use std::fs::File; - use mockall::predicate::*; use mockall::Sequence; + use mockall::predicate::*; use tempfile::TempDir; use super::*; diff --git a/src/query/storages/common/cache/src/providers/disk_cache_builder.rs b/src/query/storages/common/cache/src/providers/disk_cache_builder.rs index 98d6ebe65f89c..60c0b323442ed 100644 --- a/src/query/storages/common/cache/src/providers/disk_cache_builder.rs +++ b/src/query/storages/common/cache/src/providers/disk_cache_builder.rs @@ -27,9 +27,9 @@ use databend_common_metrics::cache::*; use log::error; use log::info; -use crate::providers::LruDiskCacheHolder; use crate::CacheAccessor; use crate::LruDiskCacheBuilder; +use crate::providers::LruDiskCacheHolder; struct CacheItem { key: String, diff --git a/src/query/storages/common/cache/src/providers/hybrid_cache.rs b/src/query/storages/common/cache/src/providers/hybrid_cache.rs index e41b21be3fcd2..feb19dab60059 100644 --- a/src/query/storages/common/cache/src/providers/hybrid_cache.rs +++ b/src/query/storages/common/cache/src/providers/hybrid_cache.rs @@ -265,12 +265,12 @@ mod tests { use parquet::data_type::AsBytes; use tempfile::TempDir; - use crate::providers::hybrid_cache::HybridCacheExt; use crate::CacheAccessor; use crate::CacheValue; use crate::DiskCacheBuilder; use crate::HybridCache; use crate::InMemoryLruCache; + use crate::providers::hybrid_cache::HybridCacheExt; // Custom test data type for serialization/deserialization testing #[derive(Clone, Debug, PartialEq)] diff --git a/src/query/storages/common/cache/src/providers/memory_cache.rs b/src/query/storages/common/cache/src/providers/memory_cache.rs index a1e941fce07e3..b8de82e8724e7 100644 --- a/src/query/storages/common/cache/src/providers/memory_cache.rs +++ b/src/query/storages/common/cache/src/providers/memory_cache.rs @@ -18,8 +18,8 @@ use databend_common_cache::Cache; use databend_common_cache::LruCache; use parking_lot::RwLock; -use crate::caches::CacheValue; use crate::Unit; +use crate::caches::CacheValue; pub struct InMemoryLruCache>> { unit: Unit, diff --git a/src/query/storages/common/cache/src/providers/mod.rs b/src/query/storages/common/cache/src/providers/mod.rs index 1b61490ffcded..d1195646c30db 100644 --- a/src/query/storages/common/cache/src/providers/mod.rs +++ b/src/query/storages/common/cache/src/providers/mod.rs @@ -17,12 +17,12 @@ mod disk_cache_builder; mod hybrid_cache; mod memory_cache; -pub use disk_cache::io_result::Error as DiskCacheError; -pub use disk_cache::io_result::Result as DiskCacheResult; pub use disk_cache::DiskCacheKey; pub use disk_cache::LruDiskCache; pub use disk_cache::LruDiskCacheBuilder; pub use disk_cache::LruDiskCacheHolder; +pub use disk_cache::io_result::Error as DiskCacheError; +pub use disk_cache::io_result::Result as DiskCacheResult; pub use disk_cache_builder::DiskCacheAccessor; pub use disk_cache_builder::DiskCacheBuilder; pub use disk_cache_builder::TableDataCacheKey; diff --git a/src/query/storages/common/cache/src/read/cached_reader.rs b/src/query/storages/common/cache/src/read/cached_reader.rs index 29c9bec53daba..1e82bc1638010 100644 --- a/src/query/storages/common/cache/src/read/cached_reader.rs +++ b/src/query/storages/common/cache/src/read/cached_reader.rs @@ -19,9 +19,9 @@ use databend_common_exception::Result; use databend_common_metrics::cache::*; use super::loader::LoadParams; -use crate::caches::CacheValue; use crate::CacheAccessor; use crate::Loader; +use crate::caches::CacheValue; /// A cache-aware reader pub struct CacheReader { diff --git a/src/query/storages/common/cache/src/read/cached_ttl_reader.rs b/src/query/storages/common/cache/src/read/cached_ttl_reader.rs index 015c9bda6794f..144444086f5c3 100644 --- a/src/query/storages/common/cache/src/read/cached_ttl_reader.rs +++ b/src/query/storages/common/cache/src/read/cached_ttl_reader.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::sync::atomic::AtomicBool; use std::sync::Arc; +use std::sync::atomic::AtomicBool; use std::time::Duration; use std::time::Instant; @@ -22,9 +22,9 @@ use databend_common_exception::Result; use databend_common_metrics::cache::*; use super::loader::LoadParams; -use crate::caches::CacheValue; use crate::CacheAccessor; use crate::Loader; +use crate::caches::CacheValue; pub trait TTLValue { fn is_expired(&self, ttl: &Duration) -> bool; diff --git a/src/query/storages/common/cache/src/read/readers.rs b/src/query/storages/common/cache/src/read/readers.rs index 6fde3c44ab075..133c0a3438f9f 100644 --- a/src/query/storages/common/cache/src/read/readers.rs +++ b/src/query/storages/common/cache/src/read/readers.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::InMemoryLruCache; use crate::providers::HybridCache; use crate::read::cached_reader::CacheReader; use crate::read::cached_ttl_reader::CacheTTLReader; -use crate::InMemoryLruCache; pub type HybridCacheReader = CacheReader>; pub type InMemoryCacheReader = CacheReader>; diff --git a/src/query/storages/common/cache/src/temp_dir.rs b/src/query/storages/common/cache/src/temp_dir.rs index 384467d770814..75c05565ae562 100644 --- a/src/query/storages/common/cache/src/temp_dir.rs +++ b/src/query/storages/common/cache/src/temp_dir.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::hash_map::Entry; use std::collections::HashMap; +use std::collections::hash_map::Entry; use std::fmt::Debug; use std::fs; use std::hash::Hash; @@ -22,11 +22,11 @@ use std::io::ErrorKind; use std::ops::Deref; use std::ops::Drop; use std::path::Path; -use std::sync::atomic::AtomicUsize; -use std::sync::atomic::Ordering; use std::sync::Arc; use std::sync::Mutex; use std::sync::Once; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; use databend_common_base::base::Alignment; use databend_common_base::base::GlobalInstance; diff --git a/src/query/storages/common/index/benches/build_from_block.rs b/src/query/storages/common/index/benches/build_from_block.rs index 4897bbbaee617..d4f170d2e0aad 100644 --- a/src/query/storages/common/index/benches/build_from_block.rs +++ b/src/query/storages/common/index/benches/build_from_block.rs @@ -22,22 +22,22 @@ fn main() { mod dummy { use std::ops::Deref; - use databend_common_expression::types::number::NumberColumn; - use databend_common_expression::types::string::StringColumnBuilder; + use databend_common_expression::Column; + use databend_common_expression::FunctionContext; use databend_common_expression::types::AccessType; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::UInt64Type; - use databend_common_expression::Column; - use databend_common_expression::FunctionContext; + use databend_common_expression::types::number::NumberColumn; + use databend_common_expression::types::string::StringColumnBuilder; + use databend_storages_common_index::BloomIndex; use databend_storages_common_index::filters::Filter; use databend_storages_common_index::filters::FilterBuilder; use databend_storages_common_index::filters::Xor8Builder; - use databend_storages_common_index::BloomIndex; - use rand::prelude::random; - use rand::rngs::StdRng; use rand::Rng; use rand::SeedableRng; + use rand::prelude::random; + use rand::rngs::StdRng; /// Benchmark building BlockFilter from DataBlock. /// @@ -57,7 +57,10 @@ mod dummy { let column = rand_i64_column(1_000_000); let mut builder = Xor8Builder::create(); - (0..column.len()).for_each(|i| builder.add_key(unsafe { &column.index_unchecked(i) })); + (0..column.len()).for_each(|i| { + let value = unsafe { column.index_unchecked(i) }; + builder.add_key(&value); + }); let filter = builder.build().unwrap(); for i in 0..column.len() { @@ -66,7 +69,10 @@ mod dummy { } bencher.bench(|| { let mut builder = Xor8Builder::create(); - (0..column.len()).for_each(|i| builder.add_key(unsafe { &column.index_unchecked(i) })); + (0..column.len()).for_each(|i| { + let value = unsafe { column.index_unchecked(i) }; + builder.add_key(&value); + }); let _filter = divan::black_box(builder.build().unwrap()); }); } @@ -76,7 +82,10 @@ mod dummy { let column = rand_str_column(1_000_000, 32); let mut builder = Xor8Builder::create(); - (0..column.len()).for_each(|i| builder.add_key(unsafe { &column.index_unchecked(i) })); + (0..column.len()).for_each(|i| { + let value = unsafe { column.index_unchecked(i) }; + builder.add_key(&value); + }); let filter = builder.build().unwrap(); for i in 0..column.len() { @@ -86,7 +95,10 @@ mod dummy { bencher.bench(|| { let mut builder = Xor8Builder::create(); - (0..column.len()).for_each(|i| builder.add_key(unsafe { &column.index_unchecked(i) })); + (0..column.len()).for_each(|i| { + let value = unsafe { column.index_unchecked(i) }; + builder.add_key(&value); + }); let _filter = divan::black_box(builder.build().unwrap()); }) } @@ -171,7 +183,7 @@ mod dummy { let seed: u64 = random(); let mut rng = StdRng::seed_from_u64(seed); - let keys: Vec = (0..n).map(|_| rng.gen::()).collect(); + let keys: Vec = (0..n).map(|_| rng.r#gen::()).collect(); Column::Number(NumberColumn::Int64(keys.into())) } diff --git a/src/query/storages/common/index/src/bloom_index.rs b/src/query/storages/common/index/src/bloom_index.rs index f3dd6deb06a16..aa10c5864f040 100644 --- a/src/query/storages/common/index/src/bloom_index.rs +++ b/src/query/storages/common/index/src/bloom_index.rs @@ -24,12 +24,29 @@ use bytes::Bytes; use databend_common_ast::Span; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::BlockEntry; +use databend_common_expression::Column; +use databend_common_expression::ColumnBuilder; +use databend_common_expression::ColumnId; +use databend_common_expression::ConstantFolder; +use databend_common_expression::DataBlock; +use databend_common_expression::Domain; +use databend_common_expression::Expr; +use databend_common_expression::ExprVisitor; +use databend_common_expression::FieldIndex; +use databend_common_expression::FunctionContext; +use databend_common_expression::LikePattern; +use databend_common_expression::Scalar; +use databend_common_expression::ScalarRef; +use databend_common_expression::TableDataType; +use databend_common_expression::TableField; +use databend_common_expression::TableSchema; +use databend_common_expression::TableSchemaRef; +use databend_common_expression::Value; use databend_common_expression::converts::datavalues::scalar_to_datavalue; use databend_common_expression::eval_function; use databend_common_expression::expr::*; use databend_common_expression::generate_like_pattern; -use databend_common_expression::types::boolean::BooleanDomain; -use databend_common_expression::types::nullable::NullableDomain; use databend_common_expression::types::AccessType; use databend_common_expression::types::AnyType; use databend_common_expression::types::BinaryType; @@ -46,30 +63,13 @@ use databend_common_expression::types::StringType; use databend_common_expression::types::TimestampType; use databend_common_expression::types::UInt64Type; use databend_common_expression::types::ValueType; +use databend_common_expression::types::boolean::BooleanDomain; +use databend_common_expression::types::nullable::NullableDomain; use databend_common_expression::visit_expr; use databend_common_expression::with_number_mapped_type; -use databend_common_expression::BlockEntry; -use databend_common_expression::Column; -use databend_common_expression::ColumnBuilder; -use databend_common_expression::ColumnId; -use databend_common_expression::ConstantFolder; -use databend_common_expression::DataBlock; -use databend_common_expression::Domain; -use databend_common_expression::Expr; -use databend_common_expression::ExprVisitor; -use databend_common_expression::FieldIndex; -use databend_common_expression::FunctionContext; -use databend_common_expression::LikePattern; -use databend_common_expression::Scalar; -use databend_common_expression::ScalarRef; -use databend_common_expression::TableDataType; -use databend_common_expression::TableField; -use databend_common_expression::TableSchema; -use databend_common_expression::TableSchemaRef; -use databend_common_expression::Value; +use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_functions::scalars::CityHasher64; use databend_common_functions::scalars::DFHash; -use databend_common_functions::BUILTIN_FUNCTIONS; use databend_storages_common_table_meta::meta::SingleColumnMeta; use databend_storages_common_table_meta::meta::StatisticsOfColumns; use databend_storages_common_table_meta::meta::Versioned; @@ -79,6 +79,7 @@ use serde::Deserialize; use serde::Serialize; use super::eliminate_cast::is_injective_cast; +use crate::Index; use crate::eliminate_cast::cast_const; use crate::filters::BlockBloomFilterIndexVersion; use crate::filters::BlockFilter; @@ -91,7 +92,6 @@ use crate::filters::V2BloomBlock; use crate::filters::Xor8Builder; use crate::filters::Xor8Filter; use crate::statistics_to_domain; -use crate::Index; const NGRAM_HASH_SEED: u64 = 1575457558; @@ -900,11 +900,14 @@ where T: EqVisitor if id.name() == "like" { // patterns like `Column like ` - if let [Expr::ColumnRef(ColumnRef { - id, - data_type: column_type, - .. - }), Expr::Constant(Constant { scalar, .. })] = args.as_slice() + if let [ + Expr::ColumnRef(ColumnRef { + id, + data_type: column_type, + .. + }), + Expr::Constant(Constant { scalar, .. }), + ] = args.as_slice() { if let Some(pattern) = scalar.as_string() { match generate_like_pattern(pattern.as_bytes(), 1) { @@ -939,24 +942,30 @@ where T: EqVisitor } else { result = match args.as_slice() { // patterns like `Column = `, ` = Column` - [Expr::ColumnRef(ColumnRef { - id, - data_type: column_type, - .. - }), Expr::Constant(Constant { - scalar, - data_type: scalar_type, - .. - })] - | [Expr::Constant(Constant { - scalar, - data_type: scalar_type, - .. - }), Expr::ColumnRef(ColumnRef { - id, - data_type: column_type, - .. - })] => { + [ + Expr::ColumnRef(ColumnRef { + id, + data_type: column_type, + .. + }), + Expr::Constant(Constant { + scalar, + data_type: scalar_type, + .. + }), + ] + | [ + Expr::Constant(Constant { + scalar, + data_type: scalar_type, + .. + }), + Expr::ColumnRef(ColumnRef { + id, + data_type: column_type, + .. + }), + ] => { // decimal don't respect datatype equal // debug_assert_eq!(scalar_type, column_type); // If the visitor returns a new expression, then replace with the current expression. @@ -968,52 +977,62 @@ where T: EqVisitor } } // patterns like `MapColumn[] = `, ` = MapColumn[]` - [Expr::FunctionCall(FunctionCall { id, args, .. }), Expr::Constant(Constant { - scalar, - data_type: scalar_type, - .. - })] - | [Expr::Constant(Constant { - scalar, - data_type: scalar_type, - .. - }), Expr::FunctionCall(FunctionCall { id, args, .. })] - if id.name() == "get" => - { + [ + Expr::FunctionCall(FunctionCall { id, args, .. }), + Expr::Constant(Constant { + scalar, + data_type: scalar_type, + .. + }), + ] + | [ + Expr::Constant(Constant { + scalar, + data_type: scalar_type, + .. + }), + Expr::FunctionCall(FunctionCall { id, args, .. }), + ] if id.name() == "get" => { self.0 .enter_map_column(*span, args, scalar, scalar_type, return_type, false)? } // patterns like `CAST(MapColumn[] as X) = `, ` = CAST(MapColumn[] as X)` - [Expr::Cast(Cast { - expr: - box Expr::FunctionCall(FunctionCall { - id, - args, - return_type, - .. - }), - dest_type, - .. - }), Expr::Constant(Constant { - scalar, - data_type: scalar_type, - .. - })] - | [Expr::Constant(Constant { - scalar, - data_type: scalar_type, - .. - }), Expr::Cast(Cast { - expr: - box Expr::FunctionCall(FunctionCall { - id, - args, - return_type, - .. - }), - dest_type, - .. - })] if id.name() == "get" => { + [ + Expr::Cast(Cast { + expr: + box Expr::FunctionCall(FunctionCall { + id, + args, + return_type, + .. + }), + dest_type, + .. + }), + Expr::Constant(Constant { + scalar, + data_type: scalar_type, + .. + }), + ] + | [ + Expr::Constant(Constant { + scalar, + data_type: scalar_type, + .. + }), + Expr::Cast(Cast { + expr: + box Expr::FunctionCall(FunctionCall { + id, + args, + return_type, + .. + }), + dest_type, + .. + }), + ] if id.name() == "get" => { // Only support cast variant value in map to string value if return_type.remove_nullable() != DataType::Variant || dest_type.remove_nullable() != DataType::String @@ -1035,18 +1054,24 @@ where T: EqVisitor self.0.enter_cast(cast, constant, false)? } - [func @ Expr::FunctionCall(FunctionCall { - id, - args, - return_type: dest_type, - .. - }), Expr::Constant(constant)] - | [Expr::Constant(constant), func @ Expr::FunctionCall(FunctionCall { - id, - args, - return_type: dest_type, - .. - })] if id.name().starts_with("to_") + [ + func @ Expr::FunctionCall(FunctionCall { + id, + args, + return_type: dest_type, + .. + }), + Expr::Constant(constant), + ] + | [ + Expr::Constant(constant), + func @ Expr::FunctionCall(FunctionCall { + id, + args, + return_type: dest_type, + .. + }), + ] if id.name().starts_with("to_") && args.len() == 1 && func.contains_column_ref() => { diff --git a/src/query/storages/common/index/src/eliminate_cast.rs b/src/query/storages/common/index/src/eliminate_cast.rs index d38f3152151be..85eb2716f1ee8 100644 --- a/src/query/storages/common/index/src/eliminate_cast.rs +++ b/src/query/storages/common/index/src/eliminate_cast.rs @@ -15,16 +15,16 @@ use std::collections::HashMap; use databend_common_ast::Span; -use databend_common_expression::expr::*; -use databend_common_expression::type_check::check_function; -use databend_common_expression::types::DataType; -use databend_common_expression::visit_expr; use databend_common_expression::ConstantFolder; use databend_common_expression::Domain; use databend_common_expression::ExprVisitor; use databend_common_expression::FunctionContext; use databend_common_expression::FunctionRegistry; use databend_common_expression::Scalar; +use databend_common_expression::expr::*; +use databend_common_expression::type_check::check_function; +use databend_common_expression::types::DataType; +use databend_common_expression::visit_expr; use databend_common_functions::BUILTIN_FUNCTIONS; pub(super) fn is_injective_cast(src: &DataType, dest: &DataType) -> bool { diff --git a/src/query/storages/common/index/src/filters/bloom_filter.rs b/src/query/storages/common/index/src/filters/bloom_filter.rs index f03380c946c4a..14ef8f4f837be 100644 --- a/src/query/storages/common/index/src/filters/bloom_filter.rs +++ b/src/query/storages/common/index/src/filters/bloom_filter.rs @@ -23,9 +23,9 @@ use databend_common_exception::ErrorCode; use databend_common_expression::types::DataType; use databend_common_functions::scalars::CityHasher64; +use crate::Index; use crate::filters::Filter; use crate::filters::FilterBuilder; -use crate::Index; type UnderType = u64; diff --git a/src/query/storages/common/index/src/filters/xor8/block_filter_versions.rs b/src/query/storages/common/index/src/filters/xor8/block_filter_versions.rs index 8fb092dab0895..61d0a51a9993b 100644 --- a/src/query/storages/common/index/src/filters/xor8/block_filter_versions.rs +++ b/src/query/storages/common/index/src/filters/xor8/block_filter_versions.rs @@ -15,8 +15,8 @@ use std::marker::PhantomData; use databend_common_exception::ErrorCode; -use databend_storages_common_table_meta::meta::testify_version; use databend_storages_common_table_meta::meta::Versioned; +use databend_storages_common_table_meta::meta::testify_version; use crate::filters::BlockFilter; diff --git a/src/query/storages/common/index/src/filters/xor8/mod.rs b/src/query/storages/common/index/src/filters/xor8/mod.rs index 60e85cc816a29..88553683fe45c 100644 --- a/src/query/storages/common/index/src/filters/xor8/mod.rs +++ b/src/query/storages/common/index/src/filters/xor8/mod.rs @@ -26,10 +26,10 @@ use databend_common_exception::ErrorCode; pub use xor8_filter::Xor8Builder; pub use xor8_filter::Xor8Filter; -pub use crate::filters::bloom_filter::BloomBuilder; -pub use crate::filters::bloom_filter::BloomFilter; use crate::filters::Filter; use crate::filters::FilterBuilder; +pub use crate::filters::bloom_filter::BloomBuilder; +pub use crate::filters::bloom_filter::BloomFilter; #[derive(PartialEq)] pub enum FilterImpl { diff --git a/src/query/storages/common/index/src/filters/xor8/xor8_filter.rs b/src/query/storages/common/index/src/filters/xor8/xor8_filter.rs index a4b4b174b4185..04bf61d0928fe 100644 --- a/src/query/storages/common/index/src/filters/xor8/xor8_filter.rs +++ b/src/query/storages/common/index/src/filters/xor8/xor8_filter.rs @@ -25,9 +25,9 @@ use databend_common_expression::types::DataType; use parquet::data_type::AsBytes; use xorfilter::Xor8; +use crate::Index; use crate::filters::Filter; use crate::filters::FilterBuilder; -use crate::Index; /// A builder that builds a xor8 filter. /// diff --git a/src/query/storages/common/index/src/hnsw_index/common/bitpacking.rs b/src/query/storages/common/index/src/hnsw_index/common/bitpacking.rs index 54015bbed6b3b..e77cbedeecf25 100644 --- a/src/query/storages/common/index/src/hnsw_index/common/bitpacking.rs +++ b/src/query/storages/common/index/src/hnsw_index/common/bitpacking.rs @@ -258,10 +258,10 @@ mod tests { use num_traits::ConstZero; use num_traits::PrimInt; use num_traits::Unsigned; - use rand::distributions::uniform::SampleUniform; - use rand::rngs::StdRng; use rand::Rng as _; use rand::SeedableRng as _; + use rand::distributions::uniform::SampleUniform; + use rand::rngs::StdRng; use super::*; diff --git a/src/query/storages/common/index/src/hnsw_index/common/bitpacking_links.rs b/src/query/storages/common/index/src/hnsw_index/common/bitpacking_links.rs index 388a0753add57..c56971918781f 100644 --- a/src/query/storages/common/index/src/hnsw_index/common/bitpacking_links.rs +++ b/src/query/storages/common/index/src/hnsw_index/common/bitpacking_links.rs @@ -13,9 +13,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::bitpacking::packed_bits; use super::bitpacking::BitReader; use super::bitpacking::BitWriter; +use super::bitpacking::packed_bits; /// To simplify value counting, each value should be at least one byte. /// Otherwise the count could would be ambiguous, e.g., a 2-byte slice of 5-bit @@ -82,7 +82,7 @@ pub fn iterate_packed_links( links: &[u8], bits_per_unsorted: u8, sorted_count: usize, -) -> PackedLinksIterator { +) -> PackedLinksIterator<'_> { let mut reader = BitReader::new(links); let mut remaining_bits = links.len() * u8::BITS as usize; diff --git a/src/query/storages/common/index/src/hnsw_index/common/bitpacking_ordered.rs b/src/query/storages/common/index/src/hnsw_index/common/bitpacking_ordered.rs index 667185ebad02a..9999f79145b95 100644 --- a/src/query/storages/common/index/src/hnsw_index/common/bitpacking_ordered.rs +++ b/src/query/storages/common/index/src/hnsw_index/common/bitpacking_ordered.rs @@ -56,15 +56,15 @@ use std::ops::RangeInclusive; use thiserror::Error; -use zerocopy::little_endian::U64; use zerocopy::FromBytes; use zerocopy::Immutable; use zerocopy::IntoBytes; use zerocopy::KnownLayout; +use zerocopy::little_endian::U64; +use super::bitpacking::BitWriter; use super::bitpacking::make_bitmask; use super::bitpacking::packed_bits; -use super::bitpacking::BitWriter; /// The size of the tail padding. /// These extra 7 bytes after the last chunk allows the decompressor to safely diff --git a/src/query/storages/common/index/src/hnsw_index/entry_points.rs b/src/query/storages/common/index/src/hnsw_index/entry_points.rs index cb9970583f867..bae5a7123004c 100644 --- a/src/query/storages/common/index/src/hnsw_index/entry_points.rs +++ b/src/query/storages/common/index/src/hnsw_index/entry_points.rs @@ -130,8 +130,8 @@ impl EntryPoints { #[cfg(test)] mod tests { - use rand::thread_rng; use rand::Rng; + use rand::thread_rng; use super::*; diff --git a/src/query/storages/common/index/src/hnsw_index/graph_layers.rs b/src/query/storages/common/index/src/hnsw_index/graph_layers.rs index bc5e31f53a224..dd27e819f7cdd 100644 --- a/src/query/storages/common/index/src/hnsw_index/graph_layers.rs +++ b/src/query/storages/common/index/src/hnsw_index/graph_layers.rs @@ -61,7 +61,7 @@ pub struct GraphLayers { } pub trait GraphLayersBase { - fn get_visited_list_from_pool(&self) -> VisitedListHandle; + fn get_visited_list_from_pool(&self) -> VisitedListHandle<'_>; fn links_map(&self, point_id: PointOffsetType, level: usize, f: F) where F: FnMut(PointOffsetType); @@ -171,7 +171,7 @@ pub trait GraphLayersBase { } impl GraphLayersBase for GraphLayers { - fn get_visited_list_from_pool(&self) -> VisitedListHandle { + fn get_visited_list_from_pool(&self) -> VisitedListHandle<'_> { self.visited_pool.get(self.links.num_points()) } @@ -181,11 +181,7 @@ impl GraphLayersBase for GraphLayers { } fn get_m(&self, level: usize) -> usize { - if level == 0 { - self.m0 - } else { - self.m - } + if level == 0 { self.m0 } else { self.m } } } diff --git a/src/query/storages/common/index/src/hnsw_index/graph_layers_builder.rs b/src/query/storages/common/index/src/hnsw_index/graph_layers_builder.rs index c226dcc146746..d1c2018fa2982 100644 --- a/src/query/storages/common/index/src/hnsw_index/graph_layers_builder.rs +++ b/src/query/storages/common/index/src/hnsw_index/graph_layers_builder.rs @@ -26,8 +26,8 @@ use databend_common_exception::Result; use parking_lot::Mutex; use parking_lot::MutexGuard; use parking_lot::RwLock; -use rand::distributions::Uniform; use rand::Rng; +use rand::distributions::Uniform; use super::graph_layers::GraphLayerData; use super::graph_links::GraphLinksFormat; @@ -70,7 +70,7 @@ pub struct GraphLayersBuilder { } impl GraphLayersBase for GraphLayersBuilder { - fn get_visited_list_from_pool(&self) -> VisitedListHandle { + fn get_visited_list_from_pool(&self) -> VisitedListHandle<'_> { self.visited_pool.get(self.num_points()) } @@ -86,17 +86,13 @@ impl GraphLayersBase for GraphLayersBuilder { } fn get_m(&self, level: usize) -> usize { - if level == 0 { - self.m0 - } else { - self.m - } + if level == 0 { self.m0 } else { self.m } } } impl GraphLayersBuilder { #[allow(dead_code)] - pub fn get_entry_points(&self) -> MutexGuard { + pub fn get_entry_points(&self) -> MutexGuard<'_, EntryPoints> { self.entry_points.lock() } diff --git a/src/query/storages/common/index/src/hnsw_index/graph_links.rs b/src/query/storages/common/index/src/hnsw_index/graph_links.rs index 7574de694a412..3e44f381be3e1 100644 --- a/src/query/storages/common/index/src/hnsw_index/graph_links.rs +++ b/src/query/storages/common/index/src/hnsw_index/graph_links.rs @@ -79,7 +79,7 @@ impl GraphLinks { Self::try_new(data.to_vec(), |x| GraphLinksView::load(x, format)) } - fn view(&self) -> &GraphLinksView { + fn view(&self) -> &GraphLinksView<'_> { self.borrow_dependent() } @@ -105,7 +105,7 @@ impl GraphLinks { } #[inline] - pub fn links(&self, point_id: PointOffsetType, level: usize) -> LinksIterator { + pub fn links(&self, point_id: PointOffsetType, level: usize) -> LinksIterator<'_> { self.view().links(point_id, level) } diff --git a/src/query/storages/common/index/src/hnsw_index/graph_links/header.rs b/src/query/storages/common/index/src/hnsw_index/graph_links/header.rs index 3a538a75e9543..6f1f6a57a1aad 100644 --- a/src/query/storages/common/index/src/hnsw_index/graph_links/header.rs +++ b/src/query/storages/common/index/src/hnsw_index/graph_links/header.rs @@ -13,11 +13,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use zerocopy::little_endian::U64 as LittleU64; use zerocopy::FromBytes; use zerocopy::Immutable; use zerocopy::IntoBytes; use zerocopy::KnownLayout; +use zerocopy::little_endian::U64 as LittleU64; use crate::hnsw_index::common::bitpacking_ordered; diff --git a/src/query/storages/common/index/src/hnsw_index/graph_links/serializer.rs b/src/query/storages/common/index/src/hnsw_index/graph_links/serializer.rs index 0942eeeab942a..a8392e361e7ea 100644 --- a/src/query/storages/common/index/src/hnsw_index/graph_links/serializer.rs +++ b/src/query/storages/common/index/src/hnsw_index/graph_links/serializer.rs @@ -19,17 +19,17 @@ use std::mem::size_of; use std::mem::take; use itertools::Either; -use zerocopy::little_endian::U64 as LittleU64; use zerocopy::IntoBytes as AsBytes; +use zerocopy::little_endian::U64 as LittleU64; -use super::header::HeaderCompressed; -use super::header::HeaderPlain; -use super::header::HEADER_VERSION_COMPRESSED; use super::GraphLinks; use super::GraphLinksFormat; +use super::header::HEADER_VERSION_COMPRESSED; +use super::header::HeaderCompressed; +use super::header::HeaderPlain; use crate::hnsw_index::common::bitpacking::packed_bits; -use crate::hnsw_index::common::bitpacking_links::pack_links; use crate::hnsw_index::common::bitpacking_links::MIN_BITS_PER_VALUE; +use crate::hnsw_index::common::bitpacking_links::pack_links; use crate::hnsw_index::common::bitpacking_ordered; use crate::hnsw_index::common::types::PointOffsetType; use crate::hnsw_index::common::zeros::WriteZerosExt; diff --git a/src/query/storages/common/index/src/hnsw_index/graph_links/view.rs b/src/query/storages/common/index/src/hnsw_index/graph_links/view.rs index e466c18d54fb8..04b48b26159c6 100644 --- a/src/query/storages/common/index/src/hnsw_index/graph_links/view.rs +++ b/src/query/storages/common/index/src/hnsw_index/graph_links/view.rs @@ -19,18 +19,18 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use itertools::Either; use itertools::Itertools as _; -use zerocopy::native_endian::U64 as NativeU64; use zerocopy::FromBytes; use zerocopy::Immutable; +use zerocopy::native_endian::U64 as NativeU64; +use super::GraphLinksFormat; +use super::header::HEADER_VERSION_COMPRESSED; use super::header::HeaderCompressed; use super::header::HeaderPlain; -use super::header::HEADER_VERSION_COMPRESSED; -use super::GraphLinksFormat; use crate::hnsw_index::common::bitpacking::packed_bits; -use crate::hnsw_index::common::bitpacking_links::iterate_packed_links; -use crate::hnsw_index::common::bitpacking_links::PackedLinksIterator; use crate::hnsw_index::common::bitpacking_links::MIN_BITS_PER_VALUE; +use crate::hnsw_index::common::bitpacking_links::PackedLinksIterator; +use crate::hnsw_index::common::bitpacking_links::iterate_packed_links; use crate::hnsw_index::common::bitpacking_ordered; use crate::hnsw_index::common::types::PointOffsetType; @@ -66,14 +66,14 @@ pub(super) enum CompressionInfo<'a> { } impl GraphLinksView<'_> { - pub(super) fn load(data: &[u8], format: GraphLinksFormat) -> Result { + pub(super) fn load(data: &[u8], format: GraphLinksFormat) -> Result> { match format { GraphLinksFormat::Compressed => Self::load_compressed(data), GraphLinksFormat::Plain => Self::load_plain(data), } } - fn load_plain(data: &[u8]) -> Result { + fn load_plain(data: &[u8]) -> Result> { let (header, data) = HeaderPlain::ref_from_prefix(data).map_err(|_| error_unsufficent_size())?; let (level_offsets, data) = @@ -89,7 +89,7 @@ impl GraphLinksView<'_> { }) } - fn load_compressed(data: &[u8]) -> Result { + fn load_compressed(data: &[u8]) -> Result> { let (header, data) = HeaderCompressed::ref_from_prefix(data).map_err(|_| error_unsufficent_size())?; debug_assert_eq!(header.version.get(), HEADER_VERSION_COMPRESSED); @@ -118,7 +118,7 @@ impl GraphLinksView<'_> { }) } - pub(super) fn links(&self, point_id: PointOffsetType, level: usize) -> LinksIterator { + pub(super) fn links(&self, point_id: PointOffsetType, level: usize) -> LinksIterator<'_> { let idx = if level == 0 { point_id as usize } else { diff --git a/src/query/storages/common/index/src/hnsw_index/hnsw.rs b/src/query/storages/common/index/src/hnsw_index/hnsw.rs index 801809d981c1b..ce93c426912d0 100644 --- a/src/query/storages/common/index/src/hnsw_index/hnsw.rs +++ b/src/query/storages/common/index/src/hnsw_index/hnsw.rs @@ -17,23 +17,23 @@ use std::sync::atomic::AtomicBool; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::Buffer; -use databend_common_expression::types::DataType; -use databend_common_expression::types::VectorColumn; -use databend_common_expression::types::VectorColumnBuilder; -use databend_common_expression::types::VectorScalar; -use databend_common_expression::types::F32; use databend_common_expression::BlockEntry; use databend_common_expression::Column; use databend_common_expression::ColumnId; use databend_common_expression::Scalar; use databend_common_expression::TableDataType; use databend_common_expression::TableField; +use databend_common_expression::types::Buffer; +use databend_common_expression::types::DataType; +use databend_common_expression::types::F32; +use databend_common_expression::types::VectorColumn; +use databend_common_expression::types::VectorColumnBuilder; +use databend_common_expression::types::VectorScalar; use log::error; use rand::thread_rng; +use rayon::ThreadPoolBuilder; use rayon::iter::IntoParallelIterator; use rayon::prelude::*; -use rayon::ThreadPoolBuilder; use crate::hnsw_index::common::types::PointOffsetType; use crate::hnsw_index::common::types::ScoredPointOffset; @@ -45,10 +45,10 @@ use crate::hnsw_index::point_scorer::FilteredScorer; use crate::hnsw_index::point_scorer::OriginalRawScorer; use crate::hnsw_index::point_scorer::QuantizedRawScorer; use crate::hnsw_index::point_scorer::RawScorer; -use crate::hnsw_index::quantization::encoded_vectors::EncodedVectors; use crate::hnsw_index::quantization::DistanceType; use crate::hnsw_index::quantization::EncodedVectorsU8; use crate::hnsw_index::quantization::VectorParameters; +use crate::hnsw_index::quantization::encoded_vectors::EncodedVectors; pub const SINGLE_THREADED_HNSW_BUILD_THRESHOLD: usize = 256; diff --git a/src/query/storages/common/index/src/hnsw_index/point_scorer.rs b/src/query/storages/common/index/src/hnsw_index/point_scorer.rs index 1c1c901103809..a8d210d4729ad 100644 --- a/src/query/storages/common/index/src/hnsw_index/point_scorer.rs +++ b/src/query/storages/common/index/src/hnsw_index/point_scorer.rs @@ -13,18 +13,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_expression::types::VectorScalarRef; -use databend_common_expression::types::F32; use databend_common_expression::Column; use databend_common_expression::ScalarRef; +use databend_common_expression::types::F32; +use databend_common_expression::types::VectorScalarRef; +use crate::DistanceType; use crate::hnsw_index::common::types::PointOffsetType; use crate::hnsw_index::common::types::ScoreType; use crate::hnsw_index::common::types::ScoredPointOffset; -use crate::hnsw_index::quantization::encoded_vectors::EncodedVectors; use crate::hnsw_index::quantization::EncodedQueryU8; use crate::hnsw_index::quantization::EncodedVectorsU8; -use crate::DistanceType; +use crate::hnsw_index::quantization::encoded_vectors::EncodedVectors; pub enum RawScorer<'a> { Original(OriginalRawScorer<'a>), diff --git a/src/query/storages/common/index/src/hnsw_index/quantization/encoded_vectors_u8.rs b/src/query/storages/common/index/src/hnsw_index/quantization/encoded_vectors_u8.rs index 29cc2da494c0c..fb7c1a39ff854 100644 --- a/src/query/storages/common/index/src/hnsw_index/quantization/encoded_vectors_u8.rs +++ b/src/query/storages/common/index/src/hnsw_index/quantization/encoded_vectors_u8.rs @@ -23,10 +23,10 @@ use serde::Serialize; use crate::hnsw_index::quantization::encoded_storage::EncodedStorage; use crate::hnsw_index::quantization::encoded_storage::EncodedStorageBuilder; -use crate::hnsw_index::quantization::encoded_vectors::validate_vector_parameters; use crate::hnsw_index::quantization::encoded_vectors::DistanceType; use crate::hnsw_index::quantization::encoded_vectors::EncodedVectors; use crate::hnsw_index::quantization::encoded_vectors::VectorParameters; +use crate::hnsw_index::quantization::encoded_vectors::validate_vector_parameters; use crate::hnsw_index::quantization::quantile::find_min_max_from_iter; use crate::hnsw_index::quantization::quantile::find_quantile_interval; diff --git a/src/query/storages/common/index/src/hnsw_index/quantization/quantile.rs b/src/query/storages/common/index/src/hnsw_index/quantization/quantile.rs index fca620aec2b11..23e4c4277c2a6 100644 --- a/src/query/storages/common/index/src/hnsw_index/quantization/quantile.rs +++ b/src/query/storages/common/index/src/hnsw_index/quantization/quantile.rs @@ -15,9 +15,9 @@ use feistel_permutation_rs::DefaultBuildHasher; use feistel_permutation_rs::Permutation; -use rand::rngs::SmallRng; use rand::Rng; use rand::SeedableRng; +use rand::rngs::SmallRng; pub const QUANTILE_SAMPLE_SIZE: usize = 100_000; @@ -49,7 +49,7 @@ pub(crate) fn find_quantile_interval<'a>( let slice_size = std::cmp::min(count, QUANTILE_SAMPLE_SIZE); let mut rng = SmallRng::from_entropy(); - let seed: u64 = rng.gen(); + let seed: u64 = rng.r#gen(); let permutor = Permutation::new(count as u64, seed, DefaultBuildHasher::new()); let mut selected_vectors: Vec = permutor .iter() diff --git a/src/query/storages/common/index/src/hnsw_index/visited_pool.rs b/src/query/storages/common/index/src/hnsw_index/visited_pool.rs index 57b5dd9097fba..e48b53838273b 100644 --- a/src/query/storages/common/index/src/hnsw_index/visited_pool.rs +++ b/src/query/storages/common/index/src/hnsw_index/visited_pool.rs @@ -126,7 +126,7 @@ impl VisitedPool { } } - pub fn get(&self, num_points: usize) -> VisitedListHandle { + pub fn get(&self, num_points: usize) -> VisitedListHandle<'_> { // If there are more concurrent requests, a new temporary list is created dynamically. // This limit is implemented to prevent memory leakage. match self.pool.write().pop() { diff --git a/src/query/storages/common/index/src/inverted_index.rs b/src/query/storages/common/index/src/inverted_index.rs index c825af9987640..8ef2dd2c3daae 100644 --- a/src/query/storages/common/index/src/inverted_index.rs +++ b/src/query/storages/common/index/src/inverted_index.rs @@ -54,22 +54,22 @@ use bytes::Bytes; use crc32fast::Hasher; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::F32; use databend_common_expression::Scalar; use databend_common_expression::TableDataType; use databend_common_expression::TableField; -use databend_storages_common_table_meta::meta::testify_version; +use databend_common_expression::types::F32; use databend_storages_common_table_meta::meta::SingleColumnMeta; use databend_storages_common_table_meta::meta::Versioned; +use databend_storages_common_table_meta::meta::testify_version; +use levenshtein_automata::DFA; use levenshtein_automata::Distance; use levenshtein_automata::LevenshteinAutomatonBuilder; -use levenshtein_automata::DFA; use log::warn; use parquet::format::FileMetaData; use roaring::RoaringTreemap; -use tantivy::directory::error::DeleteError; -use tantivy::directory::error::OpenReadError; -use tantivy::directory::error::OpenWriteError; +use tantivy::Directory; +use tantivy::Term; +use tantivy::Version; use tantivy::directory::AntiCallToken; use tantivy::directory::FileHandle; use tantivy::directory::FileSlice; @@ -78,6 +78,9 @@ use tantivy::directory::TerminatingWrite; use tantivy::directory::WatchCallback; use tantivy::directory::WatchHandle; use tantivy::directory::WritePtr; +use tantivy::directory::error::DeleteError; +use tantivy::directory::error::OpenReadError; +use tantivy::directory::error::OpenWriteError; use tantivy::fieldnorm::FieldNormReader; use tantivy::positions::PositionReader; use tantivy::postings::BlockSegmentPostings; @@ -98,9 +101,6 @@ use tantivy::query::QueryClone; use tantivy::query::TermQuery; use tantivy::schema::Field; use tantivy::version; -use tantivy::Directory; -use tantivy::Term; -use tantivy::Version; use tantivy_common::BinarySerializable; use tantivy_common::HasLen; use tantivy_common::VInt; @@ -1570,6 +1570,7 @@ impl Directory for InvertedIndexDirectory { impl Versioned<0> for InvertedIndexFile {} +#[allow(dead_code)] pub enum InvertedIndexFileVersion { V0(PhantomData), } diff --git a/src/query/storages/common/index/src/lib.rs b/src/query/storages/common/index/src/lib.rs index 3ad4d62d94944..a32f2be5dae78 100644 --- a/src/query/storages/common/index/src/lib.rs +++ b/src/query/storages/common/index/src/lib.rs @@ -12,7 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(clippy::uninlined_format_args)] +#![allow( + clippy::cloned_ref_to_slice_refs, + clippy::collapsible_if, + clippy::manual_is_multiple_of, + clippy::needless_range_loop, + clippy::uninlined_format_args +)] #![feature(box_patterns)] #![feature(never_type)] @@ -39,14 +45,14 @@ pub use hnsw_index::ScoredPointOffset; pub use hnsw_index::VectorIndexFile; pub use hnsw_index::VectorIndexMeta; pub use index::Index; -pub use inverted_index::build_tantivy_footer; -pub use inverted_index::extract_component_fields; -pub use inverted_index::extract_fsts; pub use inverted_index::DocIdsCollector; pub use inverted_index::InvertedIndexDirectory; pub use inverted_index::InvertedIndexFile; pub use inverted_index::InvertedIndexMeta; pub use inverted_index::TermReader; +pub use inverted_index::build_tantivy_footer; +pub use inverted_index::extract_component_fields; +pub use inverted_index::extract_fsts; pub use page_index::PageIndex; -pub use range_index::statistics_to_domain; pub use range_index::RangeIndex; +pub use range_index::statistics_to_domain; diff --git a/src/query/storages/common/index/src/page_index.rs b/src/query/storages/common/index/src/page_index.rs index 61555aa50784f..fa599f8ac2912 100644 --- a/src/query/storages/common/index/src/page_index.rs +++ b/src/query/storages/common/index/src/page_index.rs @@ -19,9 +19,6 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::expr::*; -use databend_common_expression::types::DataType; -use databend_common_expression::visit_expr; use databend_common_expression::Constant; use databend_common_expression::ConstantFolder; use databend_common_expression::DataField; @@ -30,6 +27,9 @@ use databend_common_expression::Domain; use databend_common_expression::FunctionContext; use databend_common_expression::Scalar; use databend_common_expression::TableSchemaRef; +use databend_common_expression::expr::*; +use databend_common_expression::types::DataType; +use databend_common_expression::visit_expr; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_storages_common_table_meta::meta::ClusterStatistics; use databend_storages_common_table_meta::meta::ColumnStatistics; diff --git a/src/query/storages/common/index/src/range_index.rs b/src/query/storages/common/index/src/range_index.rs index 60d4da9655cc4..04466b3c1db56 100644 --- a/src/query/storages/common/index/src/range_index.rs +++ b/src/query/storages/common/index/src/range_index.rs @@ -16,14 +16,17 @@ use std::borrow::Cow; use std::collections::HashMap; use databend_common_exception::Result; +use databend_common_expression::ColumnId; +use databend_common_expression::Constant; +use databend_common_expression::ConstantFolder; +use databend_common_expression::Domain; +use databend_common_expression::Expr; +use databend_common_expression::FunctionContext; +use databend_common_expression::Scalar; +use databend_common_expression::TableDataType; +use databend_common_expression::TableSchemaRef; use databend_common_expression::is_internal_column; use databend_common_expression::is_stream_column; -use databend_common_expression::types::decimal::Decimal128Type; -use databend_common_expression::types::decimal::Decimal256Type; -use databend_common_expression::types::decimal::DecimalDomain; -use databend_common_expression::types::nullable::NullableDomain; -use databend_common_expression::types::number::SimpleDomain; -use databend_common_expression::types::string::StringDomain; use databend_common_expression::types::AccessType; use databend_common_expression::types::ArgType; use databend_common_expression::types::DataType; @@ -33,17 +36,14 @@ use databend_common_expression::types::DecimalScalar; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::NumberType; use databend_common_expression::types::TimestampType; +use databend_common_expression::types::decimal::Decimal128Type; +use databend_common_expression::types::decimal::Decimal256Type; +use databend_common_expression::types::decimal::DecimalDomain; +use databend_common_expression::types::nullable::NullableDomain; +use databend_common_expression::types::number::SimpleDomain; +use databend_common_expression::types::string::StringDomain; use databend_common_expression::visit_expr; use databend_common_expression::with_number_mapped_type; -use databend_common_expression::ColumnId; -use databend_common_expression::Constant; -use databend_common_expression::ConstantFolder; -use databend_common_expression::Domain; -use databend_common_expression::Expr; -use databend_common_expression::FunctionContext; -use databend_common_expression::Scalar; -use databend_common_expression::TableDataType; -use databend_common_expression::TableSchemaRef; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_storages_common_table_meta::meta::ColumnStatistics; use databend_storages_common_table_meta::meta::StatisticsOfColumns; diff --git a/src/query/storages/common/index/tests/it/bloom_pruner.rs b/src/query/storages/common/index/tests/it/bloom_pruner.rs index 731ad3c286bc3..caa0d0d4cf75b 100644 --- a/src/query/storages/common/index/tests/it/bloom_pruner.rs +++ b/src/query/storages/common/index/tests/it/bloom_pruner.rs @@ -17,24 +17,6 @@ use std::collections::HashMap; use std::io::Write; use std::sync::Arc; -use databend_common_expression::type_check; -use databend_common_expression::type_check::check_function; -use databend_common_expression::types::map::KvColumn; -use databend_common_expression::types::map::KvPair; -use databend_common_expression::types::AnyType; -use databend_common_expression::types::ArrayColumn; -use databend_common_expression::types::Buffer; -use databend_common_expression::types::DataType; -use databend_common_expression::types::DateType; -use databend_common_expression::types::Int16Type; -use databend_common_expression::types::Int32Type; -use databend_common_expression::types::Int8Type; -use databend_common_expression::types::MapType; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::NumberScalar; -use databend_common_expression::types::StringType; -use databend_common_expression::types::UInt8Type; -use databend_common_expression::types::VariantType; use databend_common_expression::BlockEntry; use databend_common_expression::Column; use databend_common_expression::ColumnRef; @@ -50,14 +32,32 @@ use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::Value; -use databend_common_functions::test_utils::parse_raw_expr; +use databend_common_expression::type_check; +use databend_common_expression::type_check::check_function; +use databend_common_expression::types::AnyType; +use databend_common_expression::types::ArrayColumn; +use databend_common_expression::types::Buffer; +use databend_common_expression::types::DataType; +use databend_common_expression::types::DateType; +use databend_common_expression::types::Int8Type; +use databend_common_expression::types::Int16Type; +use databend_common_expression::types::Int32Type; +use databend_common_expression::types::MapType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::NumberScalar; +use databend_common_expression::types::StringType; +use databend_common_expression::types::UInt8Type; +use databend_common_expression::types::VariantType; +use databend_common_expression::types::map::KvColumn; +use databend_common_expression::types::map::KvPair; use databend_common_functions::BUILTIN_FUNCTIONS; -use databend_storages_common_index::filters::Xor8Filter; +use databend_common_functions::test_utils::parse_raw_expr; use databend_storages_common_index::BloomIndex; use databend_storages_common_index::BloomIndexBuilder; use databend_storages_common_index::FilterEvalResult; use databend_storages_common_index::Index; use databend_storages_common_index::NgramArgs; +use databend_storages_common_index::filters::Xor8Filter; use databend_storages_common_table_meta::meta::ColumnStatistics; use goldenfile::Mint; diff --git a/src/query/storages/common/index/tests/it/eliminate_cast.rs b/src/query/storages/common/index/tests/it/eliminate_cast.rs index 75290df97b879..b1e7294f9ccc5 100644 --- a/src/query/storages/common/index/tests/it/eliminate_cast.rs +++ b/src/query/storages/common/index/tests/it/eliminate_cast.rs @@ -14,15 +14,15 @@ use std::io::Write; +use databend_common_expression::Domain; +use databend_common_expression::Expr; +use databend_common_expression::Scalar; use databend_common_expression::type_check; use databend_common_expression::types::ArgType; use databend_common_expression::types::DataType; use databend_common_expression::types::Int32Type; -use databend_common_expression::Domain; -use databend_common_expression::Expr; -use databend_common_expression::Scalar; -use databend_common_functions::test_utils::parse_raw_expr; use databend_common_functions::BUILTIN_FUNCTIONS; +use databend_common_functions::test_utils::parse_raw_expr; use databend_storages_common_index::eliminate_cast; use goldenfile::Mint; diff --git a/src/query/storages/common/index/tests/it/page_pruner.rs b/src/query/storages/common/index/tests/it/page_pruner.rs index 3dd4a9740d4fa..eb49807f94d8c 100644 --- a/src/query/storages/common/index/tests/it/page_pruner.rs +++ b/src/query/storages/common/index/tests/it/page_pruner.rs @@ -16,14 +16,14 @@ use std::io::Write; use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::types::ArgType; -use databend_common_expression::types::Int32Type; -use databend_common_expression::types::NumberDataType; use databend_common_expression::FunctionContext; use databend_common_expression::Scalar; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; +use databend_common_expression::types::ArgType; +use databend_common_expression::types::Int32Type; +use databend_common_expression::types::NumberDataType; use databend_storages_common_index::PageIndex; use databend_storages_common_table_meta::meta::ClusterStatistics; use goldenfile::Mint; diff --git a/src/query/storages/common/index/tests/it/range_pruner.rs b/src/query/storages/common/index/tests/it/range_pruner.rs index 88f3d2baeec79..a228de95ad57d 100644 --- a/src/query/storages/common/index/tests/it/range_pruner.rs +++ b/src/query/storages/common/index/tests/it/range_pruner.rs @@ -16,15 +16,15 @@ use std::io::Write; use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::types::ArgType; -use databend_common_expression::types::DataType; -use databend_common_expression::types::Int32Type; -use databend_common_expression::types::NumberDataType; use databend_common_expression::FunctionContext; use databend_common_expression::Scalar; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; +use databend_common_expression::types::ArgType; +use databend_common_expression::types::DataType; +use databend_common_expression::types::Int32Type; +use databend_common_expression::types::NumberDataType; use databend_storages_common_index::RangeIndex; use databend_storages_common_table_meta::meta::ColumnStatistics; use databend_storages_common_table_meta::meta::StatisticsOfColumns; diff --git a/src/query/storages/common/index/tests/it/xor8.rs b/src/query/storages/common/index/tests/it/xor8.rs index 7a874833f452c..0917ddf0598e1 100644 --- a/src/query/storages/common/index/tests/it/xor8.rs +++ b/src/query/storages/common/index/tests/it/xor8.rs @@ -13,16 +13,16 @@ // limitations under the License. use databend_common_exception::Result; -use databend_common_expression::types::number::NumberScalar; use databend_common_expression::ScalarRef; +use databend_common_expression::types::number::NumberScalar; use databend_storages_common_index::filters::Filter; use databend_storages_common_index::filters::FilterBuilder; use databend_storages_common_index::filters::Xor8Builder; use databend_storages_common_index::filters::Xor8Filter; -use rand::prelude::random; -use rand::rngs::StdRng; use rand::Rng; use rand::SeedableRng; +use rand::prelude::random; +use rand::rngs::StdRng; #[test] fn test_xor_bitmap_u64() -> Result<()> { @@ -31,7 +31,7 @@ fn test_xor_bitmap_u64() -> Result<()> { let size = 8 * numbers; let mut rng = StdRng::seed_from_u64(seed); - let keys: Vec = (0..numbers).map(|_| rng.gen::()).collect(); + let keys: Vec = (0..numbers).map(|_| rng.r#gen::()).collect(); let mut builder = Xor8Builder::create(); builder.add_keys(&keys); @@ -66,7 +66,7 @@ fn test_xor_bitmap_bool() -> Result<()> { let size = numbers; let mut rng = StdRng::seed_from_u64(seed); - let keys: Vec = (0..numbers).map(|_| rng.gen::() % 2 == 0).collect(); + let keys: Vec = (0..numbers).map(|_| rng.r#gen::() % 2 == 0).collect(); let mut builder = Xor8Builder::create(); builder.add_keys(&keys); @@ -184,7 +184,7 @@ fn test_xor_bitmap_data_block() -> Result<()> { let size = 8 * numbers; let mut rng = StdRng::seed_from_u64(seed); - let keys: Vec = (0..numbers).map(|_| rng.gen::()).collect(); + let keys: Vec = (0..numbers).map(|_| rng.r#gen::()).collect(); let mut builder = Xor8Builder::create(); keys.iter() diff --git a/src/query/storages/common/io/src/merge_io_reader.rs b/src/query/storages/common/io/src/merge_io_reader.rs index 13f5a0353938e..9c9caf4e63dc6 100644 --- a/src/query/storages/common/io/src/merge_io_reader.rs +++ b/src/query/storages/common/io/src/merge_io_reader.rs @@ -25,9 +25,9 @@ use databend_common_metrics::storage::*; use futures::future::try_join_all; use opendal::Operator; -use crate::merge_io_result::OwnerMemory; use crate::MergeIOReadResult; use crate::ReadSettings; +use crate::merge_io_result::OwnerMemory; pub struct MergeIOReader {} diff --git a/src/query/storages/common/pruner/src/block_meta.rs b/src/query/storages/common/pruner/src/block_meta.rs index e54db7df3e4bb..be0610676880b 100644 --- a/src/query/storages/common/pruner/src/block_meta.rs +++ b/src/query/storages/common/pruner/src/block_meta.rs @@ -18,11 +18,11 @@ use std::ops::Range; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::number::F32; use databend_common_expression::BlockMetaInfo; use databend_common_expression::BlockMetaInfoDowncast; use databend_common_expression::BlockMetaInfoPtr; use databend_common_expression::ColumnId; +use databend_common_expression::types::number::F32; use databend_storages_common_table_meta::meta::VirtualColumnMeta; #[derive(serde::Serialize, serde::Deserialize, Clone, Debug, Default, PartialEq, Eq)] diff --git a/src/query/storages/common/pruner/src/internal_column_pruner.rs b/src/query/storages/common/pruner/src/internal_column_pruner.rs index a84852756541c..54d1956befe6e 100644 --- a/src/query/storages/common/pruner/src/internal_column_pruner.rs +++ b/src/query/storages/common/pruner/src/internal_column_pruner.rs @@ -15,15 +15,15 @@ use std::collections::HashMap; use std::sync::Arc; -use databend_common_expression::types::string::StringDomain; +use databend_common_expression::BLOCK_NAME_COL_NAME; use databend_common_expression::Constant; use databend_common_expression::ConstantFolder; use databend_common_expression::Domain; use databend_common_expression::Expr; use databend_common_expression::FunctionContext; -use databend_common_expression::Scalar; -use databend_common_expression::BLOCK_NAME_COL_NAME; use databend_common_expression::SEGMENT_NAME_COL_NAME; +use databend_common_expression::Scalar; +use databend_common_expression::types::string::StringDomain; use databend_common_functions::BUILTIN_FUNCTIONS; /// Only support `_segment_name` and `_block_name` now. diff --git a/src/query/storages/common/pruner/src/limiter_pruner.rs b/src/query/storages/common/pruner/src/limiter_pruner.rs index 8b2c4c9f1ee34..0ce1ca294d35a 100644 --- a/src/query/storages/common/pruner/src/limiter_pruner.rs +++ b/src/query/storages/common/pruner/src/limiter_pruner.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::Arc; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; -use std::sync::Arc; pub trait Limiter { fn exceeded(&self) -> bool; diff --git a/src/query/storages/common/pruner/src/topn_pruner.rs b/src/query/storages/common/pruner/src/topn_pruner.rs index 0b291da08827c..92809572d685d 100644 --- a/src/query/storages/common/pruner/src/topn_pruner.rs +++ b/src/query/storages/common/pruner/src/topn_pruner.rs @@ -16,11 +16,11 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::number::F32; use databend_common_expression::RemoteExpr; +use databend_common_expression::SEARCH_SCORE_COL_NAME; use databend_common_expression::TableDataType; use databend_common_expression::TableSchemaRef; -use databend_common_expression::SEARCH_SCORE_COL_NAME; +use databend_common_expression::types::number::F32; use databend_storages_common_table_meta::meta::BlockMeta; use crate::BlockMetaIndex; @@ -314,13 +314,13 @@ fn block_score_range(scores: &[F32]) -> Option<(F32, F32)> { mod tests { use std::collections::HashMap; - use databend_common_expression::types::number::NumberDataType; - use databend_common_expression::types::DataType; use databend_common_expression::ColumnId; + use databend_common_expression::SEARCH_SCORE_COL_NAME; use databend_common_expression::Scalar; use databend_common_expression::TableField; use databend_common_expression::TableSchema; - use databend_common_expression::SEARCH_SCORE_COL_NAME; + use databend_common_expression::types::DataType; + use databend_common_expression::types::number::NumberDataType; use databend_storages_common_table_meta::meta::ColumnMeta; use databend_storages_common_table_meta::meta::ColumnStatistics; use databend_storages_common_table_meta::meta::Compression; diff --git a/src/query/storages/common/session/src/temp_table.rs b/src/query/storages/common/session/src/temp_table.rs index 4c4948994fd83..ec8989d0ce154 100644 --- a/src/query/storages/common/session/src/temp_table.rs +++ b/src/query/storages/common/session/src/temp_table.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::hash_map::Entry; use std::collections::BTreeMap; use std::collections::HashMap; +use std::collections::hash_map::Entry; use std::sync::Arc; use databend_common_exception::ErrorCode; @@ -41,14 +41,14 @@ use databend_common_meta_app::schema::UpdateTempTableReq; use databend_common_meta_app::schema::UpsertTableOptionReply; use databend_common_meta_app::schema::UpsertTableOptionReq; use databend_common_meta_types::SeqV; -use databend_common_storage::init_operator; use databend_common_storage::DataOperator; -use databend_storages_common_blocks::memory::InMemoryDataKey; +use databend_common_storage::init_operator; use databend_storages_common_blocks::memory::IN_MEMORY_DATA; +use databend_storages_common_blocks::memory::InMemoryDataKey; use databend_storages_common_table_meta::meta::parse_storage_prefix; use databend_storages_common_table_meta::table::OPT_KEY_DATABASE_ID; -use databend_storages_common_table_meta::table_id_ranges::is_temp_table_id; use databend_storages_common_table_meta::table_id_ranges::TEMP_TBL_ID_BEGIN; +use databend_storages_common_table_meta::table_id_ranges::is_temp_table_id; use log::info; use opendal::Operator; use parking_lot::Mutex; @@ -467,8 +467,8 @@ pub async fn drop_all_temp_tables( let num_mem_table = mem_tbl_ids.len(); info!( - "[TEMP TABLE] session={user_name_session_id} starting cleanup, reason = {reason}, {} fuse table, {} mem table." - , num_fuse_table, num_mem_table + "[TEMP TABLE] session={user_name_session_id} starting cleanup, reason = {reason}, {} fuse table, {} mem table.", + num_fuse_table, num_mem_table ); // Clean up each fuse table directory individually with the correct operator diff --git a/src/query/storages/common/stage/src/lib.rs b/src/query/storages/common/stage/src/lib.rs index 5a1208d43f6b9..def1f33b0930a 100644 --- a/src/query/storages/common/stage/src/lib.rs +++ b/src/query/storages/common/stage/src/lib.rs @@ -1,4 +1,3 @@ -#![feature(box_patterns)] // Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![feature(box_patterns)] +#![allow(clippy::cloned_ref_to_slice_refs)] + mod read; mod write; diff --git a/src/query/storages/common/stage/src/read/columnar/arrow_to_variant.rs b/src/query/storages/common/stage/src/read/columnar/arrow_to_variant.rs index eb21107274b2c..af522d2e7ddb5 100644 --- a/src/query/storages/common/stage/src/read/columnar/arrow_to_variant.rs +++ b/src/query/storages/common/stage/src/read/columnar/arrow_to_variant.rs @@ -13,12 +13,12 @@ // limitations under the License. use arrow_array::RecordBatch; -use databend_common_expression::types::binary::BinaryColumnBuilder; -use databend_common_expression::types::variant::cast_scalar_to_variant; use databend_common_expression::Column; use databend_common_expression::DataBlock; use databend_common_expression::DataSchema; use databend_common_expression::TableDataType; +use databend_common_expression::types::binary::BinaryColumnBuilder; +use databend_common_expression::types::variant::cast_scalar_to_variant; use jiff::tz::TimeZone; pub fn read_record_batch_to_variant_column( diff --git a/src/query/storages/common/stage/src/read/columnar/internal_columns.rs b/src/query/storages/common/stage/src/read/columnar/internal_columns.rs index d98e56c01aa38..cde638c2c6148 100644 --- a/src/query/storages/common/stage/src/read/columnar/internal_columns.rs +++ b/src/query/storages/common/stage/src/read/columnar/internal_columns.rs @@ -13,11 +13,11 @@ // limitations under the License. use databend_common_catalog::plan::InternalColumnType; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberColumnBuilder; use databend_common_expression::Column; use databend_common_expression::DataBlock; use databend_common_expression::Scalar; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberColumnBuilder; pub fn add_internal_columns( internal_columns: &[InternalColumnType], diff --git a/src/query/storages/common/stage/src/read/columnar/projection.rs b/src/query/storages/common/stage/src/read/columnar/projection.rs index 659278ee5f0d4..f4c49a5dbf279 100644 --- a/src/query/storages/common/stage/src/read/columnar/projection.rs +++ b/src/query/storages/common/stage/src/read/columnar/projection.rs @@ -13,10 +13,6 @@ // limitations under the License. use databend_common_exception::ErrorCode; -use databend_common_expression::type_check::check_cast; -use databend_common_expression::type_check::check_function; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberScalar; use databend_common_expression::ColumnRef; use databend_common_expression::Constant; use databend_common_expression::Expr; @@ -25,6 +21,10 @@ use databend_common_expression::Scalar; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; +use databend_common_expression::type_check::check_cast; +use databend_common_expression::type_check::check_function; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberScalar; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::principal::NullAs; use databend_common_meta_app::principal::StageFileFormatType; @@ -195,7 +195,9 @@ pub fn project_columnar( match &default_exprs[i] { RemoteDefaultExpr::RemoteExpr(expr) => expr.as_expr(&BUILTIN_FUNCTIONS), RemoteDefaultExpr::Sequence(_) => { - return Err(ErrorCode::BadDataValueType("not supported yet: fill missing column with sequence as default")); + return Err(ErrorCode::BadDataValueType( + "not supported yet: fill missing column with sequence as default", + )); } } } diff --git a/src/query/storages/common/stage/src/read/schema.rs b/src/query/storages/common/stage/src/read/schema.rs index fb19d5c0f4d69..3f170bb3038cf 100644 --- a/src/query/storages/common/stage/src/read/schema.rs +++ b/src/query/storages/common/stage/src/read/schema.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_expression::types::NumberDataType; use databend_common_expression::TableDataType; use databend_common_expression::TableSchema; +use databend_common_expression::types::NumberDataType; pub fn schema_date_time_to_int(schema: &mut TableSchema) { schema diff --git a/src/query/storages/common/table_meta/src/lib.rs b/src/query/storages/common/table_meta/src/lib.rs index 8fbe75653d64f..32a9457e6dc06 100644 --- a/src/query/storages/common/table_meta/src/lib.rs +++ b/src/query/storages/common/table_meta/src/lib.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(clippy::uninlined_format_args)] +#![allow(clippy::collapsible_if, clippy::uninlined_format_args)] #![feature(box_into_inner)] #![feature(iter_order_by)] diff --git a/src/query/storages/common/table_meta/src/meta/column_oriented_segment/mod.rs b/src/query/storages/common/table_meta/src/meta/column_oriented_segment/mod.rs index 86453a71543fc..9c8d578b565d9 100644 --- a/src/query/storages/common/table_meta/src/meta/column_oriented_segment/mod.rs +++ b/src/query/storages/common/table_meta/src/meta/column_oriented_segment/mod.rs @@ -19,12 +19,6 @@ mod schema; mod segment; mod segment_builder; pub use block_meta::AbstractBlockMeta; -pub use schema::block_level_field_names; -pub use schema::col_meta_type; -pub use schema::col_stats_type; -pub use schema::meta_name; -pub use schema::segment_schema; -pub use schema::stat_name; pub use schema::BLOCK_SIZE; pub use schema::BLOOM_FILTER_INDEX_LOCATION; pub use schema::BLOOM_FILTER_INDEX_SIZE; @@ -38,8 +32,14 @@ pub use schema::LOCATION_FORMAT_VERSION; pub use schema::LOCATION_PATH; pub use schema::NGRAM_FILTER_INDEX_SIZE; pub use schema::ROW_COUNT; -pub use segment::deserialize_column_oriented_segment; +pub use schema::block_level_field_names; +pub use schema::col_meta_type; +pub use schema::col_stats_type; +pub use schema::meta_name; +pub use schema::segment_schema; +pub use schema::stat_name; pub use segment::AbstractSegment; pub use segment::ColumnOrientedSegment; +pub use segment::deserialize_column_oriented_segment; pub use segment_builder::ColumnOrientedSegmentBuilder; pub use segment_builder::SegmentBuilder; diff --git a/src/query/storages/common/table_meta/src/meta/column_oriented_segment/schema.rs b/src/query/storages/common/table_meta/src/meta/column_oriented_segment/schema.rs index dbba08ce7c622..1a2e182176b8a 100644 --- a/src/query/storages/common/table_meta/src/meta/column_oriented_segment/schema.rs +++ b/src/query/storages/common/table_meta/src/meta/column_oriented_segment/schema.rs @@ -14,11 +14,11 @@ use std::collections::HashSet; -use databend_common_expression::types::NumberDataType; use databend_common_expression::ColumnId; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; +use databend_common_expression::types::NumberDataType; use crate::meta::supported_stat_type; pub const ROW_COUNT: &str = "row_count"; diff --git a/src/query/storages/common/table_meta/src/meta/column_oriented_segment/segment.rs b/src/query/storages/common/table_meta/src/meta/column_oriented_segment/segment.rs index abe19d7e642a2..ddcca6eca44ae 100644 --- a/src/query/storages/common/table_meta/src/meta/column_oriented_segment/segment.rs +++ b/src/query/storages/common/table_meta/src/meta/column_oriented_segment/segment.rs @@ -20,40 +20,40 @@ use arrow::datatypes::Schema; use bytes::Bytes; use databend_common_column::binview::BinaryViewColumnGeneric; use databend_common_exception::Result; -use databend_common_expression::types::Buffer; use databend_common_expression::Column; use databend_common_expression::ColumnId; use databend_common_expression::DataBlock; use databend_common_expression::DataSchema; use databend_common_expression::TableDataType; use databend_common_expression::TableSchema; -use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; +use databend_common_expression::types::Buffer; use parquet::arrow::ArrowWriter; use parquet::arrow::ProjectionMask; +use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; use parquet::file::metadata::ParquetMetaDataReader; use parquet::file::properties::WriterProperties; -use super::block_meta::AbstractBlockMeta; -use super::block_meta::ColumnOrientedBlockMeta; -use super::meta_name; -use super::stat_name; use super::BLOCK_SIZE; use super::BLOOM_FILTER_INDEX_SIZE; use super::COMPRESSION; use super::FILE_SIZE; use super::LOCATION_PATH; use super::ROW_COUNT; -use crate::meta::column_oriented_segment::LOCATION; -use crate::meta::format::compress; -use crate::meta::format::decode; -use crate::meta::format::decompress; -use crate::meta::format::encode; +use super::block_meta::AbstractBlockMeta; +use super::block_meta::ColumnOrientedBlockMeta; +use super::meta_name; +use super::stat_name; use crate::meta::BlockMeta; use crate::meta::CompactSegmentInfo; use crate::meta::MetaCompression; use crate::meta::MetaEncoding; use crate::meta::SegmentInfo; use crate::meta::Statistics; +use crate::meta::column_oriented_segment::LOCATION; +use crate::meta::format::compress; +use crate::meta::format::decode; +use crate::meta::format::decompress; +use crate::meta::format::encode; pub trait AbstractSegment: Send + Sync + 'static + Sized { type BlockMeta: AbstractBlockMeta; diff --git a/src/query/storages/common/table_meta/src/meta/column_oriented_segment/segment_builder.rs b/src/query/storages/common/table_meta/src/meta/column_oriented_segment/segment_builder.rs index 4790e08400c82..df4aca1250acc 100644 --- a/src/query/storages/common/table_meta/src/meta/column_oriented_segment/segment_builder.rs +++ b/src/query/storages/common/table_meta/src/meta/column_oriented_segment/segment_builder.rs @@ -17,14 +17,6 @@ use std::cmp::Ordering; use std::collections::HashMap; use databend_common_exception::Result; -use databend_common_expression::types::BinaryType; -use databend_common_expression::types::DataType; -use databend_common_expression::types::Int64Type; -use databend_common_expression::types::MutableBitmap; -use databend_common_expression::types::NullableColumn; -use databend_common_expression::types::StringType; -use databend_common_expression::types::UInt64Type; -use databend_common_expression::types::UInt8Type; use databend_common_expression::BlockThresholds; use databend_common_expression::Column; use databend_common_expression::ColumnBuilder; @@ -35,13 +27,19 @@ use databend_common_expression::Scalar; use databend_common_expression::TableDataType; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRef; +use databend_common_expression::types::BinaryType; +use databend_common_expression::types::DataType; +use databend_common_expression::types::Int64Type; +use databend_common_expression::types::MutableBitmap; +use databend_common_expression::types::NullableColumn; +use databend_common_expression::types::StringType; +use databend_common_expression::types::UInt8Type; +use databend_common_expression::types::UInt64Type; use databend_common_functions::aggregates::eval_aggr; +use super::AbstractSegment; use super::schema::segment_schema; use super::segment::ColumnOrientedSegment; -use super::AbstractSegment; -use crate::meta::format::encode; -use crate::meta::supported_stat_type; use crate::meta::AdditionalStatsMeta; use crate::meta::BlockMeta; use crate::meta::ClusterStatistics; @@ -50,6 +48,8 @@ use crate::meta::Location; use crate::meta::MetaEncoding; use crate::meta::Statistics; use crate::meta::VirtualBlockMeta; +use crate::meta::format::encode; +use crate::meta::supported_stat_type; pub trait SegmentBuilder: Send + Sync + 'static { type Segment: AbstractSegment; diff --git a/src/query/storages/common/table_meta/src/meta/format.rs b/src/query/storages/common/table_meta/src/meta/format.rs index 66bdd67ffe3d0..aaef9054ee606 100644 --- a/src/query/storages/common/table_meta/src/meta/format.rs +++ b/src/query/storages/common/table_meta/src/meta/format.rs @@ -22,9 +22,9 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_frozen_api::FrozenAPI; use databend_common_io::prelude::BinaryRead; -use serde::de::DeserializeOwned; use serde::Deserialize; use serde::Serialize; +use serde::de::DeserializeOwned; #[cfg(feature = "dev")] use snap::raw::Decoder as SnapDecoder; #[cfg(feature = "dev")] diff --git a/src/query/storages/common/table_meta/src/meta/mod.rs b/src/query/storages/common/table_meta/src/meta/mod.rs index e3e7c5b7491fe..2aa4337b298de 100644 --- a/src/query/storages/common/table_meta/src/meta/mod.rs +++ b/src/query/storages/common/table_meta/src/meta/mod.rs @@ -29,31 +29,31 @@ mod versions; pub use compression::Compression; // table meta types of current version pub use current::*; -pub use format::decode; -pub(crate) use format::load_json; pub use format::MetaCompression; pub use format::MetaEncoding; pub use format::NUM_BLOCK_ID_BITS; +pub use format::decode; +pub(crate) use format::load_json; pub use statistics::*; // export legacy versioned table meta types locally, // currently, used by versioned readers only pub(crate) use testing::*; +pub use utils::SnapshotTimestampValidationContext; +pub use utils::TEMP_TABLE_STORAGE_PREFIX; +pub use utils::TableMetaTimestamps; +pub use utils::VACUUM2_OBJECT_KEY_PREFIX; pub use utils::parse_storage_prefix; pub use utils::trim_object_prefix; pub use utils::try_extract_uuid_str_from_path; pub use utils::uuid_from_date_time; -pub use utils::SnapshotTimestampValidationContext; -pub use utils::TableMetaTimestamps; -pub use utils::TEMP_TABLE_STORAGE_PREFIX; -pub use utils::VACUUM2_OBJECT_KEY_PREFIX; pub(crate) use utils::*; pub use v0::ColumnMeta as ColumnMetaV0; -pub use versions::testify_version; pub use versions::SegmentInfoVersion; pub use versions::SegmentStatisticsVersion; pub use versions::SnapshotVersion; pub use versions::TableSnapshotStatisticsVersion; pub use versions::Versioned; +pub use versions::testify_version; // - export legacy versioned table meta types for testing purposes // currently, only used by crate `test_kits` diff --git a/src/query/storages/common/table_meta/src/meta/statistics.rs b/src/query/storages/common/table_meta/src/meta/statistics.rs index 590561271d72b..14b916ddb0ec6 100644 --- a/src/query/storages/common/table_meta/src/meta/statistics.rs +++ b/src/query/storages/common/table_meta/src/meta/statistics.rs @@ -17,18 +17,18 @@ use std::io::Cursor; use databend_common_base::base::uuid::Uuid; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::ColumnId; +use databend_common_expression::types::DataType; use databend_common_frozen_api::FrozenAPI; use databend_common_storage::MetaHLL; use serde::Deserialize; use serde::Serialize; +use crate::meta::ColumnStatistics; +use crate::meta::SegmentStatistics; use crate::meta::format::compress; use crate::meta::format::encode; use crate::meta::format::read_and_deserialize; -use crate::meta::ColumnStatistics; -use crate::meta::SegmentStatistics; pub type FormatVersion = u64; pub type SnapshotId = Uuid; diff --git a/src/query/storages/common/table_meta/src/meta/utils.rs b/src/query/storages/common/table_meta/src/meta/utils.rs index dfba6e672c00d..9ddcfc3b206cf 100644 --- a/src/query/storages/common/table_meta/src/meta/utils.rs +++ b/src/query/storages/common/table_meta/src/meta/utils.rs @@ -29,10 +29,10 @@ use databend_common_base::base::uuid::Uuid; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use crate::table::table_storage_prefix; use crate::table::OPT_KEY_DATABASE_ID; use crate::table::OPT_KEY_STORAGE_PREFIX; use crate::table::OPT_KEY_TEMP_PREFIX; +use crate::table::table_storage_prefix; pub const TEMP_TABLE_STORAGE_PREFIX: &str = "_tmp_tbl"; use crate::meta::TableSnapshot; @@ -188,9 +188,9 @@ mod tests { use chrono::Duration; use databend_common_base::base::uuid::Uuid; + use crate::meta::VACUUM2_OBJECT_KEY_PREFIX; use crate::meta::trim_object_prefix; use crate::meta::try_extract_uuid_str_from_path; - use crate::meta::VACUUM2_OBJECT_KEY_PREFIX; #[test] fn test_trim_vacuum2_object_prefix() { diff --git a/src/query/storages/common/table_meta/src/meta/v0/common.rs b/src/query/storages/common/table_meta/src/meta/v0/common.rs index c5541723eca1e..d309c2d70e885 100644 --- a/src/query/storages/common/table_meta/src/meta/v0/common.rs +++ b/src/query/storages/common/table_meta/src/meta/v0/common.rs @@ -18,6 +18,7 @@ use databend_common_expression::ColumnId; use databend_common_expression::Scalar; #[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Eq)] +#[allow(dead_code)] pub struct ColumnStatistics { pub min: Scalar, pub max: Scalar, @@ -30,6 +31,7 @@ pub struct ColumnStatistics { } #[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Eq)] +#[allow(dead_code)] pub struct ClusterStatistics { #[serde(default = "default_cluster_key_id")] pub cluster_key_id: u32, @@ -40,15 +42,18 @@ pub struct ClusterStatistics { pub level: i32, } +#[allow(dead_code)] fn default_cluster_key_id() -> u32 { 0 } +#[allow(dead_code)] fn default_level() -> i32 { 0 } #[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Eq, Default)] +#[allow(dead_code)] pub struct Statistics { pub row_count: u64, pub block_count: u64, diff --git a/src/query/storages/common/table_meta/src/meta/v1/segment.rs b/src/query/storages/common/table_meta/src/meta/v1/segment.rs index 9923c410884b5..553e2a42a851f 100644 --- a/src/query/storages/common/table_meta/src/meta/v1/segment.rs +++ b/src/query/storages/common/table_meta/src/meta/v1/segment.rs @@ -19,14 +19,14 @@ use databend_common_expression::ColumnId; use serde::Deserialize; use serde::Serialize; -use crate::meta::v0::statistics::ClusterStatistics; -use crate::meta::v0::statistics::ColumnStatistics; -use crate::meta::v0::statistics::Statistics; -use crate::meta::v0::ColumnMeta; use crate::meta::Compression; use crate::meta::FormatVersion; use crate::meta::Location; use crate::meta::Versioned; +use crate::meta::v0::ColumnMeta; +use crate::meta::v0::statistics::ClusterStatistics; +use crate::meta::v0::statistics::ColumnStatistics; +use crate::meta::v0::statistics::Statistics; /// A segment comprises one or more blocks #[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] diff --git a/src/query/storages/common/table_meta/src/meta/v1/snapshot.rs b/src/query/storages/common/table_meta/src/meta/v1/snapshot.rs index e72a9aa39678e..7e11f139be243 100644 --- a/src/query/storages/common/table_meta/src/meta/v1/snapshot.rs +++ b/src/query/storages/common/table_meta/src/meta/v1/snapshot.rs @@ -20,13 +20,13 @@ use serde::Deserialize; use serde::Serialize; use super::super::v0::statistics::Statistics; -use crate::meta::monotonically_increased_timestamp; -use crate::meta::trim_timestamp_to_milli_second; use crate::meta::ClusterKey; use crate::meta::FormatVersion; use crate::meta::Location; use crate::meta::SnapshotId; use crate::meta::Versioned; +use crate::meta::monotonically_increased_timestamp; +use crate::meta::trim_timestamp_to_milli_second; #[derive(Serialize, Deserialize, Clone, Debug)] pub struct TableSnapshot { diff --git a/src/query/storages/common/table_meta/src/meta/v2/segment.rs b/src/query/storages/common/table_meta/src/meta/v2/segment.rs index b3b791c45b092..61fc3c6423a66 100644 --- a/src/query/storages/common/table_meta/src/meta/v2/segment.rs +++ b/src/query/storages/common/table_meta/src/meta/v2/segment.rs @@ -18,22 +18,20 @@ use std::sync::Arc; use chrono::DateTime; use chrono::Utc; -use databend_common_expression::types::NumberDataType; use databend_common_expression::BlockMetaInfo; use databend_common_expression::BlockMetaInfoDowncast; use databend_common_expression::ColumnId; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::VariantDataType; -use databend_common_frozen_api::frozen_api; +use databend_common_expression::types::NumberDataType; use databend_common_frozen_api::FrozenAPI; +use databend_common_frozen_api::frozen_api; use databend_common_native::ColumnMeta as NativeColumnMeta; use enum_as_inner::EnumAsInner; use serde::Deserialize; use serde::Serialize; -use crate::meta::v0; -use crate::meta::v1; use crate::meta::BlockHLLState; use crate::meta::ClusterStatistics; use crate::meta::ColumnStatistics; @@ -42,6 +40,8 @@ use crate::meta::FormatVersion; use crate::meta::Location; use crate::meta::Statistics; use crate::meta::Versioned; +use crate::meta::v0; +use crate::meta::v1; /// A segment comprises one or more blocks #[frozen_api("e19aba63")] diff --git a/src/query/storages/common/table_meta/src/meta/v2/segment_statistics.rs b/src/query/storages/common/table_meta/src/meta/v2/segment_statistics.rs index 3a103f655aaf6..db24d180cde3f 100644 --- a/src/query/storages/common/table_meta/src/meta/v2/segment_statistics.rs +++ b/src/query/storages/common/table_meta/src/meta/v2/segment_statistics.rs @@ -15,20 +15,20 @@ use std::io::Read; use databend_common_exception::Result; -use databend_common_frozen_api::frozen_api; use databend_common_frozen_api::FrozenAPI; +use databend_common_frozen_api::frozen_api; use databend_common_io::prelude::BinaryRead; use serde::Deserialize; use serde::Serialize; -use crate::meta::format::compress; -use crate::meta::format::encode; -use crate::meta::format::read_and_deserialize; use crate::meta::FormatVersion; use crate::meta::MetaCompression; use crate::meta::MetaEncoding; use crate::meta::RawBlockHLL; use crate::meta::Versioned; +use crate::meta::format::compress; +use crate::meta::format::encode; +use crate::meta::format::read_and_deserialize; #[frozen_api("99795401")] #[derive(Serialize, Deserialize, Clone, Debug, FrozenAPI)] diff --git a/src/query/storages/common/table_meta/src/meta/v2/snapshot.rs b/src/query/storages/common/table_meta/src/meta/v2/snapshot.rs index df262012e522b..b9d50957f01d4 100644 --- a/src/query/storages/common/table_meta/src/meta/v2/snapshot.rs +++ b/src/query/storages/common/table_meta/src/meta/v2/snapshot.rs @@ -14,21 +14,21 @@ use chrono::DateTime; use chrono::Utc; -use databend_common_expression::converts::datavalues::from_schema; use databend_common_expression::TableSchema; +use databend_common_expression::converts::datavalues::from_schema; use serde::Deserialize; use serde::Serialize; -use crate::meta::monotonically_increased_timestamp; -use crate::meta::trim_timestamp_to_milli_second; -use crate::meta::v0; -use crate::meta::v1; use crate::meta::ClusterKey; use crate::meta::FormatVersion; use crate::meta::Location; use crate::meta::SnapshotId; use crate::meta::Statistics; use crate::meta::Versioned; +use crate::meta::monotonically_increased_timestamp; +use crate::meta::trim_timestamp_to_milli_second; +use crate::meta::v0; +use crate::meta::v1; #[derive(Serialize, Deserialize, Clone, Debug)] pub struct TableSnapshot { diff --git a/src/query/storages/common/table_meta/src/meta/v2/statistics.rs b/src/query/storages/common/table_meta/src/meta/v2/statistics.rs index 13f00d4c9bf5b..d74596375bf69 100644 --- a/src/query/storages/common/table_meta/src/meta/v2/statistics.rs +++ b/src/query/storages/common/table_meta/src/meta/v2/statistics.rs @@ -16,21 +16,21 @@ use std::collections::HashMap; use std::fmt; use std::marker::PhantomData; -use databend_common_expression::converts::datavalues::from_scalar; -use databend_common_expression::converts::meta::IndexScalar; -use databend_common_expression::types::DataType; use databend_common_expression::ColumnId; use databend_common_expression::Scalar; use databend_common_expression::TableDataType; use databend_common_expression::TableField; +use databend_common_expression::converts::datavalues::from_scalar; +use databend_common_expression::converts::meta::IndexScalar; +use databend_common_expression::types::DataType; use databend_common_frozen_api::FrozenAPI; use log::info; use serde::de::Error; -use crate::meta::supported_stat_type; -use crate::meta::v0; use crate::meta::Location; use crate::meta::RawBlockHLL; +use crate::meta::supported_stat_type; +use crate::meta::v0; #[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Eq, FrozenAPI)] pub struct ColumnStatistics { diff --git a/src/query/storages/common/table_meta/src/meta/v2/table_snapshot_statistics.rs b/src/query/storages/common/table_meta/src/meta/v2/table_snapshot_statistics.rs index ecaf619b983f4..823db5534aa64 100644 --- a/src/query/storages/common/table_meta/src/meta/v2/table_snapshot_statistics.rs +++ b/src/query/storages/common/table_meta/src/meta/v2/table_snapshot_statistics.rs @@ -19,10 +19,10 @@ use databend_common_storage::MetaHLL12; use serde::Deserialize; use serde::Serialize; -use crate::meta::v1; use crate::meta::FormatVersion; use crate::meta::SnapshotId; use crate::meta::Versioned; +use crate::meta::v1; #[derive(Serialize, Deserialize, Clone, Debug)] pub struct TableSnapshotStatistics { diff --git a/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs b/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs index ccc22fdbdb600..230358f1be02d 100644 --- a/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs +++ b/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs @@ -14,9 +14,9 @@ use std::collections::HashMap; -use databend_common_expression::converts::meta::LegacyScalar; use databend_common_expression::ColumnId; use databend_common_expression::Scalar; +use databend_common_expression::converts::meta::LegacyScalar; use serde::Deserialize; use serde::Serialize; diff --git a/src/query/storages/common/table_meta/src/meta/v3/frozen/statistics.rs b/src/query/storages/common/table_meta/src/meta/v3/frozen/statistics.rs index 7ff6cdbf34a42..31103ec2f61f4 100644 --- a/src/query/storages/common/table_meta/src/meta/v3/frozen/statistics.rs +++ b/src/query/storages/common/table_meta/src/meta/v3/frozen/statistics.rs @@ -14,9 +14,9 @@ use std::collections::HashMap; -use databend_common_expression::converts::meta::LegacyScalar; use databend_common_expression::ColumnId; use databend_common_expression::Scalar; +use databend_common_expression::converts::meta::LegacyScalar; // the following types are supposed to be frozen diff --git a/src/query/storages/common/table_meta/src/meta/v3/segment.rs b/src/query/storages/common/table_meta/src/meta/v3/segment.rs index ae7ab5ccbc8e0..9d7da772da87b 100644 --- a/src/query/storages/common/table_meta/src/meta/v3/segment.rs +++ b/src/query/storages/common/table_meta/src/meta/v3/segment.rs @@ -20,12 +20,12 @@ use serde::Deserialize; use serde::Serialize; use super::frozen; -use crate::meta::format::decode_segment_header; -use crate::meta::format::read_and_deserialize; -use crate::meta::format::SegmentHeader; use crate::meta::FormatVersion; use crate::meta::MetaEncoding; use crate::meta::Versioned; +use crate::meta::format::SegmentHeader; +use crate::meta::format::decode_segment_header; +use crate::meta::format::read_and_deserialize; /// A segment comprises one or more blocks /// The structure of the segment is the same as that of v2, but the serialization and deserialization methods are different diff --git a/src/query/storages/common/table_meta/src/meta/v3/snapshot.rs b/src/query/storages/common/table_meta/src/meta/v3/snapshot.rs index 8dceb59c269de..f58d5b015ba10 100644 --- a/src/query/storages/common/table_meta/src/meta/v3/snapshot.rs +++ b/src/query/storages/common/table_meta/src/meta/v3/snapshot.rs @@ -23,7 +23,6 @@ use serde::Deserialize; use serde::Serialize; use super::frozen; -use crate::meta::format::read_and_deserialize; use crate::meta::ClusterKey; use crate::meta::FormatVersion; use crate::meta::Location; @@ -31,6 +30,7 @@ use crate::meta::MetaCompression; use crate::meta::MetaEncoding; use crate::meta::SnapshotId; use crate::meta::Versioned; +use crate::meta::format::read_and_deserialize; /// The structure of the segment is the same as that of v2, but the serialization and deserialization methods are different #[derive(Serialize, Deserialize)] diff --git a/src/query/storages/common/table_meta/src/meta/v3/table_snapshot_statistics.rs b/src/query/storages/common/table_meta/src/meta/v3/table_snapshot_statistics.rs index 8bcd744487e9f..dc36614303719 100644 --- a/src/query/storages/common/table_meta/src/meta/v3/table_snapshot_statistics.rs +++ b/src/query/storages/common/table_meta/src/meta/v3/table_snapshot_statistics.rs @@ -20,11 +20,11 @@ use databend_common_storage::MetaHLL12; use serde::Deserialize; use serde::Serialize; -use crate::meta::v1; -use crate::meta::v2; use crate::meta::FormatVersion; use crate::meta::SnapshotId; use crate::meta::Versioned; +use crate::meta::v1; +use crate::meta::v2; #[derive(Serialize, Deserialize, Clone, Debug)] pub struct TableSnapshotStatistics { diff --git a/src/query/storages/common/table_meta/src/meta/v4/segment.rs b/src/query/storages/common/table_meta/src/meta/v4/segment.rs index 77b6757df8d3a..f00e4dfddedab 100644 --- a/src/query/storages/common/table_meta/src/meta/v4/segment.rs +++ b/src/query/storages/common/table_meta/src/meta/v4/segment.rs @@ -18,25 +18,25 @@ use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_frozen_api::frozen_api; use databend_common_frozen_api::FrozenAPI; +use databend_common_frozen_api::frozen_api; use serde::Deserialize; use serde::Serialize; use super::super::v2; use super::super::v3; +use crate::meta::FormatVersion; +use crate::meta::MetaEncoding; +use crate::meta::Statistics; +use crate::meta::Versioned; +use crate::meta::format::MAX_SEGMENT_BLOCK_NUMBER; +use crate::meta::format::MetaCompression; +use crate::meta::format::SegmentHeader; use crate::meta::format::compress; use crate::meta::format::decode_segment_header; use crate::meta::format::encode; use crate::meta::format::read_and_deserialize; -use crate::meta::format::MetaCompression; -use crate::meta::format::SegmentHeader; -use crate::meta::format::MAX_SEGMENT_BLOCK_NUMBER; use crate::meta::v2::BlockMeta; -use crate::meta::FormatVersion; -use crate::meta::MetaEncoding; -use crate::meta::Statistics; -use crate::meta::Versioned; /// A segment comprises one or more blocks /// The structure of the segment is the same as that of v2, but the serialization and deserialization methods are different diff --git a/src/query/storages/common/table_meta/src/meta/v4/snapshot.rs b/src/query/storages/common/table_meta/src/meta/v4/snapshot.rs index 4f6ea9c3306c6..34c195d6c54f7 100644 --- a/src/query/storages/common/table_meta/src/meta/v4/snapshot.rs +++ b/src/query/storages/common/table_meta/src/meta/v4/snapshot.rs @@ -23,20 +23,12 @@ use chrono::Utc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::TableSchema; -use databend_common_frozen_api::frozen_api; use databend_common_frozen_api::FrozenAPI; +use databend_common_frozen_api::frozen_api; use databend_common_io::prelude::BinaryRead; use serde::Deserialize; use serde::Serialize; -use crate::meta::format::compress; -use crate::meta::format::encode; -use crate::meta::format::read_and_deserialize; -use crate::meta::format::MetaCompression; -use crate::meta::monotonically_increased_timestamp; -use crate::meta::uuid_from_date_time; -use crate::meta::v2; -use crate::meta::v3; use crate::meta::ClusterKey; use crate::meta::FormatVersion; use crate::meta::Location; @@ -45,6 +37,14 @@ use crate::meta::SnapshotId; use crate::meta::Statistics; use crate::meta::TableMetaTimestamps; use crate::meta::Versioned; +use crate::meta::format::MetaCompression; +use crate::meta::format::compress; +use crate::meta::format::encode; +use crate::meta::format::read_and_deserialize; +use crate::meta::monotonically_increased_timestamp; +use crate::meta::uuid_from_date_time; +use crate::meta::v2; +use crate::meta::v3; use crate::readers::snapshot_reader::TableSnapshotAccessor; #[frozen_api("dba4542b")] diff --git a/src/query/storages/common/table_meta/src/meta/v4/table_snapshot_statistics.rs b/src/query/storages/common/table_meta/src/meta/v4/table_snapshot_statistics.rs index 0b4ab7f409d1d..3919ea7500614 100644 --- a/src/query/storages/common/table_meta/src/meta/v4/table_snapshot_statistics.rs +++ b/src/query/storages/common/table_meta/src/meta/v4/table_snapshot_statistics.rs @@ -15,17 +15,17 @@ use std::collections::HashMap; use databend_common_expression::ColumnId; -use databend_common_frozen_api::frozen_api; use databend_common_frozen_api::FrozenAPI; +use databend_common_frozen_api::frozen_api; use databend_common_storage::Histogram; use databend_common_storage::MetaHLL; -use crate::meta::v1; -use crate::meta::v2; -use crate::meta::v3; use crate::meta::FormatVersion; use crate::meta::SnapshotId; use crate::meta::Versioned; +use crate::meta::v1; +use crate::meta::v2; +use crate::meta::v3; #[frozen_api("446eb231")] #[derive(serde::Serialize, serde::Deserialize, Clone, Debug, FrozenAPI)] diff --git a/src/query/storages/common/table_meta/src/readers/segment_reader.rs b/src/query/storages/common/table_meta/src/readers/segment_reader.rs index ca3b178badd47..2305a5b9234a2 100644 --- a/src/query/storages/common/table_meta/src/readers/segment_reader.rs +++ b/src/query/storages/common/table_meta/src/readers/segment_reader.rs @@ -17,11 +17,11 @@ use std::io::Read; use databend_common_exception::Result; use databend_common_expression::TableSchemaRef; -use crate::meta::load_json; use crate::meta::CompactSegmentInfo; use crate::meta::SegmentInfo; use crate::meta::SegmentInfoV3; use crate::meta::SegmentInfoVersion; +use crate::meta::load_json; use crate::readers::VersionedReader; impl VersionedReader for (SegmentInfoVersion, TableSchemaRef) { diff --git a/src/query/storages/common/table_meta/src/readers/snapshot_reader.rs b/src/query/storages/common/table_meta/src/readers/snapshot_reader.rs index 21a98831151e5..774c411fe0af4 100644 --- a/src/query/storages/common/table_meta/src/readers/snapshot_reader.rs +++ b/src/query/storages/common/table_meta/src/readers/snapshot_reader.rs @@ -18,7 +18,6 @@ use std::sync::Arc; use databend_common_exception::Result; use databend_common_expression::TableSchema; -use crate::meta::load_json; use crate::meta::AdditionalStatsMeta; use crate::meta::FormatVersion; use crate::meta::Location; @@ -28,6 +27,7 @@ use crate::meta::Statistics; use crate::meta::TableSnapshot; use crate::meta::TableSnapshotV2; use crate::meta::TableSnapshotV3; +use crate::meta::load_json; use crate::readers::VersionedReader; impl VersionedReader for SnapshotVersion { diff --git a/src/query/storages/common/table_meta/src/readers/versioned_reader.rs b/src/query/storages/common/table_meta/src/readers/versioned_reader.rs index 525b1932a9fae..83103a7b713bc 100644 --- a/src/query/storages/common/table_meta/src/readers/versioned_reader.rs +++ b/src/query/storages/common/table_meta/src/readers/versioned_reader.rs @@ -16,11 +16,11 @@ use std::io::Read; use databend_common_exception::Result; -use crate::meta::load_json; use crate::meta::SegmentStatistics; use crate::meta::SegmentStatisticsVersion; use crate::meta::TableSnapshotStatistics; use crate::meta::TableSnapshotStatisticsVersion; +use crate::meta::load_json; pub trait VersionedReader { type TargetType; diff --git a/src/query/storages/delta/Cargo.toml b/src/query/storages/delta/Cargo.toml index 92b52eec41801..762617ea6fd83 100644 --- a/src/query/storages/delta/Cargo.toml +++ b/src/query/storages/delta/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "databend-common-storages-delta" version = { workspace = true } -edition = "2021" +edition = "2024" authors = ["Databend Authors "] license = "Apache-2.0" publish = false diff --git a/src/query/storages/delta/src/arrow56_conversion.rs b/src/query/storages/delta/src/arrow56_conversion.rs index 321aa1cdf861e..0437a82629788 100644 --- a/src/query/storages/delta/src/arrow56_conversion.rs +++ b/src/query/storages/delta/src/arrow56_conversion.rs @@ -22,7 +22,6 @@ use arrow_schema::Field as ArrowField; use arrow_schema::Schema as ArrowSchema; use arrow_schema::SchemaRef as ArrowSchemaRef; use arrow_schema::TimeUnit; -use deltalake::kernel::error::Error; use deltalake::kernel::ArrayType; use deltalake::kernel::DataType; use deltalake::kernel::MapType; @@ -30,6 +29,7 @@ use deltalake::kernel::MetadataValue; use deltalake::kernel::PrimitiveType; use deltalake::kernel::StructField; use deltalake::kernel::StructType; +use deltalake::kernel::error::Error; use itertools::Itertools; pub(crate) const LIST_ARRAY_ROOT: &str = "element"; diff --git a/src/query/storages/delta/src/lib.rs b/src/query/storages/delta/src/lib.rs index 87f63f01a8731..7286622c19523 100644 --- a/src/query/storages/delta/src/lib.rs +++ b/src/query/storages/delta/src/lib.rs @@ -13,7 +13,7 @@ // limitations under the License. #![feature(impl_trait_in_assoc_type)] -#![allow(clippy::diverging_sub_expression)] +#![allow(clippy::collapsible_if, clippy::diverging_sub_expression)] mod partition; mod table; diff --git a/src/query/storages/delta/src/table.rs b/src/query/storages/delta/src/table.rs index af8f0421422ae..1f1ca5306c05b 100644 --- a/src/query/storages/delta/src/table.rs +++ b/src/query/storages/delta/src/table.rs @@ -50,8 +50,8 @@ use databend_common_storages_parquet::ParquetSourceType; use databend_storages_common_pruner::partition_prunner::FetchPartitionScalars; use databend_storages_common_pruner::partition_prunner::PartitionPruner; use databend_storages_common_table_meta::table::OPT_KEY_ENGINE_META; -use deltalake::kernel::Add; use deltalake::DeltaTableBuilder; +use deltalake::kernel::Add; use object_store_opendal::OpendalStore; use serde::Deserialize; use serde::Serialize; diff --git a/src/query/storages/delta/src/table_source.rs b/src/query/storages/delta/src/table_source.rs index bd18b82ae032b..ade5ccb5a1990 100644 --- a/src/query/storages/delta/src/table_source.rs +++ b/src/query/storages/delta/src/table_source.rs @@ -22,7 +22,6 @@ use databend_common_base::runtime::profile::ProfileStatisticsName; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::BlockEntry; use databend_common_expression::DataBlock; use databend_common_expression::DataSchema; @@ -30,6 +29,7 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::FieldIndex; use databend_common_expression::Scalar; use databend_common_expression::TableField; +use databend_common_expression::types::DataType; use databend_common_pipeline::core::Event; use databend_common_pipeline::core::OutputPort; use databend_common_pipeline::core::Processor; diff --git a/src/query/storages/factory/Cargo.toml b/src/query/storages/factory/Cargo.toml index f673d63945976..ecab65dba25ec 100644 --- a/src/query/storages/factory/Cargo.toml +++ b/src/query/storages/factory/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "databend-common-storages-factory" version = "0.1.0" -edition = "2021" +edition = "2024" authors = ["Databend Authors "] license = "Apache-2.0" publish = false diff --git a/src/query/storages/factory/src/storage_factory.rs b/src/query/storages/factory/src/storage_factory.rs index ee09d59cb881f..7e6052d1fd8f3 100644 --- a/src/query/storages/factory/src/storage_factory.rs +++ b/src/query/storages/factory/src/storage_factory.rs @@ -21,16 +21,16 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::storage::S3StorageClass; -use databend_common_storages_basic::view_table::ViewTable; use databend_common_storages_basic::MemoryTable; use databend_common_storages_basic::NullTable; use databend_common_storages_basic::RandomTable; +use databend_common_storages_basic::view_table::ViewTable; use databend_common_storages_delta::DeltaTable; use databend_common_storages_iceberg::IcebergTable; use databend_common_storages_stream::stream_table::StreamTable; -use crate::fuse::FuseTable; use crate::Table; +use crate::fuse::FuseTable; pub trait StorageCreator: Send + Sync { fn try_create(&self, table_info: TableInfo, disable_refresh: bool) -> Result>; diff --git a/src/query/storages/fuse/benches/bench.rs b/src/query/storages/fuse/benches/bench.rs index 10fe5133c6075..e52099d49062e 100644 --- a/src/query/storages/fuse/benches/bench.rs +++ b/src/query/storages/fuse/benches/bench.rs @@ -40,9 +40,9 @@ mod dummy { use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRef; use databend_common_native::read::NativeColumnsReader; - use databend_common_storages_fuse::io::serialize_block; - use databend_common_storages_fuse::io::WriteSettings; use databend_common_storages_fuse::FuseStorageFormat; + use databend_common_storages_fuse::io::WriteSettings; + use databend_common_storages_fuse::io::serialize_block; use databend_storages_common_table_meta::table::TableCompression; use divan::counter::BytesCount; use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; @@ -119,9 +119,9 @@ mod dummy { let bs = a.slice(meta.offset as usize..(meta.offset + meta.total_len()) as usize); let col = reader - .batch_read_column(vec![bs.as_ref()], f.data_type.clone(), vec![meta - .pages - .clone()]) + .batch_read_column(vec![bs.as_ref()], f.data_type.clone(), vec![ + meta.pages.clone(), + ]) .unwrap(); columns.push(col); diff --git a/src/query/storages/fuse/src/fuse_part.rs b/src/query/storages/fuse/src/fuse_part.rs index 702807ae63f13..3345544b6ea55 100644 --- a/src/query/storages/fuse/src/fuse_part.rs +++ b/src/query/storages/fuse/src/fuse_part.rs @@ -13,8 +13,8 @@ // limitations under the License. use std::any::Any; -use std::collections::hash_map::DefaultHasher; use std::collections::HashMap; +use std::collections::hash_map::DefaultHasher; use std::hash::Hash; use std::hash::Hasher; use std::ops::Range; diff --git a/src/query/storages/fuse/src/fuse_table.rs b/src/query/storages/fuse/src/fuse_table.rs index dc8c9ea11d75b..36a0a0ede9bec 100644 --- a/src/query/storages/fuse/src/fuse_table.rs +++ b/src/query/storages/fuse/src/fuse_table.rs @@ -34,7 +34,6 @@ use databend_common_catalog::plan::Partitions; use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::plan::ReclusterParts; use databend_common_catalog::plan::StreamColumn; -use databend_common_catalog::table::is_temp_table_by_table_info; use databend_common_catalog::table::Bound; use databend_common_catalog::table::ColumnRange; use databend_common_catalog::table::ColumnStatisticsProvider; @@ -42,20 +41,21 @@ use databend_common_catalog::table::CompactionLimits; use databend_common_catalog::table::DistributionLevel; use databend_common_catalog::table::NavigationDescriptor; use databend_common_catalog::table::TimeNavigation; +use databend_common_catalog::table::is_temp_table_by_table_info; use databend_common_catalog::table_context::TableContext; use databend_common_config::GlobalConfig; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::BlockThresholds; use databend_common_expression::ColumnId; -use databend_common_expression::RemoteExpr; -use databend_common_expression::TableField; -use databend_common_expression::TableSchema; use databend_common_expression::ORIGIN_BLOCK_ID_COL_NAME; use databend_common_expression::ORIGIN_BLOCK_ROW_NUM_COL_NAME; use databend_common_expression::ORIGIN_VERSION_COL_NAME; +use databend_common_expression::RemoteExpr; +use databend_common_expression::TableField; +use databend_common_expression::TableSchema; use databend_common_expression::VECTOR_SCORE_COLUMN_ID; +use databend_common_expression::types::DataType; use databend_common_io::constants::DEFAULT_BLOCK_BUFFER_SIZE; use databend_common_io::constants::DEFAULT_BLOCK_COMPRESSED_SIZE; use databend_common_io::constants::DEFAULT_BLOCK_PER_SEGMENT; @@ -66,22 +66,20 @@ use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; use databend_common_meta_app::schema::UpdateStreamMetaReq; use databend_common_meta_app::schema::UpsertTableCopiedFileReq; -use databend_common_meta_app::storage::set_s3_storage_class; use databend_common_meta_app::storage::S3StorageClass; use databend_common_meta_app::storage::StorageParams; +use databend_common_meta_app::storage::set_s3_storage_class; use databend_common_pipeline::core::Pipeline; +use databend_common_sql::ApproxDistinctColumns; +use databend_common_sql::BloomIndexColumns; use databend_common_sql::binder::STREAM_COLUMN_FACTORY; use databend_common_sql::parse_cluster_keys; use databend_common_sql::plans::TruncateMode; -use databend_common_sql::ApproxDistinctColumns; -use databend_common_sql::BloomIndexColumns; -use databend_common_storage::init_operator; use databend_common_storage::StorageMetrics; use databend_common_storage::StorageMetricsLayer; +use databend_common_storage::init_operator; use databend_storages_common_cache::LoadParams; use databend_storages_common_io::Files; -use databend_storages_common_table_meta::meta::decode_column_hll; -use databend_storages_common_table_meta::meta::parse_storage_prefix; use databend_storages_common_table_meta::meta::ClusterKey; use databend_storages_common_table_meta::meta::CompactSegmentInfo; use databend_storages_common_table_meta::meta::SnapshotId; @@ -89,9 +87,10 @@ use databend_storages_common_table_meta::meta::TableMetaTimestamps; use databend_storages_common_table_meta::meta::TableSnapshot; use databend_storages_common_table_meta::meta::TableSnapshotStatistics; use databend_storages_common_table_meta::meta::Versioned; +use databend_storages_common_table_meta::meta::decode_column_hll; +use databend_storages_common_table_meta::meta::parse_storage_prefix; use databend_storages_common_table_meta::table::ChangeType; use databend_storages_common_table_meta::table::ClusterType; -use databend_storages_common_table_meta::table::TableCompression; use databend_storages_common_table_meta::table::OPT_KEY_APPROX_DISTINCT_COLUMNS; use databend_storages_common_table_meta::table::OPT_KEY_BLOOM_INDEX_COLUMNS; use databend_storages_common_table_meta::table::OPT_KEY_CHANGE_TRACKING; @@ -103,6 +102,7 @@ use databend_storages_common_table_meta::table::OPT_KEY_SNAPSHOT_LOCATION_FIXED_ use databend_storages_common_table_meta::table::OPT_KEY_STORAGE_FORMAT; use databend_storages_common_table_meta::table::OPT_KEY_TABLE_ATTACHED_DATA_URI; use databend_storages_common_table_meta::table::OPT_KEY_TABLE_COMPRESSION; +use databend_storages_common_table_meta::table::TableCompression; use futures_util::TryStreamExt; use itertools::Itertools; use log::info; @@ -110,23 +110,6 @@ use log::warn; use opendal::Operator; use parking_lot::Mutex; -use crate::fuse_column::FuseTableColumnStatisticsProvider; -use crate::fuse_type::FuseTableType; -use crate::io::MetaReaders; -use crate::io::SegmentsIO; -use crate::io::TableMetaLocationGenerator; -use crate::io::TableSnapshotReader; -use crate::io::WriteSettings; -use crate::operations::load_last_snapshot_hint; -use crate::operations::ChangesDesc; -use crate::operations::SnapshotHint; -use crate::statistics::reduce_block_statistics; -use crate::statistics::Trim; -use crate::FuseSegmentFormat; -use crate::FuseStorageFormat; -use crate::NavigationPoint; -use crate::Table; -use crate::TableStatistics; use crate::DEFAULT_ROW_PER_PAGE; use crate::FUSE_OPT_KEY_ATTACH_COLUMN_IDS; use crate::FUSE_OPT_KEY_BLOCK_IN_MEM_SIZE_THRESHOLD; @@ -137,6 +120,23 @@ use crate::FUSE_OPT_KEY_ENABLE_PARQUET_DICTIONARY; use crate::FUSE_OPT_KEY_FILE_SIZE; use crate::FUSE_OPT_KEY_ROW_PER_BLOCK; use crate::FUSE_OPT_KEY_ROW_PER_PAGE; +use crate::FuseSegmentFormat; +use crate::FuseStorageFormat; +use crate::NavigationPoint; +use crate::Table; +use crate::TableStatistics; +use crate::fuse_column::FuseTableColumnStatisticsProvider; +use crate::fuse_type::FuseTableType; +use crate::io::MetaReaders; +use crate::io::SegmentsIO; +use crate::io::TableMetaLocationGenerator; +use crate::io::TableSnapshotReader; +use crate::io::WriteSettings; +use crate::operations::ChangesDesc; +use crate::operations::SnapshotHint; +use crate::operations::load_last_snapshot_hint; +use crate::statistics::Trim; +use crate::statistics::reduce_block_statistics; #[derive(Clone)] pub struct FuseTable { @@ -680,15 +680,16 @@ impl FuseTable { info!( "extracting snapshot location of table {} with id {:?} from the last snapshot hint file.", - table_info.desc, - table_info.ident + table_info.desc, table_info.ident ); let snapshot_hint = Self::refresh_schema_from_hint(operator, storage_prefix)?; info!( "extracted snapshot location [{:?}] of table {}, with id {:?} from the last snapshot hint file.", - snapshot_hint.as_ref().map(|(hint, _)| &hint.snapshot_full_path), + snapshot_hint + .as_ref() + .map(|(hint, _)| &hint.snapshot_full_path), table_info.desc, table_info.ident ); @@ -953,11 +954,7 @@ impl Table for FuseTable { } Err(e) if e.code() == ErrorCode::TABLE_HISTORICAL_DATA_NOT_FOUND => { warn!("navigate failed: {:?}", e); - if dry_run { - Ok(Some(vec![])) - } else { - Ok(None) - } + if dry_run { Ok(Some(vec![])) } else { Ok(None) } } Err(e) => Err(e), } diff --git a/src/query/storages/fuse/src/io/locations.rs b/src/query/storages/fuse/src/io/locations.rs index a98de49cdb430..06c196536da11 100644 --- a/src/query/storages/fuse/src/io/locations.rs +++ b/src/query/storages/fuse/src/io/locations.rs @@ -16,32 +16,32 @@ use std::marker::PhantomData; use databend_common_exception::Result; use databend_common_expression::DataBlock; -use databend_storages_common_table_meta::meta::trim_object_prefix; -use databend_storages_common_table_meta::meta::uuid_from_date_time; use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::SegmentInfo; use databend_storages_common_table_meta::meta::SegmentStatistics; use databend_storages_common_table_meta::meta::SnapshotVersion; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use databend_storages_common_table_meta::meta::TableSnapshotStatisticsVersion; -use databend_storages_common_table_meta::meta::Versioned; use databend_storages_common_table_meta::meta::VACUUM2_OBJECT_KEY_PREFIX; +use databend_storages_common_table_meta::meta::Versioned; +use databend_storages_common_table_meta::meta::trim_object_prefix; +use databend_storages_common_table_meta::meta::uuid_from_date_time; use uuid::Uuid; use uuid::Version; -use crate::constants::FUSE_TBL_BLOCK_PREFIX; -use crate::constants::FUSE_TBL_SEGMENT_PREFIX; -use crate::constants::FUSE_TBL_SNAPSHOT_PREFIX; -use crate::constants::FUSE_TBL_SNAPSHOT_STATISTICS_PREFIX; -use crate::constants::FUSE_TBL_VIRTUAL_BLOCK_PREFIX; -use crate::index::filters::BlockFilter; -use crate::index::InvertedIndexFile; use crate::FUSE_TBL_AGG_INDEX_PREFIX; use crate::FUSE_TBL_INVERTED_INDEX_PREFIX; use crate::FUSE_TBL_LAST_SNAPSHOT_HINT_V2; use crate::FUSE_TBL_SEGMENT_STATISTICS_PREFIX; use crate::FUSE_TBL_VECTOR_INDEX_PREFIX; use crate::FUSE_TBL_XOR_BLOOM_INDEX_PREFIX; +use crate::constants::FUSE_TBL_BLOCK_PREFIX; +use crate::constants::FUSE_TBL_SEGMENT_PREFIX; +use crate::constants::FUSE_TBL_SNAPSHOT_PREFIX; +use crate::constants::FUSE_TBL_SNAPSHOT_STATISTICS_PREFIX; +use crate::constants::FUSE_TBL_VIRTUAL_BLOCK_PREFIX; +use crate::index::InvertedIndexFile; +use crate::index::filters::BlockFilter; static SNAPSHOT_V0: SnapshotVersion = SnapshotVersion::V0(PhantomData); static SNAPSHOT_V1: SnapshotVersion = SnapshotVersion::V1(PhantomData); static SNAPSHOT_V2: SnapshotVersion = SnapshotVersion::V2(PhantomData); diff --git a/src/query/storages/fuse/src/io/mod.rs b/src/query/storages/fuse/src/io/mod.rs index e13b252e29b15..ac0c40baefc22 100644 --- a/src/query/storages/fuse/src/io/mod.rs +++ b/src/query/storages/fuse/src/io/mod.rs @@ -36,12 +36,6 @@ pub use segments::SegmentsIO; pub use segments::SerializedSegment; pub use snapshots::SnapshotLiteExtended; pub use snapshots::SnapshotsIO; -pub use write::build_column_hlls; -pub(crate) use write::create_index_schema; -pub(crate) use write::create_inverted_index_builders; -pub(crate) use write::create_tokenizer_manager; -pub use write::serialize_block; -pub use write::write_data; pub use write::BlockBuilder; pub use write::BlockSerialization; pub use write::BlockWriter; @@ -50,10 +44,16 @@ pub use write::BloomIndexState; pub use write::CachedMetaWriter; pub use write::InvertedIndexBuilder; pub use write::InvertedIndexWriter; +pub use write::MAX_BLOCK_UNCOMPRESSED_SIZE; pub use write::MetaWriter; pub(crate) use write::StreamBlockBuilder; pub(crate) use write::StreamBlockProperties; pub use write::VectorIndexBuilder; pub use write::VirtualColumnBuilder; pub use write::WriteSettings; -pub use write::MAX_BLOCK_UNCOMPRESSED_SIZE; +pub use write::build_column_hlls; +pub(crate) use write::create_index_schema; +pub(crate) use write::create_inverted_index_builders; +pub(crate) use write::create_tokenizer_manager; +pub use write::serialize_block; +pub use write::write_data; diff --git a/src/query/storages/fuse/src/io/read/agg_index/agg_index_reader.rs b/src/query/storages/fuse/src/io/read/agg_index/agg_index_reader.rs index ead67dd81e7fe..fb9f24122a24d 100644 --- a/src/query/storages/fuse/src/io/read/agg_index/agg_index_reader.rs +++ b/src/query/storages/fuse/src/io/read/agg_index/agg_index_reader.rs @@ -18,14 +18,14 @@ use databend_common_catalog::plan::AggIndexInfo; use databend_common_catalog::plan::AggIndexMeta; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::BooleanType; -use databend_common_expression::types::DataType; use databend_common_expression::BlockEntry; use databend_common_expression::DataBlock; use databend_common_expression::Evaluator; use databend_common_expression::Expr; use databend_common_expression::FunctionContext; use databend_common_expression::Scalar; +use databend_common_expression::types::BooleanType; +use databend_common_expression::types::DataType; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_storages_common_table_meta::table::TableCompression; use opendal::Operator; diff --git a/src/query/storages/fuse/src/io/read/agg_index/agg_index_reader_native.rs b/src/query/storages/fuse/src/io/read/agg_index/agg_index_reader_native.rs index c79d841af06d0..d66c980dd3017 100644 --- a/src/query/storages/fuse/src/io/read/agg_index/agg_index_reader_native.rs +++ b/src/query/storages/fuse/src/io/read/agg_index/agg_index_reader_native.rs @@ -21,8 +21,8 @@ use databend_storages_common_table_meta::meta::ColumnMeta; use log::debug; use super::AggIndexReader; -use crate::io::NativeSourceData; use crate::FuseBlockPartInfo; +use crate::io::NativeSourceData; impl AggIndexReader { pub async fn read_native_data(&self, loc: &str) -> Option { @@ -41,12 +41,11 @@ impl AggIndexReader { return None; } let num_rows = metadata[0].pages.iter().map(|p| p.num_values).sum(); - debug_assert!(metadata.iter().all(|c| c - .pages - .iter() - .map(|p| p.num_values) - .sum::() - == num_rows)); + debug_assert!( + metadata + .iter() + .all(|c| c.pages.iter().map(|p| p.num_values).sum::() == num_rows) + ); let columns_meta = metadata .into_iter() .enumerate() @@ -95,9 +94,11 @@ impl AggIndexReader { self.reader.data_schema(), ))); } - debug_assert!(all_columns_arrays - .iter() - .all(|a| a.len() == all_columns_arrays[0].len())); + debug_assert!( + all_columns_arrays + .iter() + .all(|a| a.len() == all_columns_arrays[0].len()) + ); let page_num = all_columns_arrays[0].len(); let mut blocks = Vec::with_capacity(page_num); diff --git a/src/query/storages/fuse/src/io/read/agg_index/agg_index_reader_parquet.rs b/src/query/storages/fuse/src/io/read/agg_index/agg_index_reader_parquet.rs index 970a7a9925e2a..d70e6366cffa6 100644 --- a/src/query/storages/fuse/src/io/read/agg_index/agg_index_reader_parquet.rs +++ b/src/query/storages/fuse/src/io/read/agg_index/agg_index_reader_parquet.rs @@ -20,9 +20,9 @@ use databend_storages_common_io::ReadSettings; use log::debug; use super::AggIndexReader; -use crate::io::read::utils::build_columns_meta; use crate::BlockReadResult; use crate::FuseBlockPartInfo; +use crate::io::read::utils::build_columns_meta; impl AggIndexReader { pub async fn read_parquet_data_by_merge_io( diff --git a/src/query/storages/fuse/src/io/read/block/block_reader.rs b/src/query/storages/fuse/src/io/read/block/block_reader.rs index 7ddb31d8d44b4..840815a611c0a 100644 --- a/src/query/storages/fuse/src/io/read/block/block_reader.rs +++ b/src/query/storages/fuse/src/io/read/block/block_reader.rs @@ -22,13 +22,13 @@ use databend_common_catalog::plan::Projection; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::ColumnId; use databend_common_expression::DataField; use databend_common_expression::DataSchema; use databend_common_expression::FieldIndex; use databend_common_expression::Scalar; use databend_common_expression::TableSchemaRef; +use databend_common_expression::types::DataType; use databend_common_native::read::NativeColumnsReader; use databend_common_sql::DefaultExprBinder; use databend_common_storage::ColumnNode; diff --git a/src/query/storages/fuse/src/io/read/block/block_reader_deserialize.rs b/src/query/storages/fuse/src/io/read/block/block_reader_deserialize.rs index 30226c46930ec..7d4aebdedaffc 100644 --- a/src/query/storages/fuse/src/io/read/block/block_reader_deserialize.rs +++ b/src/query/storages/fuse/src/io/read/block/block_reader_deserialize.rs @@ -22,16 +22,16 @@ use databend_common_expression::ColumnId; use databend_common_expression::DataBlock; use databend_storages_common_cache::SizedColumnArray; use databend_storages_common_io::ReadSettings; -use databend_storages_common_table_meta::meta::column_oriented_segment::BlockReadInfo; use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::ColumnMeta; use databend_storages_common_table_meta::meta::Compression; +use databend_storages_common_table_meta::meta::column_oriented_segment::BlockReadInfo; use super::BlockReader; -use crate::io::read::block::block_reader_merge_io::DataItem; use crate::BlockReadResult; use crate::FuseBlockPartInfo; use crate::FuseStorageFormat; +use crate::io::read::block::block_reader_merge_io::DataItem; pub enum DeserializedArray<'a> { Cached(&'a Arc), diff --git a/src/query/storages/fuse/src/io/read/block/block_reader_merge_io.rs b/src/query/storages/fuse/src/io/read/block/block_reader_merge_io.rs index e3265a65849a1..595d7fc483b47 100644 --- a/src/query/storages/fuse/src/io/read/block/block_reader_merge_io.rs +++ b/src/query/storages/fuse/src/io/read/block/block_reader_merge_io.rs @@ -52,7 +52,7 @@ impl BlockReadResult { } } - pub fn columns_chunks(&self) -> Result> { + pub fn columns_chunks(&self) -> Result>> { let mut res = HashMap::with_capacity(self.merge_io_result.columns_chunk_offsets.len()); // merge column data fetched from object storage diff --git a/src/query/storages/fuse/src/io/read/block/block_reader_merge_io_async.rs b/src/query/storages/fuse/src/io/read/block/block_reader_merge_io_async.rs index 0e6ad1028a106..5355030654e1b 100644 --- a/src/query/storages/fuse/src/io/read/block/block_reader_merge_io_async.rs +++ b/src/query/storages/fuse/src/io/read/block/block_reader_merge_io_async.rs @@ -28,8 +28,8 @@ use databend_storages_common_io::MergeIOReader; use databend_storages_common_io::ReadSettings; use databend_storages_common_table_meta::meta::ColumnMeta; -use crate::io::BlockReader; use crate::BlockReadResult; +use crate::io::BlockReader; impl BlockReader { #[async_backtrace::framed] diff --git a/src/query/storages/fuse/src/io/read/block/block_reader_native.rs b/src/query/storages/fuse/src/io/read/block/block_reader_native.rs index b528ebf0ee1b1..5d8b4c5c2a77c 100644 --- a/src/query/storages/fuse/src/io/read/block/block_reader_native.rs +++ b/src/query/storages/fuse/src/io/read/block/block_reader_native.rs @@ -27,9 +27,9 @@ use databend_common_expression::ColumnId; use databend_common_expression::DataBlock; use databend_common_expression::DataSchema; use databend_common_metrics::storage::*; -use databend_common_native::read::reader::read_meta_async; -use databend_common_native::read::reader::NativeReader; use databend_common_native::read::NativeReadBuf; +use databend_common_native::read::reader::NativeReader; +use databend_common_native::read::reader::read_meta_async; use databend_storages_common_io::ReadSettings; use databend_storages_common_table_meta::meta::ColumnMeta; use opendal::Operator; diff --git a/src/query/storages/fuse/src/io/read/block/block_reader_native_deserialize.rs b/src/query/storages/fuse/src/io/read/block/block_reader_native_deserialize.rs index f583ae24802fd..1f8ca52c19590 100644 --- a/src/query/storages/fuse/src/io/read/block/block_reader_native_deserialize.rs +++ b/src/query/storages/fuse/src/io/read/block/block_reader_native_deserialize.rs @@ -24,8 +24,8 @@ use databend_common_expression::DataBlock; use databend_common_expression::TableField; use databend_common_expression::Value; use databend_common_metrics::storage::*; -use databend_common_native::read::reader::NativeReader; use databend_common_native::read::ColumnIter; +use databend_common_native::read::reader::NativeReader; use databend_common_storage::ColumnNode; use databend_storages_common_cache::CacheAccessor; use databend_storages_common_cache::CacheManager; @@ -35,9 +35,9 @@ use opendal::Buffer; use super::block_reader_deserialize::DeserializedArray; use super::block_reader_deserialize::FieldDeserializationContext; -use crate::io::read::block::block_reader_merge_io::DataItem; use crate::io::BlockReader; use crate::io::NativeReaderExt; +use crate::io::read::block::block_reader_merge_io::DataItem; impl BlockReader { /// Deserialize column chunks data from native format to DataBlock. diff --git a/src/query/storages/fuse/src/io/read/block/parquet/deserialize.rs b/src/query/storages/fuse/src/io/read/block/parquet/deserialize.rs index 22907c2d03210..a1a855921f598 100644 --- a/src/query/storages/fuse/src/io/read/block/parquet/deserialize.rs +++ b/src/query/storages/fuse/src/io/read/block/parquet/deserialize.rs @@ -19,10 +19,10 @@ use arrow_schema::Schema; use databend_common_expression::ColumnId; use databend_common_expression::TableSchema; use databend_storages_common_table_meta::meta::Compression; -use parquet::arrow::arrow_reader::ParquetRecordBatchReader; -use parquet::arrow::parquet_to_arrow_field_levels; use parquet::arrow::ArrowSchemaConverter; use parquet::arrow::ProjectionMask; +use parquet::arrow::arrow_reader::ParquetRecordBatchReader; +use parquet::arrow::parquet_to_arrow_field_levels; use parquet::basic::Compression as ParquetCompression; use crate::io::read::block::block_reader_merge_io::DataItem; diff --git a/src/query/storages/fuse/src/io/read/block/parquet/mod.rs b/src/query/storages/fuse/src/io/read/block/parquet/mod.rs index 781bb843c3f8f..416ce48595eba 100644 --- a/src/query/storages/fuse/src/io/read/block/parquet/mod.rs +++ b/src/query/storages/fuse/src/io/read/block/parquet/mod.rs @@ -37,8 +37,8 @@ mod deserialize; pub use adapter::RowGroupImplBuilder; pub use deserialize::column_chunks_to_record_batch; -use crate::io::read::block::block_reader_merge_io::DataItem; use crate::io::BlockReader; +use crate::io::read::block::block_reader_merge_io::DataItem; impl BlockReader { pub(crate) fn deserialize_parquet_chunks( diff --git a/src/query/storages/fuse/src/io/read/bloom/block_filter_reader.rs b/src/query/storages/fuse/src/io/read/bloom/block_filter_reader.rs index ec1e100168a2f..c741b497d7366 100644 --- a/src/query/storages/fuse/src/io/read/bloom/block_filter_reader.rs +++ b/src/query/storages/fuse/src/io/read/bloom/block_filter_reader.rs @@ -30,8 +30,8 @@ use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_storages_common_cache::LoadParams; -use databend_storages_common_index::filters::FilterImpl; use databend_storages_common_index::BloomIndexMeta; +use databend_storages_common_index::filters::FilterImpl; use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::SingleColumnMeta; use futures_util::future::try_join_all; @@ -41,8 +41,8 @@ use parquet::schema::types::SchemaDescPtr; use crate::index::filters::BlockBloomFilterIndexVersion; use crate::index::filters::BlockFilter; -use crate::io::read::bloom::column_filter_reader::BloomColumnFilterReader; use crate::io::MetaReaders; +use crate::io::read::bloom::column_filter_reader::BloomColumnFilterReader; #[async_trait::async_trait] pub trait BloomBlockFilterReader { async fn read_block_filter( diff --git a/src/query/storages/fuse/src/io/read/bloom/column_filter_reader.rs b/src/query/storages/fuse/src/io/read/bloom/column_filter_reader.rs index 2920fe1c9548d..afce67c444fc1 100644 --- a/src/query/storages/fuse/src/io/read/bloom/column_filter_reader.rs +++ b/src/query/storages/fuse/src/io/read/bloom/column_filter_reader.rs @@ -27,9 +27,9 @@ use databend_storages_common_index::filters::Filter; use databend_storages_common_index::filters::FilterImpl; use databend_storages_common_table_meta::meta::SingleColumnMeta; use opendal::Operator; +use parquet::arrow::ProjectionMask; use parquet::arrow::arrow_reader::ParquetRecordBatchReader; use parquet::arrow::parquet_to_arrow_field_levels; -use parquet::arrow::ProjectionMask; use parquet::basic::Compression as ParquetCompression; use parquet::schema::types::SchemaDescPtr; diff --git a/src/query/storages/fuse/src/io/read/inverted_index/inverted_index_loader.rs b/src/query/storages/fuse/src/io/read/inverted_index/inverted_index_loader.rs index 4904da1fbe069..1cebb5bbb04fd 100644 --- a/src/query/storages/fuse/src/io/read/inverted_index/inverted_index_loader.rs +++ b/src/query/storages/fuse/src/io/read/inverted_index/inverted_index_loader.rs @@ -40,14 +40,14 @@ use databend_storages_common_table_meta::meta::SingleColumnMeta; use databend_storages_common_table_meta::table::TableCompression; use log::info; use opendal::Operator; -use parquet::arrow::arrow_reader::ParquetRecordBatchReader; -use parquet::arrow::parquet_to_arrow_field_levels; use parquet::arrow::ArrowSchemaConverter; use parquet::arrow::ProjectionMask; +use parquet::arrow::arrow_reader::ParquetRecordBatchReader; +use parquet::arrow::parquet_to_arrow_field_levels; use crate::index::InvertedIndexFile; -use crate::io::read::block::parquet::RowGroupImplBuilder; use crate::io::MetaReaders; +use crate::io::read::block::parquet::RowGroupImplBuilder; const INDEX_COLUMN_NAMES: [&str; 8] = [ "fast", diff --git a/src/query/storages/fuse/src/io/read/inverted_index/inverted_index_reader.rs b/src/query/storages/fuse/src/io/read/inverted_index/inverted_index_reader.rs index 38a17e30fa2cd..ef0514dd1bc9d 100644 --- a/src/query/storages/fuse/src/io/read/inverted_index/inverted_index_reader.rs +++ b/src/query/storages/fuse/src/io/read/inverted_index/inverted_index_reader.rs @@ -24,6 +24,7 @@ use databend_common_metrics::storage::metrics_inc_block_inverted_index_search_mi use databend_storages_common_io::ReadSettings; use databend_storages_common_table_meta::meta::SingleColumnMeta; use opendal::Operator; +use tantivy::Index; use tantivy::collector::DocSetCollector; use tantivy::collector::TopDocs; use tantivy::directory::FileSlice; @@ -36,7 +37,6 @@ use tantivy::query::QueryClone; use tantivy::schema::IndexRecordOption; use tantivy::termdict::TermInfoStore; use tantivy::tokenizer::TokenizerManager; -use tantivy::Index; use tantivy_common::BinarySerializable; use tantivy_fst::raw::Fst; diff --git a/src/query/storages/fuse/src/io/read/meta/mod.rs b/src/query/storages/fuse/src/io/read/meta/mod.rs index 258b61bbb6650..ce776a78db6fc 100644 --- a/src/query/storages/fuse/src/io/read/meta/mod.rs +++ b/src/query/storages/fuse/src/io/read/meta/mod.rs @@ -14,8 +14,8 @@ mod meta_readers; -pub use meta_readers::bytes_reader; pub use meta_readers::CompactSegmentInfoReader; pub use meta_readers::MetaReaders; pub use meta_readers::SegmentStatsReader; pub use meta_readers::TableSnapshotReader; +pub use meta_readers::bytes_reader; diff --git a/src/query/storages/fuse/src/io/read/mod.rs b/src/query/storages/fuse/src/io/read/mod.rs index ae7c578de3a19..fc4e1df277f82 100644 --- a/src/query/storages/fuse/src/io/read/mod.rs +++ b/src/query/storages/fuse/src/io/read/mod.rs @@ -34,15 +34,15 @@ pub use inverted_index::InvertedIndexReader; pub use meta::CompactSegmentInfoReader; pub use meta::MetaReaders; pub use meta::TableSnapshotReader; -pub use segment_reader::read_column_oriented_segment; pub use segment_reader::ColumnOrientedSegmentReader; pub use segment_reader::RowOrientedSegmentReader; pub use segment_reader::SegmentReader; +pub use segment_reader::read_column_oriented_segment; pub use snapshot_history_reader::SnapshotHistoryReader; pub use statistics::*; pub use utils::build_columns_meta; +pub use vector_index::VectorIndexReader; pub use vector_index::load_vector_index_files; pub use vector_index::load_vector_index_meta; -pub use vector_index::VectorIndexReader; pub use virtual_column::VirtualBlockReadResult; pub use virtual_column::VirtualColumnReader; diff --git a/src/query/storages/fuse/src/io/read/segment_reader.rs b/src/query/storages/fuse/src/io/read/segment_reader.rs index d024b267e5903..a193de7ef2750 100644 --- a/src/query/storages/fuse/src/io/read/segment_reader.rs +++ b/src/query/storages/fuse/src/io/read/segment_reader.rs @@ -19,14 +19,14 @@ use databend_common_exception::Result; use databend_common_expression::TableSchemaRef; use databend_storages_common_cache::CacheAccessor; use databend_storages_common_cache::CacheManager; -use databend_storages_common_table_meta::meta::column_oriented_segment::deserialize_column_oriented_segment; +use databend_storages_common_table_meta::meta::CompactSegmentInfo; +use databend_storages_common_table_meta::meta::Location; +use databend_storages_common_table_meta::meta::SegmentInfo; use databend_storages_common_table_meta::meta::column_oriented_segment::AbstractSegment; use databend_storages_common_table_meta::meta::column_oriented_segment::ColumnOrientedSegment; use databend_storages_common_table_meta::meta::column_oriented_segment::ColumnOrientedSegmentBuilder; use databend_storages_common_table_meta::meta::column_oriented_segment::SegmentBuilder; -use databend_storages_common_table_meta::meta::CompactSegmentInfo; -use databend_storages_common_table_meta::meta::Location; -use databend_storages_common_table_meta::meta::SegmentInfo; +use databend_storages_common_table_meta::meta::column_oriented_segment::deserialize_column_oriented_segment; use opendal::Operator; use super::meta::bytes_reader; diff --git a/src/query/storages/fuse/src/io/read/snapshot_history_reader.rs b/src/query/storages/fuse/src/io/read/snapshot_history_reader.rs index 79dd6382efbf2..f0374e80c34d4 100644 --- a/src/query/storages/fuse/src/io/read/snapshot_history_reader.rs +++ b/src/query/storages/fuse/src/io/read/snapshot_history_reader.rs @@ -50,7 +50,7 @@ impl SnapshotHistoryReader for TableSnapshotReader { ) -> TableSnapshotStream { let stream = stream::try_unfold( (self, location_gen, Some((location, format_version))), - |(reader, gen, next)| async move { + |(reader, loc_gen, next)| async move { if let Some((loc, ver)) = next { let load_params = LoadParams { location: loc, @@ -78,13 +78,13 @@ impl SnapshotHistoryReader for TableSnapshotReader { if let Some((prev_id, prev_version)) = snapshot.prev_snapshot_id { let new_ver = prev_version; let new_loc = - gen.snapshot_location_from_uuid(&prev_id, prev_version)?; + loc_gen.snapshot_location_from_uuid(&prev_id, prev_version)?; Ok(Some(( (snapshot, ver), - (reader, gen, Some((new_loc, new_ver))), + (reader, loc_gen, Some((new_loc, new_ver))), ))) } else { - Ok(Some(((snapshot, ver), (reader, gen, None)))) + Ok(Some(((snapshot, ver), (reader, loc_gen, None)))) } } Ok(None) => Ok(None), diff --git a/src/query/storages/fuse/src/io/read/vector_index/vector_index_loader.rs b/src/query/storages/fuse/src/io/read/vector_index/vector_index_loader.rs index ea8378f38a130..6545866053bc4 100644 --- a/src/query/storages/fuse/src/io/read/vector_index/vector_index_loader.rs +++ b/src/query/storages/fuse/src/io/read/vector_index/vector_index_loader.rs @@ -36,14 +36,14 @@ use databend_storages_common_io::MergeIOReader; use databend_storages_common_io::ReadSettings; use databend_storages_common_table_meta::table::TableCompression; use opendal::Operator; -use parquet::arrow::arrow_reader::ParquetRecordBatchReader; -use parquet::arrow::parquet_to_arrow_field_levels; use parquet::arrow::ArrowSchemaConverter; use parquet::arrow::ProjectionMask; +use parquet::arrow::arrow_reader::ParquetRecordBatchReader; +use parquet::arrow::parquet_to_arrow_field_levels; use crate::index::VectorIndexFile; -use crate::io::read::block::parquet::RowGroupImplBuilder; use crate::io::MetaReaders; +use crate::io::read::block::parquet::RowGroupImplBuilder; #[async_trait::async_trait] trait InRuntime diff --git a/src/query/storages/fuse/src/io/read/virtual_column/virtual_column_reader_parquet.rs b/src/query/storages/fuse/src/io/read/virtual_column/virtual_column_reader_parquet.rs index 5e55c735f4994..7a0208552cb89 100644 --- a/src/query/storages/fuse/src/io/read/virtual_column/virtual_column_reader_parquet.rs +++ b/src/query/storages/fuse/src/io/read/virtual_column/virtual_column_reader_parquet.rs @@ -16,14 +16,14 @@ use std::collections::HashSet; use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::eval_function; -use databend_common_expression::types::DataType; use databend_common_expression::Column; use databend_common_expression::ColumnId; use databend_common_expression::DataBlock; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRef; use databend_common_expression::Value; +use databend_common_expression::eval_function; +use databend_common_expression::types::DataType; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_storages_common_io::MergeIOReader; use databend_storages_common_io::ReadSettings; @@ -31,8 +31,8 @@ use databend_storages_common_pruner::VirtualBlockMetaIndex; use databend_storages_common_table_meta::meta::Compression; use super::VirtualColumnReader; -use crate::io::read::block::parquet::column_chunks_to_record_batch; use crate::BlockReadResult; +use crate::io::read::block::parquet::column_chunks_to_record_batch; pub struct VirtualBlockReadResult { pub num_rows: usize, diff --git a/src/query/storages/fuse/src/io/write/block_statistics_writer.rs b/src/query/storages/fuse/src/io/write/block_statistics_writer.rs index eb1b68dc2cef0..af708495092b4 100644 --- a/src/query/storages/fuse/src/io/write/block_statistics_writer.rs +++ b/src/query/storages/fuse/src/io/write/block_statistics_writer.rs @@ -23,9 +23,9 @@ use databend_common_expression::FieldIndex; use databend_common_expression::TableField; use databend_storages_common_table_meta::meta::BlockHLL; -use crate::io::write::stream::create_column_ndv_estimator; use crate::io::write::stream::ColumnNDVEstimator; use crate::io::write::stream::ColumnNDVEstimatorOps; +use crate::io::write::stream::create_column_ndv_estimator; pub fn build_column_hlls( block: &DataBlock, diff --git a/src/query/storages/fuse/src/io/write/block_writer.rs b/src/query/storages/fuse/src/io/write/block_writer.rs index 5a03e64978c34..dfb7a73e8d897 100644 --- a/src/query/storages/fuse/src/io/write/block_writer.rs +++ b/src/query/storages/fuse/src/io/write/block_writer.rs @@ -12,16 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::hash_map::Entry; use std::collections::BTreeMap; use std::collections::HashMap; +use std::collections::hash_map::Entry; use std::sync::Arc; use std::time::Instant; use chrono::Utc; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::local_block_meta_serde; use databend_common_expression::BlockMetaInfo; use databend_common_expression::Column; use databend_common_expression::ColumnId; @@ -29,6 +28,7 @@ use databend_common_expression::DataBlock; use databend_common_expression::FieldIndex; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; +use databend_common_expression::local_block_meta_serde; use databend_common_io::constants::DEFAULT_BLOCK_BUFFER_SIZE; use databend_common_metrics::storage::metrics_inc_block_index_write_milliseconds; use databend_common_metrics::storage::metrics_inc_block_index_write_nums; @@ -46,7 +46,6 @@ use databend_common_metrics::storage::metrics_inc_block_write_nums; use databend_common_native::write::NativeWriter; use databend_storages_common_blocks::blocks_to_parquet_with_stats; use databend_storages_common_index::NgramArgs; -use databend_storages_common_table_meta::meta::encode_column_hll; use databend_storages_common_table_meta::meta::BlockHLLState; use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::ClusterStatistics; @@ -54,23 +53,24 @@ use databend_storages_common_table_meta::meta::ColumnMeta; use databend_storages_common_table_meta::meta::ExtendedBlockMeta; use databend_storages_common_table_meta::meta::StatisticsOfColumns; use databend_storages_common_table_meta::meta::TableMetaTimestamps; +use databend_storages_common_table_meta::meta::encode_column_hll; use databend_storages_common_table_meta::table::TableCompression; use opendal::Operator; +use crate::FuseStorageFormat; +use crate::io::BloomIndexState; +use crate::io::TableMetaLocationGenerator; use crate::io::build_column_hlls; -use crate::io::write::virtual_column_builder::VirtualColumnBuilder; -use crate::io::write::virtual_column_builder::VirtualColumnState; use crate::io::write::InvertedIndexBuilder; use crate::io::write::InvertedIndexState; use crate::io::write::VectorIndexBuilder; use crate::io::write::VectorIndexState; use crate::io::write::WriteSettings; -use crate::io::BloomIndexState; -use crate::io::TableMetaLocationGenerator; +use crate::io::write::virtual_column_builder::VirtualColumnBuilder; +use crate::io::write::virtual_column_builder::VirtualColumnState; use crate::operations::column_parquet_metas; -use crate::statistics::gen_columns_statistics; use crate::statistics::ClusterStatsGenerator; -use crate::FuseStorageFormat; +use crate::statistics::gen_columns_statistics; pub fn serialize_block( write_settings: &WriteSettings, diff --git a/src/query/storages/fuse/src/io/write/bloom_index_writer.rs b/src/query/storages/fuse/src/io/write/bloom_index_writer.rs index 34f9e192143f4..a7cc4642d7ce6 100644 --- a/src/query/storages/fuse/src/io/write/bloom_index_writer.rs +++ b/src/query/storages/fuse/src/io/write/bloom_index_writer.rs @@ -26,19 +26,19 @@ use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; use databend_common_io::constants::DEFAULT_BLOCK_INDEX_BUFFER_SIZE; use databend_storages_common_blocks::blocks_to_parquet; -use databend_storages_common_index::filters::BlockFilter; use databend_storages_common_index::BloomIndex; use databend_storages_common_index::BloomIndexBuilder; use databend_storages_common_index::NgramArgs; +use databend_storages_common_index::filters::BlockFilter; use databend_storages_common_io::ReadSettings; -use databend_storages_common_table_meta::meta::column_oriented_segment::BlockReadInfo; use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::Versioned; +use databend_storages_common_table_meta::meta::column_oriented_segment::BlockReadInfo; use databend_storages_common_table_meta::table::TableCompression; use opendal::Operator; -use crate::io::BlockReader; use crate::FuseStorageFormat; +use crate::io::BlockReader; #[derive(Debug)] pub struct BloomIndexState { diff --git a/src/query/storages/fuse/src/io/write/inverted_index_writer.rs b/src/query/storages/fuse/src/io/write/inverted_index_writer.rs index a596e38bd0bb1..5f5ff17c9fc2a 100644 --- a/src/query/storages/fuse/src/io/write/inverted_index_writer.rs +++ b/src/query/storages/fuse/src/io/write/inverted_index_writer.rs @@ -20,7 +20,6 @@ use std::time::Instant; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::BlockEntry; use databend_common_expression::DataBlock; use databend_common_expression::DataField; @@ -32,6 +31,7 @@ use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::DataType; use databend_common_io::constants::DEFAULT_BLOCK_BUFFER_SIZE; use databend_common_io::constants::DEFAULT_BLOCK_INDEX_BUFFER_SIZE; use databend_common_meta_app::schema::TableIndexType; @@ -40,10 +40,14 @@ use databend_common_metrics::storage::metrics_inc_block_inverted_index_generate_ use databend_storages_common_blocks::blocks_to_parquet; use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::table::TableCompression; -use jsonb::from_raw_jsonb; use jsonb::RawJsonb; +use jsonb::from_raw_jsonb; use log::debug; use log::info; +use tantivy::Directory; +use tantivy::IndexBuilder; +use tantivy::IndexSettings; +use tantivy::IndexWriter; use tantivy::index::SegmentComponent; use tantivy::indexer::UserOperation; use tantivy::schema::Field; @@ -61,10 +65,6 @@ use tantivy::tokenizer::Stemmer; use tantivy::tokenizer::StopWordFilter; use tantivy::tokenizer::TextAnalyzer; use tantivy::tokenizer::TokenizerManager; -use tantivy::Directory; -use tantivy::IndexBuilder; -use tantivy::IndexSettings; -use tantivy::IndexWriter; use tantivy_jieba::JiebaTokenizer; use crate::index::build_tantivy_footer; diff --git a/src/query/storages/fuse/src/io/write/mod.rs b/src/query/storages/fuse/src/io/write/mod.rs index 368cc31ae1497..fcad4c956ee9d 100644 --- a/src/query/storages/fuse/src/io/write/mod.rs +++ b/src/query/storages/fuse/src/io/write/mod.rs @@ -22,21 +22,21 @@ mod vector_index_writer; mod virtual_column_builder; mod write_settings; -pub use block_statistics_writer::build_column_hlls; pub use block_statistics_writer::BlockStatsBuilder; -pub use block_writer::serialize_block; -pub use block_writer::write_data; +pub use block_statistics_writer::build_column_hlls; pub use block_writer::BlockBuilder; pub use block_writer::BlockSerialization; pub use block_writer::BlockWriter; +pub use block_writer::serialize_block; +pub use block_writer::write_data; pub use bloom_index_writer::BloomIndexRebuilder; pub use bloom_index_writer::BloomIndexState; -pub(crate) use inverted_index_writer::create_index_schema; -pub(crate) use inverted_index_writer::create_inverted_index_builders; -pub(crate) use inverted_index_writer::create_tokenizer_manager; pub use inverted_index_writer::InvertedIndexBuilder; pub(crate) use inverted_index_writer::InvertedIndexState; pub use inverted_index_writer::InvertedIndexWriter; +pub(crate) use inverted_index_writer::create_index_schema; +pub(crate) use inverted_index_writer::create_inverted_index_builders; +pub(crate) use inverted_index_writer::create_tokenizer_manager; pub use meta_writer::CachedMetaWriter; pub use meta_writer::MetaWriter; pub(crate) use stream::StreamBlockBuilder; @@ -44,5 +44,5 @@ pub(crate) use stream::StreamBlockProperties; pub use vector_index_writer::VectorIndexBuilder; pub(crate) use vector_index_writer::VectorIndexState; pub use virtual_column_builder::VirtualColumnBuilder; -pub use write_settings::WriteSettings; pub use write_settings::MAX_BLOCK_UNCOMPRESSED_SIZE; +pub use write_settings::WriteSettings; diff --git a/src/query/storages/fuse/src/io/write/stream/block_builder.rs b/src/query/storages/fuse/src/io/write/stream/block_builder.rs index 36369c41a031b..7f8d20d9475b1 100644 --- a/src/query/storages/fuse/src/io/write/stream/block_builder.rs +++ b/src/query/storages/fuse/src/io/write/stream/block_builder.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::hash_map::Entry; use std::collections::BTreeMap; use std::collections::HashMap; use std::collections::HashSet; +use std::collections::hash_map::Entry; use std::mem; use std::sync::Arc; @@ -25,7 +25,6 @@ use chrono::Utc; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::BlockThresholds; use databend_common_expression::Column; use databend_common_expression::ColumnId; @@ -35,13 +34,14 @@ use databend_common_expression::FieldIndex; use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRef; +use databend_common_expression::types::DataType; use databend_common_io::constants::DEFAULT_BLOCK_BUFFER_SIZE; use databend_common_meta_app::schema::TableIndex; use databend_common_native::write::NativeWriter; use databend_common_native::write::WriteOptions; use databend_common_sql::executor::physical_plans::MutationKind; -use databend_storages_common_blocks::build_parquet_writer_properties; use databend_storages_common_blocks::NdvProvider; +use databend_storages_common_blocks::build_parquet_writer_properties; use databend_storages_common_index::BloomIndex; use databend_storages_common_index::BloomIndexBuilder; use databend_storages_common_index::Index; @@ -55,13 +55,8 @@ use databend_storages_common_table_meta::table::TableCompression; use parquet::arrow::ArrowWriter; use parquet::format::FileMetaData; -use crate::io::create_inverted_index_builders; -use crate::io::write::stream::block_builder::ArrowParquetWriter::Initialized; -use crate::io::write::stream::cluster_statistics::ClusterStatisticsBuilder; -use crate::io::write::stream::cluster_statistics::ClusterStatisticsState; -use crate::io::write::stream::ColumnStatisticsState; -use crate::io::write::BlockStatsBuilder; -use crate::io::write::InvertedIndexState; +use crate::FuseStorageFormat; +use crate::FuseTable; use crate::io::BlockSerialization; use crate::io::BloomIndexState; use crate::io::InvertedIndexBuilder; @@ -70,9 +65,14 @@ use crate::io::TableMetaLocationGenerator; use crate::io::VectorIndexBuilder; use crate::io::VirtualColumnBuilder; use crate::io::WriteSettings; +use crate::io::create_inverted_index_builders; +use crate::io::write::BlockStatsBuilder; +use crate::io::write::InvertedIndexState; +use crate::io::write::stream::ColumnStatisticsState; +use crate::io::write::stream::block_builder::ArrowParquetWriter::Initialized; +use crate::io::write::stream::cluster_statistics::ClusterStatisticsBuilder; +use crate::io::write::stream::cluster_statistics::ClusterStatisticsState; use crate::operations::column_parquet_metas; -use crate::FuseStorageFormat; -use crate::FuseTable; pub struct UninitializedArrowWriter { write_settings: WriteSettings, diff --git a/src/query/storages/fuse/src/io/write/stream/cluster_statistics.rs b/src/query/storages/fuse/src/io/write/stream/cluster_statistics.rs index c397694572bab..58c2264347408 100644 --- a/src/query/storages/fuse/src/io/write/stream/cluster_statistics.rs +++ b/src/query/storages/fuse/src/io/write/stream/cluster_statistics.rs @@ -25,8 +25,8 @@ use databend_common_expression::Expr; use databend_common_expression::FunctionContext; use databend_common_expression::Scalar; use databend_common_expression::TableSchemaRef; -use databend_common_functions::aggregates::eval_aggr; use databend_common_functions::BUILTIN_FUNCTIONS; +use databend_common_functions::aggregates::eval_aggr; use databend_common_sql::evaluator::BlockOperator; use databend_storages_common_table_meta::meta::ClusterStatistics; use databend_storages_common_table_meta::table::ClusterType; diff --git a/src/query/storages/fuse/src/io/write/stream/column_ndv_estimator.rs b/src/query/storages/fuse/src/io/write/stream/column_ndv_estimator.rs index 8784094357e78..2d8826613abd0 100644 --- a/src/query/storages/fuse/src/io/write/stream/column_ndv_estimator.rs +++ b/src/query/storages/fuse/src/io/write/stream/column_ndv_estimator.rs @@ -15,31 +15,31 @@ use std::hash::Hash; use std::marker::PhantomData; -use databend_common_expression::types::boolean::TrueIdxIter; -use databend_common_expression::types::timestamp_tz::TimestampTzType; +use databend_common_expression::Column; +use databend_common_expression::SELECTIVITY_THRESHOLD; +use databend_common_expression::ScalarRef; use databend_common_expression::types::DataType; use databend_common_expression::types::DateType; +use databend_common_expression::types::Decimal64Type; use databend_common_expression::types::Decimal128Type; use databend_common_expression::types::Decimal256Type; -use databend_common_expression::types::Decimal64Type; use databend_common_expression::types::Float32Type; use databend_common_expression::types::Float64Type; +use databend_common_expression::types::Int8Type; use databend_common_expression::types::Int16Type; use databend_common_expression::types::Int32Type; use databend_common_expression::types::Int64Type; -use databend_common_expression::types::Int8Type; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::StringType; use databend_common_expression::types::TimestampType; +use databend_common_expression::types::UInt8Type; use databend_common_expression::types::UInt16Type; use databend_common_expression::types::UInt32Type; use databend_common_expression::types::UInt64Type; -use databend_common_expression::types::UInt8Type; use databend_common_expression::types::ValueType; +use databend_common_expression::types::boolean::TrueIdxIter; +use databend_common_expression::types::timestamp_tz::TimestampTzType; use databend_common_expression::with_number_type; -use databend_common_expression::Column; -use databend_common_expression::ScalarRef; -use databend_common_expression::SELECTIVITY_THRESHOLD; use databend_common_storage::MetaHLL; use enum_dispatch::enum_dispatch; diff --git a/src/query/storages/fuse/src/io/write/stream/column_statistics_builder.rs b/src/query/storages/fuse/src/io/write/stream/column_statistics_builder.rs index 5459a3d90316a..4bee006f62afa 100644 --- a/src/query/storages/fuse/src/io/write/stream/column_statistics_builder.rs +++ b/src/query/storages/fuse/src/io/write/stream/column_statistics_builder.rs @@ -17,33 +17,33 @@ use std::marker::PhantomData; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::boolean::TrueIdxIter; -use databend_common_expression::types::timestamp_tz::TimestampTzType; +use databend_common_expression::Column; +use databend_common_expression::SELECTIVITY_THRESHOLD; +use databend_common_expression::Scalar; +use databend_common_expression::ScalarRef; use databend_common_expression::types::DataType; use databend_common_expression::types::DateType; use databend_common_expression::types::Decimal; +use databend_common_expression::types::Decimal64Type; use databend_common_expression::types::Decimal128Type; use databend_common_expression::types::Decimal256Type; -use databend_common_expression::types::Decimal64Type; use databend_common_expression::types::Float32Type; use databend_common_expression::types::Float64Type; +use databend_common_expression::types::Int8Type; use databend_common_expression::types::Int16Type; use databend_common_expression::types::Int32Type; use databend_common_expression::types::Int64Type; -use databend_common_expression::types::Int8Type; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::StringType; use databend_common_expression::types::TimestampType; +use databend_common_expression::types::UInt8Type; use databend_common_expression::types::UInt16Type; use databend_common_expression::types::UInt32Type; use databend_common_expression::types::UInt64Type; -use databend_common_expression::types::UInt8Type; use databend_common_expression::types::ValueType; +use databend_common_expression::types::boolean::TrueIdxIter; +use databend_common_expression::types::timestamp_tz::TimestampTzType; use databend_common_expression::with_number_type; -use databend_common_expression::Column; -use databend_common_expression::Scalar; -use databend_common_expression::ScalarRef; -use databend_common_expression::SELECTIVITY_THRESHOLD; use databend_storages_common_table_meta::meta::ColumnStatistics; use enum_dispatch::enum_dispatch; diff --git a/src/query/storages/fuse/src/io/write/stream/column_statistics_state.rs b/src/query/storages/fuse/src/io/write/stream/column_statistics_state.rs index df5304e4e5afc..f462de3dc83e7 100644 --- a/src/query/storages/fuse/src/io/write/stream/column_statistics_state.rs +++ b/src/query/storages/fuse/src/io/write/stream/column_statistics_state.rs @@ -15,19 +15,19 @@ use std::collections::HashMap; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::ColumnId; use databend_common_expression::DataBlock; use databend_common_expression::TableSchemaRef; use databend_common_expression::Value; +use databend_common_expression::types::DataType; use databend_storages_common_table_meta::meta::StatisticsOfColumns; -use crate::io::write::stream::create_column_ndv_estimator; -use crate::io::write::stream::create_column_stats_builder; use crate::io::write::stream::ColumnNDVEstimator; use crate::io::write::stream::ColumnNDVEstimatorOps; use crate::io::write::stream::ColumnStatisticsBuilder; use crate::io::write::stream::ColumnStatsOps; +use crate::io::write::stream::create_column_ndv_estimator; +use crate::io::write::stream::create_column_stats_builder; use crate::statistics::traverse_values_dfs; pub struct ColumnStatisticsState { @@ -124,16 +124,16 @@ impl ColumnStatisticsState { mod tests { use std::sync::Arc; - use databend_common_expression::types::BinaryType; - use databend_common_expression::types::Int64Type; - use databend_common_expression::types::NumberDataType; - use databend_common_expression::types::StringType; - use databend_common_expression::types::UInt64Type; use databend_common_expression::Column; use databend_common_expression::FromData; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; + use databend_common_expression::types::BinaryType; + use databend_common_expression::types::Int64Type; + use databend_common_expression::types::NumberDataType; + use databend_common_expression::types::StringType; + use databend_common_expression::types::UInt64Type; use databend_storages_common_index::Index; use databend_storages_common_index::RangeIndex; diff --git a/src/query/storages/fuse/src/io/write/stream/mod.rs b/src/query/storages/fuse/src/io/write/stream/mod.rs index f0c7365b5ba01..441f3e395e4a3 100644 --- a/src/query/storages/fuse/src/io/write/stream/mod.rs +++ b/src/query/storages/fuse/src/io/write/stream/mod.rs @@ -20,10 +20,10 @@ mod column_statistics_state; pub(crate) use block_builder::StreamBlockBuilder; pub(crate) use block_builder::StreamBlockProperties; -pub(crate) use column_ndv_estimator::create_column_ndv_estimator; pub(crate) use column_ndv_estimator::ColumnNDVEstimator; pub(crate) use column_ndv_estimator::ColumnNDVEstimatorOps; -pub(crate) use column_statistics_builder::create_column_stats_builder; +pub(crate) use column_ndv_estimator::create_column_ndv_estimator; pub(crate) use column_statistics_builder::ColumnStatisticsBuilder; pub(crate) use column_statistics_builder::ColumnStatsOps; +pub(crate) use column_statistics_builder::create_column_stats_builder; pub(crate) use column_statistics_state::ColumnStatisticsState; diff --git a/src/query/storages/fuse/src/io/write/virtual_column_builder.rs b/src/query/storages/fuse/src/io/write/virtual_column_builder.rs index 9b1325f6a3785..3854aea2ccefb 100644 --- a/src/query/storages/fuse/src/io/write/virtual_column_builder.rs +++ b/src/query/storages/fuse/src/io/write/virtual_column_builder.rs @@ -23,17 +23,6 @@ use databend_common_catalog::table_context::TableContext; use databend_common_column::types::months_days_micros; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::infer_schema_type; -use databend_common_expression::types::binary::BinaryColumnBuilder; -use databend_common_expression::types::i256; -use databend_common_expression::types::DataType; -use databend_common_expression::types::Decimal; -use databend_common_expression::types::DecimalScalar; -use databend_common_expression::types::DecimalSize; -use databend_common_expression::types::MutableBitmap; -use databend_common_expression::types::NullableColumn; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::NumberScalar; use databend_common_expression::Column; use databend_common_expression::ColumnBuilder; use databend_common_expression::ColumnId; @@ -44,8 +33,19 @@ use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; -use databend_common_expression::VariantDataType; use databend_common_expression::VIRTUAL_COLUMNS_LIMIT; +use databend_common_expression::VariantDataType; +use databend_common_expression::infer_schema_type; +use databend_common_expression::types::DataType; +use databend_common_expression::types::Decimal; +use databend_common_expression::types::DecimalScalar; +use databend_common_expression::types::DecimalSize; +use databend_common_expression::types::MutableBitmap; +use databend_common_expression::types::NullableColumn; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::NumberScalar; +use databend_common_expression::types::binary::BinaryColumnBuilder; +use databend_common_expression::types::i256; use databend_common_hashtable::StackHashMap; use databend_common_io::constants::DEFAULT_BLOCK_INDEX_BUFFER_SIZE; use databend_common_license::license::Feature; @@ -56,23 +56,23 @@ use databend_storages_common_table_meta::meta::DraftVirtualColumnMeta; use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::StatisticsOfColumns; use databend_storages_common_table_meta::meta::VirtualColumnMeta; -use jsonb::keypath::KeyPath as JsonbKeyPath; -use jsonb::keypath::KeyPaths as JsonbKeyPaths; use jsonb::Date as JsonbDate; +use jsonb::Decimal64 as JsonbDecimal64; use jsonb::Decimal128 as JsonbDecimal128; use jsonb::Decimal256 as JsonbDecimal256; -use jsonb::Decimal64 as JsonbDecimal64; use jsonb::Interval as JsonbInterval; use jsonb::Number as JsonbNumber; use jsonb::RawJsonb; use jsonb::Timestamp as JsonbTimestamp; use jsonb::Value as JsonbValue; +use jsonb::keypath::KeyPath as JsonbKeyPath; +use jsonb::keypath::KeyPaths as JsonbKeyPaths; use parquet::format::FileMetaData; use siphasher::sip128::Hasher128; use siphasher::sip128::SipHasher24; -use crate::io::write::WriteSettings; use crate::io::TableMetaLocationGenerator; +use crate::io::write::WriteSettings; use crate::statistics::gen_columns_statistics; const SAMPLE_ROWS: usize = 10; @@ -114,7 +114,7 @@ impl VirtualColumnBuilder { .unwrap_or_default() { return Err(ErrorCode::VirtualColumnError( - "Virtual column is an experimental feature, `set enable_experimental_virtual_column=1` to use this feature." + "Virtual column is an experimental feature, `set enable_experimental_virtual_column=1` to use this feature.", )); } @@ -127,7 +127,9 @@ impl VirtualColumnBuilder { } } if variant_fields.is_empty() { - return Err(ErrorCode::VirtualColumnError("Virtual column only support variant type, but this table don't have variant type fields")); + return Err(ErrorCode::VirtualColumnError( + "Virtual column only support variant type, but this table don't have variant type fields", + )); } let mut virtual_paths = Vec::with_capacity(variant_fields.len()); for _ in 0..variant_fields.len() { diff --git a/src/query/storages/fuse/src/io/write/write_settings.rs b/src/query/storages/fuse/src/io/write/write_settings.rs index 9408ee33c26c8..87baa8c3ecd18 100644 --- a/src/query/storages/fuse/src/io/write/write_settings.rs +++ b/src/query/storages/fuse/src/io/write/write_settings.rs @@ -15,8 +15,8 @@ use databend_common_io::constants::DEFAULT_BLOCK_PER_SEGMENT; use databend_storages_common_table_meta::table::TableCompression; -use crate::FuseStorageFormat; use crate::DEFAULT_ROW_PER_PAGE; +use crate::FuseStorageFormat; pub const MAX_BLOCK_UNCOMPRESSED_SIZE: usize = 1024 * 1024 * 400; diff --git a/src/query/storages/fuse/src/lib.rs b/src/query/storages/fuse/src/lib.rs index c79ccb2b655be..87193cbec0df0 100644 --- a/src/query/storages/fuse/src/lib.rs +++ b/src/query/storages/fuse/src/lib.rs @@ -12,11 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(clippy::uninlined_format_args)] +#![allow( + clippy::cloned_ref_to_slice_refs, + clippy::collapsible_if, + clippy::let_and_return, + clippy::unnecessary_unwrap, + clippy::uninlined_format_args +)] #![allow(clippy::useless_asref)] #![feature(type_alias_impl_trait)] #![feature(iter_order_by)] -#![feature(let_chains)] #![feature(impl_trait_in_assoc_type)] #![feature(int_roundings)] #![feature(iterator_try_reduce)] @@ -49,10 +54,10 @@ pub use fuse_part::FuseBlockPartInfo; pub use fuse_part::FuseLazyPartInfo; pub use fuse_table::FuseTable; pub use fuse_table::RetentionPolicy; -pub use fuse_type::segment_format_from_location; pub use fuse_type::FuseSegmentFormat; pub use fuse_type::FuseStorageFormat; pub use fuse_type::FuseTableType; +pub use fuse_type::segment_format_from_location; pub use io::BlockReadResult; pub use pruning::SegmentLocation; pub use retry::commit_with_backoff; diff --git a/src/query/storages/fuse/src/operations/agg_index_sink.rs b/src/query/storages/fuse/src/operations/agg_index_sink.rs index 341e4af2c6fbe..182457586dd2d 100644 --- a/src/query/storages/fuse/src/operations/agg_index_sink.rs +++ b/src/query/storages/fuse/src/operations/agg_index_sink.rs @@ -18,10 +18,10 @@ use std::time::Instant; use async_trait::async_trait; use databend_common_exception::Result; -use databend_common_expression::types::StringType; use databend_common_expression::BlockRowIndex; use databend_common_expression::DataBlock; use databend_common_expression::TableSchemaRef; +use databend_common_expression::types::StringType; use databend_common_metrics::storage::*; use databend_common_pipeline::core::InputPort; use databend_common_pipeline::core::ProcessorPtr; diff --git a/src/query/storages/fuse/src/operations/analyze/analyze_ndv_meta.rs b/src/query/storages/fuse/src/operations/analyze/analyze_ndv_meta.rs index 377db12556abd..9c90d74e5890f 100644 --- a/src/query/storages/fuse/src/operations/analyze/analyze_ndv_meta.rs +++ b/src/query/storages/fuse/src/operations/analyze/analyze_ndv_meta.rs @@ -16,10 +16,10 @@ use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; -use databend_common_expression::local_block_meta_serde; use databend_common_expression::BlockMetaInfo; use databend_common_expression::BlockMetaInfoPtr; use databend_common_expression::ColumnId; +use databend_common_expression::local_block_meta_serde; use databend_common_storage::MetaHLL; #[derive(Clone)] diff --git a/src/query/storages/fuse/src/operations/analyze/analyze_state_sink.rs b/src/query/storages/fuse/src/operations/analyze/analyze_state_sink.rs index 2d18d716477c6..4dcb47818a2f5 100644 --- a/src/query/storages/fuse/src/operations/analyze/analyze_state_sink.rs +++ b/src/query/storages/fuse/src/operations/analyze/analyze_state_sink.rs @@ -39,7 +39,6 @@ use databend_common_storage::Histogram; use databend_common_storage::HistogramBucket; use databend_common_storage::MetaHLL; use databend_storages_common_cache::Partitions; -use databend_storages_common_table_meta::meta::encode_column_hll; use databend_storages_common_table_meta::meta::AdditionalStatsMeta; use databend_storages_common_table_meta::meta::ClusterStatistics; use databend_storages_common_table_meta::meta::SegmentInfo; @@ -47,15 +46,16 @@ use databend_storages_common_table_meta::meta::SnapshotId; use databend_storages_common_table_meta::meta::StatisticsOfColumns; use databend_storages_common_table_meta::meta::TableSnapshot; use databend_storages_common_table_meta::meta::TableSnapshotStatistics; +use databend_storages_common_table_meta::meta::encode_column_hll; +use crate::FuseLazyPartInfo; +use crate::FuseTable; use crate::io::SegmentsIO; use crate::operations::analyze::AnalyzeCollectNDVSource; use crate::operations::analyze::AnalyzeNDVMeta; use crate::statistics::reduce_block_statistics; use crate::statistics::reduce_cluster_statistics; use crate::statistics::reducers::reduce_virtual_column_statistics; -use crate::FuseLazyPartInfo; -use crate::FuseTable; impl FuseTable { pub fn do_analyze( diff --git a/src/query/storages/fuse/src/operations/analyze/collect_ndv_source.rs b/src/query/storages/fuse/src/operations/analyze/collect_ndv_source.rs index 3e58be6651aa9..252f89133597e 100644 --- a/src/query/storages/fuse/src/operations/analyze/collect_ndv_source.rs +++ b/src/query/storages/fuse/src/operations/analyze/collect_ndv_source.rs @@ -43,8 +43,6 @@ use databend_storages_common_cache::LoadParams; use databend_storages_common_cache::SegmentStatistics; use databend_storages_common_index::RangeIndex; use databend_storages_common_io::ReadSettings; -use databend_storages_common_table_meta::meta::decode_column_hll; -use databend_storages_common_table_meta::meta::encode_column_hll; use databend_storages_common_table_meta::meta::AdditionalStatsMeta; use databend_storages_common_table_meta::meta::BlockHLL; use databend_storages_common_table_meta::meta::Location; @@ -52,20 +50,22 @@ use databend_storages_common_table_meta::meta::RawBlockHLL; use databend_storages_common_table_meta::meta::SegmentInfo; use databend_storages_common_table_meta::meta::Statistics; use databend_storages_common_table_meta::meta::Versioned; +use databend_storages_common_table_meta::meta::decode_column_hll; +use databend_storages_common_table_meta::meta::encode_column_hll; use opendal::Operator; -use crate::io::build_column_hlls; -use crate::io::read::meta::SegmentStatsReader; +use crate::FuseLazyPartInfo; +use crate::FuseStorageFormat; +use crate::FuseTable; use crate::io::BlockReader; use crate::io::CachedMetaWriter; use crate::io::CompactSegmentInfoReader; use crate::io::MetaReaders; use crate::io::TableMetaLocationGenerator; +use crate::io::build_column_hlls; +use crate::io::read::meta::SegmentStatsReader; use crate::operations::acquire_task_permit; use crate::operations::analyze::AnalyzeNDVMeta; -use crate::FuseLazyPartInfo; -use crate::FuseStorageFormat; -use crate::FuseTable; struct SegmentWithHLL { segment_location: Location, @@ -296,7 +296,7 @@ impl Processor for AnalyzeCollectNDVSource { return Err(ErrorCode::Internal(format!( "Invalid state reached in sync process: {:?}. This is a bug.", state - ))) + ))); } } Ok(()) @@ -416,7 +416,7 @@ impl Processor for AnalyzeCollectNDVSource { return Err(ErrorCode::Internal(format!( "Invalid state reached in async process: {:?}. This is a bug.", state - ))) + ))); } } Ok(()) diff --git a/src/query/storages/fuse/src/operations/append.rs b/src/query/storages/fuse/src/operations/append.rs index c0d29847e287e..3427897f42128 100644 --- a/src/query/storages/fuse/src/operations/append.rs +++ b/src/query/storages/fuse/src/operations/append.rs @@ -27,22 +27,22 @@ use databend_common_expression::LimitType; use databend_common_expression::SortColumnDescription; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_pipeline::core::Pipeline; +use databend_common_pipeline_transforms::TransformPipelineHelper; use databend_common_pipeline_transforms::blocks::CompoundBlockOperator; use databend_common_pipeline_transforms::build_compact_block_pipeline; use databend_common_pipeline_transforms::create_dummy_item; use databend_common_pipeline_transforms::sorts::TransformSortPartial; -use databend_common_pipeline_transforms::TransformPipelineHelper; use databend_common_sql::evaluator::BlockOperator; use databend_common_sql::executor::physical_plans::MutationKind; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use databend_storages_common_table_meta::table::ClusterType; +use crate::FuseTable; use crate::io::StreamBlockProperties; use crate::operations::TransformBlockBuilder; use crate::operations::TransformBlockWriter; use crate::operations::TransformSerializeBlock; use crate::statistics::ClusterStatsGenerator; -use crate::FuseTable; impl FuseTable { pub fn do_append_data( diff --git a/src/query/storages/fuse/src/operations/changes.rs b/src/query/storages/fuse/src/operations/changes.rs index 819292a67e2c4..100c3a26fc639 100644 --- a/src/query/storages/fuse/src/operations/changes.rs +++ b/src/query/storages/fuse/src/operations/changes.rs @@ -18,36 +18,36 @@ use std::sync::Arc; use std::time::Instant; use chrono::Utc; -use databend_common_catalog::plan::block_id_from_location; use databend_common_catalog::plan::PartStatistics; use databend_common_catalog::plan::Partitions; use databend_common_catalog::plan::PartitionsShuffleKind; use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::plan::StreamTablePart; +use databend_common_catalog::plan::block_id_from_location; use databend_common_catalog::table::NavigationPoint; use databend_common_catalog::table::Table; use databend_common_catalog::table::TableStatistics; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::decimal::Decimal128Type; +use databend_common_expression::BASE_BLOCK_IDS_COL_NAME; use databend_common_expression::RemoteExpr; use databend_common_expression::Scalar; -use databend_common_expression::BASE_BLOCK_IDS_COL_NAME; +use databend_common_expression::types::decimal::Decimal128Type; use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::SegmentInfo; use databend_storages_common_table_meta::meta::TableSnapshot; use databend_storages_common_table_meta::table::ChangeType; -use databend_storages_common_table_meta::table::StreamMode; use databend_storages_common_table_meta::table::OPT_KEY_CHANGE_TRACKING_BEGIN_VER; use databend_storages_common_table_meta::table::OPT_KEY_TABLE_VER; +use databend_storages_common_table_meta::table::StreamMode; use log::info; +use crate::FuseTable; use crate::io::SegmentsIO; use crate::io::SnapshotsIO; use crate::pruning::FusePruner; -use crate::FuseTable; #[derive(Clone)] pub struct ChangesDesc { diff --git a/src/query/storages/fuse/src/operations/commit.rs b/src/query/storages/fuse/src/operations/commit.rs index ac3b386254071..9a648611ce1f1 100644 --- a/src/query/storages/fuse/src/operations/commit.rs +++ b/src/query/storages/fuse/src/operations/commit.rs @@ -42,8 +42,6 @@ use databend_common_sql::executor::physical_plans::MutationKind; use databend_storages_common_cache::CacheAccessor; use databend_storages_common_cache::CachedObject; use databend_storages_common_cache::LoadParams; -use databend_storages_common_table_meta::meta::decode_column_hll; -use databend_storages_common_table_meta::meta::merge_column_hll_mut; use databend_storages_common_table_meta::meta::BlockHLL; use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::SegmentInfo; @@ -53,6 +51,8 @@ use databend_storages_common_table_meta::meta::TableMetaTimestamps; use databend_storages_common_table_meta::meta::TableSnapshot; use databend_storages_common_table_meta::meta::TableSnapshotStatistics; use databend_storages_common_table_meta::meta::Versioned; +use databend_storages_common_table_meta::meta::decode_column_hll; +use databend_storages_common_table_meta::meta::merge_column_hll_mut; use databend_storages_common_table_meta::readers::snapshot_reader::TableSnapshotAccessor; use databend_storages_common_table_meta::table::OPT_KEY_LEGACY_SNAPSHOT_LOC; use databend_storages_common_table_meta::table::OPT_KEY_SNAPSHOT_LOCATION; @@ -61,21 +61,21 @@ use log::debug; use log::info; use opendal::Operator; +use super::TableMutationAggregator; use super::decorate_snapshot; use super::new_serialize_segment_processor; -use super::TableMutationAggregator; +use crate::FuseTable; use crate::io::MetaReaders; use crate::io::MetaWriter; use crate::io::SegmentsIO; use crate::io::TableMetaLocationGenerator; +use crate::operations::SnapshotHintWriter; use crate::operations::common::AppendGenerator; use crate::operations::common::CommitSink; use crate::operations::common::ConflictResolveContext; use crate::operations::set_backoff; -use crate::operations::SnapshotHintWriter; -use crate::statistics::merge_statistics; use crate::statistics::TableStatsGenerator; -use crate::FuseTable; +use crate::statistics::merge_statistics; impl FuseTable { #[async_backtrace::framed] diff --git a/src/query/storages/fuse/src/operations/common/generators/append_generator.rs b/src/query/storages/fuse/src/operations/common/generators/append_generator.rs index 5d7668e4acf30..aa136bf99a10e 100644 --- a/src/query/storages/fuse/src/operations/common/generators/append_generator.rs +++ b/src/query/storages/fuse/src/operations/common/generators/append_generator.rs @@ -19,11 +19,11 @@ use std::sync::Arc; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::ColumnId; use databend_common_expression::Scalar; use databend_common_expression::TableDataType; use databend_common_expression::TableSchema; +use databend_common_expression::types::DataType; use databend_common_meta_app::schema::TableInfo; use databend_common_sql::DefaultExprBinder; use databend_storages_common_table_meta::meta::ColumnStatistics; @@ -35,8 +35,8 @@ use log::warn; use crate::operations::common::ConflictResolveContext; use crate::operations::common::SnapshotGenerator; use crate::operations::common::SnapshotMerged; -use crate::statistics::reducers::merge_statistics_mut; use crate::statistics::TableStatsGenerator; +use crate::statistics::reducers::merge_statistics_mut; #[derive(Clone)] pub struct AppendGenerator { diff --git a/src/query/storages/fuse/src/operations/common/generators/mod.rs b/src/query/storages/fuse/src/operations/common/generators/mod.rs index 06b7ab46a8e74..6df1d9038aad9 100644 --- a/src/query/storages/fuse/src/operations/common/generators/mod.rs +++ b/src/query/storages/fuse/src/operations/common/generators/mod.rs @@ -23,7 +23,7 @@ pub use conflict_resolve_context::ConflictResolveContext; pub use conflict_resolve_context::SnapshotChanges; pub use conflict_resolve_context::SnapshotMerged; pub use mutation_generator::MutationGenerator; +pub use snapshot_generator::SnapshotGenerator; pub use snapshot_generator::decorate_snapshot; pub(crate) use snapshot_generator::set_compaction_num_block_hint; -pub use snapshot_generator::SnapshotGenerator; pub use truncate_generator::TruncateGenerator; diff --git a/src/query/storages/fuse/src/operations/common/generators/mutation_generator.rs b/src/query/storages/fuse/src/operations/common/generators/mutation_generator.rs index 5a1fe6ea06971..00a672c8cdd1c 100644 --- a/src/query/storages/fuse/src/operations/common/generators/mutation_generator.rs +++ b/src/query/storages/fuse/src/operations/common/generators/mutation_generator.rs @@ -27,9 +27,9 @@ use log::info; use crate::operations::common::ConflictResolveContext; use crate::operations::common::SnapshotGenerator; +use crate::statistics::TableStatsGenerator; use crate::statistics::merge_statistics; use crate::statistics::reducers::deduct_statistics_mut; -use crate::statistics::TableStatsGenerator; #[derive(Clone)] pub struct MutationGenerator { diff --git a/src/query/storages/fuse/src/operations/common/processors/mod.rs b/src/query/storages/fuse/src/operations/common/processors/mod.rs index 32ec57c3bfa33..6251dd5eebf78 100644 --- a/src/query/storages/fuse/src/operations/common/processors/mod.rs +++ b/src/query/storages/fuse/src/operations/common/processors/mod.rs @@ -29,5 +29,5 @@ pub use transform_constraint_verify::TransformConstraintVerify; pub use transform_merge_commit_meta::TransformMergeCommitMeta; pub use transform_mutation_aggregator::TableMutationAggregator; pub use transform_serialize_block::TransformSerializeBlock; -pub use transform_serialize_segment::new_serialize_segment_processor; pub use transform_serialize_segment::TransformSerializeSegment; +pub use transform_serialize_segment::new_serialize_segment_processor; diff --git a/src/query/storages/fuse/src/operations/common/processors/multi_table_insert_commit.rs b/src/query/storages/fuse/src/operations/common/processors/multi_table_insert_commit.rs index 822855356df21..88d09a2527207 100644 --- a/src/query/storages/fuse/src/operations/common/processors/multi_table_insert_commit.rs +++ b/src/query/storages/fuse/src/operations/common/processors/multi_table_insert_commit.rs @@ -41,13 +41,13 @@ use log::debug; use log::error; use log::info; -use crate::operations::set_backoff; -use crate::operations::set_compaction_num_block_hint; +use crate::FuseTable; use crate::operations::AppendGenerator; use crate::operations::CommitMeta; use crate::operations::SnapshotGenerator; use crate::operations::TransformMergeCommitMeta; -use crate::FuseTable; +use crate::operations::set_backoff; +use crate::operations::set_compaction_num_block_hint; pub struct CommitMultiTableInsert { commit_metas: HashMap, diff --git a/src/query/storages/fuse/src/operations/common/processors/sink_commit.rs b/src/query/storages/fuse/src/operations/common/processors/sink_commit.rs index 59caafebef139..b87f4b70f07eb 100644 --- a/src/query/storages/fuse/src/operations/common/processors/sink_commit.rs +++ b/src/query/storages/fuse/src/operations/common/processors/sink_commit.rs @@ -21,8 +21,8 @@ use std::sync::Arc; use std::time::Duration; use std::time::Instant; -use backoff::backoff::Backoff; use backoff::ExponentialBackoff; +use backoff::backoff::Backoff; use databend_common_base::base::GlobalInstance; use databend_common_catalog::table::Table; use databend_common_catalog::table::TableExt; @@ -56,20 +56,20 @@ use log::error; use log::info; use opendal::Operator; +use crate::FUSE_OPT_KEY_ENABLE_AUTO_ANALYZE; +use crate::FUSE_OPT_KEY_ENABLE_AUTO_VACUUM; +use crate::FuseTable; use crate::io::TableMetaLocationGenerator; -use crate::operations::set_backoff; -use crate::operations::set_compaction_num_block_hint; -use crate::operations::vacuum::vacuum_table; use crate::operations::AppendGenerator; use crate::operations::CommitMeta; use crate::operations::MutationGenerator; use crate::operations::SnapshotGenerator; use crate::operations::TransformMergeCommitMeta; use crate::operations::TruncateGenerator; +use crate::operations::set_backoff; +use crate::operations::set_compaction_num_block_hint; +use crate::operations::vacuum::vacuum_table; use crate::statistics::TableStatsGenerator; -use crate::FuseTable; -use crate::FUSE_OPT_KEY_ENABLE_AUTO_ANALYZE; -use crate::FUSE_OPT_KEY_ENABLE_AUTO_VACUUM; enum State { None, @@ -222,9 +222,9 @@ where F: SnapshotGenerator + Send + Sync + 'static snapshot_gen .as_any() .downcast_ref::() - .is_some_and(|gen| { + .is_some_and(|g| { matches!( - gen.mutation_kind, + g.mutation_kind, MutationKind::Update | MutationKind::Delete | MutationKind::MergeInto @@ -315,14 +315,14 @@ where F: SnapshotGenerator + Send + Sync + 'static snapshot_gen .as_any() .downcast_ref::() - .is_some_and(|gen| matches!(gen.mode(), TruncateMode::DropAll)) + .is_some_and(|g| matches!(g.mode(), TruncateMode::DropAll)) } fn need_truncate(&self) -> bool { self.snapshot_gen .as_any() .downcast_ref::() - .is_some_and(|gen| !matches!(gen.mode(), TruncateMode::Delete)) + .is_some_and(|g| !matches!(g.mode(), TruncateMode::Delete)) } fn is_append_only_txn(&self) -> bool { diff --git a/src/query/storages/fuse/src/operations/common/processors/transform_block_writer.rs b/src/query/storages/fuse/src/operations/common/processors/transform_block_writer.rs index 28c0be2933a65..ce67c53bc9bf1 100644 --- a/src/query/storages/fuse/src/operations/common/processors/transform_block_writer.rs +++ b/src/query/storages/fuse/src/operations/common/processors/transform_block_writer.rs @@ -35,13 +35,13 @@ use databend_common_sql::executor::physical_plans::MutationKind; use databend_common_storage::MutationStatus; use opendal::Operator; +use crate::FuseTable; use crate::io::BlockSerialization; use crate::io::BlockWriter; use crate::io::StreamBlockBuilder; use crate::io::StreamBlockProperties; use crate::operations::MutationLogEntry; use crate::operations::MutationLogs; -use crate::FuseTable; enum State { Consume, diff --git a/src/query/storages/fuse/src/operations/common/processors/transform_constraint_verify.rs b/src/query/storages/fuse/src/operations/common/processors/transform_constraint_verify.rs index 9aab09248016e..12634d6067b4a 100644 --- a/src/query/storages/fuse/src/operations/common/processors/transform_constraint_verify.rs +++ b/src/query/storages/fuse/src/operations/common/processors/transform_constraint_verify.rs @@ -56,7 +56,11 @@ impl AsyncAccumulatingTransform for TransformConstraintVerify { match evaluator.run(expr)? { Value::Scalar(scalar) => { if !matches!(scalar.as_boolean(), Some(true)) { - return Err(ErrorCode::ConstraintError(format!("CHECK constraint failed on table {} with constraint {name} expression {}", self.table_name, expr.sql_display()))); + return Err(ErrorCode::ConstraintError(format!( + "CHECK constraint failed on table {} with constraint {name} expression {}", + self.table_name, + expr.sql_display() + ))); } } Value::Column(column) => { @@ -64,7 +68,11 @@ impl AsyncAccumulatingTransform for TransformConstraintVerify { .iter() .any(|scalar| !matches!(scalar.as_boolean(), Some(true))) { - return Err(ErrorCode::ConstraintError(format!("CHECK constraint failed on table {} with constraint {name} expression {}", self.table_name, expr.sql_display()))); + return Err(ErrorCode::ConstraintError(format!( + "CHECK constraint failed on table {} with constraint {name} expression {}", + self.table_name, + expr.sql_display() + ))); } } } diff --git a/src/query/storages/fuse/src/operations/common/processors/transform_mutation_aggregator.rs b/src/query/storages/fuse/src/operations/common/processors/transform_mutation_aggregator.rs index 26aee7148135e..767a4a429a412 100644 --- a/src/query/storages/fuse/src/operations/common/processors/transform_mutation_aggregator.rs +++ b/src/query/storages/fuse/src/operations/common/processors/transform_mutation_aggregator.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::hash_map::Entry; use std::collections::BTreeMap; use std::collections::HashMap; +use std::collections::hash_map::Entry; use std::sync::Arc; use std::time::Instant; @@ -31,7 +31,6 @@ use databend_common_expression::VirtualDataSchema; use databend_common_pipeline_transforms::processors::AsyncAccumulatingTransform; use databend_common_sql::executor::physical_plans::MutationKind; use databend_storages_common_cache::SegmentStatistics; -use databend_storages_common_table_meta::meta::merge_column_hll_mut; use databend_storages_common_table_meta::meta::AdditionalStatsMeta; use databend_storages_common_table_meta::meta::BlockHLL; use databend_storages_common_table_meta::meta::BlockHLLState; @@ -45,6 +44,7 @@ use databend_storages_common_table_meta::meta::Statistics; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use databend_storages_common_table_meta::meta::Versioned; use databend_storages_common_table_meta::meta::VirtualBlockMeta; +use databend_storages_common_table_meta::meta::merge_column_hll_mut; use databend_storages_common_table_meta::table::ClusterType; use itertools::Itertools; use log::debug; @@ -52,10 +52,11 @@ use log::info; use log::warn; use opendal::Operator; -use crate::io::read::read_segment_stats; +use crate::FuseTable; use crate::io::CachedMetaWriter; use crate::io::SegmentsIO; use crate::io::TableMetaLocationGenerator; +use crate::io::read::read_segment_stats; use crate::operations::common::CommitMeta; use crate::operations::common::ConflictResolveContext; use crate::operations::common::MutationLogEntry; @@ -64,11 +65,10 @@ use crate::operations::common::SnapshotChanges; use crate::operations::common::SnapshotMerged; use crate::operations::mutation::BlockIndex; use crate::operations::mutation::SegmentIndex; +use crate::statistics::VirtualColumnAccumulator; use crate::statistics::reducers::merge_statistics_mut; use crate::statistics::reducers::reduce_block_metas; use crate::statistics::sort_by_cluster_stats; -use crate::statistics::VirtualColumnAccumulator; -use crate::FuseTable; pub struct TableMutationAggregator { ctx: Arc, diff --git a/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs b/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs index b4d59326a1803..926954639263a 100644 --- a/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs +++ b/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs @@ -38,19 +38,19 @@ use databend_storages_common_index::RangeIndex; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use opendal::Operator; -use crate::io::create_inverted_index_builders; +use crate::FuseTable; use crate::io::BlockBuilder; use crate::io::BlockSerialization; use crate::io::BlockWriter; use crate::io::VectorIndexBuilder; use crate::io::VirtualColumnBuilder; +use crate::io::create_inverted_index_builders; use crate::operations::common::BlockMetaIndex; use crate::operations::common::MutationLogEntry; use crate::operations::common::MutationLogs; use crate::operations::mutation::ClusterStatsGenType; use crate::operations::mutation::SerializeDataMeta; use crate::statistics::ClusterStatsGenerator; -use crate::FuseTable; #[allow(clippy::large_enum_variant)] enum State { diff --git a/src/query/storages/fuse/src/operations/common/processors/transform_serialize_segment.rs b/src/query/storages/fuse/src/operations/common/processors/transform_serialize_segment.rs index fe10c06ea0c1d..5a62c459d28c6 100644 --- a/src/query/storages/fuse/src/operations/common/processors/transform_serialize_segment.rs +++ b/src/query/storages/fuse/src/operations/common/processors/transform_serialize_segment.rs @@ -29,7 +29,6 @@ use databend_common_pipeline::core::InputPort; use databend_common_pipeline::core::OutputPort; use databend_common_pipeline::core::Processor; use databend_common_pipeline::core::ProcessorPtr; -use databend_storages_common_table_meta::meta::column_oriented_segment::*; use databend_storages_common_table_meta::meta::AdditionalStatsMeta; use databend_storages_common_table_meta::meta::BlockHLL; use databend_storages_common_table_meta::meta::ExtendedBlockMeta; @@ -38,17 +37,18 @@ use databend_storages_common_table_meta::meta::SegmentStatistics; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use databend_storages_common_table_meta::meta::Versioned; use databend_storages_common_table_meta::meta::VirtualBlockMeta; +use databend_storages_common_table_meta::meta::column_oriented_segment::*; use log::info; use opendal::Operator; +use crate::FuseSegmentFormat; +use crate::FuseTable; use crate::io::TableMetaLocationGenerator; use crate::operations::common::MutationLogEntry; use crate::operations::common::MutationLogs; use crate::statistics::ColumnHLLAccumulator; use crate::statistics::RowOrientedSegmentBuilder; use crate::statistics::VirtualColumnAccumulator; -use crate::FuseSegmentFormat; -use crate::FuseTable; enum State { None, diff --git a/src/query/storages/fuse/src/operations/compact.rs b/src/query/storages/fuse/src/operations/compact.rs index 9e03211b5ea29..29091eb495575 100644 --- a/src/query/storages/fuse/src/operations/compact.rs +++ b/src/query/storages/fuse/src/operations/compact.rs @@ -22,12 +22,12 @@ use databend_common_expression::FieldIndex; use databend_common_io::constants::DEFAULT_BLOCK_PER_SEGMENT; use databend_storages_common_table_meta::meta::TableSnapshot; -use crate::operations::mutation::BlockCompactMutator; -use crate::operations::mutation::SegmentCompactMutator; +use crate::FUSE_OPT_KEY_BLOCK_PER_SEGMENT; use crate::FuseTable; use crate::Table; use crate::TableContext; -use crate::FUSE_OPT_KEY_BLOCK_PER_SEGMENT; +use crate::operations::mutation::BlockCompactMutator; +use crate::operations::mutation::SegmentCompactMutator; #[derive(Clone)] pub struct CompactOptions { diff --git a/src/query/storages/fuse/src/operations/gc.rs b/src/query/storages/fuse/src/operations/gc.rs index d61456bf6fe18..94be8e63a5b7e 100644 --- a/src/query/storages/fuse/src/operations/gc.rs +++ b/src/query/storages/fuse/src/operations/gc.rs @@ -30,29 +30,29 @@ use databend_storages_common_cache::LoadParams; use databend_storages_common_index::BloomIndexMeta; use databend_storages_common_index::InvertedIndexMeta; use databend_storages_common_io::Files; -use databend_storages_common_table_meta::meta::column_oriented_segment::ColumnOrientedSegment; -use databend_storages_common_table_meta::meta::column_oriented_segment::BLOOM_FILTER_INDEX_LOCATION; -use databend_storages_common_table_meta::meta::column_oriented_segment::LOCATION; use databend_storages_common_table_meta::meta::CompactSegmentInfo; use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::SegmentInfo; use databend_storages_common_table_meta::meta::TableSnapshot; use databend_storages_common_table_meta::meta::TableSnapshotStatistics; +use databend_storages_common_table_meta::meta::column_oriented_segment::BLOOM_FILTER_INDEX_LOCATION; +use databend_storages_common_table_meta::meta::column_oriented_segment::ColumnOrientedSegment; +use databend_storages_common_table_meta::meta::column_oriented_segment::LOCATION; use log::error; use log::info; use log::warn; +use crate::FUSE_TBL_SNAPSHOT_PREFIX; +use crate::FuseTable; use crate::index::InvertedIndexFile; -use crate::io::read::ColumnOrientedSegmentReader; -use crate::io::read::RowOrientedSegmentReader; use crate::io::InvertedIndexReader; use crate::io::MetaReaders; use crate::io::SegmentsIO; use crate::io::SnapshotLiteExtended; use crate::io::SnapshotsIO; use crate::io::TableMetaLocationGenerator; -use crate::FuseTable; -use crate::FUSE_TBL_SNAPSHOT_PREFIX; +use crate::io::read::ColumnOrientedSegmentReader; +use crate::io::read::RowOrientedSegmentReader; impl FuseTable { pub async fn do_purge( diff --git a/src/query/storages/fuse/src/operations/inverted_index.rs b/src/query/storages/fuse/src/operations/inverted_index.rs index 33a1924c01fd3..30bcacd5edae6 100644 --- a/src/query/storages/fuse/src/operations/inverted_index.rs +++ b/src/query/storages/fuse/src/operations/inverted_index.rs @@ -14,9 +14,9 @@ use std::collections::BTreeMap; use std::collections::VecDeque; +use std::sync::Arc; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; -use std::sync::Arc; use std::time::Instant; use async_trait::async_trait; @@ -47,13 +47,13 @@ use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::Location; use opendal::Operator; -use crate::io::write_data; +use crate::FuseStorageFormat; +use crate::FuseTable; use crate::io::BlockReader; use crate::io::InvertedIndexWriter; use crate::io::MetaReaders; use crate::io::TableMetaLocationGenerator; -use crate::FuseStorageFormat; -use crate::FuseTable; +use crate::io::write_data; impl FuseTable { // The big picture of refresh inverted index into pipeline: diff --git a/src/query/storages/fuse/src/operations/merge.rs b/src/query/storages/fuse/src/operations/merge.rs index 2ac87907b3453..6c2905680f514 100644 --- a/src/query/storages/fuse/src/operations/merge.rs +++ b/src/query/storages/fuse/src/operations/merge.rs @@ -27,11 +27,11 @@ use databend_storages_common_table_meta::meta::TableMetaTimestamps; use super::merge_into::MatchedAggregator; use super::mutation::SegmentIndex; -use crate::io::create_inverted_index_builders; +use crate::FuseTable; use crate::io::BlockBuilder; use crate::io::VectorIndexBuilder; +use crate::io::create_inverted_index_builders; use crate::statistics::ClusterStatsGenerator; -use crate::FuseTable; impl FuseTable { // todo: (JackTan25) add pipeline picture diff --git a/src/query/storages/fuse/src/operations/merge_into/mutator/delete_by_expr_mutator.rs b/src/query/storages/fuse/src/operations/merge_into/mutator/delete_by_expr_mutator.rs index d99536df05a53..763ef810ee12f 100644 --- a/src/query/storages/fuse/src/operations/merge_into/mutator/delete_by_expr_mutator.rs +++ b/src/query/storages/fuse/src/operations/merge_into/mutator/delete_by_expr_mutator.rs @@ -13,8 +13,6 @@ // limitations under the License. use databend_common_exception::Result; -use databend_common_expression::types::BooleanType; -use databend_common_expression::types::DataType; use databend_common_expression::Constant; use databend_common_expression::DataBlock; use databend_common_expression::Evaluator; @@ -22,6 +20,8 @@ use databend_common_expression::Expr; use databend_common_expression::FunctionContext; use databend_common_expression::Scalar; use databend_common_expression::Value; +use databend_common_expression::types::BooleanType; +use databend_common_expression::types::DataType; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_sql::executor::cast_expr_to_non_null_boolean; diff --git a/src/query/storages/fuse/src/operations/merge_into/mutator/matched_mutator.rs b/src/query/storages/fuse/src/operations/merge_into/mutator/matched_mutator.rs index 3bbc2e873264d..f38f9616de528 100644 --- a/src/query/storages/fuse/src/operations/merge_into/mutator/matched_mutator.rs +++ b/src/query/storages/fuse/src/operations/merge_into/mutator/matched_mutator.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::hash_map::Entry; use std::collections::HashMap; use std::collections::HashSet; +use std::collections::hash_map::Entry; use std::sync::Arc; use std::time::Instant; @@ -22,20 +22,20 @@ use ahash::AHashMap; use databend_common_base::base::tokio::sync::Semaphore; use databend_common_base::runtime::GlobalIORuntime; use databend_common_base::runtime::TrySpawn; +use databend_common_catalog::plan::Projection; use databend_common_catalog::plan::build_origin_block_row_num; use databend_common_catalog::plan::gen_mutation_stream_meta; use databend_common_catalog::plan::split_prefix; use databend_common_catalog::plan::split_row_id; -use databend_common_catalog::plan::Projection; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::BlockMetaInfoDowncast; +use databend_common_expression::DataBlock; use databend_common_expression::types::DataType; use databend_common_expression::types::MutableBitmap; use databend_common_expression::types::NumberDataType; -use databend_common_expression::BlockMetaInfoDowncast; -use databend_common_expression::DataBlock; use databend_common_metrics::storage::*; use databend_common_sql::StreamContext; use databend_common_storage::MutationStatus; @@ -48,12 +48,14 @@ use itertools::Itertools; use log::info; use opendal::Operator; +use crate::FuseTable; use crate::io::BlockBuilder; use crate::io::BlockReader; use crate::io::BlockWriter; use crate::io::CompactSegmentInfoReader; use crate::io::MetaReaders; use crate::io::WriteSettings; +use crate::operations::BlockMetaIndex; use crate::operations::acquire_task_permit; use crate::operations::common::MutationLogEntry; use crate::operations::common::MutationLogs; @@ -61,8 +63,6 @@ use crate::operations::merge_into::processors::RowIdKind; use crate::operations::mutation::BlockIndex; use crate::operations::mutation::SegmentIndex; use crate::operations::read_block; -use crate::operations::BlockMetaIndex; -use crate::FuseTable; struct AggregationContext { data_accessor: Operator, @@ -322,8 +322,8 @@ impl MatchedAggregator { // the row_id is generated by block_id, not block_idx,reference to fill_internal_column_meta() let block_meta = segment_info.blocks[block_idx].clone(); - let update_modified_offsets = &item.1 .0; - let delete_modified_offsets = &item.1 .1; + let update_modified_offsets = &item.1.0; + let delete_modified_offsets = &item.1.1; let modified_offsets: HashSet = update_modified_offsets .union(delete_modified_offsets) .cloned() diff --git a/src/query/storages/fuse/src/operations/merge_into/mutator/split_by_expr_mutator.rs b/src/query/storages/fuse/src/operations/merge_into/mutator/split_by_expr_mutator.rs index 92a95b595d197..15da09636fa98 100644 --- a/src/query/storages/fuse/src/operations/merge_into/mutator/split_by_expr_mutator.rs +++ b/src/query/storages/fuse/src/operations/merge_into/mutator/split_by_expr_mutator.rs @@ -13,13 +13,13 @@ // limitations under the License. use databend_common_exception::Result; -use databend_common_expression::eval_function; -use databend_common_expression::types::BooleanType; -use databend_common_expression::types::DataType; use databend_common_expression::DataBlock; use databend_common_expression::Evaluator; use databend_common_expression::Expr; use databend_common_expression::FunctionContext; +use databend_common_expression::eval_function; +use databend_common_expression::types::BooleanType; +use databend_common_expression::types::DataType; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_sql::executor::cast_expr_to_non_null_boolean; diff --git a/src/query/storages/fuse/src/operations/merge_into/mutator/update_by_expr_mutator.rs b/src/query/storages/fuse/src/operations/merge_into/mutator/update_by_expr_mutator.rs index ad24cc0569f1c..cbe2b52810886 100644 --- a/src/query/storages/fuse/src/operations/merge_into/mutator/update_by_expr_mutator.rs +++ b/src/query/storages/fuse/src/operations/merge_into/mutator/update_by_expr_mutator.rs @@ -15,8 +15,6 @@ use std::collections::HashMap; use databend_common_exception::Result; -use databend_common_expression::types::BooleanType; -use databend_common_expression::types::DataType; use databend_common_expression::BlockEntry; use databend_common_expression::Constant; use databend_common_expression::DataBlock; @@ -26,6 +24,8 @@ use databend_common_expression::FieldIndex; use databend_common_expression::FunctionContext; use databend_common_expression::RemoteExpr; use databend_common_expression::Value; +use databend_common_expression::types::BooleanType; +use databend_common_expression::types::DataType; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_sql::evaluator::BlockOperator; use databend_common_sql::executor::cast_expr_to_non_null_boolean; diff --git a/src/query/storages/fuse/src/operations/merge_into/mutator/utils.rs b/src/query/storages/fuse/src/operations/merge_into/mutator/utils.rs index e92d9db436fd4..08d3bdf75a433 100644 --- a/src/query/storages/fuse/src/operations/merge_into/mutator/utils.rs +++ b/src/query/storages/fuse/src/operations/merge_into/mutator/utils.rs @@ -13,14 +13,14 @@ // limitations under the License. use databend_common_exception::Result; -use databend_common_expression::eval_function; -use databend_common_expression::types::AnyType; -use databend_common_expression::types::BooleanType; -use databend_common_expression::types::DataType; use databend_common_expression::Evaluator; use databend_common_expression::Expr; use databend_common_expression::FunctionContext; use databend_common_expression::Value; +use databend_common_expression::eval_function; +use databend_common_expression::types::AnyType; +use databend_common_expression::types::BooleanType; +use databend_common_expression::types::DataType; use databend_common_functions::BUILTIN_FUNCTIONS; pub(crate) fn get_and( diff --git a/src/query/storages/fuse/src/operations/merge_into/processors/processor_merge_into_matched_and_split.rs b/src/query/storages/fuse/src/operations/merge_into/processors/processor_merge_into_matched_and_split.rs index fb46c5259ad32..b862e053e3426 100644 --- a/src/query/storages/fuse/src/operations/merge_into/processors/processor_merge_into_matched_and_split.rs +++ b/src/query/storages/fuse/src/operations/merge_into/processors/processor_merge_into_matched_and_split.rs @@ -19,8 +19,6 @@ use std::time::Instant; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::type_check::check; -use databend_common_expression::types::BooleanType; use databend_common_expression::BlockMetaInfo; use databend_common_expression::BlockMetaInfoDowncast; use databend_common_expression::DataBlock; @@ -29,6 +27,8 @@ use databend_common_expression::FieldIndex; use databend_common_expression::RawExpr; use databend_common_expression::RemoteExpr; use databend_common_expression::Value; +use databend_common_expression::type_check::check; +use databend_common_expression::types::BooleanType; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_metrics::storage::*; use databend_common_pipeline::core::Event; @@ -37,15 +37,15 @@ use databend_common_pipeline::core::OutputPort; use databend_common_pipeline::core::PipeItem; use databend_common_pipeline::core::Processor; use databend_common_pipeline::core::ProcessorPtr; +use databend_common_sql::IndexType; use databend_common_sql::evaluator::BlockOperator; use databend_common_sql::executor::physical_plans::MatchExpr; -use databend_common_sql::IndexType; use databend_common_storage::MutationStatus; +use crate::operations::BlockMetaIndex; use crate::operations::common::MutationLogs; use crate::operations::merge_into::mutator::DeleteByExprMutator; use crate::operations::merge_into::mutator::UpdateByExprMutator; -use crate::operations::BlockMetaIndex; #[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq)] pub struct SourceFullMatched; diff --git a/src/query/storages/fuse/src/operations/merge_into/processors/processor_merge_into_not_matched.rs b/src/query/storages/fuse/src/operations/merge_into/processors/processor_merge_into_not_matched.rs index 1d642ba11c997..8425c436bc925 100644 --- a/src/query/storages/fuse/src/operations/merge_into/processors/processor_merge_into_not_matched.rs +++ b/src/query/storages/fuse/src/operations/merge_into/processors/processor_merge_into_not_matched.rs @@ -32,13 +32,13 @@ use databend_common_pipeline::core::OutputPort; use databend_common_pipeline::core::PipeItem; use databend_common_pipeline::core::Processor; use databend_common_pipeline::core::ProcessorPtr; -use databend_common_sql::evaluator::BlockOperator; use databend_common_sql::ColumnSet; +use databend_common_sql::evaluator::BlockOperator; use databend_common_storage::MutationStatus; use itertools::Itertools; -use crate::operations::merge_into::mutator::SplitByExprMutator; use crate::operations::BlockMetaIndex; +use crate::operations::merge_into::mutator::SplitByExprMutator; // (source_schema,condition,values_exprs) pub type UnMatchedExprs = Vec<(DataSchemaRef, Option, Vec)>; diff --git a/src/query/storages/fuse/src/operations/merge_into/processors/processor_merge_into_split.rs b/src/query/storages/fuse/src/operations/merge_into/processors/processor_merge_into_split.rs index e5dba860f6c61..4d2e9875107e9 100644 --- a/src/query/storages/fuse/src/operations/merge_into/processors/processor_merge_into_split.rs +++ b/src/query/storages/fuse/src/operations/merge_into/processors/processor_merge_into_split.rs @@ -27,8 +27,8 @@ use databend_common_pipeline::core::PipeItem; use databend_common_pipeline::core::Processor; use databend_common_pipeline::core::ProcessorPtr; -use crate::operations::merge_into::mutator::MutationSplitMutator; use crate::operations::BlockMetaIndex; +use crate::operations::merge_into::mutator::MutationSplitMutator; // There are two kinds of usage for this processor: // 1. we will receive a probed datablock from join, and split it by rowid into matched block and unmatched block diff --git a/src/query/storages/fuse/src/operations/mutation/meta/mutation_meta.rs b/src/query/storages/fuse/src/operations/mutation/meta/mutation_meta.rs index c1e3949247b4e..dbd2fb55fdcdd 100644 --- a/src/query/storages/fuse/src/operations/mutation/meta/mutation_meta.rs +++ b/src/query/storages/fuse/src/operations/mutation/meta/mutation_meta.rs @@ -14,16 +14,16 @@ use std::sync::Arc; -use databend_common_expression::local_block_meta_serde; use databend_common_expression::BlockMetaInfo; use databend_common_expression::BlockMetaInfoDowncast; +use databend_common_expression::local_block_meta_serde; use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::ClusterStatistics; +use crate::BlockReadResult; use crate::operations::common::BlockMetaIndex; use crate::operations::mutation::CompactExtraInfo; use crate::operations::mutation::DeletedSegmentInfo; -use crate::BlockReadResult; #[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq)] pub enum SerializeDataMeta { diff --git a/src/query/storages/fuse/src/operations/mutation/mutator/block_compact_mutator.rs b/src/query/storages/fuse/src/operations/mutation/mutator/block_compact_mutator.rs index ab029f2ed6208..91c9fc4e1e9f3 100644 --- a/src/query/storages/fuse/src/operations/mutation/mutator/block_compact_mutator.rs +++ b/src/query/storages/fuse/src/operations/mutation/mutator/block_compact_mutator.rs @@ -27,9 +27,9 @@ use databend_common_catalog::plan::Partitions; use databend_common_catalog::plan::PartitionsShuffleKind; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::is_stream_column_id; use databend_common_expression::BlockThresholds; use databend_common_expression::ColumnId; +use databend_common_expression::is_stream_column_id; use databend_common_metrics::storage::*; use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::CompactSegmentInfo; @@ -38,8 +38,10 @@ use databend_storages_common_table_meta::meta::Statistics; use log::info; use opendal::Operator; -use crate::io::read::read_segment_stats; +use crate::TableContext; use crate::io::SegmentsIO; +use crate::io::read::read_segment_stats; +use crate::operations::CompactOptions; use crate::operations::acquire_task_permit; use crate::operations::common::BlockMetaIndex; use crate::operations::mutation::BlockIndex; @@ -48,10 +50,8 @@ use crate::operations::mutation::CompactExtraInfo; use crate::operations::mutation::CompactLazyPartInfo; use crate::operations::mutation::CompactTaskInfo; use crate::operations::mutation::SegmentIndex; -use crate::operations::CompactOptions; use crate::statistics::reducers::merge_statistics_mut; use crate::statistics::sort_by_cluster_stats; -use crate::TableContext; #[derive(Clone)] pub struct BlockCompactMutator { diff --git a/src/query/storages/fuse/src/operations/mutation/mutator/recluster_mutator.rs b/src/query/storages/fuse/src/operations/mutation/mutator/recluster_mutator.rs index 4422a29bb2d6f..64ab11f415a51 100644 --- a/src/query/storages/fuse/src/operations/mutation/mutator/recluster_mutator.rs +++ b/src/query/storages/fuse/src/operations/mutation/mutator/recluster_mutator.rs @@ -19,8 +19,8 @@ use std::collections::HashMap; use std::collections::HashSet; use std::sync::Arc; -use databend_common_base::runtime::execute_futures_in_parallel; use databend_common_base::runtime::GLOBAL_MEM_STAT; +use databend_common_base::runtime::execute_futures_in_parallel; use databend_common_catalog::plan::Partitions; use databend_common_catalog::plan::PartitionsShuffleKind; use databend_common_catalog::plan::ReclusterParts; @@ -28,11 +28,11 @@ use databend_common_catalog::plan::ReclusterTask; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::compare_scalars; -use databend_common_expression::types::DataType; use databend_common_expression::BlockThresholds; use databend_common_expression::Scalar; use databend_common_expression::TableSchemaRef; +use databend_common_expression::compare_scalars; +use databend_common_expression::types::DataType; use databend_common_storage::ColumnNodes; use databend_storages_common_cache::LoadParams; use databend_storages_common_pruner::BlockMetaIndex; @@ -42,26 +42,26 @@ use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::RawBlockHLL; use databend_storages_common_table_meta::meta::Statistics; use databend_storages_common_table_meta::meta::TableSnapshot; +use fastrace::Span; use fastrace::func_path; use fastrace::future::FutureExt; -use fastrace::Span; use indexmap::IndexSet; use log::debug; use log::warn; use opendal::Operator; +use crate::DEFAULT_AVG_DEPTH_THRESHOLD; +use crate::FUSE_OPT_KEY_ROW_AVG_DEPTH_THRESHOLD; +use crate::FuseTable; +use crate::SegmentLocation; use crate::io::MetaReaders; -use crate::operations::common::BlockMetaIndex as BlockIndex; -use crate::operations::mutation::mutator::block_compact_mutator::CompactLimitState; -use crate::operations::mutation::SegmentCompactChecker; use crate::operations::BlockCompactMutator; use crate::operations::CompactLazyPartInfo; +use crate::operations::common::BlockMetaIndex as BlockIndex; +use crate::operations::mutation::SegmentCompactChecker; +use crate::operations::mutation::mutator::block_compact_mutator::CompactLimitState; use crate::statistics::reducers::merge_statistics_mut; use crate::statistics::sort_by_cluster_stats; -use crate::FuseTable; -use crate::SegmentLocation; -use crate::DEFAULT_AVG_DEPTH_THRESHOLD; -use crate::FUSE_OPT_KEY_ROW_AVG_DEPTH_THRESHOLD; pub enum ReclusterMode { Recluster, diff --git a/src/query/storages/fuse/src/operations/mutation/mutator/segment_compact_mutator.rs b/src/query/storages/fuse/src/operations/mutation/mutator/segment_compact_mutator.rs index 6daa79bbe20d7..e3b22cd244860 100644 --- a/src/query/storages/fuse/src/operations/mutation/mutator/segment_compact_mutator.rs +++ b/src/query/storages/fuse/src/operations/mutation/mutator/segment_compact_mutator.rs @@ -27,15 +27,15 @@ use databend_storages_common_table_meta::meta::Versioned; use log::info; use opendal::Operator; -use crate::io::read::read_segment_stats_in_parallel; +use crate::FuseTable; +use crate::TableContext; use crate::io::CachedMetaWriter; use crate::io::SegmentsIO; use crate::io::TableMetaLocationGenerator; +use crate::io::read::read_segment_stats_in_parallel; use crate::operations::CompactOptions; use crate::statistics::reducers::merge_statistics_mut; use crate::statistics::sort_by_cluster_stats; -use crate::FuseTable; -use crate::TableContext; #[derive(Default)] pub struct SegmentCompactionState { diff --git a/src/query/storages/fuse/src/operations/mutation/processors/compact_source.rs b/src/query/storages/fuse/src/operations/mutation/processors/compact_source.rs index a2b9cc81a157a..4a54fda0b3b22 100644 --- a/src/query/storages/fuse/src/operations/mutation/processors/compact_source.rs +++ b/src/query/storages/fuse/src/operations/mutation/processors/compact_source.rs @@ -28,13 +28,13 @@ use databend_common_pipeline_transforms::processors::UnknownMode; use databend_common_sql::StreamContext; use databend_storages_common_io::ReadSettings; +use crate::FuseStorageFormat; use crate::io::BlockReader; use crate::operations::ClusterStatsGenType; use crate::operations::CompactBlockPartInfo; use crate::operations::CompactSourceMeta; use crate::operations::SerializeBlock; use crate::operations::SerializeDataMeta; -use crate::FuseStorageFormat; pub struct CompactSource { ctx: Arc, diff --git a/src/query/storages/fuse/src/operations/mutation/processors/mutation_source.rs b/src/query/storages/fuse/src/operations/mutation/processors/mutation_source.rs index 9620e2aa00905..58064424d5ed4 100644 --- a/src/query/storages/fuse/src/operations/mutation/processors/mutation_source.rs +++ b/src/query/storages/fuse/src/operations/mutation/processors/mutation_source.rs @@ -17,19 +17,19 @@ use std::ops::Not; use std::sync::Arc; use databend_common_base::base::ProgressValues; +use databend_common_catalog::plan::PartInfoPtr; use databend_common_catalog::plan::build_origin_block_row_num; use databend_common_catalog::plan::gen_mutation_stream_meta; -use databend_common_catalog::plan::PartInfoPtr; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::BooleanType; -use databend_common_expression::types::DataType; use databend_common_expression::BlockMetaInfoPtr; use databend_common_expression::DataBlock; use databend_common_expression::Evaluator; use databend_common_expression::Expr; use databend_common_expression::Value; +use databend_common_expression::types::BooleanType; +use databend_common_expression::types::DataType; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_pipeline::core::Event; use databend_common_pipeline::core::OutputPort; @@ -39,6 +39,8 @@ use databend_common_sql::evaluator::BlockOperator; use databend_common_storage::MutationStatus; use databend_storages_common_io::ReadSettings; +use crate::BlockReadResult; +use crate::FuseStorageFormat; use crate::fuse_part::FuseBlockPartInfo; use crate::io::BlockReader; use crate::operations::common::BlockMetaIndex; @@ -46,8 +48,6 @@ use crate::operations::mutation::ClusterStatsGenType; use crate::operations::mutation::Mutation; use crate::operations::mutation::SerializeBlock; use crate::operations::mutation::SerializeDataMeta; -use crate::BlockReadResult; -use crate::FuseStorageFormat; #[derive(Debug, Clone, PartialEq)] pub enum MutationAction { diff --git a/src/query/storages/fuse/src/operations/mutation_source.rs b/src/query/storages/fuse/src/operations/mutation_source.rs index 10071fa8f38c0..a01f3b69f8458 100644 --- a/src/query/storages/fuse/src/operations/mutation_source.rs +++ b/src/query/storages/fuse/src/operations/mutation_source.rs @@ -36,15 +36,15 @@ use databend_storages_common_pruner::RangePruner; use databend_storages_common_table_meta::meta::StatisticsOfColumns; use databend_storages_common_table_meta::meta::TableSnapshot; +use crate::FuseLazyPartInfo; +use crate::FuseTable; +use crate::SegmentLocation; use crate::operations::mutation::Mutation; use crate::operations::mutation::MutationAction; use crate::operations::mutation::MutationPartInfo; use crate::operations::mutation::MutationSource; -use crate::pruning::create_segment_location_vector; use crate::pruning::FusePruner; -use crate::FuseLazyPartInfo; -use crate::FuseTable; -use crate::SegmentLocation; +use crate::pruning::create_segment_location_vector; impl FuseTable { #[async_backtrace::framed] diff --git a/src/query/storages/fuse/src/operations/navigate.rs b/src/query/storages/fuse/src/operations/navigate.rs index 0eba172650d63..20b973b4d3357 100644 --- a/src/query/storages/fuse/src/operations/navigate.rs +++ b/src/query/storages/fuse/src/operations/navigate.rs @@ -32,13 +32,13 @@ use futures::TryStreamExt; use log::info; use opendal::EntryMode; +use crate::FUSE_TBL_SNAPSHOT_PREFIX; +use crate::FuseTable; use crate::fuse_table::RetentionPolicy; use crate::io::MetaReaders; use crate::io::SnapshotHistoryReader; use crate::io::SnapshotsIO; use crate::io::TableMetaLocationGenerator; -use crate::FuseTable; -use crate::FUSE_TBL_SNAPSHOT_PREFIX; impl FuseTable { #[fastrace::trace] @@ -437,11 +437,13 @@ impl FuseTable { let meta = op.stat(de.path()).await?; meta.last_modified() }; - let location = de.path().to_string(); if let Some(modified) = modified { - if f(location.clone(), modified) { - file_list.push((location, modified)); + let utc_modified = DateTime::from_timestamp_nanos( + modified.into_inner().as_nanosecond() as i64, + ); + if f(location.clone(), utc_modified) { + file_list.push((location, utc_modified)); } } } diff --git a/src/query/storages/fuse/src/operations/read/block_partition_meta.rs b/src/query/storages/fuse/src/operations/read/block_partition_meta.rs index 96f4594b2ba0d..91269afb54e70 100644 --- a/src/query/storages/fuse/src/operations/read/block_partition_meta.rs +++ b/src/query/storages/fuse/src/operations/read/block_partition_meta.rs @@ -16,9 +16,9 @@ use std::fmt::Debug; use std::fmt::Formatter; use databend_common_catalog::plan::PartInfoPtr; -use databend_common_expression::local_block_meta_serde; use databend_common_expression::BlockMetaInfo; use databend_common_expression::BlockMetaInfoPtr; +use databend_common_expression::local_block_meta_serde; pub struct BlockPartitionMeta { pub part_ptr: Vec, diff --git a/src/query/storages/fuse/src/operations/read/fuse_rows_fetcher.rs b/src/query/storages/fuse/src/operations/read/fuse_rows_fetcher.rs index fbaf190782e0e..6adf92692be90 100644 --- a/src/query/storages/fuse/src/operations/read/fuse_rows_fetcher.rs +++ b/src/query/storages/fuse/src/operations/read/fuse_rows_fetcher.rs @@ -16,20 +16,20 @@ use std::any::Any; use std::collections::HashMap; use std::sync::Arc; -use databend_common_catalog::plan::split_row_id; use databend_common_catalog::plan::DataSourcePlan; use databend_common_catalog::plan::Projection; +use databend_common_catalog::plan::split_row_id; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::nullable::NullableColumn; +use databend_common_expression::BlockEntry; +use databend_common_expression::Column; +use databend_common_expression::DataBlock; use databend_common_expression::types::Bitmap; use databend_common_expression::types::Buffer; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; -use databend_common_expression::BlockEntry; -use databend_common_expression::Column; -use databend_common_expression::DataBlock; +use databend_common_expression::types::nullable::NullableColumn; use databend_common_pipeline::core::Event; use databend_common_pipeline::core::InputPort; use databend_common_pipeline::core::OutputPort; diff --git a/src/query/storages/fuse/src/operations/read/fuse_source.rs b/src/query/storages/fuse/src/operations/read/fuse_source.rs index a6a594fb7cb2a..7dc5cabc92f47 100644 --- a/src/query/storages/fuse/src/operations/read/fuse_source.rs +++ b/src/query/storages/fuse/src/operations/read/fuse_source.rs @@ -13,8 +13,8 @@ // limitations under the License. use std::collections::VecDeque; -use std::sync::atomic::AtomicU64; use std::sync::Arc; +use std::sync::atomic::AtomicU64; use async_channel::Receiver; use databend_common_catalog::plan::DataSourcePlan; @@ -36,13 +36,13 @@ use crate::fuse_part::FuseBlockPartInfo; use crate::io::AggIndexReader; use crate::io::BlockReader; use crate::io::VirtualColumnReader; +use crate::operations::read::DeserializeDataTransform; +use crate::operations::read::NativeDeserializeDataTransform; +use crate::operations::read::TransformRuntimeFilterWait; use crate::operations::read::block_partition_receiver_source::BlockPartitionReceiverSource; use crate::operations::read::block_partition_source::BlockPartitionSource; use crate::operations::read::native_data_transform_reader::ReadNativeDataTransform; use crate::operations::read::parquet_data_transform_reader::ReadParquetDataTransform; -use crate::operations::read::DeserializeDataTransform; -use crate::operations::read::NativeDeserializeDataTransform; -use crate::operations::read::TransformRuntimeFilterWait; #[allow(clippy::too_many_arguments)] pub fn build_fuse_native_source_pipeline( diff --git a/src/query/storages/fuse/src/operations/read/native_data_source_deserializer.rs b/src/query/storages/fuse/src/operations/read/native_data_source_deserializer.rs index 5e4a840142d93..476d6f1a4446d 100644 --- a/src/query/storages/fuse/src/operations/read/native_data_source_deserializer.rs +++ b/src/query/storages/fuse/src/operations/read/native_data_source_deserializer.rs @@ -32,10 +32,6 @@ use databend_common_catalog::runtime_filter_info::RuntimeFilterEntry; use databend_common_catalog::runtime_filter_info::RuntimeFilterStats; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::filter_helper::FilterHelpers; -use databend_common_expression::types::BooleanType; -use databend_common_expression::types::DataType; -use databend_common_expression::types::MutableBitmap; use databend_common_expression::BlockEntry; use databend_common_expression::BlockMetaInfoDowncast; use databend_common_expression::Column; @@ -51,6 +47,10 @@ use databend_common_expression::FunctionContext; use databend_common_expression::Scalar; use databend_common_expression::TopKSorter; use databend_common_expression::Value; +use databend_common_expression::filter_helper::FilterHelpers; +use databend_common_expression::types::BooleanType; +use databend_common_expression::types::DataType; +use databend_common_expression::types::MutableBitmap; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_metrics::storage::*; use databend_common_native::read::ColumnIter; @@ -64,12 +64,12 @@ use roaring::RoaringTreemap; use super::native_data_source::NativeDataSource; use super::util::add_data_block_meta; use super::util::need_reserve_block_info; +use crate::DEFAULT_ROW_PER_PAGE; use crate::fuse_part::FuseBlockPartInfo; use crate::io::AggIndexReader; use crate::io::BlockReader; use crate::operations::read::data_source_with_meta::DataSourceWithMeta; use crate::pruning::ExprBloomFilter; -use crate::DEFAULT_ROW_PER_PAGE; /// A helper struct to store the intermediate state while reading a native partition. #[derive(Default)] diff --git a/src/query/storages/fuse/src/operations/read/native_data_transform_reader.rs b/src/query/storages/fuse/src/operations/read/native_data_transform_reader.rs index 975dbd5a30ac5..5b1e30fb430f1 100644 --- a/src/query/storages/fuse/src/operations/read/native_data_transform_reader.rs +++ b/src/query/storages/fuse/src/operations/read/native_data_transform_reader.rs @@ -30,6 +30,7 @@ use databend_common_sql::IndexType; use log::debug; use super::native_data_source::NativeDataSource; +use crate::FuseBlockPartInfo; use crate::io::AggIndexReader; use crate::io::BlockReader; use crate::io::TableMetaLocationGenerator; @@ -37,7 +38,6 @@ use crate::operations::read::block_partition_meta::BlockPartitionMeta; use crate::operations::read::data_source_with_meta::DataSourceWithMeta; use crate::pruning::ExprRuntimePruner; use crate::pruning::RuntimeFilterExpr; -use crate::FuseBlockPartInfo; pub struct ReadNativeDataTransform { func_ctx: FunctionContext, diff --git a/src/query/storages/fuse/src/operations/read/parquet_data_source_deserializer.rs b/src/query/storages/fuse/src/operations/read/parquet_data_source_deserializer.rs index 9b7fe8d41d602..d329701ffe443 100644 --- a/src/query/storages/fuse/src/operations/read/parquet_data_source_deserializer.rs +++ b/src/query/storages/fuse/src/operations/read/parquet_data_source_deserializer.rs @@ -28,15 +28,15 @@ use databend_common_catalog::runtime_filter_info::RuntimeFilterEntry; use databend_common_catalog::runtime_filter_info::RuntimeFilterStats; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::Bitmap; -use databend_common_expression::types::DataType; -use databend_common_expression::types::MutableBitmap; use databend_common_expression::BlockMetaInfoDowncast; use databend_common_expression::DataBlock; use databend_common_expression::DataField; use databend_common_expression::DataSchema; use databend_common_expression::FieldIndex; use databend_common_expression::Scalar; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::DataType; +use databend_common_expression::types::MutableBitmap; use databend_common_metrics::storage::*; use databend_common_pipeline::core::Event; use databend_common_pipeline::core::InputPort; diff --git a/src/query/storages/fuse/src/operations/read/parquet_data_transform_reader.rs b/src/query/storages/fuse/src/operations/read/parquet_data_transform_reader.rs index 5bf7afb1a3f4c..58ac2470a080c 100644 --- a/src/query/storages/fuse/src/operations/read/parquet_data_transform_reader.rs +++ b/src/query/storages/fuse/src/operations/read/parquet_data_transform_reader.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::Arc; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; -use std::sync::Arc; use std::time::Instant; use databend_common_base::runtime::profile::Profile; @@ -233,7 +233,10 @@ impl AsyncTransform for ReadParquetDataTransform { if unfinished_processors_count == 1 { let blocks_total = self.stats.blocks_total.load(Ordering::Relaxed); let blocks_pruned = self.stats.blocks_pruned.load(Ordering::Relaxed); - info!("RUNTIME-FILTER: AsyncReadParquetDataTransform finished, scan_id: {}, blocks_total: {}, blocks_pruned: {}", self.scan_id, blocks_total, blocks_pruned); + info!( + "RUNTIME-FILTER: AsyncReadParquetDataTransform finished, scan_id: {}, blocks_total: {}, blocks_pruned: {}", + self.scan_id, blocks_total, blocks_pruned + ); } Ok(()) } diff --git a/src/query/storages/fuse/src/operations/read/parquet_rows_fetcher.rs b/src/query/storages/fuse/src/operations/read/parquet_rows_fetcher.rs index aa6e4f0cffd80..a3630c77f820d 100644 --- a/src/query/storages/fuse/src/operations/read/parquet_rows_fetcher.rs +++ b/src/query/storages/fuse/src/operations/read/parquet_rows_fetcher.rs @@ -12,18 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::hash_map::Entry; use std::collections::HashMap; +use std::collections::hash_map::Entry; use std::future::Future; use std::sync::Arc; use databend_common_base::runtime::spawn; +use databend_common_catalog::plan::Projection; use databend_common_catalog::plan::block_id_in_segment; use databend_common_catalog::plan::block_idx_in_segment; use databend_common_catalog::plan::compute_row_id_prefix; use databend_common_catalog::plan::split_prefix; use databend_common_catalog::plan::split_row_id; -use databend_common_catalog::plan::Projection; use databend_common_catalog::table::Table; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -46,12 +46,12 @@ use itertools::Itertools; use super::fuse_rows_fetcher::RowsFetchMetadata; use super::fuse_rows_fetcher::RowsFetcher; -use crate::io::BlockReader; -use crate::io::CompactSegmentInfoReader; -use crate::io::MetaReaders; use crate::BlockReadResult; use crate::FuseBlockPartInfo; use crate::FuseTable; +use crate::io::BlockReader; +use crate::io::CompactSegmentInfoReader; +use crate::io::MetaReaders; pub struct RowsFetchMetadataImpl { // Average bytes per row @@ -251,7 +251,7 @@ impl ParquetRowsFetcher { metadata: Arc, final_index: u32, take_indices: Vec, - ) -> impl Future> { + ) -> impl Future> + use<> { { let settings = self.settings; let reader = self.reader.clone(); diff --git a/src/query/storages/fuse/src/operations/read/util.rs b/src/query/storages/fuse/src/operations/read/util.rs index ae80ab1bb0b5f..9bfd71c8b080e 100644 --- a/src/query/storages/fuse/src/operations/read/util.rs +++ b/src/query/storages/fuse/src/operations/read/util.rs @@ -15,8 +15,8 @@ use std::sync::Arc; use databend_common_catalog::merge_into_join::MergeIntoJoinType; -use databend_common_catalog::plan::gen_mutation_stream_meta; use databend_common_catalog::plan::InternalColumnMeta; +use databend_common_catalog::plan::gen_mutation_stream_meta; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; use databend_common_expression::BlockMetaInfoPtr; @@ -24,8 +24,8 @@ use databend_common_expression::DataBlock; use databend_common_expression::Scalar; use roaring::RoaringTreemap; -use crate::operations::BlockMetaIndex; use crate::FuseBlockPartInfo; +use crate::operations::BlockMetaIndex; pub fn need_reserve_block_info(ctx: Arc, table_idx: usize) -> (bool, bool) { let merge_into_join = ctx.get_merge_into_join(); diff --git a/src/query/storages/fuse/src/operations/read_data.rs b/src/query/storages/fuse/src/operations/read_data.rs index 534d8ec2c46dd..e73002a82e7d8 100644 --- a/src/query/storages/fuse/src/operations/read_data.rs +++ b/src/query/storages/fuse/src/operations/read_data.rs @@ -28,15 +28,15 @@ use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; use databend_common_pipeline::core::Pipeline; +use crate::FuseLazyPartInfo; +use crate::FuseStorageFormat; +use crate::FuseTable; +use crate::SegmentLocation; use crate::io::AggIndexReader; use crate::io::BlockReader; use crate::io::VirtualColumnReader; use crate::operations::read::build_fuse_parquet_source_pipeline; use crate::operations::read::fuse_source::build_fuse_native_source_pipeline; -use crate::FuseLazyPartInfo; -use crate::FuseStorageFormat; -use crate::FuseTable; -use crate::SegmentLocation; impl FuseTable { pub fn create_block_reader( diff --git a/src/query/storages/fuse/src/operations/read_partitions.rs b/src/query/storages/fuse/src/operations/read_partitions.rs index a1ee6558fc0bb..76648c5e3da66 100644 --- a/src/query/storages/fuse/src/operations/read_partitions.rs +++ b/src/query/storages/fuse/src/operations/read_partitions.rs @@ -55,8 +55,8 @@ use databend_storages_common_index::BloomIndex; use databend_storages_common_index::NgramArgs; use databend_storages_common_pruner::BlockMetaIndex; use databend_storages_common_pruner::TopNPruner; -use databend_storages_common_table_meta::meta::column_oriented_segment::meta_name; -use databend_storages_common_table_meta::meta::column_oriented_segment::stat_name; +use databend_storages_common_table_meta::meta::BlockMeta; +use databend_storages_common_table_meta::meta::ColumnStatistics; use databend_storages_common_table_meta::meta::column_oriented_segment::BLOCK_SIZE; use databend_storages_common_table_meta::meta::column_oriented_segment::BLOOM_FILTER_INDEX_LOCATION; use databend_storages_common_table_meta::meta::column_oriented_segment::BLOOM_FILTER_INDEX_SIZE; @@ -68,8 +68,8 @@ use databend_storages_common_table_meta::meta::column_oriented_segment::INVERTED use databend_storages_common_table_meta::meta::column_oriented_segment::LOCATION; use databend_storages_common_table_meta::meta::column_oriented_segment::NGRAM_FILTER_INDEX_SIZE; use databend_storages_common_table_meta::meta::column_oriented_segment::ROW_COUNT; -use databend_storages_common_table_meta::meta::BlockMeta; -use databend_storages_common_table_meta::meta::ColumnStatistics; +use databend_storages_common_table_meta::meta::column_oriented_segment::meta_name; +use databend_storages_common_table_meta::meta::column_oriented_segment::stat_name; use databend_storages_common_table_meta::table::ChangeType; use databend_storages_common_table_meta::table::ClusterType; use itertools::Itertools; @@ -78,15 +78,18 @@ use opendal::Operator; use sha2::Digest; use sha2::Sha256; +use crate::FuseLazyPartInfo; +use crate::FuseSegmentFormat; +use crate::FuseTable; use crate::fuse_part::FuseBlockPartInfo; use crate::io::BloomIndexRebuilder; -use crate::pruning::create_segment_location_vector; -use crate::pruning::table_sample; use crate::pruning::BlockPruner; use crate::pruning::FusePruner; use crate::pruning::SegmentLocation; use crate::pruning::SegmentPruner; use crate::pruning::VectorIndexPruner; +use crate::pruning::create_segment_location_vector; +use crate::pruning::table_sample; use crate::pruning_pipeline::AsyncBlockPruneTransform; use crate::pruning_pipeline::ColumnOrientedBlockPruneSink; use crate::pruning_pipeline::ExtractSegmentTransform; @@ -101,9 +104,6 @@ use crate::pruning_pipeline::SyncBlockPruneTransform; use crate::pruning_pipeline::TopNPruneTransform; use crate::pruning_pipeline::VectorIndexPruneTransform; use crate::segment_format_from_location; -use crate::FuseLazyPartInfo; -use crate::FuseSegmentFormat; -use crate::FuseTable; const DEFAULT_GRAM_SIZE: usize = 3; const DEFAULT_BLOOM_SIZE: u64 = 1024 * 1024; diff --git a/src/query/storages/fuse/src/operations/recluster.rs b/src/query/storages/fuse/src/operations/recluster.rs index 4ce55ee9b9052..79f1c5c2fbfdf 100644 --- a/src/query/storages/fuse/src/operations/recluster.rs +++ b/src/query/storages/fuse/src/operations/recluster.rs @@ -16,8 +16,8 @@ use std::sync::Arc; use std::time::Instant; use databend_common_base::base::tokio::select; -use databend_common_base::base::tokio::sync::mpsc; use databend_common_base::base::tokio::sync::Semaphore; +use databend_common_base::base::tokio::sync::mpsc; use databend_common_base::runtime::GlobalIORuntime; use databend_common_base::runtime::TrySpawn; use databend_common_catalog::plan::PushDownInfo; @@ -36,14 +36,14 @@ use databend_storages_common_table_meta::table::ClusterType; use log::warn; use opendal::Operator; +use crate::FuseTable; +use crate::SegmentLocation; +use crate::operations::ReclusterMutator; use crate::operations::acquire_task_permit; use crate::operations::mutation::ReclusterMode; -use crate::operations::ReclusterMutator; -use crate::pruning::create_segment_location_vector; use crate::pruning::PruningContext; use crate::pruning::SegmentPruner; -use crate::FuseTable; -use crate::SegmentLocation; +use crate::pruning::create_segment_location_vector; impl FuseTable { #[async_backtrace::framed] diff --git a/src/query/storages/fuse/src/operations/replace.rs b/src/query/storages/fuse/src/operations/replace.rs index 2ff0febf3c169..3419589fa6053 100644 --- a/src/query/storages/fuse/src/operations/replace.rs +++ b/src/query/storages/fuse/src/operations/replace.rs @@ -27,10 +27,10 @@ use databend_storages_common_table_meta::meta::BlockSlotDescription; use databend_storages_common_table_meta::meta::Location; use rand::prelude::SliceRandom; +use crate::FuseTable; use crate::io::BlockBuilder; use crate::operations::mutation::SegmentIndex; use crate::operations::replace_into::ReplaceIntoOperationAggregator; -use crate::FuseTable; impl FuseTable { // The big picture of the replace into pipeline: diff --git a/src/query/storages/fuse/src/operations/replace_into/mutator/column_hash.rs b/src/query/storages/fuse/src/operations/replace_into/mutator/column_hash.rs index 8275f1c21df00..2b4a715c256b2 100644 --- a/src/query/storages/fuse/src/operations/replace_into/mutator/column_hash.rs +++ b/src/query/storages/fuse/src/operations/replace_into/mutator/column_hash.rs @@ -16,20 +16,20 @@ use std::hash::Hasher; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::decimal::DecimalScalar; -use databend_common_expression::types::AnyType; -use databend_common_expression::types::NumberScalar; use databend_common_expression::ScalarRef; use databend_common_expression::Value; +use databend_common_expression::types::AnyType; +use databend_common_expression::types::NumberScalar; +use databend_common_expression::types::decimal::DecimalScalar; use siphasher::sip128; use siphasher::sip128::Hasher128; pub(crate) trait RowScalarValue { - fn row_scalar(&self, idx: usize) -> Result; + fn row_scalar(&self, idx: usize) -> Result>; } impl RowScalarValue for Value { - fn row_scalar(&self, idx: usize) -> Result { + fn row_scalar(&self, idx: usize) -> Result> { match self { Value::Scalar(v) => Ok(v.as_ref()), Value::Column(c) => c.index(idx).ok_or_else(|| { diff --git a/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_mutator.rs b/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_mutator.rs index dd56da8ffdad0..caaf3cad66226 100644 --- a/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_mutator.rs +++ b/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_mutator.rs @@ -13,8 +13,8 @@ // limitations under the License. use std::cmp::Ordering; -use std::collections::hash_map::Entry; use std::collections::HashMap; +use std::collections::hash_map::Entry; use std::iter::once; use ahash::HashSet; @@ -22,9 +22,6 @@ use ahash::HashSetExt; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::AnyType; -use databend_common_expression::types::DataType; -use databend_common_expression::types::MutableBitmap; use databend_common_expression::Column; use databend_common_expression::ColumnId; use databend_common_expression::DataBlock; @@ -37,8 +34,11 @@ use databend_common_expression::Scalar; use databend_common_expression::ScalarRef; use databend_common_expression::TableSchema; use databend_common_expression::Value; -use databend_common_functions::aggregates::eval_aggr; +use databend_common_expression::types::AnyType; +use databend_common_expression::types::DataType; +use databend_common_expression::types::MutableBitmap; use databend_common_functions::BUILTIN_FUNCTIONS; +use databend_common_functions::aggregates::eval_aggr; use databend_common_metrics::storage::*; use databend_common_sql::executor::physical_plans::OnConflictField; use databend_storages_common_index::BloomIndex; @@ -48,8 +48,8 @@ use log::info; use crate::operations::replace_into::meta::DeletionByColumn; use crate::operations::replace_into::meta::ReplaceIntoOperation; use crate::operations::replace_into::meta::UniqueKeyDigest; -use crate::operations::replace_into::mutator::column_hash::row_hash_of_columns; use crate::operations::replace_into::mutator::column_hash::RowScalarValue; +use crate::operations::replace_into::mutator::column_hash::row_hash_of_columns; // Replace is somehow a simplified merge_into, which // - do insertion for "matched" branch @@ -441,10 +441,9 @@ impl Partitioner { let row_bloom_hashes: Vec> = column_bloom_hashes .iter() .filter_map(|(hashes, validity)| match validity { - Some(v) if v.null_count() != 0 => { - v.get(row_idx) - .map(|v| if v { hashes.get(row_idx) } else { None }) - } + Some(v) if v.null_count() != 0 => v + .get(row_idx) + .map(|v| if v { hashes.get(row_idx) } else { None }), _ => Some(hashes.get(row_idx)), }) .collect(); @@ -488,9 +487,9 @@ fn on_conflict_key_column_values( #[cfg(test)] mod tests { + use databend_common_expression::FromData; use databend_common_expression::types::NumberType; use databend_common_expression::types::StringType; - use databend_common_expression::FromData; use super::*; diff --git a/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs b/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs index 6b735e3b4c1d7..7829194f4a737 100644 --- a/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs +++ b/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs @@ -20,14 +20,12 @@ use ahash::AHashMap; use databend_common_base::base::tokio::sync::Semaphore; use databend_common_base::runtime::GlobalIORuntime; use databend_common_base::runtime::TrySpawn; -use databend_common_catalog::plan::gen_mutation_stream_meta; use databend_common_catalog::plan::Projection; +use databend_common_catalog::plan::gen_mutation_stream_meta; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::MutableBitmap; -use databend_common_expression::types::UInt64Type; use databend_common_expression::Column; use databend_common_expression::ColumnId; use databend_common_expression::ComputedExpr; @@ -35,17 +33,19 @@ use databend_common_expression::DataBlock; use databend_common_expression::FieldIndex; use databend_common_expression::FromData; use databend_common_expression::Scalar; +use databend_common_expression::types::MutableBitmap; +use databend_common_expression::types::UInt64Type; use databend_common_metrics::storage::*; +use databend_common_sql::StreamContext; use databend_common_sql::evaluator::BlockOperator; use databend_common_sql::executor::physical_plans::OnConflictField; -use databend_common_sql::StreamContext; use databend_storages_common_cache::BlockMetaCache; use databend_storages_common_cache::CacheAccessor; use databend_storages_common_cache::CacheManager; use databend_storages_common_cache::LoadParams; +use databend_storages_common_index::BloomIndex; use databend_storages_common_index::filters::Filter; use databend_storages_common_index::filters::FilterImpl; -use databend_storages_common_index::BloomIndex; use databend_storages_common_io::ReadSettings; use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::BlockSlotDescription; @@ -56,13 +56,14 @@ use log::info; use log::warn; use opendal::Operator; -use crate::io::read::bloom::block_filter_reader::BloomBlockFilterReader; +use crate::FuseTable; use crate::io::BlockBuilder; use crate::io::BlockReader; use crate::io::BlockWriter; use crate::io::CompactSegmentInfoReader; use crate::io::MetaReaders; use crate::io::WriteSettings; +use crate::io::read::bloom::block_filter_reader::BloomBlockFilterReader; use crate::operations::acquire_task_permit; use crate::operations::common::BlockMetaIndex; use crate::operations::common::MutationLogEntry; @@ -73,9 +74,8 @@ use crate::operations::read_block; use crate::operations::replace_into::meta::DeletionByColumn; use crate::operations::replace_into::meta::ReplaceIntoOperation; use crate::operations::replace_into::meta::UniqueKeyDigest; -use crate::operations::replace_into::mutator::row_hash_of_columns; use crate::operations::replace_into::mutator::DeletionAccumulator; -use crate::FuseTable; +use crate::operations::replace_into::mutator::row_hash_of_columns; struct AggregationContext { segment_locations: AHashMap, @@ -788,11 +788,11 @@ impl AggregationContext { #[cfg(test)] mod tests { - use databend_common_expression::types::NumberDataType; - use databend_common_expression::types::NumberScalar; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; + use databend_common_expression::types::NumberDataType; + use databend_common_expression::types::NumberScalar; use super::*; diff --git a/src/query/storages/fuse/src/operations/replace_into/processors/processor_replace_into.rs b/src/query/storages/fuse/src/operations/replace_into/processors/processor_replace_into.rs index 5105a94db6026..45db4547d7ee8 100644 --- a/src/query/storages/fuse/src/operations/replace_into/processors/processor_replace_into.rs +++ b/src/query/storages/fuse/src/operations/replace_into/processors/processor_replace_into.rs @@ -20,7 +20,6 @@ use std::time::Instant; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::BooleanType; use databend_common_expression::ColumnId; use databend_common_expression::DataBlock; use databend_common_expression::Evaluator; @@ -29,6 +28,7 @@ use databend_common_expression::FieldIndex; use databend_common_expression::RemoteExpr; use databend_common_expression::TableSchema; use databend_common_expression::Value; +use databend_common_expression::types::BooleanType; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_metrics::storage::*; use databend_common_pipeline::core::Event; diff --git a/src/query/storages/fuse/src/operations/revert.rs b/src/query/storages/fuse/src/operations/revert.rs index bc2729d193aed..f3e92c13570ec 100644 --- a/src/query/storages/fuse/src/operations/revert.rs +++ b/src/query/storages/fuse/src/operations/revert.rs @@ -22,8 +22,8 @@ use databend_common_exception::Result; use databend_common_meta_app::schema::UpdateTableMetaReq; use databend_common_meta_types::MatchSeq; -use crate::operations::SnapshotHintWriter; use crate::FuseTable; +use crate::operations::SnapshotHintWriter; impl FuseTable { #[async_backtrace::framed] diff --git a/src/query/storages/fuse/src/operations/snapshot_hint.rs b/src/query/storages/fuse/src/operations/snapshot_hint.rs index a41d93d2f90a1..71eedf1d54aba 100644 --- a/src/query/storages/fuse/src/operations/snapshot_hint.rs +++ b/src/query/storages/fuse/src/operations/snapshot_hint.rs @@ -29,9 +29,9 @@ use opendal::Operator; use serde::Deserialize; use serde::Serialize; -use crate::io::TableMetaLocationGenerator; use crate::FUSE_TBL_LAST_SNAPSHOT_HINT; use crate::FUSE_TBL_LAST_SNAPSHOT_HINT_V2; +use crate::io::TableMetaLocationGenerator; pub struct SnapshotHintWriter<'a> { ctx: &'a dyn TableContext, diff --git a/src/query/storages/fuse/src/operations/table_index.rs b/src/query/storages/fuse/src/operations/table_index.rs index 3009fb32017cd..32d0f5ae81e45 100644 --- a/src/query/storages/fuse/src/operations/table_index.rs +++ b/src/query/storages/fuse/src/operations/table_index.rs @@ -23,7 +23,6 @@ use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::local_block_meta_serde; use databend_common_expression::BlockMetaInfo; use databend_common_expression::BlockMetaInfoDowncast; use databend_common_expression::DataBlock; @@ -31,6 +30,7 @@ use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRef; +use databend_common_expression::local_block_meta_serde; use databend_common_meta_app::schema::TableIndex; use databend_common_meta_app::schema::TableIndexType; use databend_common_meta_app::schema::TableMeta; @@ -56,29 +56,29 @@ use databend_storages_common_table_meta::meta::Versioned; use log::info; use opendal::Operator; -use crate::index::filters::BlockFilter; -use crate::index::filters::Filter; +use crate::FuseStorageFormat; +use crate::FuseTable; use crate::index::BloomIndex; use crate::index::BloomIndexBuilder; use crate::index::NgramArgs; -use crate::io::read::bloom::block_filter_reader::load_bloom_filter_by_columns; -use crate::io::read::bloom::block_filter_reader::load_index_meta; -use crate::io::read::load_vector_index_meta; -use crate::io::read::read_segment_stats; +use crate::index::filters::BlockFilter; +use crate::index::filters::Filter; use crate::io::BlockReader; use crate::io::BlockWriter; use crate::io::BloomIndexState; use crate::io::MetaReaders; use crate::io::TableMetaLocationGenerator; use crate::io::VectorIndexBuilder; +use crate::io::read::bloom::block_filter_reader::load_bloom_filter_by_columns; +use crate::io::read::bloom::block_filter_reader::load_index_meta; +use crate::io::read::load_vector_index_meta; +use crate::io::read::read_segment_stats; use crate::operations::BlockMetaIndex; use crate::operations::CommitSink; use crate::operations::MutationGenerator; use crate::operations::MutationLogEntry; use crate::operations::MutationLogs; use crate::operations::TableMutationAggregator; -use crate::FuseStorageFormat; -use crate::FuseTable; pub async fn do_refresh_table_index( fuse_table: &FuseTable, diff --git a/src/query/storages/fuse/src/operations/truncate.rs b/src/query/storages/fuse/src/operations/truncate.rs index 1543f6c75b387..dbb4059a77cd0 100644 --- a/src/query/storages/fuse/src/operations/truncate.rs +++ b/src/query/storages/fuse/src/operations/truncate.rs @@ -24,11 +24,11 @@ use databend_common_pipeline::sources::OneBlockSource; use databend_common_sql::plans::TruncateMode; use databend_storages_common_table_meta::meta::TableSnapshot; +use crate::FuseTable; use crate::operations::common::CommitMeta; use crate::operations::common::CommitSink; use crate::operations::common::ConflictResolveContext; use crate::operations::common::TruncateGenerator; -use crate::FuseTable; impl FuseTable { #[inline] diff --git a/src/query/storages/fuse/src/operations/util.rs b/src/query/storages/fuse/src/operations/util.rs index 9a27492e70aa0..c5573b9175223 100644 --- a/src/query/storages/fuse/src/operations/util.rs +++ b/src/query/storages/fuse/src/operations/util.rs @@ -32,8 +32,8 @@ use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::ColumnMeta; use databend_storages_common_table_meta::meta::SingleColumnMeta; -use crate::io::BlockReader; use crate::FuseStorageFormat; +use crate::io::BlockReader; const OCC_DEFAULT_BACKOFF_INIT_DELAY_MS: Duration = Duration::from_millis(5); const OCC_DEFAULT_BACKOFF_MAX_DELAY_MS: Duration = Duration::from_millis(20 * 1000); diff --git a/src/query/storages/fuse/src/pruning/block_pruner.rs b/src/query/storages/fuse/src/pruning/block_pruner.rs index 0056c49a534b6..f45cb89240804 100644 --- a/src/query/storages/fuse/src/pruning/block_pruner.rs +++ b/src/query/storages/fuse/src/pruning/block_pruner.rs @@ -22,8 +22,8 @@ use databend_common_base::base::tokio::sync::OwnedSemaphorePermit; use databend_common_catalog::plan::block_id_in_segment; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::F32; use databend_common_expression::BLOCK_NAME_COL_NAME; +use databend_common_expression::types::F32; use databend_common_metrics::storage::*; use databend_storages_common_pruner::BlockMetaIndex; use databend_storages_common_pruner::VirtualBlockMetaIndex; diff --git a/src/query/storages/fuse/src/pruning/bloom_pruner.rs b/src/query/storages/fuse/src/pruning/bloom_pruner.rs index b2563748e2d3c..71807f9db971a 100644 --- a/src/query/storages/fuse/src/pruning/bloom_pruner.rs +++ b/src/query/storages/fuse/src/pruning/bloom_pruner.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::hash_map::Entry; use std::collections::HashMap; use std::collections::HashSet; +use std::collections::hash_map::Entry; use std::sync::Arc; use databend_common_exception::ErrorCode; @@ -28,13 +28,13 @@ use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRef; use databend_common_expression::Value; use databend_common_sql::BloomIndexColumns; -use databend_storages_common_index::filters::BlockFilter; use databend_storages_common_index::BloomIndex; use databend_storages_common_index::FilterEvalResult; use databend_storages_common_index::NgramArgs; -use databend_storages_common_table_meta::meta::column_oriented_segment::BlockReadInfo; +use databend_storages_common_index::filters::BlockFilter; use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::StatisticsOfColumns; +use databend_storages_common_table_meta::meta::column_oriented_segment::BlockReadInfo; use log::info; use log::warn; use opendal::Operator; diff --git a/src/query/storages/fuse/src/pruning/expr_bloom_filter.rs b/src/query/storages/fuse/src/pruning/expr_bloom_filter.rs index 44f9df6194bf2..23d4409c8ba51 100644 --- a/src/query/storages/fuse/src/pruning/expr_bloom_filter.rs +++ b/src/query/storages/fuse/src/pruning/expr_bloom_filter.rs @@ -14,10 +14,10 @@ use databend_common_catalog::sbbf::Sbbf; use databend_common_exception::Result; -use databend_common_expression::hash_util::hash_by_method_for_bloom; -use databend_common_expression::types::MutableBitmap; use databend_common_expression::Column; use databend_common_expression::DataBlock; +use databend_common_expression::hash_util::hash_by_method_for_bloom; +use databend_common_expression::types::MutableBitmap; pub struct ExprBloomFilter<'a> { filter: &'a Sbbf, diff --git a/src/query/storages/fuse/src/pruning/expr_runtime_pruner.rs b/src/query/storages/fuse/src/pruning/expr_runtime_pruner.rs index 5b6fb836dda02..e3b8f250cd67b 100644 --- a/src/query/storages/fuse/src/pruning/expr_runtime_pruner.rs +++ b/src/query/storages/fuse/src/pruning/expr_runtime_pruner.rs @@ -113,7 +113,9 @@ impl ExprRuntimePruner { } } } else { - info!("Can't prune the partition by runtime filter, because there is no statistics for the partition"); + info!( + "Can't prune the partition by runtime filter, because there is no statistics for the partition" + ); } } diff --git a/src/query/storages/fuse/src/pruning/fuse_pruner.rs b/src/query/storages/fuse/src/pruning/fuse_pruner.rs index 81d699588eefa..3e82ad5d487d0 100644 --- a/src/query/storages/fuse/src/pruning/fuse_pruner.rs +++ b/src/query/storages/fuse/src/pruning/fuse_pruner.rs @@ -24,8 +24,8 @@ use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::RemoteExpr; -use databend_common_expression::TableSchemaRef; use databend_common_expression::SEGMENT_NAME_COL_NAME; +use databend_common_expression::TableSchemaRef; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_metrics::storage::metrics_inc_blocks_topn_pruning_after; use databend_common_metrics::storage::metrics_inc_blocks_topn_pruning_before; @@ -62,7 +62,6 @@ use rand::thread_rng; use crate::io::BloomIndexRebuilder; use crate::operations::DeletedSegmentInfo; -use crate::pruning::segment_pruner::SegmentPruner; use crate::pruning::BlockPruner; use crate::pruning::BloomPruner; use crate::pruning::BloomPrunerCreator; @@ -73,6 +72,7 @@ use crate::pruning::PruningCostKind; use crate::pruning::SegmentLocation; use crate::pruning::VectorIndexPruner; use crate::pruning::VirtualColumnPruner; +use crate::pruning::segment_pruner::SegmentPruner; const SMALL_DATASET_SAMPLE_THRESHOLD: usize = 100; diff --git a/src/query/storages/fuse/src/pruning/inverted_index_pruner.rs b/src/query/storages/fuse/src/pruning/inverted_index_pruner.rs index 703ec1c65dd71..ad97f760b8eb1 100644 --- a/src/query/storages/fuse/src/pruning/inverted_index_pruner.rs +++ b/src/query/storages/fuse/src/pruning/inverted_index_pruner.rs @@ -28,11 +28,11 @@ use tantivy::schema::Field; use tantivy::schema::IndexRecordOption; use tantivy::tokenizer::TokenizerManager; +use crate::TableContext; +use crate::io::TableMetaLocationGenerator; use crate::io::create_index_schema; use crate::io::create_tokenizer_manager; use crate::io::read::InvertedIndexReader; -use crate::io::TableMetaLocationGenerator; -use crate::TableContext; // Each block file has a corresponding index file, // the data in the index file is generated by tantivy. diff --git a/src/query/storages/fuse/src/pruning/mod.rs b/src/query/storages/fuse/src/pruning/mod.rs index c35c4b2044730..dd19cfe96aa38 100644 --- a/src/query/storages/fuse/src/pruning/mod.rs +++ b/src/query/storages/fuse/src/pruning/mod.rs @@ -30,13 +30,13 @@ pub use bloom_pruner::BloomPrunerCreator; pub use expr_bloom_filter::ExprBloomFilter; pub use expr_runtime_pruner::ExprRuntimePruner; pub use expr_runtime_pruner::RuntimeFilterExpr; -pub use fuse_pruner::table_sample; pub use fuse_pruner::FusePruner; pub use fuse_pruner::PruningContext; -pub use inverted_index_pruner::create_inverted_index_query; +pub use fuse_pruner::table_sample; pub use inverted_index_pruner::InvertedIndexPruner; -pub use pruner_location::create_segment_location_vector; +pub use inverted_index_pruner::create_inverted_index_query; pub use pruner_location::SegmentLocation; +pub use pruner_location::create_segment_location_vector; pub use pruning_statistics::FusePruningStatistics; pub use pruning_statistics::PruningCostController; pub use pruning_statistics::PruningCostKind; diff --git a/src/query/storages/fuse/src/pruning/pruning_statistics.rs b/src/query/storages/fuse/src/pruning/pruning_statistics.rs index 015d82dcbfa21..99a72f84464f6 100644 --- a/src/query/storages/fuse/src/pruning/pruning_statistics.rs +++ b/src/query/storages/fuse/src/pruning/pruning_statistics.rs @@ -14,10 +14,10 @@ use std::array; use std::future::Future; +use std::sync::Arc; use std::sync::atomic::AtomicU64; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; -use std::sync::Arc; use std::time::Duration; use std::time::Instant; diff --git a/src/query/storages/fuse/src/pruning/segment_pruner.rs b/src/query/storages/fuse/src/pruning/segment_pruner.rs index d4acc540e5131..3a8e4d404e86d 100644 --- a/src/query/storages/fuse/src/pruning/segment_pruner.rs +++ b/src/query/storages/fuse/src/pruning/segment_pruner.rs @@ -18,8 +18,8 @@ use std::sync::Arc; use databend_common_exception::Result; use databend_common_expression::TableSchemaRef; use databend_common_metrics::storage::*; -use databend_storages_common_table_meta::meta::column_oriented_segment::AbstractSegment; use databend_storages_common_table_meta::meta::CompactSegmentInfo; +use databend_storages_common_table_meta::meta::column_oriented_segment::AbstractSegment; use crate::io::read::SegmentReader; use crate::pruning::PruningContext; diff --git a/src/query/storages/fuse/src/pruning/vector_index_pruner.rs b/src/query/storages/fuse/src/pruning/vector_index_pruner.rs index a61a1c444df05..e842b4bad3b41 100644 --- a/src/query/storages/fuse/src/pruning/vector_index_pruner.rs +++ b/src/query/storages/fuse/src/pruning/vector_index_pruner.rs @@ -28,9 +28,6 @@ use databend_common_catalog::plan::Filters; use databend_common_catalog::plan::VectorIndexInfo; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::Buffer; -use databend_common_expression::types::NumberColumn; -use databend_common_expression::types::F32; use databend_common_expression::BlockEntry; use databend_common_expression::Column; use databend_common_expression::DataBlock; @@ -40,6 +37,9 @@ use databend_common_expression::FunctionContext; use databend_common_expression::RemoteExpr; use databend_common_expression::TableSchemaRef; use databend_common_expression::VECTOR_SCORE_COL_NAME; +use databend_common_expression::types::Buffer; +use databend_common_expression::types::F32; +use databend_common_expression::types::NumberColumn; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_metrics::storage::metrics_inc_block_vector_index_pruning_milliseconds; use databend_common_metrics::storage::metrics_inc_blocks_vector_index_pruning_after; diff --git a/src/query/storages/fuse/src/pruning/virtual_column_pruner.rs b/src/query/storages/fuse/src/pruning/virtual_column_pruner.rs index 3c79832efc8af..dfb154e3bb58e 100644 --- a/src/query/storages/fuse/src/pruning/virtual_column_pruner.rs +++ b/src/query/storages/fuse/src/pruning/virtual_column_pruner.rs @@ -62,7 +62,8 @@ impl VirtualColumnPruner { { // If the number of rows is not same, the virtual column data has been corrupted, ignore the data. if virtual_column_meta.num_values != row_count { - warn!("Virtual column row count mismatch, location: {}, block rows: {}, virtual column rows: {}", + warn!( + "Virtual column row count mismatch, location: {}, block rows: {}, virtual column rows: {}", virtual_block_meta.virtual_location.0, row_count, virtual_column_meta.num_values diff --git a/src/query/storages/fuse/src/pruning_pipeline/block_metas_meta.rs b/src/query/storages/fuse/src/pruning_pipeline/block_metas_meta.rs index f294abd736e24..acdc0fc84691a 100644 --- a/src/query/storages/fuse/src/pruning_pipeline/block_metas_meta.rs +++ b/src/query/storages/fuse/src/pruning_pipeline/block_metas_meta.rs @@ -16,9 +16,9 @@ use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; -use databend_common_expression::local_block_meta_serde; use databend_common_expression::BlockMetaInfo; use databend_common_expression::BlockMetaInfoPtr; +use databend_common_expression::local_block_meta_serde; use databend_storages_common_table_meta::meta::BlockMeta; use crate::SegmentLocation; diff --git a/src/query/storages/fuse/src/pruning_pipeline/block_prune_result_meta.rs b/src/query/storages/fuse/src/pruning_pipeline/block_prune_result_meta.rs index 4320a80b05c81..4d1c28dfb77e7 100644 --- a/src/query/storages/fuse/src/pruning_pipeline/block_prune_result_meta.rs +++ b/src/query/storages/fuse/src/pruning_pipeline/block_prune_result_meta.rs @@ -16,9 +16,9 @@ use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; -use databend_common_expression::local_block_meta_serde; use databend_common_expression::BlockMetaInfo; use databend_common_expression::BlockMetaInfoPtr; +use databend_common_expression::local_block_meta_serde; use databend_storages_common_pruner::BlockMetaIndex; use databend_storages_common_table_meta::meta::BlockMeta; diff --git a/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs b/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs index 83d07677232c4..8ef21157566dd 100644 --- a/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs +++ b/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs @@ -18,30 +18,30 @@ use std::sync::Arc; use async_channel::Sender; use chrono::DateTime; use databend_common_base::base::tokio::sync::OwnedSemaphorePermit; -use databend_common_catalog::plan::block_id_in_segment; use databend_common_catalog::plan::PartInfoPtr; +use databend_common_catalog::plan::block_id_in_segment; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::BLOCK_NAME_COL_NAME; use databend_common_expression::BlockMetaInfoDowncast; use databend_common_expression::ColumnId; use databend_common_expression::DataBlock; use databend_common_expression::ScalarRef; -use databend_common_expression::BLOCK_NAME_COL_NAME; use databend_common_pipeline::core::InputPort; use databend_common_pipeline::core::ProcessorPtr; use databend_common_pipeline::sinks::AsyncSink; use databend_common_pipeline::sinks::AsyncSinker; use databend_storages_common_pruner::BlockMetaIndex; -use databend_storages_common_table_meta::meta::column_oriented_segment::*; use databend_storages_common_table_meta::meta::ColumnMeta; use databend_storages_common_table_meta::meta::ColumnMetaV0; use databend_storages_common_table_meta::meta::ColumnStatistics; use databend_storages_common_table_meta::meta::Compression; +use databend_storages_common_table_meta::meta::column_oriented_segment::*; use futures_util::future; use super::PrunedColumnOrientedSegmentMeta; -use crate::pruning::BlockPruner; use crate::FuseBlockPartInfo; +use crate::pruning::BlockPruner; pub struct ColumnOrientedBlockPruneSink { block_pruner: Arc, diff --git a/src/query/storages/fuse/src/pruning_pipeline/lazy_segment_meta.rs b/src/query/storages/fuse/src/pruning_pipeline/lazy_segment_meta.rs index 5d40cdd5d9b70..57ddeb2e5de50 100644 --- a/src/query/storages/fuse/src/pruning_pipeline/lazy_segment_meta.rs +++ b/src/query/storages/fuse/src/pruning_pipeline/lazy_segment_meta.rs @@ -15,9 +15,9 @@ use std::fmt::Debug; use std::fmt::Formatter; -use databend_common_expression::local_block_meta_serde; use databend_common_expression::BlockMetaInfo; use databend_common_expression::BlockMetaInfoPtr; +use databend_common_expression::local_block_meta_serde; use crate::SegmentLocation; diff --git a/src/query/storages/fuse/src/pruning_pipeline/lazy_segment_receiver_source.rs b/src/query/storages/fuse/src/pruning_pipeline/lazy_segment_receiver_source.rs index 5b9261fd5ca04..465285e140ab2 100644 --- a/src/query/storages/fuse/src/pruning_pipeline/lazy_segment_receiver_source.rs +++ b/src/query/storages/fuse/src/pruning_pipeline/lazy_segment_receiver_source.rs @@ -23,8 +23,8 @@ use databend_common_pipeline::core::ProcessorPtr; use databend_common_pipeline::sources::AsyncSource; use databend_common_pipeline::sources::AsyncSourcer; -use crate::pruning_pipeline::LazySegmentMeta; use crate::SegmentLocation; +use crate::pruning_pipeline::LazySegmentMeta; pub struct LazySegmentReceiverSource { pub meta_receiver: Receiver, diff --git a/src/query/storages/fuse/src/pruning_pipeline/pruned_segment_meta.rs b/src/query/storages/fuse/src/pruning_pipeline/pruned_segment_meta.rs index 088b9f5dcbe86..a320e2950ce13 100644 --- a/src/query/storages/fuse/src/pruning_pipeline/pruned_segment_meta.rs +++ b/src/query/storages/fuse/src/pruning_pipeline/pruned_segment_meta.rs @@ -16,16 +16,16 @@ use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; -use databend_common_expression::local_block_meta_serde; use databend_common_expression::BlockMetaInfo; use databend_common_expression::BlockMetaInfoPtr; -use databend_storages_common_table_meta::meta::column_oriented_segment::*; +use databend_common_expression::local_block_meta_serde; use databend_storages_common_table_meta::meta::CompactSegmentInfo; +use databend_storages_common_table_meta::meta::column_oriented_segment::*; +use crate::SegmentLocation; use crate::io::read::ColumnOrientedSegmentReader; use crate::io::read::RowOrientedSegmentReader; use crate::io::read::SegmentReader; -use crate::SegmentLocation; pub struct PrunedCompactSegmentMeta { pub segments: (SegmentLocation, Arc), diff --git a/src/query/storages/fuse/src/pruning_pipeline/send_part_info_sink.rs b/src/query/storages/fuse/src/pruning_pipeline/send_part_info_sink.rs index 202dc78e3c125..2a12acc383873 100644 --- a/src/query/storages/fuse/src/pruning_pipeline/send_part_info_sink.rs +++ b/src/query/storages/fuse/src/pruning_pipeline/send_part_info_sink.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::Arc; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; -use std::sync::Arc; use async_channel::Sender; use databend_common_catalog::plan::PartInfoPtr; @@ -43,9 +43,9 @@ use databend_storages_common_pruner::BlockMetaIndex; use databend_storages_common_table_meta::meta::BlockMeta; use parking_lot::Mutex; +use crate::FuseTable; use crate::pruning::FusePruner; use crate::pruning_pipeline::block_prune_result_meta::BlockPruneResult; -use crate::FuseTable; pub struct SendPartCache { partitions: Partitions, diff --git a/src/query/storages/fuse/src/retry/commit.rs b/src/query/storages/fuse/src/retry/commit.rs index 1833d1f1a1283..a9791d0e27e02 100644 --- a/src/query/storages/fuse/src/retry/commit.rs +++ b/src/query/storages/fuse/src/retry/commit.rs @@ -27,18 +27,18 @@ use databend_common_meta_app::schema::UpdateTableMetaReq; use databend_common_meta_types::MatchSeq; use databend_storages_common_cache::Table; use databend_storages_common_cache::TableSnapshot; +use databend_storages_common_table_meta::meta::Versioned; use databend_storages_common_table_meta::meta::decode_column_hll; use databend_storages_common_table_meta::meta::encode_column_hll; use databend_storages_common_table_meta::meta::merge_column_hll; -use databend_storages_common_table_meta::meta::Versioned; use databend_storages_common_table_meta::readers::snapshot_reader::TableSnapshotAccessor; use log::info; use super::diff::SegmentsDiff; +use crate::FuseTable; use crate::operations::set_backoff; use crate::statistics::merge_statistics; use crate::statistics::reducers::deduct_statistics; -use crate::FuseTable; const FUSE_ENGINE: &str = "FUSE"; @@ -288,9 +288,9 @@ async fn try_rebuild_req( if txn_begin_timestamp < latest_snapshot_timestamp { return Err(ErrorCode::UnresolvableConflict(format!( - "Unresolvable conflict detected for table {} while resolving conflicts: txn started with logical timestamp {}, which is less than the latest table timestamp {}. Transaction must be aborted.", - tid, txn_begin_timestamp, latest_snapshot_timestamp - ))); + "Unresolvable conflict detected for table {} while resolving conflicts: txn started with logical timestamp {}, which is less than the latest table timestamp {}. Transaction must be aborted.", + tid, txn_begin_timestamp, latest_snapshot_timestamp + ))); } } } @@ -344,9 +344,10 @@ fn retry_too_many_msg( format!( "Transaction aborted after retries({} times, {} ms). The table_ids that failed to update: {:?}", retries, - Instant::now() - .duration_since(start_time) - .as_millis(), - update_failed_tbls.into_iter().map(|(tid, _, _)| tid).collect::>() + Instant::now().duration_since(start_time).as_millis(), + update_failed_tbls + .into_iter() + .map(|(tid, _, _)| tid) + .collect::>() ) } diff --git a/src/query/storages/fuse/src/statistics/accumulator.rs b/src/query/storages/fuse/src/statistics/accumulator.rs index 6f71c49da1577..ce01027ab10b9 100644 --- a/src/query/storages/fuse/src/statistics/accumulator.rs +++ b/src/query/storages/fuse/src/statistics/accumulator.rs @@ -27,9 +27,6 @@ use databend_common_expression::VirtualDataField; use databend_common_expression::VirtualDataSchema; use databend_common_license::license::Feature; use databend_common_license::license_manager::LicenseManagerSwitch; -use databend_storages_common_table_meta::meta::column_oriented_segment::*; -use databend_storages_common_table_meta::meta::encode_column_hll; -use databend_storages_common_table_meta::meta::merge_column_hll_mut; use databend_storages_common_table_meta::meta::AdditionalStatsMeta; use databend_storages_common_table_meta::meta::BlockHLL; use databend_storages_common_table_meta::meta::BlockHLLState; @@ -39,6 +36,9 @@ use databend_storages_common_table_meta::meta::RawBlockHLL; use databend_storages_common_table_meta::meta::SegmentInfo; use databend_storages_common_table_meta::meta::SegmentStatistics; use databend_storages_common_table_meta::meta::VirtualColumnMeta; +use databend_storages_common_table_meta::meta::column_oriented_segment::*; +use databend_storages_common_table_meta::meta::encode_column_hll; +use databend_storages_common_table_meta::meta::merge_column_hll_mut; #[derive(Default)] pub struct RowOrientedSegmentBuilder { diff --git a/src/query/storages/fuse/src/statistics/column_statistic.rs b/src/query/storages/fuse/src/statistics/column_statistic.rs index a14a8d75095e5..f3bf623d91f7f 100644 --- a/src/query/storages/fuse/src/statistics/column_statistic.rs +++ b/src/query/storages/fuse/src/statistics/column_statistic.rs @@ -15,16 +15,16 @@ use std::collections::HashMap; use databend_common_exception::Result; -use databend_common_expression::types::AccessType; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberScalar; -use databend_common_expression::types::NumberType; use databend_common_expression::Column; use databend_common_expression::ColumnId; use databend_common_expression::DataBlock; use databend_common_expression::Scalar; use databend_common_expression::TableSchemaRef; use databend_common_expression::Value; +use databend_common_expression::types::AccessType; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberScalar; +use databend_common_expression::types::NumberType; use databend_common_functions::aggregates::eval_aggr; use databend_storages_common_index::Index; use databend_storages_common_index::RangeIndex; @@ -258,9 +258,9 @@ impl Trim for Scalar { mod tests { use databend_common_expression::Scalar; - use crate::statistics::Trim; use crate::statistics::END_OF_UNICODE_RANGE; use crate::statistics::STATS_STRING_PREFIX_LEN; + use crate::statistics::Trim; #[test] fn test_trim_max() { diff --git a/src/query/storages/fuse/src/statistics/mod.rs b/src/query/storages/fuse/src/statistics/mod.rs index 59a3ed6d010d1..a14a07d4a9df5 100644 --- a/src/query/storages/fuse/src/statistics/mod.rs +++ b/src/query/storages/fuse/src/statistics/mod.rs @@ -22,14 +22,14 @@ mod traverse; pub use accumulator::ColumnHLLAccumulator; pub use accumulator::RowOrientedSegmentBuilder; pub use accumulator::VirtualColumnAccumulator; -pub use cluster_statistics::sort_by_cluster_stats; pub use cluster_statistics::ClusterStatsGenerator; +pub use cluster_statistics::sort_by_cluster_stats; +pub use column_statistic::END_OF_UNICODE_RANGE; +pub use column_statistic::STATS_STRING_PREFIX_LEN; +pub use column_statistic::Trim; pub use column_statistic::calc_column_distinct_of_values; pub use column_statistic::gen_columns_statistics; pub use column_statistic::scalar_min_max; -pub use column_statistic::Trim; -pub use column_statistic::END_OF_UNICODE_RANGE; -pub use column_statistic::STATS_STRING_PREFIX_LEN; pub use reducers::merge_statistics; pub use reducers::reduce_block_metas; pub use reducers::reduce_block_statistics; diff --git a/src/query/storages/fuse/src/statistics/reducers.rs b/src/query/storages/fuse/src/statistics/reducers.rs index 9580f729cb3d1..8c947f7fb96a4 100644 --- a/src/query/storages/fuse/src/statistics/reducers.rs +++ b/src/query/storages/fuse/src/statistics/reducers.rs @@ -16,10 +16,10 @@ use std::borrow::Borrow; use std::collections::HashMap; use std::collections::HashSet; -use databend_common_expression::types::DataType; use databend_common_expression::BlockThresholds; use databend_common_expression::ColumnId; use databend_common_expression::Scalar; +use databend_common_expression::types::DataType; use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::ClusterStatistics; use databend_storages_common_table_meta::meta::ColumnStatistics; diff --git a/src/query/storages/fuse/src/statistics/table_statistics.rs b/src/query/storages/fuse/src/statistics/table_statistics.rs index 58ffadaac9bd0..5b29b9bbae57a 100644 --- a/src/query/storages/fuse/src/statistics/table_statistics.rs +++ b/src/query/storages/fuse/src/statistics/table_statistics.rs @@ -15,9 +15,9 @@ use std::collections::HashMap; use databend_common_expression::ColumnId; -use databend_storages_common_table_meta::meta::encode_column_hll; use databend_storages_common_table_meta::meta::AdditionalStatsMeta; use databend_storages_common_table_meta::meta::BlockHLL; +use databend_storages_common_table_meta::meta::encode_column_hll; #[derive(Clone, Default)] pub struct TableStatsGenerator { diff --git a/src/query/storages/fuse/src/statistics/traverse.rs b/src/query/storages/fuse/src/statistics/traverse.rs index 5681c281df8ae..7153510596e58 100644 --- a/src/query/storages/fuse/src/statistics/traverse.rs +++ b/src/query/storages/fuse/src/statistics/traverse.rs @@ -12,10 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_expression::types::map::KvPair; -use databend_common_expression::types::AccessType; -use databend_common_expression::types::AnyType; -use databend_common_expression::types::DataType; use databend_common_expression::BlockEntry; use databend_common_expression::Column; use databend_common_expression::ColumnBuilder; @@ -23,6 +19,10 @@ use databend_common_expression::ColumnId; use databend_common_expression::Scalar; use databend_common_expression::TableField; use databend_common_expression::Value; +use databend_common_expression::types::AccessType; +use databend_common_expression::types::AnyType; +use databend_common_expression::types::DataType; +use databend_common_expression::types::map::KvPair; use databend_storages_common_index::Index; use databend_storages_common_index::RangeIndex; diff --git a/src/query/storages/fuse/src/table_functions/clustering_information.rs b/src/query/storages/fuse/src/table_functions/clustering_information.rs index 9e58c1cf0cb6d..7918bfe95a90f 100644 --- a/src/query/storages/fuse/src/table_functions/clustering_information.rs +++ b/src/query/storages/fuse/src/table_functions/clustering_information.rs @@ -24,11 +24,6 @@ use databend_common_catalog::plan::DataSourcePlan; use databend_common_catalog::table_args::TableArgs; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::compare_scalars; -use databend_common_expression::types::DataType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::TimestampType; -use databend_common_expression::types::VariantType; use databend_common_expression::BlockEntry; use databend_common_expression::ConstantFolder; use databend_common_expression::DataBlock; @@ -41,6 +36,11 @@ use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::compare_scalars; +use databend_common_expression::types::DataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::TimestampType; +use databend_common_expression::types::VariantType; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_sql::analyze_cluster_keys; use databend_storages_common_index::statistics_to_domain; @@ -52,14 +52,14 @@ use jsonb::Value as JsonbValue; use log::warn; use serde::Serialize; +use crate::FuseTable; +use crate::Table; use crate::io::SegmentsIO; use crate::sessions::TableContext; -use crate::table_functions::parse_db_tb_opt_args; -use crate::table_functions::string_literal; use crate::table_functions::SimpleArgFunc; use crate::table_functions::SimpleArgFuncTemplate; -use crate::FuseTable; -use crate::Table; +use crate::table_functions::parse_db_tb_opt_args; +use crate::table_functions::string_literal; pub struct ClusteringInformationArgs { database_name: String, diff --git a/src/query/storages/fuse/src/table_functions/clustering_statistics.rs b/src/query/storages/fuse/src/table_functions/clustering_statistics.rs index 640a1d09dd4a4..5443e0f8eecfb 100644 --- a/src/query/storages/fuse/src/table_functions/clustering_statistics.rs +++ b/src/query/storages/fuse/src/table_functions/clustering_statistics.rs @@ -19,10 +19,6 @@ use databend_common_catalog::table::Table; use databend_common_catalog::table_args::TableArgs; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::string::StringColumnBuilder; -use databend_common_expression::types::Int32Type; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; use databend_common_expression::Column; use databend_common_expression::DataBlock; use databend_common_expression::FromData; @@ -32,16 +28,20 @@ use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::Int32Type; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::string::StringColumnBuilder; use databend_storages_common_table_meta::meta::SegmentInfo; use databend_storages_common_table_meta::meta::TableSnapshot; +use crate::FuseTable; use crate::io::SegmentsIO; use crate::sessions::TableContext; -use crate::table_functions::parse_db_tb_args; -use crate::table_functions::string_literal; use crate::table_functions::SimpleArgFunc; use crate::table_functions::SimpleArgFuncTemplate; -use crate::FuseTable; +use crate::table_functions::parse_db_tb_args; +use crate::table_functions::string_literal; pub struct ClusteringStatsArgs { database_name: String, diff --git a/src/query/storages/fuse/src/table_functions/function_template/fuse_table_meta_func.rs b/src/query/storages/fuse/src/table_functions/function_template/fuse_table_meta_func.rs index 9c5aac272a259..1b91e030b78ae 100644 --- a/src/query/storages/fuse/src/table_functions/function_template/fuse_table_meta_func.rs +++ b/src/query/storages/fuse/src/table_functions/function_template/fuse_table_meta_func.rs @@ -26,13 +26,13 @@ use databend_common_expression::TableSchemaRef; use databend_storages_common_table_meta::meta::TableSnapshot; use futures_util::TryStreamExt; +use crate::FuseTable; use crate::io::MetaReaders; use crate::io::SnapshotHistoryReader; -use crate::table_functions::parse_db_tb_opt_args; -use crate::table_functions::string_literal; use crate::table_functions::SimpleArgFunc; use crate::table_functions::SimpleArgFuncTemplate; -use crate::FuseTable; +use crate::table_functions::parse_db_tb_opt_args; +use crate::table_functions::string_literal; pub struct TableMetaFuncCommonArgs { pub database_name: String, diff --git a/src/query/storages/fuse/src/table_functions/function_template/simple_func_template.rs b/src/query/storages/fuse/src/table_functions/function_template/simple_func_template.rs index 69d4c897ccce5..3670bb0d38ec3 100644 --- a/src/query/storages/fuse/src/table_functions/function_template/simple_func_template.rs +++ b/src/query/storages/fuse/src/table_functions/function_template/simple_func_template.rs @@ -34,10 +34,10 @@ use databend_common_pipeline::core::ProcessorPtr; use databend_common_pipeline::sources::AsyncSource; use databend_common_pipeline::sources::AsyncSourcer; +use crate::Table; use crate::sessions::TableContext; use crate::table_functions::TableArgs; use crate::table_functions::TableFunction; -use crate::Table; #[derive(serde::Serialize, serde::Deserialize, PartialEq, Eq)] struct PlaceHolder; diff --git a/src/query/storages/fuse/src/table_functions/fuse_amend.rs b/src/query/storages/fuse/src/table_functions/fuse_amend.rs index 60af158643cda..a456a6d4a036b 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_amend.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_amend.rs @@ -18,24 +18,24 @@ use databend_common_catalog::catalog_kind::CATALOG_DEFAULT; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::StringType; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::StringType; use databend_common_license::license::Feature; use databend_common_license::license_manager::LicenseManagerSwitch; -use databend_enterprise_fail_safe::get_fail_safe_handler; use databend_enterprise_fail_safe::FailSafeHandlerWrapper; +use databend_enterprise_fail_safe::get_fail_safe_handler; +use crate::FuseTable; use crate::sessions::TableContext; -use crate::table_functions::parse_db_tb_args; -use crate::table_functions::string_literal; use crate::table_functions::SimpleTableFunc; use crate::table_functions::TableArgs; -use crate::FuseTable; +use crate::table_functions::parse_db_tb_args; +use crate::table_functions::string_literal; struct AmendTableArgs { database_name: String, diff --git a/src/query/storages/fuse/src/table_functions/fuse_block.rs b/src/query/storages/fuse/src/table_functions/fuse_block.rs index eaecb5425c809..39666d75fee2b 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_block.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_block.rs @@ -16,11 +16,6 @@ use std::sync::Arc; use databend_common_catalog::table::Table; use databend_common_exception::Result; -use databend_common_expression::types::string::StringColumnBuilder; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::TimestampType; -use databend_common_expression::types::UInt64Type; use databend_common_expression::BlockEntry; use databend_common_expression::Column; use databend_common_expression::DataBlock; @@ -29,14 +24,19 @@ use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::TimestampType; +use databend_common_expression::types::UInt64Type; +use databend_common_expression::types::string::StringColumnBuilder; use databend_storages_common_table_meta::meta::SegmentInfo; use databend_storages_common_table_meta::meta::TableSnapshot; +use crate::FuseTable; use crate::io::SegmentsIO; use crate::sessions::TableContext; -use crate::table_functions::function_template::TableMetaFunc; use crate::table_functions::TableMetaFuncTemplate; -use crate::FuseTable; +use crate::table_functions::function_template::TableMetaFunc; pub struct FuseBlock; pub type FuseBlockFunc = TableMetaFuncTemplate; diff --git a/src/query/storages/fuse/src/table_functions/fuse_column.rs b/src/query/storages/fuse/src/table_functions/fuse_column.rs index c81fe93b5fd9b..140f3224028d3 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_column.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_column.rs @@ -17,12 +17,6 @@ use std::sync::Arc; use databend_common_catalog::table::Table; use databend_common_exception::Result; -use databend_common_expression::types::string::StringColumnBuilder; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::TimestampType; -use databend_common_expression::types::UInt32Type; -use databend_common_expression::types::UInt64Type; use databend_common_expression::BlockEntry; use databend_common_expression::Column; use databend_common_expression::DataBlock; @@ -31,18 +25,24 @@ use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::TimestampType; +use databend_common_expression::types::UInt32Type; +use databend_common_expression::types::UInt64Type; +use databend_common_expression::types::string::StringColumnBuilder; +use databend_storages_common_table_meta::meta::TableSnapshot; use databend_storages_common_table_meta::meta::column_oriented_segment::AbstractBlockMeta; use databend_storages_common_table_meta::meta::column_oriented_segment::AbstractSegment; -use databend_storages_common_table_meta::meta::TableSnapshot; +use crate::FuseTable; +use crate::io::SegmentsIO; use crate::io::read::ColumnOrientedSegmentReader; use crate::io::read::RowOrientedSegmentReader; use crate::io::read::SegmentReader; -use crate::io::SegmentsIO; use crate::sessions::TableContext; -use crate::table_functions::function_template::TableMetaFunc; use crate::table_functions::TableMetaFuncTemplate; -use crate::FuseTable; +use crate::table_functions::function_template::TableMetaFunc; pub struct FuseColumn; pub type FuseColumnFunc = TableMetaFuncTemplate; diff --git a/src/query/storages/fuse/src/table_functions/fuse_dump_snapshot.rs b/src/query/storages/fuse/src/table_functions/fuse_dump_snapshot.rs index fb88d9e19f538..fc2ca31a5d3e8 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_dump_snapshot.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_dump_snapshot.rs @@ -19,22 +19,22 @@ use databend_common_catalog::plan::DataSourcePlan; use databend_common_catalog::table_args::TableArgs; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::StringType; -use databend_common_expression::types::VariantType; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::StringType; +use databend_common_expression::types::VariantType; +use crate::FuseTable; use crate::io::MetaReaders; use crate::io::TableMetaLocationGenerator; use crate::sessions::TableContext; +use crate::table_functions::SimpleTableFunc; use crate::table_functions::parse_db_tb_args; use crate::table_functions::string_literal; -use crate::table_functions::SimpleTableFunc; -use crate::FuseTable; pub struct FuseDumpSnapshotsArgs { database_name: String, diff --git a/src/query/storages/fuse/src/table_functions/fuse_encoding.rs b/src/query/storages/fuse/src/table_functions/fuse_encoding.rs index 884a997287066..eb4ecb66a6c4b 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_encoding.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_encoding.rs @@ -21,13 +21,6 @@ use databend_common_catalog::table::Table; use databend_common_catalog::table_args::TableArgs; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::expr::*; -use databend_common_expression::types::nullable::NullableColumnBuilder; -use databend_common_expression::types::string::StringColumnBuilder; -use databend_common_expression::types::BooleanType; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::UInt32Type; use databend_common_expression::Column; use databend_common_expression::DataBlock; use databend_common_expression::Evaluator; @@ -41,24 +34,31 @@ use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::expr::*; +use databend_common_expression::types::BooleanType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::UInt32Type; +use databend_common_expression::types::nullable::NullableColumnBuilder; +use databend_common_expression::types::string::StringColumnBuilder; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_native::read::reader::NativeReader; -use databend_common_native::stat::stat_simple; use databend_common_native::stat::ColumnInfo; use databend_common_native::stat::PageBody; +use databend_common_native::stat::stat_simple; use databend_storages_common_io::MergeIOReader; use databend_storages_common_io::ReadSettings; use databend_storages_common_table_meta::meta::SegmentInfo; +use crate::BlockReadResult; +use crate::FuseStorageFormat; +use crate::FuseTable; use crate::io::SegmentsIO; use crate::sessions::TableContext; -use crate::table_functions::parse_db_tb_col_args; -use crate::table_functions::string_literal; use crate::table_functions::SimpleArgFunc; use crate::table_functions::SimpleArgFuncTemplate; -use crate::BlockReadResult; -use crate::FuseStorageFormat; -use crate::FuseTable; +use crate::table_functions::parse_db_tb_col_args; +use crate::table_functions::string_literal; pub struct FuseEncodingArgs { database_name: String, diff --git a/src/query/storages/fuse/src/table_functions/fuse_segment.rs b/src/query/storages/fuse/src/table_functions/fuse_segment.rs index 338f37ec0922e..4a907d0537bfa 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_segment.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_segment.rs @@ -16,23 +16,23 @@ use std::sync::Arc; use databend_common_catalog::table::Table; use databend_common_exception::Result; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::UInt64Type; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::UInt64Type; use databend_storages_common_table_meta::meta::CompactSegmentInfo; use databend_storages_common_table_meta::meta::TableSnapshot; +use crate::FuseTable; use crate::io::SegmentsIO; use crate::sessions::TableContext; use crate::table_functions::TableMetaFunc; use crate::table_functions::TableMetaFuncTemplate; -use crate::FuseTable; pub struct FuseSegment; diff --git a/src/query/storages/fuse/src/table_functions/fuse_snapshot.rs b/src/query/storages/fuse/src/table_functions/fuse_snapshot.rs index 80e5e78e9e639..54c11efb90665 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_snapshot.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_snapshot.rs @@ -19,26 +19,26 @@ use databend_common_catalog::plan::DataSourcePlan; use databend_common_catalog::table_args::TableArgs; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::number::UInt64Type; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::TimestampType; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::TimestampType; +use databend_common_expression::types::number::UInt64Type; use databend_storages_common_table_meta::meta::TableSnapshotLite; use log::info; +use crate::FuseTable; use crate::io::SnapshotsIO; use crate::io::TableMetaLocationGenerator; use crate::sessions::TableContext; +use crate::table_functions::SimpleTableFunc; use crate::table_functions::parse_db_tb_args; use crate::table_functions::string_literal; -use crate::table_functions::SimpleTableFunc; -use crate::FuseTable; pub struct FuseSnapshotArgs { database_name: String, @@ -105,7 +105,7 @@ impl FuseSnapshotFunc { inverted_index_size.push(s.inverted_index_size); vector_index_size.push(s.vector_index_size); virtual_column_size.push(s.virtual_column_size); - timestamps.push(s.timestamp.map(|dt| (dt.timestamp_micros()))); + timestamps.push(s.timestamp.map(|dt| dt.timestamp_micros())); current_snapshot_version = ver; } diff --git a/src/query/storages/fuse/src/table_functions/fuse_statistic.rs b/src/query/storages/fuse/src/table_functions/fuse_statistic.rs index 590eaec2ded6b..193c603f853ff 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_statistic.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_statistic.rs @@ -17,14 +17,11 @@ use std::sync::Arc; use databend_common_catalog::catalog_kind::CATALOG_DEFAULT; use databend_common_catalog::plan::DataSourcePlan; -use databend_common_catalog::table_args::string_value; use databend_common_catalog::table_args::TableArgs; +use databend_common_catalog::table_args::string_value; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::UInt64Type; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::TableDataType; @@ -32,14 +29,17 @@ use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; -use databend_storages_common_table_meta::meta::decode_column_hll; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::UInt64Type; use databend_storages_common_table_meta::meta::Statistics; use databend_storages_common_table_meta::meta::TableSnapshotStatistics; +use databend_storages_common_table_meta::meta::decode_column_hll; -use crate::table_functions::string_literal; +use crate::FuseTable; use crate::table_functions::SimpleArgFunc; use crate::table_functions::SimpleArgFuncTemplate; -use crate::FuseTable; +use crate::table_functions::string_literal; pub struct FuseStatsArgs { catalog_name: Option, diff --git a/src/query/storages/fuse/src/table_functions/fuse_time_travel_size.rs b/src/query/storages/fuse/src/table_functions/fuse_time_travel_size.rs index 31c3d6fca4f67..5a242960f453d 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_time_travel_size.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_time_travel_size.rs @@ -21,27 +21,27 @@ use databend_common_catalog::table_args::TableArgs; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::BooleanType; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::UInt64Type; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::BooleanType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::UInt64Type; use futures_util::TryStreamExt; use log::info; use opendal::Operator; use super::parse_opt_opt_args; +use crate::FUSE_OPT_KEY_DATA_RETENTION_PERIOD_IN_HOURS; +use crate::FuseTable; use crate::io::SnapshotsIO; -use crate::table_functions::string_literal; use crate::table_functions::SimpleArgFunc; use crate::table_functions::SimpleArgFuncTemplate; -use crate::FuseTable; -use crate::FUSE_OPT_KEY_DATA_RETENTION_PERIOD_IN_HOURS; +use crate::table_functions::string_literal; pub struct FuseTimeTravelSizeArgs { pub database_name: Option, diff --git a/src/query/storages/fuse/src/table_functions/fuse_vacuum_drop_aggregating_index.rs b/src/query/storages/fuse/src/table_functions/fuse_vacuum_drop_aggregating_index.rs index ce10c35e2e0c0..8f560305619e7 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_vacuum_drop_aggregating_index.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_vacuum_drop_aggregating_index.rs @@ -18,14 +18,14 @@ use chrono::Duration; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::UInt64Type; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::UInt64Type; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use log::info; diff --git a/src/query/storages/fuse/src/table_functions/fuse_vacuum_drop_inverted_index.rs b/src/query/storages/fuse/src/table_functions/fuse_vacuum_drop_inverted_index.rs index c33c71f2e9afd..89f8c46c50d54 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_vacuum_drop_inverted_index.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_vacuum_drop_inverted_index.rs @@ -18,15 +18,15 @@ use chrono::Duration; use databend_common_catalog::plan::DataSourcePlan; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::UInt64Type; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::UInt64Type; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use log::info; diff --git a/src/query/storages/fuse/src/table_functions/fuse_vacuum_temporary_table.rs b/src/query/storages/fuse/src/table_functions/fuse_vacuum_temporary_table.rs index 1f5a868fa6405..f7e3e3cb476f3 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_vacuum_temporary_table.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_vacuum_temporary_table.rs @@ -19,8 +19,6 @@ use databend_common_catalog::plan::DataSourcePlan; use databend_common_catalog::table_args::TableArgs; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::NumberScalar; -use databend_common_expression::types::StringType; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::Scalar; @@ -28,6 +26,8 @@ use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::NumberScalar; +use databend_common_expression::types::StringType; use databend_common_storage::DataOperator; use databend_common_users::UserApiProvider; use databend_storages_common_table_meta::meta::TEMP_TABLE_STORAGE_PREFIX; @@ -161,7 +161,7 @@ impl SimpleTableFunc for FuseVacuumTemporaryTable { return Err(ErrorCode::BadArguments(format!( "invalid value {:?} expect to be unsigned integer literal.", args[0] - ))) + ))); } }; Some(limit_val) diff --git a/src/query/storages/fuse/src/table_functions/fuse_virtual_column.rs b/src/query/storages/fuse/src/table_functions/fuse_virtual_column.rs index bbcb18583ebde..ae07df37ccafb 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_virtual_column.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_virtual_column.rs @@ -17,12 +17,6 @@ use std::sync::Arc; use databend_common_catalog::table::Table; use databend_common_exception::Result; -use databend_common_expression::types::string::StringColumnBuilder; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::TimestampType; -use databend_common_expression::types::UInt32Type; -use databend_common_expression::types::UInt64Type; use databend_common_expression::BlockEntry; use databend_common_expression::Column; use databend_common_expression::DataBlock; @@ -31,18 +25,24 @@ use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::TimestampType; +use databend_common_expression::types::UInt32Type; +use databend_common_expression::types::UInt64Type; +use databend_common_expression::types::string::StringColumnBuilder; +use databend_storages_common_table_meta::meta::TableSnapshot; use databend_storages_common_table_meta::meta::column_oriented_segment::AbstractBlockMeta; use databend_storages_common_table_meta::meta::column_oriented_segment::AbstractSegment; -use databend_storages_common_table_meta::meta::TableSnapshot; +use crate::FuseTable; +use crate::io::SegmentsIO; use crate::io::read::ColumnOrientedSegmentReader; use crate::io::read::RowOrientedSegmentReader; use crate::io::read::SegmentReader; -use crate::io::SegmentsIO; use crate::sessions::TableContext; -use crate::table_functions::function_template::TableMetaFunc; use crate::table_functions::TableMetaFuncTemplate; -use crate::FuseTable; +use crate::table_functions::function_template::TableMetaFunc; pub struct FuseVirtualColumn; pub type FuseVirtualColumnFunc = TableMetaFuncTemplate; diff --git a/src/query/storages/fuse/src/table_functions/mod.rs b/src/query/storages/fuse/src/table_functions/mod.rs index a568dacb0f5de..412b1813677b5 100644 --- a/src/query/storages/fuse/src/table_functions/mod.rs +++ b/src/query/storages/fuse/src/table_functions/mod.rs @@ -49,7 +49,7 @@ pub use fuse_time_travel_size::FuseTimeTravelSize; pub use fuse_time_travel_size::FuseTimeTravelSizeFunc; pub use fuse_vacuum_drop_aggregating_index::FuseVacuumDropAggregatingIndex; pub use fuse_vacuum_drop_inverted_index::FuseVacuumDropInvertedIndex; -pub use fuse_vacuum_temporary_table::vacuum_inactive_temp_tables; pub use fuse_vacuum_temporary_table::FuseVacuumTemporaryTable; +pub use fuse_vacuum_temporary_table::vacuum_inactive_temp_tables; pub use fuse_virtual_column::FuseVirtualColumnFunc; pub use set_cache_capacity::SetCacheCapacity; diff --git a/src/query/storages/fuse/src/table_functions/set_cache_capacity.rs b/src/query/storages/fuse/src/table_functions/set_cache_capacity.rs index 9c1cfc6bc9e13..7100918453100 100644 --- a/src/query/storages/fuse/src/table_functions/set_cache_capacity.rs +++ b/src/query/storages/fuse/src/table_functions/set_cache_capacity.rs @@ -18,19 +18,19 @@ use databend_common_catalog::plan::DataSourcePlan; use databend_common_catalog::table::DistributionLevel; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::StringType; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::StringType; use databend_storages_common_cache::CacheManager; -use crate::table_functions::string_literal; -use crate::table_functions::string_value; use crate::table_functions::SimpleTableFunc; use crate::table_functions::TableArgs; +use crate::table_functions::string_literal; +use crate::table_functions::string_value; #[derive(Clone)] pub struct SetCapacity { diff --git a/src/query/storages/hive/hive/Cargo.toml b/src/query/storages/hive/hive/Cargo.toml index fbb6eae0b8fd8..0f985c7c5add2 100644 --- a/src/query/storages/hive/hive/Cargo.toml +++ b/src/query/storages/hive/hive/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "databend-common-storages-hive" version = "0.1.0" -edition = "2021" +edition = "2024" authors = ["Databend Authors "] license = "Apache-2.0" publish = false diff --git a/src/query/storages/hive/hive/src/converters.rs b/src/query/storages/hive/hive/src/converters.rs index cc10d9f3a694c..54ec055aaa1ce 100644 --- a/src/query/storages/hive/hive/src/converters.rs +++ b/src/query/storages/hive/hive/src/converters.rs @@ -16,25 +16,25 @@ use std::sync::Arc; use chrono::Utc; use databend_common_exception::Result; -use databend_common_expression::types::decimal::DecimalSize; -use databend_common_expression::types::DecimalDataType; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; -use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; +use databend_common_expression::types::DecimalDataType; +use databend_common_expression::types::decimal::DecimalSize; use databend_common_meta_app::schema::CatalogInfo; use databend_common_meta_app::schema::DatabaseInfo; use databend_common_meta_app::schema::DatabaseMeta; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; +use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_app::storage::StorageParams; use databend_common_meta_app::tenant::Tenant; use databend_common_sql::resolve_type_name_by_str; use hive_metastore as hms; -use crate::hive_database::HiveDatabase; use crate::hive_database::HIVE_DATABASE_ENGINE; +use crate::hive_database::HiveDatabase; use crate::hive_table::HIVE_TABLE_ENGINE; use crate::hive_table_options::HiveTableOptions; diff --git a/src/query/storages/hive/hive/src/hive_catalog.rs b/src/query/storages/hive/hive/src/hive_catalog.rs index dfd729bde8f5f..49fe9ad36a9bd 100644 --- a/src/query/storages/hive/hive/src/hive_catalog.rs +++ b/src/query/storages/hive/hive/src/hive_catalog.rs @@ -29,8 +29,6 @@ use databend_common_catalog::table_function::TableFunction; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_meta_app::principal::UDTFServer; -use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; -use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; use databend_common_meta_app::schema::CatalogInfo; use databend_common_meta_app::schema::CatalogOption; use databend_common_meta_app::schema::CommitTableMetaReply; @@ -101,6 +99,8 @@ use databend_common_meta_app::schema::UpdateIndexReply; use databend_common_meta_app::schema::UpdateIndexReq; use databend_common_meta_app::schema::UpsertTableOptionReply; use databend_common_meta_app::schema::UpsertTableOptionReq; +use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; +use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; use databend_common_meta_app::storage::StorageParams; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_types::*; @@ -109,8 +109,8 @@ use faststr::FastStr; use hive_metastore::Partition; use hive_metastore::ThriftHiveMetastoreClient; use hive_metastore::ThriftHiveMetastoreClientBuilder; -use volo_thrift::transport::pool; use volo_thrift::MaybeException; +use volo_thrift::transport::pool; use super::hive_database::HiveDatabase; use crate::hive_table::HiveTable; diff --git a/src/query/storages/hive/hive/src/hive_partition.rs b/src/query/storages/hive/hive/src/hive_partition.rs index b8a615444ca08..94bcbdb1db00a 100644 --- a/src/query/storages/hive/hive/src/hive_partition.rs +++ b/src/query/storages/hive/hive/src/hive_partition.rs @@ -13,8 +13,8 @@ // limitations under the License. use std::any::Any; -use std::collections::hash_map::DefaultHasher; use std::collections::HashMap; +use std::collections::hash_map::DefaultHasher; use std::hash::Hash; use std::hash::Hasher; use std::sync::Arc; diff --git a/src/query/storages/hive/hive/src/hive_table.rs b/src/query/storages/hive/hive/src/hive_table.rs index 1603787f85fdc..21b2841307b52 100644 --- a/src/query/storages/hive/hive/src/hive_table.rs +++ b/src/query/storages/hive/hive/src/hive_table.rs @@ -49,8 +49,8 @@ use databend_common_pipeline::core::Pipeline; use databend_common_pipeline::core::ProcessorPtr; use databend_common_pipeline::sources::SyncSource; use databend_common_pipeline::sources::SyncSourcer; -use databend_common_storage::init_operator; use databend_common_storage::DataOperator; +use databend_common_storage::init_operator; use databend_common_storages_parquet::ParquetPruner; use databend_common_storages_parquet::ParquetReaderBuilder; use databend_common_storages_parquet::ParquetSourceType; @@ -66,10 +66,10 @@ use opendal::Operator; use super::hive_catalog::HiveCatalog; use super::hive_table_options::HiveTableOptions; -use crate::hive_table_source::HiveTableSource; -use crate::utils::HiveFetchPartitionScalars; use crate::HivePartInfo; use crate::HivePartitionFiller; +use crate::hive_table_source::HiveTableSource; +use crate::utils::HiveFetchPartitionScalars; pub const HIVE_TABLE_ENGINE: &str = "hive"; pub const HIVE_DEFAULT_PARTITION: &str = "__HIVE_DEFAULT_PARTITION__"; @@ -245,8 +245,7 @@ impl HiveTable { if partition_num < 100000 { trace!( "get {} partitions from hive metastore:{:?}", - partition_num, - partition_names + partition_num, partition_names ); } else { trace!("get {} partitions from hive metastore", partition_num); diff --git a/src/query/storages/hive/hive/src/hive_table_options.rs b/src/query/storages/hive/hive/src/hive_table_options.rs index 4981891edb514..09c4c665d54b9 100644 --- a/src/query/storages/hive/hive/src/hive_table_options.rs +++ b/src/query/storages/hive/hive/src/hive_table_options.rs @@ -52,11 +52,7 @@ impl TryFrom<&BTreeMap> for HiveTableOptions { let partition_keys = options.get(PARTITION_KEYS); let partition_keys = if let Some(partition_keys) = partition_keys { let a: Vec = partition_keys.split(' ').map(str::to_string).collect(); - if !a.is_empty() { - Some(a) - } else { - None - } + if !a.is_empty() { Some(a) } else { None } } else { None }; diff --git a/src/query/storages/hive/hive/src/hive_table_source.rs b/src/query/storages/hive/hive/src/hive_table_source.rs index f69178c20cc3f..ec4567559700c 100644 --- a/src/query/storages/hive/hive/src/hive_table_source.rs +++ b/src/query/storages/hive/hive/src/hive_table_source.rs @@ -22,7 +22,6 @@ use databend_common_base::runtime::profile::ProfileStatisticsName; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::BlockEntry; use databend_common_expression::DataBlock; use databend_common_expression::DataSchema; @@ -30,6 +29,7 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::FieldIndex; use databend_common_expression::Scalar; use databend_common_expression::TableField; +use databend_common_expression::types::DataType; use databend_common_pipeline::core::Event; use databend_common_pipeline::core::OutputPort; use databend_common_pipeline::core::Processor; diff --git a/src/query/storages/hive/hive/src/lib.rs b/src/query/storages/hive/hive/src/lib.rs index 9989e1c7bc75a..e73b8a1c3b6d7 100644 --- a/src/query/storages/hive/hive/src/lib.rs +++ b/src/query/storages/hive/hive/src/lib.rs @@ -13,7 +13,7 @@ // limitations under the License. #![feature(duration_constants)] -#![allow(clippy::uninlined_format_args)] +#![allow(clippy::collapsible_if, clippy::uninlined_format_args)] #![allow(clippy::diverging_sub_expression)] mod converters; diff --git a/src/query/storages/hive/hive/src/utils.rs b/src/query/storages/hive/hive/src/utils.rs index 8c5eedcaab8f8..a15e812ce5d09 100644 --- a/src/query/storages/hive/hive/src/utils.rs +++ b/src/query/storages/hive/hive/src/utils.rs @@ -17,9 +17,9 @@ use std::fmt::Debug; use databend_common_catalog::partition_columns::str_to_scalar; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::Scalar; use databend_common_expression::TableField; +use databend_common_expression::types::DataType; use databend_storages_common_pruner::partition_prunner::FetchPartitionScalars; use volo_thrift::MaybeException; diff --git a/src/query/storages/iceberg/Cargo.toml b/src/query/storages/iceberg/Cargo.toml index e394dc451c4e7..627464fb3cb4c 100644 --- a/src/query/storages/iceberg/Cargo.toml +++ b/src/query/storages/iceberg/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "databend-common-storages-iceberg" version = { workspace = true } -edition = "2021" +edition = "2024" authors = ["Databend Authors "] license = "Apache-2.0" publish = false diff --git a/src/query/storages/iceberg/src/cache.rs b/src/query/storages/iceberg/src/cache.rs index 5a4b00917238a..de09937ad190a 100644 --- a/src/query/storages/iceberg/src/cache.rs +++ b/src/query/storages/iceberg/src/cache.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::sync::atomic::AtomicBool; use std::sync::Arc; +use std::sync::atomic::AtomicBool; use std::time::Duration; use std::time::Instant; diff --git a/src/query/storages/iceberg/src/catalog.rs b/src/query/storages/iceberg/src/catalog.rs index 5c3013c854d31..76bdbbbb08a70 100644 --- a/src/query/storages/iceberg/src/catalog.rs +++ b/src/query/storages/iceberg/src/catalog.rs @@ -28,9 +28,8 @@ use databend_common_catalog::table_context::TableContext; use databend_common_catalog::table_function::TableFunction; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_meta_app::KeyWithTenant; use databend_common_meta_app::principal::UDTFServer; -use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; -use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; use databend_common_meta_app::schema::CatalogInfo; use databend_common_meta_app::schema::CatalogOption; use databend_common_meta_app::schema::CommitTableMetaReply; @@ -104,8 +103,9 @@ use databend_common_meta_app::schema::UpdateIndexReply; use databend_common_meta_app::schema::UpdateIndexReq; use databend_common_meta_app::schema::UpsertTableOptionReply; use databend_common_meta_app::schema::UpsertTableOptionReq; +use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; +use databend_common_meta_app::schema::dictionary_name_ident::DictionaryNameIdent; use databend_common_meta_app::tenant::Tenant; -use databend_common_meta_app::KeyWithTenant; use databend_common_meta_types::MetaId; use databend_common_meta_types::SeqV; use databend_common_users::GrantObjectVisibilityChecker; @@ -121,8 +121,8 @@ use iceberg_catalog_rest::RestCatalogConfig; use iceberg_catalog_s3tables::S3TablesCatalog; use iceberg_catalog_s3tables::S3TablesCatalogConfig; -use crate::database::IcebergDatabase; use crate::IcebergTable; +use crate::database::IcebergDatabase; pub const ICEBERG_CATALOG: &str = "iceberg"; diff --git a/src/query/storages/iceberg/src/database.rs b/src/query/storages/iceberg/src/database.rs index 8d1dc5351f41a..d745742493304 100644 --- a/src/query/storages/iceberg/src/database.rs +++ b/src/query/storages/iceberg/src/database.rs @@ -25,13 +25,12 @@ use databend_common_catalog::database::Database; use databend_common_catalog::table::Table; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::NumberDataType::Float32; -use databend_common_expression::types::NumberDataType::Float64; use databend_common_expression::TableDataType; use databend_common_expression::TableDataType::Null; use databend_common_expression::TableDataType::Timestamp; use databend_common_expression::TableSchema; -use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; +use databend_common_expression::types::NumberDataType::Float32; +use databend_common_expression::types::NumberDataType::Float64; use databend_common_meta_app::schema::CreateOption; use databend_common_meta_app::schema::CreateTableReply; use databend_common_meta_app::schema::CreateTableReq; @@ -41,21 +40,22 @@ use databend_common_meta_app::schema::DatabaseMeta; use databend_common_meta_app::schema::DropTableByIdReq; use databend_common_meta_app::schema::DropTableReply; use databend_common_meta_app::schema::TablePartition; +use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_types::SeqV; use databend_storages_common_cache::LoadParams; use educe::Educe; +use iceberg::TableCreation; +use iceberg::TableIdent; use iceberg::arrow::arrow_schema_to_schema; use iceberg::spec::PartitionSpec; use iceberg::spec::Schema as IceSchema; use iceberg::spec::Schema as IcebergSchema; use iceberg::spec::Transform; use iceberg::spec::UnboundPartitionField; -use iceberg::TableCreation; -use iceberg::TableIdent; -use crate::cache; use crate::IcebergMutableCatalog; +use crate::cache; const PARQUET_FIELD_ID_META_KEY: &str = "PARQUET:field_id"; @@ -351,7 +351,8 @@ fn build_identity_partition_spec( let origin_ty = &req_schema.field_with_name(col)?.data_type; if is_invalid_partition_type(origin_ty) { return Err(ErrorCode::TableOptionInvalid(format!( - "Partition key {} is {:?} type. Cannot set FLOAT, DOUBLE, DECIMAL, DATETIME as partition field", col, origin_ty + "Partition key {} is {:?} type. Cannot set FLOAT, DOUBLE, DECIMAL, DATETIME as partition field", + col, origin_ty ))); } diff --git a/src/query/storages/iceberg/src/iceberg_inspect.rs b/src/query/storages/iceberg/src/iceberg_inspect.rs index d2ee0a0a4c76c..338db4fe61c9a 100644 --- a/src/query/storages/iceberg/src/iceberg_inspect.rs +++ b/src/query/storages/iceberg/src/iceberg_inspect.rs @@ -21,13 +21,12 @@ use databend_common_catalog::plan::PartStatistics; use databend_common_catalog::plan::Partitions; use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table::Table; -use databend_common_catalog::table_args::parse_db_tb_args; use databend_common_catalog::table_args::TableArgs; +use databend_common_catalog::table_args::parse_db_tb_args; use databend_common_catalog::table_context::TableContext; use databend_common_catalog::table_function::TableFunction; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::NumberDataType; use databend_common_expression::DataBlock; use databend_common_expression::DataSchema; use databend_common_expression::TableDataType; @@ -35,6 +34,7 @@ use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::NumberDataType; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; diff --git a/src/query/storages/iceberg/src/lib.rs b/src/query/storages/iceberg/src/lib.rs index d2bfb6c340f98..59e368d582326 100644 --- a/src/query/storages/iceberg/src/lib.rs +++ b/src/query/storages/iceberg/src/lib.rs @@ -17,7 +17,7 @@ //! should be managed by ourselves. #![feature(impl_trait_in_assoc_type)] -#![allow(clippy::diverging_sub_expression)] +#![allow(clippy::collapsible_if, clippy::diverging_sub_expression)] pub(crate) mod cache; mod catalog; @@ -28,8 +28,8 @@ mod predicate; mod statistics; pub mod table; +pub use catalog::ICEBERG_CATALOG; pub use catalog::IcebergMutableCatalog; pub use catalog::IcebergMutableCreator; -pub use catalog::ICEBERG_CATALOG; pub use iceberg_inspect::IcebergInspectTable; pub use table::IcebergTable; diff --git a/src/query/storages/iceberg/src/predicate.rs b/src/query/storages/iceberg/src/predicate.rs index 4f9b154e76d57..ff25fe94cef12 100644 --- a/src/query/storages/iceberg/src/predicate.rs +++ b/src/query/storages/iceberg/src/predicate.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberScalar; use databend_common_expression::RemoteExpr; use databend_common_expression::Scalar; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberScalar; use iceberg::expr::Predicate; use iceberg::expr::Reference; use iceberg::spec::Datum; diff --git a/src/query/storages/iceberg/src/statistics.rs b/src/query/storages/iceberg/src/statistics.rs index fd6a075bb086b..6c06693bebf53 100644 --- a/src/query/storages/iceberg/src/statistics.rs +++ b/src/query/storages/iceberg/src/statistics.rs @@ -20,16 +20,16 @@ use databend_common_catalog::statistics::BasicColumnStatistics; use databend_common_catalog::table::ColumnStatisticsProvider; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::ColumnId; +use databend_common_expression::Scalar; +use databend_common_expression::TableField; use databend_common_expression::types::ArgType; use databend_common_expression::types::Decimal; use databend_common_expression::types::DecimalSize; -use databend_common_expression::types::Int64Type; -use databend_common_expression::types::Number; use databend_common_expression::types::F32; use databend_common_expression::types::F64; -use databend_common_expression::ColumnId; -use databend_common_expression::Scalar; -use databend_common_expression::TableField; +use databend_common_expression::types::Int64Type; +use databend_common_expression::types::Number; use databend_common_storage::Datum as DatabendDatum; use iceberg::spec::DataContentType; use iceberg::spec::Datum; diff --git a/src/query/storages/iceberg/src/table.rs b/src/query/storages/iceberg/src/table.rs index 6698f78ef15c1..e201128fb55e3 100644 --- a/src/query/storages/iceberg/src/table.rs +++ b/src/query/storages/iceberg/src/table.rs @@ -628,10 +628,10 @@ impl Table for IcebergTable { } _ => { return Err(ErrorCode::Unimplemented(format!( - "Time travel operation is not supported for the table '{}', which uses the '{}' engine.", - self.name(), - self.get_table_info().engine(), - ))); + "Time travel operation is not supported for the table '{}', which uses the '{}' engine.", + self.name(), + self.get_table_info().engine(), + ))); } }, _ => { diff --git a/src/query/storages/information_schema/src/character_sets_table.rs b/src/query/storages/information_schema/src/character_sets_table.rs index f91c3889f1bfe..8de240cd9b9e7 100644 --- a/src/query/storages/information_schema/src/character_sets_table.rs +++ b/src/query/storages/information_schema/src/character_sets_table.rs @@ -22,8 +22,8 @@ use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; use databend_common_meta_app::tenant::Tenant; -use databend_common_storages_basic::view_table::ViewTable; use databend_common_storages_basic::view_table::QUERY; +use databend_common_storages_basic::view_table::ViewTable; use databend_common_storages_system::generate_catalog_meta; const CHARACTER_SETS: &[(&str, &str, &str, u8)] = &[ diff --git a/src/query/storages/information_schema/src/columns_table.rs b/src/query/storages/information_schema/src/columns_table.rs index 7e2d6324c761b..f9f9c924be6ba 100644 --- a/src/query/storages/information_schema/src/columns_table.rs +++ b/src/query/storages/information_schema/src/columns_table.rs @@ -22,8 +22,8 @@ use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; use databend_common_meta_app::tenant::Tenant; -use databend_common_storages_basic::view_table::ViewTable; use databend_common_storages_basic::view_table::QUERY; +use databend_common_storages_basic::view_table::ViewTable; use databend_common_storages_system::generate_catalog_meta; pub struct ColumnsTable {} diff --git a/src/query/storages/information_schema/src/key_column_usage_table.rs b/src/query/storages/information_schema/src/key_column_usage_table.rs index 2946d7db7ab32..8edc8e13d69b7 100644 --- a/src/query/storages/information_schema/src/key_column_usage_table.rs +++ b/src/query/storages/information_schema/src/key_column_usage_table.rs @@ -22,8 +22,8 @@ use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; use databend_common_meta_app::tenant::Tenant; -use databend_common_storages_basic::view_table::ViewTable; use databend_common_storages_basic::view_table::QUERY; +use databend_common_storages_basic::view_table::ViewTable; use databend_common_storages_system::generate_catalog_meta; pub struct KeyColumnUsageTable {} diff --git a/src/query/storages/information_schema/src/keywords_table.rs b/src/query/storages/information_schema/src/keywords_table.rs index e356769cf3313..2f664f01ceb87 100644 --- a/src/query/storages/information_schema/src/keywords_table.rs +++ b/src/query/storages/information_schema/src/keywords_table.rs @@ -23,8 +23,8 @@ use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; use databend_common_meta_app::tenant::Tenant; -use databend_common_storages_basic::view_table::ViewTable; use databend_common_storages_basic::view_table::QUERY; +use databend_common_storages_basic::view_table::ViewTable; use databend_common_storages_system::generate_catalog_meta; pub struct KeywordsTable {} diff --git a/src/query/storages/information_schema/src/referential_constraints_table.rs b/src/query/storages/information_schema/src/referential_constraints_table.rs index 0c5b042344e88..3dcbdccf895ca 100644 --- a/src/query/storages/information_schema/src/referential_constraints_table.rs +++ b/src/query/storages/information_schema/src/referential_constraints_table.rs @@ -22,8 +22,8 @@ use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; use databend_common_meta_app::tenant::Tenant; -use databend_common_storages_basic::view_table::ViewTable; use databend_common_storages_basic::view_table::QUERY; +use databend_common_storages_basic::view_table::ViewTable; use databend_common_storages_system::generate_catalog_meta; /// A minimal INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS table. diff --git a/src/query/storages/information_schema/src/schemata_table.rs b/src/query/storages/information_schema/src/schemata_table.rs index 3f2a9aca32d31..771cab39261b3 100644 --- a/src/query/storages/information_schema/src/schemata_table.rs +++ b/src/query/storages/information_schema/src/schemata_table.rs @@ -22,8 +22,8 @@ use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; use databend_common_meta_app::tenant::Tenant; -use databend_common_storages_basic::view_table::ViewTable; use databend_common_storages_basic::view_table::QUERY; +use databend_common_storages_basic::view_table::ViewTable; use databend_common_storages_system::generate_catalog_meta; pub struct SchemataTable {} diff --git a/src/query/storages/information_schema/src/statistics_table.rs b/src/query/storages/information_schema/src/statistics_table.rs index 82e2af4ce88de..93a3eb776865e 100644 --- a/src/query/storages/information_schema/src/statistics_table.rs +++ b/src/query/storages/information_schema/src/statistics_table.rs @@ -22,8 +22,8 @@ use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; use databend_common_meta_app::tenant::Tenant; -use databend_common_storages_basic::view_table::ViewTable; use databend_common_storages_basic::view_table::QUERY; +use databend_common_storages_basic::view_table::ViewTable; use databend_common_storages_system::generate_catalog_meta; pub struct StatisticsTable {} diff --git a/src/query/storages/information_schema/src/tables_table.rs b/src/query/storages/information_schema/src/tables_table.rs index a5e1aa8207673..ef8da158e668b 100644 --- a/src/query/storages/information_schema/src/tables_table.rs +++ b/src/query/storages/information_schema/src/tables_table.rs @@ -22,8 +22,8 @@ use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; use databend_common_meta_app::tenant::Tenant; -use databend_common_storages_basic::view_table::ViewTable; use databend_common_storages_basic::view_table::QUERY; +use databend_common_storages_basic::view_table::ViewTable; use databend_common_storages_system::generate_catalog_meta; pub struct TablesTable {} diff --git a/src/query/storages/information_schema/src/views_table.rs b/src/query/storages/information_schema/src/views_table.rs index 10ca7d71f0383..63cdea707c617 100644 --- a/src/query/storages/information_schema/src/views_table.rs +++ b/src/query/storages/information_schema/src/views_table.rs @@ -22,8 +22,8 @@ use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; use databend_common_meta_app::tenant::Tenant; -use databend_common_storages_basic::view_table::ViewTable; use databend_common_storages_basic::view_table::QUERY; +use databend_common_storages_basic::view_table::ViewTable; use databend_common_storages_system::generate_catalog_meta; pub struct ViewsTable {} diff --git a/src/query/storages/orc/src/chunk_reader_impl.rs b/src/query/storages/orc/src/chunk_reader_impl.rs index 4e53b8b8938e1..a5307184a4bb5 100644 --- a/src/query/storages/orc/src/chunk_reader_impl.rs +++ b/src/query/storages/orc/src/chunk_reader_impl.rs @@ -13,10 +13,10 @@ // limitations under the License. use bytes::Bytes; -use futures_util::future::BoxFuture; use futures_util::AsyncRead; use futures_util::AsyncReadExt; use futures_util::FutureExt; +use futures_util::future::BoxFuture; use opendal::Operator; use orc_rust::reader::AsyncChunkReader; diff --git a/src/query/storages/orc/src/copy_into_table/processors/decoder.rs b/src/query/storages/orc/src/copy_into_table/processors/decoder.rs index f094f2835e172..995efce9764b7 100644 --- a/src/query/storages/orc/src/copy_into_table/processors/decoder.rs +++ b/src/query/storages/orc/src/copy_into_table/processors/decoder.rs @@ -22,9 +22,6 @@ use arrow_array::RecordBatch; use databend_common_catalog::query_kind::QueryKind; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::ArrayColumn; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NullableColumn; use databend_common_expression::BlockEntry; use databend_common_expression::BlockMetaInfoDowncast; use databend_common_expression::Column; @@ -35,6 +32,9 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::Evaluator; use databend_common_expression::Expr; use databend_common_expression::FunctionContext; +use databend_common_expression::types::ArrayColumn; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NullableColumn; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_pipeline::core::Event; use databend_common_pipeline::core::InputPort; diff --git a/src/query/storages/orc/src/lib.rs b/src/query/storages/orc/src/lib.rs index 4bc7ea245936f..621f3cf23d479 100644 --- a/src/query/storages/orc/src/lib.rs +++ b/src/query/storages/orc/src/lib.rs @@ -13,12 +13,12 @@ // limitations under the License. #![allow(internal_features)] +#![allow(clippy::collapsible_if)] #![allow(clippy::uninlined_format_args)] #![allow(clippy::useless_asref)] #![allow(clippy::diverging_sub_expression)] #![feature(try_blocks)] #![feature(impl_trait_in_assoc_type)] -#![feature(let_chains)] #![feature(core_intrinsics)] #![feature(int_roundings)] #![feature(box_patterns)] diff --git a/src/query/storages/orc/src/processors/source.rs b/src/query/storages/orc/src/processors/source.rs index 245b7507c7908..9d76ad18e5bbc 100644 --- a/src/query/storages/orc/src/processors/source.rs +++ b/src/query/storages/orc/src/processors/source.rs @@ -30,9 +30,9 @@ use databend_common_pipeline::sources::AsyncSource; use databend_common_pipeline::sources::AsyncSourcer; use databend_common_storage::OperatorRegistry; use databend_storages_common_stage::SingleFilePartition; +use orc_rust::ArrowReaderBuilder; use orc_rust::async_arrow_reader::StripeFactory; use orc_rust::projection::ProjectionMask; -use orc_rust::ArrowReaderBuilder; use crate::chunk_reader_impl::OrcChunkReader; use crate::hashable_schema::HashableSchema; diff --git a/src/query/storages/orc/src/strip.rs b/src/query/storages/orc/src/strip.rs index e28a360ae894c..2c527d2561289 100644 --- a/src/query/storages/orc/src/strip.rs +++ b/src/query/storages/orc/src/strip.rs @@ -14,8 +14,8 @@ use std::fmt::Debug; -use databend_common_expression::local_block_meta_serde; use databend_common_expression::BlockMetaInfo; +use databend_common_expression::local_block_meta_serde; use orc_rust::stripe::Stripe; use crate::hashable_schema::HashableSchema; diff --git a/src/query/storages/orc/src/table.rs b/src/query/storages/orc/src/table.rs index d3172bc13111b..d75f889eaf34d 100644 --- a/src/query/storages/orc/src/table.rs +++ b/src/query/storages/orc/src/table.rs @@ -31,18 +31,18 @@ use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::ColumnId; +use databend_common_expression::FILE_ROW_NUMBER_COLUMN_ID; +use databend_common_expression::FILENAME_COLUMN_ID; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; -use databend_common_expression::FILENAME_COLUMN_ID; -use databend_common_expression::FILE_ROW_NUMBER_COLUMN_ID; use databend_common_meta_app::principal::StageInfo; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; use databend_common_pipeline::core::Pipeline; -use databend_common_storage::init_stage_operator; use databend_common_storage::StageFileInfo; +use databend_common_storage::init_stage_operator; use databend_storages_common_table_meta::table::ChangeType; use opendal::Operator; use orc_rust::ArrowReaderBuilder; diff --git a/src/query/storages/parquet/src/copy_into_table/reader.rs b/src/query/storages/parquet/src/copy_into_table/reader.rs index a006076e17d11..f350c5e4a31f9 100644 --- a/src/query/storages/parquet/src/copy_into_table/reader.rs +++ b/src/query/storages/parquet/src/copy_into_table/reader.rs @@ -19,9 +19,9 @@ use databend_common_catalog::plan::Projection; use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::expr::*; use databend_common_expression::RemoteDefaultExpr; use databend_common_expression::TableSchemaRef; +use databend_common_expression::expr::*; use databend_common_meta_app::principal::NullAs; use databend_common_meta_app::principal::StageFileFormatType; use databend_common_storage::parquet::infer_schema_with_extension; @@ -29,10 +29,10 @@ use databend_storages_common_stage::project_columnar; use opendal::Operator; use parquet::file::metadata::FileMetaData; -use crate::parquet_reader::policy::ReadPolicyBuilder; -use crate::parquet_reader::policy::ReadPolicyImpl; use crate::parquet_reader::InMemoryRowGroup; use crate::parquet_reader::ParquetReaderBuilder; +use crate::parquet_reader::policy::ReadPolicyBuilder; +use crate::parquet_reader::policy::ReadPolicyImpl; use crate::partition::ParquetRowGroupPart; use crate::read_settings::ReadSettings; use crate::schema::arrow_to_table_schema; diff --git a/src/query/storages/parquet/src/copy_into_table/source.rs b/src/query/storages/parquet/src/copy_into_table/source.rs index 8776f9df12a1f..516ba916fb960 100644 --- a/src/query/storages/parquet/src/copy_into_table/source.rs +++ b/src/query/storages/parquet/src/copy_into_table/source.rs @@ -31,11 +31,11 @@ use databend_common_pipeline::core::Processor; use databend_common_pipeline::core::ProcessorPtr; use opendal::Operator; +use crate::ParquetPart; use crate::copy_into_table::projection::CopyProjectionEvaluator; use crate::copy_into_table::reader::RowGroupReaderForCopy; use crate::parquet_reader::policy::ReadPolicyImpl; use crate::read_settings::ReadSettings; -use crate::ParquetPart; enum State { Init, diff --git a/src/query/storages/parquet/src/copy_into_table/table.rs b/src/query/storages/parquet/src/copy_into_table/table.rs index 26e19c1681553..ae99ae2e97354 100644 --- a/src/query/storages/parquet/src/copy_into_table/table.rs +++ b/src/query/storages/parquet/src/copy_into_table/table.rs @@ -30,15 +30,15 @@ use databend_common_exception::Result; use databend_common_expression::DataSchema; use databend_common_meta_app::principal::FileFormatParams; use databend_common_pipeline::core::Pipeline; -use databend_common_storage::init_stage_operator; use databend_common_storage::FileStatus; +use databend_common_storage::init_stage_operator; use parquet::file::metadata::FileMetaData; +use crate::ParquetPart; use crate::copy_into_table::reader::RowGroupReaderForCopy; use crate::copy_into_table::source::ParquetCopySource; use crate::meta::read_metas_in_parallel_for_copy; use crate::partition::ParquetRowGroupPart; -use crate::ParquetPart; pub struct ParquetTableForCopy {} diff --git a/src/query/storages/parquet/src/lib.rs b/src/query/storages/parquet/src/lib.rs index c076e2e601eef..2ea1364155d8f 100644 --- a/src/query/storages/parquet/src/lib.rs +++ b/src/query/storages/parquet/src/lib.rs @@ -13,17 +13,20 @@ // limitations under the License. #![allow(internal_features)] -#![allow(clippy::uninlined_format_args)] +#![allow( + clippy::collapsible_if, + clippy::redundant_iter_cloned, + clippy::unnecessary_unwrap, + clippy::uninlined_format_args +)] #![allow(clippy::large_enum_variant)] #![allow(clippy::useless_asref)] #![allow(clippy::diverging_sub_expression)] #![feature(try_blocks)] #![feature(impl_trait_in_assoc_type)] -#![feature(let_chains)] #![feature(core_intrinsics)] #![feature(int_roundings)] #![feature(box_patterns)] -#![feature(result_flattening)] // FIXME: Remove this once the deprecated code is removed #![allow(deprecated)] @@ -51,12 +54,12 @@ pub use parquet_part::DeleteTask; pub use parquet_part::DeleteType; pub use parquet_part::ParquetFilePart; pub use parquet_part::ParquetPart; -pub use parquet_reader::read_all; pub use parquet_reader::InMemoryRowGroup; pub use parquet_reader::InmMemoryFile; pub use parquet_reader::ParquetFileReader; pub use parquet_reader::ParquetReaderBuilder; pub use parquet_reader::ParquetWholeFileReader; +pub use parquet_reader::read_all; pub use parquet_table::ParquetTable; pub use parquet_variant_table::ParquetVariantTable; // for it test diff --git a/src/query/storages/parquet/src/meta.rs b/src/query/storages/parquet/src/meta.rs index ae26d05843eb2..33caa614f5d5a 100644 --- a/src/query/storages/parquet/src/meta.rs +++ b/src/query/storages/parquet/src/meta.rs @@ -15,8 +15,8 @@ use std::intrinsics::unlikely; use std::sync::Arc; -use databend_common_base::runtime::execute_futures_in_parallel; use databend_common_base::runtime::GLOBAL_MEM_STAT; +use databend_common_base::runtime::execute_futures_in_parallel; use databend_common_catalog::plan::FullParquetMeta; use databend_common_exception::ErrorCode; use databend_common_exception::Result; diff --git a/src/query/storages/parquet/src/parquet_part.rs b/src/query/storages/parquet/src/parquet_part.rs index b0cc629797141..d0573dfa56b09 100644 --- a/src/query/storages/parquet/src/parquet_part.rs +++ b/src/query/storages/parquet/src/parquet_part.rs @@ -169,7 +169,7 @@ fn collect_small_file_parts( // there are no large files, so we choose a default value. max_compressed_size = 128u64 << 20; } - let mut num_small_files = small_files.len(); + let num_small_files = small_files.len(); stats.read_rows += num_small_files; let mut small_part = vec![]; let mut part_size = 0; @@ -186,9 +186,6 @@ fn collect_small_file_parts( if !small_part.is_empty() && (part_size > max_compressed_size || small_part.len() >= max_files || is_last) { - // turn small_parts into ParquetPart::ParquetSmallFiles - num_small_files -= small_part.len(); - let files = small_part .iter() .cloned() diff --git a/src/query/storages/parquet/src/parquet_reader/mod.rs b/src/query/storages/parquet/src/parquet_reader/mod.rs index c1db46cd2a42c..bc6ac6632a495 100644 --- a/src/query/storages/parquet/src/parquet_reader/mod.rs +++ b/src/query/storages/parquet/src/parquet_reader/mod.rs @@ -25,7 +25,7 @@ pub use reader::ParquetFileReader; pub use reader::ParquetReaderBuilder; pub use reader::ParquetWholeFileReader; pub use reader::RowGroupReader; -pub use row_group::cached_range_full_read; pub use row_group::InMemoryRowGroup; pub use row_group::RowGroupCore; +pub use row_group::cached_range_full_read; pub use utils::DataBlockIterator; diff --git a/src/query/storages/parquet/src/parquet_reader/predicate.rs b/src/query/storages/parquet/src/parquet_reader/predicate.rs index a0eaa11489f5c..1d04b6a831a87 100644 --- a/src/query/storages/parquet/src/parquet_reader/predicate.rs +++ b/src/query/storages/parquet/src/parquet_reader/predicate.rs @@ -19,8 +19,6 @@ use arrow_array::RecordBatch; use databend_common_catalog::plan::PrewhereInfo; use databend_common_catalog::plan::Projection; use databend_common_exception::Result; -use databend_common_expression::types::Bitmap; -use databend_common_expression::types::DataType; use databend_common_expression::BlockEntry; use databend_common_expression::DataBlock; use databend_common_expression::DataSchema; @@ -29,15 +27,17 @@ use databend_common_expression::Expr; use databend_common_expression::FunctionContext; use databend_common_expression::Scalar; use databend_common_expression::TableSchema; +use databend_common_expression::types::Bitmap; +use databend_common_expression::types::DataType; use databend_common_functions::BUILTIN_FUNCTIONS; -use parquet::arrow::parquet_to_arrow_field_levels; use parquet::arrow::FieldLevels; use parquet::arrow::ProjectionMask; +use parquet::arrow::parquet_to_arrow_field_levels; use parquet::schema::types::SchemaDescriptor; +use super::utils::FieldPaths; use super::utils::bitmap_to_boolean_array; use super::utils::transform_record_batch; -use super::utils::FieldPaths; use crate::parquet_reader::utils::compute_output_field_paths; #[derive(Debug)] diff --git a/src/query/storages/parquet/src/parquet_reader/read_policy/no_prefetch.rs b/src/query/storages/parquet/src/parquet_reader/read_policy/no_prefetch.rs index d73316e6154aa..2dcec4b41928f 100644 --- a/src/query/storages/parquet/src/parquet_reader/read_policy/no_prefetch.rs +++ b/src/query/storages/parquet/src/parquet_reader/read_policy/no_prefetch.rs @@ -18,11 +18,11 @@ use databend_common_exception::Result; use databend_common_expression::DataBlock; use databend_common_expression::DataSchema; use databend_common_expression::TopKSorter; +use parquet::arrow::FieldLevels; +use parquet::arrow::ProjectionMask; use parquet::arrow::arrow_reader::ParquetRecordBatchReader; use parquet::arrow::arrow_reader::RowSelection; use parquet::arrow::parquet_to_arrow_field_levels; -use parquet::arrow::FieldLevels; -use parquet::arrow::ProjectionMask; use parquet::schema::types::SchemaDescriptor; use super::policy::ReadPolicy; @@ -31,9 +31,9 @@ use super::policy::ReadPolicyImpl; use crate::parquet_reader::predicate::ParquetPredicate; use crate::parquet_reader::read_policy::utils::read_all; use crate::parquet_reader::row_group::InMemoryRowGroup; +use crate::parquet_reader::utils::FieldPaths; use crate::parquet_reader::utils::bitmap_to_boolean_array; use crate::parquet_reader::utils::transform_record_batch; -use crate::parquet_reader::utils::FieldPaths; use crate::transformer::RecordBatchTransformer; pub struct NoPretchPolicyBuilder { diff --git a/src/query/storages/parquet/src/parquet_reader/read_policy/predicate_and_topk.rs b/src/query/storages/parquet/src/parquet_reader/read_policy/predicate_and_topk.rs index 78e4c581ff288..f0bc953b35c1d 100644 --- a/src/query/storages/parquet/src/parquet_reader/read_policy/predicate_and_topk.rs +++ b/src/query/storages/parquet/src/parquet_reader/read_policy/predicate_and_topk.rs @@ -22,11 +22,11 @@ use databend_common_expression::DataSchema; use databend_common_expression::DataSchemaRef; use databend_common_expression::TableSchema; use databend_common_expression::TopKSorter; +use parquet::arrow::FieldLevels; +use parquet::arrow::ProjectionMask; use parquet::arrow::arrow_reader::ParquetRecordBatchReader; use parquet::arrow::arrow_reader::RowSelection; use parquet::arrow::parquet_to_arrow_field_levels; -use parquet::arrow::FieldLevels; -use parquet::arrow::ProjectionMask; use parquet::schema::types::SchemaDescriptor; use super::policy::ReadPolicy; @@ -38,10 +38,10 @@ use crate::parquet_reader::predicate::ParquetPredicate; use crate::parquet_reader::row_group::InMemoryRowGroup; use crate::parquet_reader::topk::BuiltTopK; use crate::parquet_reader::topk::ParquetTopK; +use crate::parquet_reader::utils::FieldPaths; use crate::parquet_reader::utils::bitmap_to_boolean_array; use crate::parquet_reader::utils::compute_output_field_paths; use crate::parquet_reader::utils::transform_record_batch; -use crate::parquet_reader::utils::FieldPaths; use crate::transformer::RecordBatchTransformer; pub struct PredicateAndTopkPolicyBuilder { diff --git a/src/query/storages/parquet/src/parquet_reader/read_policy/topk_only.rs b/src/query/storages/parquet/src/parquet_reader/read_policy/topk_only.rs index f570b77a1e539..1de1eb295863e 100644 --- a/src/query/storages/parquet/src/parquet_reader/read_policy/topk_only.rs +++ b/src/query/storages/parquet/src/parquet_reader/read_policy/topk_only.rs @@ -23,11 +23,11 @@ use databend_common_expression::DataSchema; use databend_common_expression::DataSchemaRef; use databend_common_expression::TableSchema; use databend_common_expression::TopKSorter; +use parquet::arrow::FieldLevels; +use parquet::arrow::ProjectionMask; use parquet::arrow::arrow_reader::ParquetRecordBatchReader; use parquet::arrow::arrow_reader::RowSelection; use parquet::arrow::parquet_to_arrow_field_levels; -use parquet::arrow::FieldLevels; -use parquet::arrow::ProjectionMask; use parquet::schema::types::SchemaDescriptor; use super::policy::ReadPolicy; @@ -39,9 +39,9 @@ use crate::parquet_reader::predicate::ParquetPredicate; use crate::parquet_reader::row_group::InMemoryRowGroup; use crate::parquet_reader::topk::BuiltTopK; use crate::parquet_reader::topk::ParquetTopK; +use crate::parquet_reader::utils::FieldPaths; use crate::parquet_reader::utils::compute_output_field_paths; use crate::parquet_reader::utils::transform_record_batch; -use crate::parquet_reader::utils::FieldPaths; use crate::transformer::RecordBatchTransformer; pub struct TopkOnlyPolicyBuilder { diff --git a/src/query/storages/parquet/src/parquet_reader/read_policy/utils.rs b/src/query/storages/parquet/src/parquet_reader/read_policy/utils.rs index 7365e8ccdd004..8153bd74eb393 100644 --- a/src/query/storages/parquet/src/parquet_reader/read_policy/utils.rs +++ b/src/query/storages/parquet/src/parquet_reader/read_policy/utils.rs @@ -16,15 +16,15 @@ use databend_common_exception::Result; use databend_common_expression::DataBlock; use databend_common_expression::DataSchema; use databend_common_expression::TopKSorter; +use parquet::arrow::FieldLevels; use parquet::arrow::arrow_reader::ParquetRecordBatchReader; use parquet::arrow::arrow_reader::RowSelection; -use parquet::arrow::FieldLevels; use crate::parquet_reader::row_group::InMemoryRowGroup; use crate::parquet_reader::topk::ParquetTopK; +use crate::parquet_reader::utils::FieldPaths; use crate::parquet_reader::utils::bitmap_to_boolean_array; use crate::parquet_reader::utils::transform_record_batch; -use crate::parquet_reader::utils::FieldPaths; pub fn read_all( data_schema: &DataSchema, diff --git a/src/query/storages/parquet/src/parquet_reader/reader/builder.rs b/src/query/storages/parquet/src/parquet_reader/reader/builder.rs index ad98f3631a12e..f902fc7167eed 100644 --- a/src/query/storages/parquet/src/parquet_reader/reader/builder.rs +++ b/src/query/storages/parquet/src/parquet_reader/reader/builder.rs @@ -28,25 +28,25 @@ use parquet::arrow::ProjectionMask; use parquet::schema::types::SchemaDescPtr; use super::RowGroupReader; -use crate::parquet_reader::policy::default_policy_builders; -use crate::parquet_reader::policy::ReadPolicyBuilder; +use crate::ParquetPruner; +use crate::ParquetSourceType; +use crate::parquet_reader::NoPretchPolicyBuilder; +use crate::parquet_reader::ParquetWholeFileReader; +use crate::parquet_reader::PredicateAndTopkPolicyBuilder; +use crate::parquet_reader::TopkOnlyPolicyBuilder; use crate::parquet_reader::policy::POLICY_NO_PREFETCH; use crate::parquet_reader::policy::POLICY_PREDICATE_AND_TOPK; use crate::parquet_reader::policy::POLICY_PREDICATE_ONLY; use crate::parquet_reader::policy::POLICY_TOPK_ONLY; -use crate::parquet_reader::predicate::build_predicate; +use crate::parquet_reader::policy::ReadPolicyBuilder; +use crate::parquet_reader::policy::default_policy_builders; use crate::parquet_reader::predicate::ParquetPredicate; -use crate::parquet_reader::topk::build_topk; +use crate::parquet_reader::predicate::build_predicate; use crate::parquet_reader::topk::BuiltTopK; -use crate::parquet_reader::utils::compute_output_field_paths; +use crate::parquet_reader::topk::build_topk; use crate::parquet_reader::utils::FieldPaths; -use crate::parquet_reader::NoPretchPolicyBuilder; -use crate::parquet_reader::ParquetWholeFileReader; -use crate::parquet_reader::PredicateAndTopkPolicyBuilder; -use crate::parquet_reader::TopkOnlyPolicyBuilder; +use crate::parquet_reader::utils::compute_output_field_paths; use crate::transformer::RecordBatchTransformer; -use crate::ParquetPruner; -use crate::ParquetSourceType; pub struct ParquetReaderBuilder<'a> { ctx: Arc, diff --git a/src/query/storages/parquet/src/parquet_reader/reader/full_reader.rs b/src/query/storages/parquet/src/parquet_reader/reader/full_reader.rs index 2b9ad8719e2e2..09cdc87cd9d39 100644 --- a/src/query/storages/parquet/src/parquet_reader/reader/full_reader.rs +++ b/src/query/storages/parquet/src/parquet_reader/reader/full_reader.rs @@ -19,39 +19,39 @@ use arrow_schema::ArrowError; use bytes::Bytes; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::DataBlock; use databend_common_expression::DataSchema; use databend_common_expression::Scalar; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; +use databend_common_expression::types::DataType; use databend_common_metrics::storage::metrics_inc_omit_filter_rowgroups; use databend_common_metrics::storage::metrics_inc_omit_filter_rows; use databend_common_storage::OperatorRegistry; -use futures::future::BoxFuture; use futures::StreamExt; use futures::TryFutureExt; +use futures::future::BoxFuture; use opendal::Reader; +use parquet::arrow::ParquetRecordBatchStreamBuilder; +use parquet::arrow::ProjectionMask; use parquet::arrow::arrow_reader::ArrowPredicateFn; use parquet::arrow::arrow_reader::ArrowReaderOptions; use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; use parquet::arrow::arrow_reader::RowFilter; use parquet::arrow::async_reader::AsyncFileReader; use parquet::arrow::async_reader::ParquetRecordBatchStream; -use parquet::arrow::ParquetRecordBatchStreamBuilder; -use parquet::arrow::ProjectionMask; use parquet::file::metadata::ParquetMetaData; use parquet::file::metadata::ParquetMetaDataReader; use parquet::schema::types::SchemaDescPtr; +use crate::ParquetPruner; use crate::meta::check_parquet_schema; +use crate::parquet_reader::DataBlockIterator; use crate::parquet_reader::predicate::ParquetPredicate; +use crate::parquet_reader::utils::FieldPaths; use crate::parquet_reader::utils::transform_record_batch; use crate::parquet_reader::utils::transform_record_batch_by_field_paths; -use crate::parquet_reader::utils::FieldPaths; -use crate::parquet_reader::DataBlockIterator; use crate::transformer::RecordBatchTransformer; -use crate::ParquetPruner; /// The reader to read a whole parquet file. pub struct ParquetWholeFileReader { diff --git a/src/query/storages/parquet/src/parquet_reader/reader/row_group_reader.rs b/src/query/storages/parquet/src/parquet_reader/reader/row_group_reader.rs index fb6bf2fdbb228..2a518426d4d19 100644 --- a/src/query/storages/parquet/src/parquet_reader/reader/row_group_reader.rs +++ b/src/query/storages/parquet/src/parquet_reader/reader/row_group_reader.rs @@ -23,10 +23,6 @@ use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::type_check::check_function; -use databend_common_expression::types::DataType; -use databend_common_expression::types::Int64Type; -use databend_common_expression::types::NumberDataType; use databend_common_expression::ColumnRef; use databend_common_expression::Constant; use databend_common_expression::Expr; @@ -38,39 +34,43 @@ use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRef; use databend_common_expression::TopKSorter; use databend_common_expression::Value; +use databend_common_expression::type_check::check_function; +use databend_common_expression::types::DataType; +use databend_common_expression::types::Int64Type; +use databend_common_expression::types::NumberDataType; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_metrics::storage::metrics_inc_omit_filter_rowgroups; use databend_common_metrics::storage::metrics_inc_omit_filter_rows; use databend_common_storage::OperatorRegistry; -use futures::future::try_join_all; use futures::StreamExt; +use futures::future::try_join_all; use opendal::Operator; use opendal::Reader; +use parquet::arrow::PARQUET_FIELD_ID_META_KEY; +use parquet::arrow::ParquetRecordBatchStreamBuilder; use parquet::arrow::arrow_reader::ArrowReaderOptions; use parquet::arrow::arrow_reader::RowSelection; use parquet::arrow::arrow_reader::RowSelector; -use parquet::arrow::ParquetRecordBatchStreamBuilder; -use parquet::arrow::PARQUET_FIELD_ID_META_KEY; use parquet::file::metadata::ParquetMetaData; use parquet::file::metadata::RowGroupMetaData; use parquet::format::PageLocation; use parquet::schema::types::SchemaDescPtr; +use crate::DeleteType; +use crate::ParquetFileReader; +use crate::ParquetReaderBuilder; +use crate::ParquetSourceType; use crate::parquet_part::DeleteTask; +use crate::parquet_reader::policy::POLICY_PREDICATE_ONLY; use crate::parquet_reader::policy::PolicyBuilders; use crate::parquet_reader::policy::PolicyType; use crate::parquet_reader::policy::ReadPolicyImpl; -use crate::parquet_reader::policy::POLICY_PREDICATE_ONLY; -use crate::parquet_reader::predicate::build_predicate; use crate::parquet_reader::predicate::ParquetPredicate; +use crate::parquet_reader::predicate::build_predicate; use crate::parquet_reader::row_group::InMemoryRowGroup; use crate::partition::ParquetRowGroupPart; use crate::read_settings::ReadSettings; use crate::transformer::RecordBatchTransformer; -use crate::DeleteType; -use crate::ParquetFileReader; -use crate::ParquetReaderBuilder; -use crate::ParquetSourceType; static DELETES_FILE_SCHEMA: LazyLock = LazyLock::new(|| { arrow_schema::Schema::new(vec![ diff --git a/src/query/storages/parquet/src/parquet_reader/reader/streaming_load_reader.rs b/src/query/storages/parquet/src/parquet_reader/reader/streaming_load_reader.rs index 3e25b6854533e..e8278b9f6ba8a 100644 --- a/src/query/storages/parquet/src/parquet_reader/reader/streaming_load_reader.rs +++ b/src/query/storages/parquet/src/parquet_reader/reader/streaming_load_reader.rs @@ -20,24 +20,24 @@ use databend_common_catalog::plan::Projection; use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::expr::*; use databend_common_expression::DataSchemaRef; use databend_common_expression::FunctionContext; use databend_common_expression::RemoteDefaultExpr; use databend_common_expression::TableSchemaRef; +use databend_common_expression::expr::*; use databend_common_meta_app::principal::NullAs; use databend_common_meta_app::principal::StageFileFormatType; use databend_common_storage::parquet::infer_schema_with_extension; use databend_storages_common_stage::project_columnar; -use opendal::services::Memory; use opendal::Operator; +use opendal::services::Memory; use parquet::file::metadata::ParquetMetaDataReader; +use crate::ParquetSourceType; use crate::copy_into_table::CopyProjectionEvaluator; use crate::parquet_reader::DataBlockIterator; use crate::parquet_reader::ParquetReaderBuilder; use crate::schema::arrow_to_table_schema; -use crate::ParquetSourceType; pub struct InmMemoryFile { file_data: Bytes, diff --git a/src/query/storages/parquet/src/parquet_reader/row_group.rs b/src/query/storages/parquet/src/parquet_reader/row_group.rs index c3506e0e3bf30..5dbe38aa946cb 100644 --- a/src/query/storages/parquet/src/parquet_reader/row_group.rs +++ b/src/query/storages/parquet/src/parquet_reader/row_group.rs @@ -25,9 +25,9 @@ use databend_storages_common_cache::CacheAccessor; use databend_storages_common_cache::CacheManager; use databend_storages_common_cache::ColumnData; use opendal::Operator; +use parquet::arrow::ProjectionMask; use parquet::arrow::arrow_reader::RowGroups; use parquet::arrow::arrow_reader::RowSelection; -use parquet::arrow::ProjectionMask; use parquet::column::page::PageIterator; use parquet::column::page::PageReader; use parquet::errors::ParquetError; @@ -322,7 +322,7 @@ impl RowGroupCore { self.column_chunks .iter() .enumerate() - .filter(|&(idx, chunk)| (chunk.is_none() && projection.leaf_included(idx))) + .filter(|&(idx, chunk)| chunk.is_none() && projection.leaf_included(idx)) .map(|(idx, _chunk)| { let column = self.metadata.meta().column(idx); let (start, length) = column.byte_range(); @@ -497,8 +497,8 @@ mod test { use arrow_schema::Schema; use bytes::Bytes; use databend_common_base::base::tokio; - use opendal::services::Memory; use opendal::Operator; + use opendal::services::Memory; use parquet::arrow::ArrowWriter; use parquet::basic::Repetition; use parquet::file::metadata::RowGroupMetaData; diff --git a/src/query/storages/parquet/src/parquet_reader/topk.rs b/src/query/storages/parquet/src/parquet_reader/topk.rs index d53b9a72a3757..23f035f420cb3 100644 --- a/src/query/storages/parquet/src/parquet_reader/topk.rs +++ b/src/query/storages/parquet/src/parquet_reader/topk.rs @@ -16,18 +16,18 @@ use std::sync::Arc; use databend_common_catalog::plan::TopK; use databend_common_exception::Result; -use databend_common_expression::types::Bitmap; use databend_common_expression::Column; use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TopKSorter; -use parquet::arrow::parquet_to_arrow_field_levels; +use databend_common_expression::types::Bitmap; use parquet::arrow::FieldLevels; use parquet::arrow::ProjectionMask; +use parquet::arrow::parquet_to_arrow_field_levels; use parquet::schema::types::SchemaDescriptor; -use super::utils::compute_output_field_paths; use super::utils::FieldPaths; +use super::utils::compute_output_field_paths; pub struct ParquetTopK { projection: ProjectionMask, diff --git a/src/query/storages/parquet/src/parquet_reader/utils.rs b/src/query/storages/parquet/src/parquet_reader/utils.rs index 0be60a4c35684..4399233fe6c36 100644 --- a/src/query/storages/parquet/src/parquet_reader/utils.rs +++ b/src/query/storages/parquet/src/parquet_reader/utils.rs @@ -17,16 +17,16 @@ use arrow_array::RecordBatch; use arrow_array::StructArray; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::Bitmap; use databend_common_expression::Column; use databend_common_expression::DataBlock; use databend_common_expression::DataField; use databend_common_expression::DataSchema; use databend_common_expression::FieldIndex; use databend_common_expression::TableSchema; -use parquet::arrow::parquet_to_arrow_schema_by_columns; +use databend_common_expression::types::Bitmap; use parquet::arrow::ArrowSchemaConverter; use parquet::arrow::ProjectionMask; +use parquet::arrow::parquet_to_arrow_schema_by_columns; use parquet::schema::types::SchemaDescriptor; pub type DataBlockIterator = Box> + Send>; diff --git a/src/query/storages/parquet/src/parquet_table/read.rs b/src/query/storages/parquet/src/parquet_table/read.rs index 4acb4bb84a91f..c6f416954eabc 100644 --- a/src/query/storages/parquet/src/parquet_table/read.rs +++ b/src/query/storages/parquet/src/parquet_table/read.rs @@ -23,12 +23,12 @@ use databend_common_expression::TableSchemaRef; use databend_common_pipeline::core::Pipeline; use super::ParquetTable; -use crate::parquet_reader::ParquetReaderBuilder; -use crate::source::ParquetSource; -use crate::utils::calc_parallelism; use crate::ParquetPart; use crate::ParquetPruner; use crate::ParquetSourceType; +use crate::parquet_reader::ParquetReaderBuilder; +use crate::source::ParquetSource; +use crate::utils::calc_parallelism; impl ParquetTable { #[inline] diff --git a/src/query/storages/parquet/src/parquet_table/table.rs b/src/query/storages/parquet/src/parquet_table/table.rs index f5971b264bf09..f834b4bd36eb5 100644 --- a/src/query/storages/parquet/src/parquet_table/table.rs +++ b/src/query/storages/parquet/src/parquet_table/table.rs @@ -35,10 +35,10 @@ use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::ColumnId; +use databend_common_expression::FILE_ROW_NUMBER_COLUMN_ID; +use databend_common_expression::FILENAME_COLUMN_ID; use databend_common_expression::TableField; use databend_common_expression::TableSchema; -use databend_common_expression::FILENAME_COLUMN_ID; -use databend_common_expression::FILE_ROW_NUMBER_COLUMN_ID; use databend_common_meta_app::principal::ParquetFileFormatParams; use databend_common_meta_app::principal::StageInfo; use databend_common_meta_app::schema::TableIdent; @@ -46,11 +46,11 @@ use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; use databend_common_pipeline::core::Pipeline; use databend_common_settings::Settings; +use databend_common_storage::StageFileInfo; +use databend_common_storage::StageFilesInfo; use databend_common_storage::init_stage_operator; use databend_common_storage::parquet::infer_schema_with_extension; use databend_common_storage::read_metadata_async; -use databend_common_storage::StageFileInfo; -use databend_common_storage::StageFilesInfo; use databend_storages_common_table_meta::table::ChangeType; use log::info; use opendal::Operator; diff --git a/src/query/storages/parquet/src/parquet_variant_table/source.rs b/src/query/storages/parquet/src/parquet_variant_table/source.rs index bc74be4c4f5d7..b68d8d7b44c48 100644 --- a/src/query/storages/parquet/src/parquet_variant_table/source.rs +++ b/src/query/storages/parquet/src/parquet_variant_table/source.rs @@ -26,12 +26,12 @@ use databend_common_catalog::query_kind::QueryKind; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::binary::BinaryColumnBuilder; use databend_common_expression::Column; use databend_common_expression::DataBlock; use databend_common_expression::DataSchema; use databend_common_expression::TableDataType; use databend_common_expression::TableSchema; +use databend_common_expression::types::binary::BinaryColumnBuilder; use databend_common_pipeline::core::Event; use databend_common_pipeline::core::OutputPort; use databend_common_pipeline::core::Processor; @@ -43,20 +43,20 @@ use databend_storages_common_stage::add_internal_columns; use databend_storages_common_stage::read_record_batch_to_variant_column; use databend_storages_common_stage::record_batch_to_variant_block; use jiff::tz::TimeZone; +use parquet::arrow::ProjectionMask; use parquet::arrow::arrow_reader::ArrowReaderOptions; use parquet::arrow::arrow_reader::ParquetRecordBatchReader; use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; use parquet::arrow::parquet_to_arrow_field_levels; use parquet::arrow::parquet_to_arrow_schema; -use parquet::arrow::ProjectionMask; +use crate::ParquetFilePart; +use crate::ParquetPart; use crate::meta::read_metadata_async_cached; -use crate::parquet_reader::cached_range_full_read; use crate::parquet_reader::InMemoryRowGroup; +use crate::parquet_reader::cached_range_full_read; use crate::read_settings::ReadSettings; use crate::schema::arrow_to_table_schema; -use crate::ParquetFilePart; -use crate::ParquetPart; enum State { Init, @@ -180,7 +180,7 @@ impl Processor for ParquetVariantSource { readers: mut vs, location, } => { - if let Some((reader, mut start_row, typ, data_schema)) = vs.front_mut() { + if let Some((reader, start_row, typ, data_schema)) = vs.front_mut() { if let Some(batch) = reader.next() { let mut block = record_batch_to_variant_block(batch?, &self.tz, typ, data_schema)?; @@ -188,7 +188,7 @@ impl Processor for ParquetVariantSource { &self.internal_columns, location.clone(), &mut block, - &mut start_row, + start_row, ); if self.is_copy { diff --git a/src/query/storages/parquet/src/row_group_serde.rs b/src/query/storages/parquet/src/row_group_serde.rs index 022165da87662..142eae7b8dde0 100644 --- a/src/query/storages/parquet/src/row_group_serde.rs +++ b/src/query/storages/parquet/src/row_group_serde.rs @@ -19,9 +19,9 @@ use databend_common_exception::ErrorCode; use parquet::file::metadata::RowGroupMetaData; use parquet::format::RowGroup; use parquet::format::SchemaElement; +use parquet::schema::types::SchemaDescriptor; use parquet::schema::types::from_thrift; use parquet::schema::types::to_thrift; -use parquet::schema::types::SchemaDescriptor; use parquet::thrift::TSerializable; use serde::Deserialize; use thrift::protocol::TCompactInputProtocol; diff --git a/src/query/storages/parquet/src/source.rs b/src/query/storages/parquet/src/source.rs index b60b70d686356..4b967f1083aa9 100644 --- a/src/query/storages/parquet/src/source.rs +++ b/src/query/storages/parquet/src/source.rs @@ -43,18 +43,18 @@ use databend_common_storage::OperatorRegistry; use databend_storages_common_stage::add_internal_columns; use parquet::arrow::parquet_to_arrow_schema; +use crate::ParquetFilePart; +use crate::ParquetPart; +use crate::ParquetReaderBuilder; use crate::meta::check_parquet_schema; use crate::meta::read_metadata_async_cached; use crate::parquet_part::DeleteTask; -use crate::parquet_reader::cached_range_full_read; -use crate::parquet_reader::policy::ReadPolicyImpl; use crate::parquet_reader::ParquetWholeFileReader; use crate::parquet_reader::RowGroupReader; +use crate::parquet_reader::cached_range_full_read; +use crate::parquet_reader::policy::ReadPolicyImpl; use crate::partition::ParquetRowGroupPart; use crate::read_settings::ReadSettings; -use crate::ParquetFilePart; -use crate::ParquetPart; -use crate::ParquetReaderBuilder; enum State { Init, @@ -214,13 +214,13 @@ impl Processor for ParquetSource { readers: mut vs, location, } => { - if let Some((reader, mut start_row)) = vs.front_mut() { + if let Some((reader, start_row)) = vs.front_mut() { if let Some(mut block) = reader.as_mut().read_block()? { add_internal_columns( &self.internal_columns, location.clone(), &mut block, - &mut start_row, + start_row, ); if self.is_copy { diff --git a/src/query/storages/parquet/src/statistics/column.rs b/src/query/storages/parquet/src/statistics/column.rs index b39a56f6844fd..57f2c7078d8f2 100644 --- a/src/query/storages/parquet/src/statistics/column.rs +++ b/src/query/storages/parquet/src/statistics/column.rs @@ -12,12 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_expression::types::i256; +use databend_common_expression::Scalar; +use databend_common_expression::TableDataType; use databend_common_expression::types::DecimalDataType; use databend_common_expression::types::DecimalScalar; use databend_common_expression::types::NumberDataType; -use databend_common_expression::Scalar; -use databend_common_expression::TableDataType; +use databend_common_expression::types::i256; use databend_storages_common_table_meta::meta::ColumnStatistics; use parquet::data_type::AsBytes; use parquet::file::statistics::Statistics; diff --git a/src/query/storages/parquet/src/statistics/page.rs b/src/query/storages/parquet/src/statistics/page.rs index 69e2b6e8b620b..f579efac8af16 100644 --- a/src/query/storages/parquet/src/statistics/page.rs +++ b/src/query/storages/parquet/src/statistics/page.rs @@ -12,12 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_expression::types::i256; +use databend_common_expression::Scalar; +use databend_common_expression::TableDataType; use databend_common_expression::types::DecimalDataType; use databend_common_expression::types::DecimalScalar; use databend_common_expression::types::NumberDataType; -use databend_common_expression::Scalar; -use databend_common_expression::TableDataType; +use databend_common_expression::types::i256; use databend_storages_common_table_meta::meta::ColumnStatistics; use parquet::data_type::AsBytes; use parquet::data_type::ByteArray; diff --git a/src/query/storages/parquet/src/statistics/utils.rs b/src/query/storages/parquet/src/statistics/utils.rs index e1fc7efa6c1a0..fedb68f64ec69 100644 --- a/src/query/storages/parquet/src/statistics/utils.rs +++ b/src/query/storages/parquet/src/statistics/utils.rs @@ -13,9 +13,9 @@ // limitations under the License. use arrow_buffer::i256; -use databend_common_expression::types::decimal::DecimalScalar; -use databend_common_expression::types::DecimalSize; use databend_common_expression::Scalar; +use databend_common_expression::types::DecimalSize; +use databend_common_expression::types::decimal::DecimalScalar; use parquet::data_type::AsBytes; use parquet::data_type::FixedLenByteArray; diff --git a/src/query/storages/parquet/src/transformer.rs b/src/query/storages/parquet/src/transformer.rs index 3cc7d66c20537..3075a832e337f 100644 --- a/src/query/storages/parquet/src/transformer.rs +++ b/src/query/storages/parquet/src/transformer.rs @@ -216,7 +216,10 @@ impl RecordBatchTransformer { let target_type = target_field.data_type(); let Some((source_field, source_index)) = source_map.get(field_id) else { - return Err(ErrorCode::TableSchemaMismatch(format!("The field with field_id: {field_id} does not exist in the source schema: {:#?}.", source))); + return Err(ErrorCode::TableSchemaMismatch(format!( + "The field with field_id: {field_id} does not exist in the source schema: {:#?}.", + source + ))); }; sources.push( @@ -254,10 +257,10 @@ mod test { use arrow_schema::Field; use arrow_schema::Fields; use arrow_schema::Schema; - use databend_common_expression::types::NumberDataType; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; + use databend_common_expression::types::NumberDataType; use parquet::arrow::PARQUET_FIELD_ID_META_KEY; use crate::transformer::RecordBatchTransformer; diff --git a/src/query/storages/stage/src/append/output.rs b/src/query/storages/stage/src/append/output.rs index 2c0c6d6321cc7..f33d486d55c4c 100644 --- a/src/query/storages/stage/src/append/output.rs +++ b/src/query/storages/stage/src/append/output.rs @@ -13,11 +13,11 @@ // limitations under the License. use databend_common_exception::Result; -use databend_common_expression::types::StringType; -use databend_common_expression::types::UInt64Type; use databend_common_expression::BlockEntry; use databend_common_expression::DataBlock; use databend_common_expression::FromData; +use databend_common_expression::types::StringType; +use databend_common_expression::types::UInt64Type; use databend_common_pipeline_transforms::processors::AccumulatingTransform; #[derive(Default)] diff --git a/src/query/storages/stage/src/append/parquet_file/block_batch.rs b/src/query/storages/stage/src/append/parquet_file/block_batch.rs index 9db4a99d56e4e..f92de91e2c086 100644 --- a/src/query/storages/stage/src/append/parquet_file/block_batch.rs +++ b/src/query/storages/stage/src/append/parquet_file/block_batch.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_expression::local_block_meta_serde; use databend_common_expression::BlockMetaInfo; use databend_common_expression::DataBlock; +use databend_common_expression::local_block_meta_serde; #[derive(Debug)] pub struct BlockBatch { diff --git a/src/query/storages/stage/src/append/parquet_file/writer_processor.rs b/src/query/storages/stage/src/append/parquet_file/writer_processor.rs index 3268b42cf0718..b2a8d18976baf 100644 --- a/src/query/storages/stage/src/append/parquet_file/writer_processor.rs +++ b/src/query/storages/stage/src/append/parquet_file/writer_processor.rs @@ -43,9 +43,9 @@ use parquet::file::properties::WriterVersion; use parquet::schema::types::ColumnPath; use super::block_batch::BlockBatch; +use crate::append::UnloadOutput; use crate::append::output::DataSummary; use crate::append::path::unload_path; -use crate::append::UnloadOutput; pub struct ParquetFileWriter { input: Arc, @@ -140,7 +140,7 @@ impl ParquetFileWriter { _ => { return Err(ErrorCode::Internal(format!( "unexpected compression {compression}" - ))) + ))); } }; let writer = create_writer( diff --git a/src/query/storages/stage/src/append/row_based_file/buffers.rs b/src/query/storages/stage/src/append/row_based_file/buffers.rs index 2bf5ffcaef5e2..ff1a962930e51 100644 --- a/src/query/storages/stage/src/append/row_based_file/buffers.rs +++ b/src/query/storages/stage/src/append/row_based_file/buffers.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_expression::local_block_meta_serde; use databend_common_expression::BlockMetaInfo; use databend_common_expression::DataBlock; +use databend_common_expression::local_block_meta_serde; #[derive(Debug)] pub struct FileOutputBuffer { diff --git a/src/query/storages/stage/src/append/row_based_file/writer_processor.rs b/src/query/storages/stage/src/append/row_based_file/writer_processor.rs index e0c96f436def2..beef171ce4a0a 100644 --- a/src/query/storages/stage/src/append/row_based_file/writer_processor.rs +++ b/src/query/storages/stage/src/append/row_based_file/writer_processor.rs @@ -32,9 +32,9 @@ use databend_storages_common_stage::CopyIntoLocationInfo; use opendal::Operator; use super::buffers::FileOutputBuffers; +use crate::append::UnloadOutput; use crate::append::output::DataSummary; use crate::append::path::unload_path; -use crate::append::UnloadOutput; pub struct RowBasedFileWriter { input: Arc, diff --git a/src/query/storages/stage/src/append/stage_sink_table.rs b/src/query/storages/stage/src/append/stage_sink_table.rs index 443da33f82a76..86aebf98f9f13 100644 --- a/src/query/storages/stage/src/append/stage_sink_table.rs +++ b/src/query/storages/stage/src/append/stage_sink_table.rs @@ -13,8 +13,8 @@ // limitations under the License. use std::any::Any; -use std::sync::atomic::AtomicUsize; use std::sync::Arc; +use std::sync::atomic::AtomicUsize; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; @@ -27,10 +27,10 @@ use databend_common_pipeline_transforms::TransformPipelineHelper; use databend_storages_common_stage::CopyIntoLocationInfo; use databend_storages_common_table_meta::meta::TableMetaTimestamps; +use crate::StageTable; use crate::append::output::SumSummaryTransform; use crate::append::parquet_file::append_data_to_parquet_files; use crate::append::row_based_file::append_data_to_row_based_files; -use crate::StageTable; pub struct StageSinkTable { info: CopyIntoLocationInfo, diff --git a/src/query/storages/stage/src/lib.rs b/src/query/storages/stage/src/lib.rs index 07304f5e39268..d32c6de8facb7 100644 --- a/src/query/storages/stage/src/lib.rs +++ b/src/query/storages/stage/src/lib.rs @@ -17,7 +17,11 @@ #![feature(impl_trait_in_assoc_type)] #![feature(box_patterns)] #![feature(iter_intersperse)] -#![allow(clippy::uninlined_format_args)] +#![allow( + clippy::collapsible_if, + clippy::needless_return, + clippy::uninlined_format_args +)] mod append; mod compression; @@ -30,9 +34,9 @@ mod transform_generating; pub use append::StageSinkTable; pub use compression::get_compression_with_path; pub use infer_schema::InferSchemaPartInfo; +pub use read::LoadContext; pub use read::row_based::BytesBatch; pub use read::row_based::BytesReader; pub use read::row_based::Decompressor; -pub use read::LoadContext; pub use stage_table::StageTable; pub use streaming_load::build_streaming_load_pipeline; diff --git a/src/query/storages/stage/src/read/avro/avro_to_jsonb.rs b/src/query/storages/stage/src/read/avro/avro_to_jsonb.rs index 5d6e0f108befb..a008a72e1c57b 100644 --- a/src/query/storages/stage/src/read/avro/avro_to_jsonb.rs +++ b/src/query/storages/stage/src/read/avro/avro_to_jsonb.rs @@ -14,10 +14,10 @@ use std::collections::BTreeMap; -use apache_avro::types::Value; use apache_avro::Schema; -use databend_common_expression::types::i256; +use apache_avro::types::Value; use databend_common_expression::types::Decimal; +use databend_common_expression::types::i256; use num_bigint::BigInt; pub(super) fn to_jsonb<'a>(value: &'a Value, schema: &Schema) -> Result, String> { @@ -117,7 +117,7 @@ pub(super) fn to_jsonb<'a>(value: &'a Value, schema: &Schema) -> Result Result<(), String> { let int64 = TableDataType::Number(NumberDataType::Int64); let int64_nullable = TableDataType::Nullable(Box::new(int64.clone())); - assert!(test_single_field( - int64.clone(), - json!(["null", "int"]), - Value::Union(0, Box::new(Value::Null)), - ScalarRef::Null - ) - .is_err()); - - assert!(test_single_field( - int64.clone(), - json!(["long", "int"]), - Value::Union(1, Box::new(Value::Int(100))), - ScalarRef::Number(NumberScalar::Int64(100)) - ) - .is_err()); - - assert!(test_single_field( - int64_nullable.clone(), - json!(["null", "int"]), - Value::Union(0, Box::new(Value::Null)), - ScalarRef::Null - ) - .is_ok()); - - assert!(test_single_field( - int64_nullable.clone(), - json!(["null", "int"]), - Value::Union(1, Box::new(Value::Int(100))), - ScalarRef::Number(NumberScalar::Int64(100)) - ) - .is_ok()); + assert!( + test_single_field( + int64.clone(), + json!(["null", "int"]), + Value::Union(0, Box::new(Value::Null)), + ScalarRef::Null + ) + .is_err() + ); + + assert!( + test_single_field( + int64.clone(), + json!(["long", "int"]), + Value::Union(1, Box::new(Value::Int(100))), + ScalarRef::Number(NumberScalar::Int64(100)) + ) + .is_err() + ); + + assert!( + test_single_field( + int64_nullable.clone(), + json!(["null", "int"]), + Value::Union(0, Box::new(Value::Null)), + ScalarRef::Null + ) + .is_ok() + ); + + assert!( + test_single_field( + int64_nullable.clone(), + json!(["null", "int"]), + Value::Union(1, Box::new(Value::Int(100))), + ScalarRef::Number(NumberScalar::Int64(100)) + ) + .is_ok() + ); Ok(()) } } diff --git a/src/query/storages/stage/src/read/avro/schema_match.rs b/src/query/storages/stage/src/read/avro/schema_match.rs index 96920fcb37ccc..542282cae643a 100644 --- a/src/query/storages/stage/src/read/avro/schema_match.rs +++ b/src/query/storages/stage/src/read/avro/schema_match.rs @@ -12,13 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +use apache_avro::Schema; use apache_avro::schema::RecordSchema; use apache_avro::schema::UnionSchema; -use apache_avro::Schema; -use databend_common_expression::types::i256; +use databend_common_expression::TableDataType; use databend_common_expression::types::Decimal; use databend_common_expression::types::NumberDataType; -use databend_common_expression::TableDataType; +use databend_common_expression::types::i256; type MatchResult = Result; @@ -153,7 +153,9 @@ impl SchemaMatcher { num_matched += 1; } else { if !self.allow_missing_field { - return Err(format!("missing field {name}. Consider add avro file format option `MISSING_FIELD_AS = FIELD_DEFAULT`")); + return Err(format!( + "missing field {name}. Consider add avro file format option `MISSING_FIELD_AS = FIELD_DEFAULT`" + )); } matched_fields[c] = if level == 0 { MatchedField::FieldDefault diff --git a/src/query/storages/stage/src/read/block_builder_state.rs b/src/query/storages/stage/src/read/block_builder_state.rs index d8d8d4bcedca9..56a275cf8c726 100644 --- a/src/query/storages/stage/src/read/block_builder_state.rs +++ b/src/query/storages/stage/src/read/block_builder_state.rs @@ -19,11 +19,11 @@ use databend_common_catalog::plan::InternalColumn; use databend_common_catalog::plan::InternalColumnType; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::NumberScalar; use databend_common_expression::Column; use databend_common_expression::ColumnBuilder; use databend_common_expression::DataBlock; use databend_common_expression::ScalarRef; +use databend_common_expression::types::NumberScalar; use databend_common_storage::FileStatus; use log::debug; diff --git a/src/query/storages/stage/src/read/load_context.rs b/src/query/storages/stage/src/read/load_context.rs index 8318f42dfacf2..2514d598c385e 100644 --- a/src/query/storages/stage/src/read/load_context.rs +++ b/src/query/storages/stage/src/read/load_context.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::sync::atomic::AtomicU64; use std::sync::Arc; +use std::sync::atomic::AtomicU64; use databend_common_ast::ast::OnErrorMode; use databend_common_catalog::plan::InternalColumn; diff --git a/src/query/storages/stage/src/read/row_based/format.rs b/src/query/storages/stage/src/read/row_based/format.rs index 53111e1e784d0..9e7e5c574b0cd 100644 --- a/src/query/storages/stage/src/read/row_based/format.rs +++ b/src/query/storages/stage/src/read/row_based/format.rs @@ -33,7 +33,7 @@ pub trait SeparatorState: Send + Sync { pub trait RowDecoder: Send + Sync { fn add(&self, block_builder: &mut BlockBuilderState, batch: RowBatchWithPosition) - -> Result<()>; + -> Result<()>; fn flush(&self, columns: Vec, _num_rows: usize) -> Vec { columns diff --git a/src/query/storages/stage/src/read/row_based/formats/csv/block_builder.rs b/src/query/storages/stage/src/read/row_based/formats/csv/block_builder.rs index 1b8dd22578ad1..85795d31fe375 100644 --- a/src/query/storages/stage/src/read/row_based/formats/csv/block_builder.rs +++ b/src/query/storages/stage/src/read/row_based/formats/csv/block_builder.rs @@ -15,11 +15,11 @@ use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::types::nullable::NullableColumnBuilder; -use databend_common_expression::types::string::StringColumnBuilder; use databend_common_expression::Column; use databend_common_expression::ColumnBuilder; use databend_common_expression::TableDataType; +use databend_common_expression::types::nullable::NullableColumnBuilder; +use databend_common_expression::types::string::StringColumnBuilder; use databend_common_formats::SeparatedTextDecoder; use databend_common_meta_app::principal::EmptyFieldAs; use databend_common_storage::FileParseError; diff --git a/src/query/storages/stage/src/read/row_based/formats/ndjson/block_builder.rs b/src/query/storages/stage/src/read/row_based/formats/ndjson/block_builder.rs index 56c396e11a118..aa0addee30203 100644 --- a/src/query/storages/stage/src/read/row_based/formats/ndjson/block_builder.rs +++ b/src/query/storages/stage/src/read/row_based/formats/ndjson/block_builder.rs @@ -215,8 +215,8 @@ fn map_json_error(err: serde_json::Error, data: &[u8], file_full_path: &str) -> #[cfg(test)] mod test { - use super::map_json_error; use super::FileParseError; + use super::map_json_error; fn decode_err(data: &str) -> String { serde_json::from_slice::(data.as_bytes()) diff --git a/src/query/storages/stage/src/read/row_based/formats/tsv/block_builder.rs b/src/query/storages/stage/src/read/row_based/formats/tsv/block_builder.rs index 4f9eb560ca821..51285ceca9a44 100644 --- a/src/query/storages/stage/src/read/row_based/formats/tsv/block_builder.rs +++ b/src/query/storages/stage/src/read/row_based/formats/tsv/block_builder.rs @@ -178,11 +178,7 @@ impl TsvDecoder { } } } - if let Some(e) = error { - Err(e) - } else { - Ok(()) - } + if let Some(e) = error { Err(e) } else { Ok(()) } } } diff --git a/src/query/storages/stage/src/stage_table.rs b/src/query/storages/stage/src/stage_table.rs index 28c40001e00ef..d84695eddde8c 100644 --- a/src/query/storages/stage/src/stage_table.rs +++ b/src/query/storages/stage/src/stage_table.rs @@ -29,14 +29,14 @@ use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::ColumnId; -use databend_common_expression::FILENAME_COLUMN_ID; use databend_common_expression::FILE_ROW_NUMBER_COLUMN_ID; +use databend_common_expression::FILENAME_COLUMN_ID; use databend_common_meta_app::principal::FileFormatParams; use databend_common_meta_app::principal::StageInfo; use databend_common_meta_app::schema::TableInfo; use databend_common_pipeline::core::Pipeline; -use databend_common_storage::init_stage_operator; use databend_common_storage::StageFileInfo; +use databend_common_storage::init_stage_operator; use databend_common_storages_orc::OrcTableForCopy; use databend_common_storages_parquet::ParquetTableForCopy; use databend_common_storages_parquet::ParquetVariantTable; diff --git a/src/query/storages/stage/src/streaming_load.rs b/src/query/storages/stage/src/streaming_load.rs index 75d43443b9c4d..b5aeeb4d6b104 100644 --- a/src/query/storages/stage/src/streaming_load.rs +++ b/src/query/storages/stage/src/streaming_load.rs @@ -19,7 +19,6 @@ use databend_common_ast::ast::OnErrorMode; use databend_common_base::base::tokio::sync::mpsc::Receiver; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::BlockMetaInfoDowncast; use databend_common_expression::BlockThresholds; use databend_common_expression::DataBlock; @@ -28,6 +27,7 @@ use databend_common_expression::DataSchemaRef; use databend_common_expression::FunctionContext; use databend_common_expression::RemoteDefaultExpr; use databend_common_expression::TableSchemaRef; +use databend_common_expression::types::DataType; use databend_common_formats::FileFormatOptionsExt; use databend_common_meta_app::principal::FileFormatParams; use databend_common_meta_app::principal::NullAs; @@ -36,11 +36,12 @@ use databend_common_meta_app::principal::StageFileCompression; use databend_common_pipeline::core::Pipeline; use databend_common_pipeline::core::ProcessorPtr; use databend_common_pipeline::sources::SyncReceiverSource; -use databend_common_pipeline_transforms::columns::TransformNullIf; use databend_common_pipeline_transforms::TransformPipelineHelper; +use databend_common_pipeline_transforms::columns::TransformNullIf; use databend_common_storages_parquet::InmMemoryFile; use parking_lot::Mutex; +use crate::BytesBatch; use crate::compression::get_compression_with_path; use crate::read::load_context::LoadContext; use crate::read::row_based::format::create_row_based_file_format; @@ -50,7 +51,6 @@ use crate::read::row_based::processors::Separator; use crate::transform_generating::DataBlockIterator; use crate::transform_generating::DataBlockIteratorBuilder; use crate::transform_generating::GeneratingTransformer; -use crate::BytesBatch; pub fn build_streaming_load_pipeline( ctx: Arc, diff --git a/src/query/storages/stream/src/stream_status_table_func.rs b/src/query/storages/stream/src/stream_status_table_func.rs index df595cbe73cb0..5c4c98682eed3 100644 --- a/src/query/storages/stream/src/stream_status_table_func.rs +++ b/src/query/storages/stream/src/stream_status_table_func.rs @@ -25,13 +25,13 @@ use databend_common_catalog::table_context::TableContext; use databend_common_catalog::table_function::TableFunction; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::BooleanType; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::BooleanType; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; diff --git a/src/query/storages/stream/src/stream_table.rs b/src/query/storages/stream/src/stream_table.rs index a43b7f6767472..2e8567bb3637f 100644 --- a/src/query/storages/stream/src/stream_table.rs +++ b/src/query/storages/stream/src/stream_table.rs @@ -30,9 +30,9 @@ use databend_common_catalog::table::TableStatistics; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::ColumnId; use databend_common_expression::BASE_BLOCK_IDS_COLUMN_ID; use databend_common_expression::BASE_ROW_ID_COLUMN_ID; +use databend_common_expression::ColumnId; use databend_common_expression::ORIGIN_BLOCK_ID_COL_NAME; use databend_common_expression::ORIGIN_BLOCK_ROW_NUM_COL_NAME; use databend_common_expression::ORIGIN_VERSION_COL_NAME; @@ -41,18 +41,18 @@ use databend_common_meta_app::storage::S3StorageClass; use databend_common_meta_app::tenant::Tenant; use databend_common_pipeline::core::Pipeline; use databend_common_sql::binder::STREAM_COLUMN_FACTORY; +use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::io::MetaReaders; use databend_common_storages_fuse::io::SnapshotHistoryReader; use databend_common_storages_fuse::io::TableMetaLocationGenerator; -use databend_common_storages_fuse::FuseTable; use databend_storages_common_table_meta::table::ChangeType; -use databend_storages_common_table_meta::table::StreamMode; use databend_storages_common_table_meta::table::OPT_KEY_DATABASE_ID; use databend_storages_common_table_meta::table::OPT_KEY_MODE; use databend_storages_common_table_meta::table::OPT_KEY_SNAPSHOT_LOCATION; use databend_storages_common_table_meta::table::OPT_KEY_SOURCE_DATABASE_ID; use databend_storages_common_table_meta::table::OPT_KEY_SOURCE_TABLE_ID; use databend_storages_common_table_meta::table::OPT_KEY_TABLE_VER; +use databend_storages_common_table_meta::table::StreamMode; use futures::TryStreamExt; pub const STREAM_ENGINE: &str = "STREAM"; diff --git a/src/query/storages/system/src/backtrace_table.rs b/src/query/storages/system/src/backtrace_table.rs index 3291434771e16..949bfba5c9de9 100644 --- a/src/query/storages/system/src/backtrace_table.rs +++ b/src/query/storages/system/src/backtrace_table.rs @@ -22,12 +22,12 @@ use databend_common_catalog::table::DistributionLevel; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::StringType; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::StringType; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; diff --git a/src/query/storages/system/src/build_options_table.rs b/src/query/storages/system/src/build_options_table.rs index 8e57ecc65d2f7..798e8a61935c9 100644 --- a/src/query/storages/system/src/build_options_table.rs +++ b/src/query/storages/system/src/build_options_table.rs @@ -18,12 +18,12 @@ use std::sync::Arc; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::StringType; -use databend_common_expression::utils::FromData; use databend_common_expression::DataBlock; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::StringType; +use databend_common_expression::utils::FromData; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; diff --git a/src/query/storages/system/src/caches_table.rs b/src/query/storages/system/src/caches_table.rs index 7a6ed66f0cb40..0dc5a086a08e2 100644 --- a/src/query/storages/system/src/caches_table.rs +++ b/src/query/storages/system/src/caches_table.rs @@ -17,14 +17,14 @@ use std::sync::Arc; use databend_common_catalog::table::DistributionLevel; use databend_common_catalog::table::Table; use databend_common_exception::Result; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::UInt64Type; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::UInt64Type; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; diff --git a/src/query/storages/system/src/catalogs_table.rs b/src/query/storages/system/src/catalogs_table.rs index b89605a2ae2ce..ec959f126ca57 100644 --- a/src/query/storages/system/src/catalogs_table.rs +++ b/src/query/storages/system/src/catalogs_table.rs @@ -19,12 +19,12 @@ use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::StringType; -use databend_common_expression::utils::FromData; use databend_common_expression::DataBlock; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::StringType; +use databend_common_expression::utils::FromData; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; diff --git a/src/query/storages/system/src/clustering_history_table.rs b/src/query/storages/system/src/clustering_history_table.rs index ab4a3a02fcb63..2dbc01f159f0a 100644 --- a/src/query/storages/system/src/clustering_history_table.rs +++ b/src/query/storages/system/src/clustering_history_table.rs @@ -13,14 +13,14 @@ // limitations under the License. use databend_common_exception::Result; -use databend_common_expression::types::number::NumberScalar; -use databend_common_expression::types::NumberDataType; use databend_common_expression::ColumnBuilder; use databend_common_expression::Scalar; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::number::NumberScalar; use crate::SystemLogElement; use crate::SystemLogQueue; diff --git a/src/query/storages/system/src/clusters_table.rs b/src/query/storages/system/src/clusters_table.rs index ba0e038ffd9aa..a984b301c6edf 100644 --- a/src/query/storages/system/src/clusters_table.rs +++ b/src/query/storages/system/src/clusters_table.rs @@ -17,15 +17,15 @@ use std::sync::Arc; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::number::NumberScalar; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; use databend_common_expression::ColumnBuilder; use databend_common_expression::DataBlock; use databend_common_expression::Scalar; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::number::NumberScalar; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; diff --git a/src/query/storages/system/src/columns_table.rs b/src/query/storages/system/src/columns_table.rs index 8b55954250e10..5e0d8ecd12bd7 100644 --- a/src/query/storages/system/src/columns_table.rs +++ b/src/query/storages/system/src/columns_table.rs @@ -22,26 +22,26 @@ use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::infer_table_schema; -use databend_common_expression::types::StringType; -use databend_common_expression::utils::FromData; use databend_common_expression::DataBlock; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::infer_table_schema; +use databend_common_expression::types::StringType; +use databend_common_expression::utils::FromData; use databend_common_functions::BUILTIN_FUNCTIONS; -use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_app::schema::CatalogInfo; use databend_common_meta_app::schema::CatalogNameIdent; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; +use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_app::tenant::Tenant; use databend_common_sql::Planner; use databend_common_storages_basic::view_table::QUERY; use databend_common_storages_basic::view_table::VIEW_ENGINE; -use databend_common_storages_stream::stream_table::StreamTable; use databend_common_storages_stream::stream_table::STREAM_ENGINE; +use databend_common_storages_stream::stream_table::StreamTable; use databend_common_users::Object; use log::warn; diff --git a/src/query/storages/system/src/configs_table.rs b/src/query/storages/system/src/configs_table.rs index 8d38d10153eb2..94f75e741bdfa 100644 --- a/src/query/storages/system/src/configs_table.rs +++ b/src/query/storages/system/src/configs_table.rs @@ -19,12 +19,12 @@ use databend_common_catalog::table_context::TableContext; use databend_common_config::Config; use databend_common_config::GlobalConfig; use databend_common_exception::Result; -use databend_common_expression::types::StringType; -use databend_common_expression::utils::FromData; use databend_common_expression::DataBlock; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::StringType; +use databend_common_expression::utils::FromData; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; diff --git a/src/query/storages/system/src/constraints_table.rs b/src/query/storages/system/src/constraints_table.rs index bdcc56716c0f1..a2dacb8c63380 100644 --- a/src/query/storages/system/src/constraints_table.rs +++ b/src/query/storages/system/src/constraints_table.rs @@ -19,13 +19,13 @@ use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::StringType; -use databend_common_expression::types::TimestampType; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::StringType; +use databend_common_expression::types::TimestampType; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; diff --git a/src/query/storages/system/src/contributors_table.rs b/src/query/storages/system/src/contributors_table.rs index 2bbf0ecb3c960..59a9af0a8ac06 100644 --- a/src/query/storages/system/src/contributors_table.rs +++ b/src/query/storages/system/src/contributors_table.rs @@ -17,12 +17,12 @@ use std::sync::Arc; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::StringType; -use databend_common_expression::utils::FromData; use databend_common_expression::DataBlock; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::StringType; +use databend_common_expression::utils::FromData; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; diff --git a/src/query/storages/system/src/credits_table.rs b/src/query/storages/system/src/credits_table.rs index 00b9b65b67b90..5c39be04f0ea5 100644 --- a/src/query/storages/system/src/credits_table.rs +++ b/src/query/storages/system/src/credits_table.rs @@ -17,12 +17,12 @@ use std::sync::Arc; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::StringType; -use databend_common_expression::utils::FromData; use databend_common_expression::DataBlock; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::StringType; +use databend_common_expression::utils::FromData; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; diff --git a/src/query/storages/system/src/databases_table.rs b/src/query/storages/system/src/databases_table.rs index 6ba461f1d0b18..8c786997c0337 100644 --- a/src/query/storages/system/src/databases_table.rs +++ b/src/query/storages/system/src/databases_table.rs @@ -20,23 +20,23 @@ use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; +use databend_common_expression::DataBlock; +use databend_common_expression::TableDataType; +use databend_common_expression::TableField; +use databend_common_expression::TableSchemaRefExt; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::StringType; use databend_common_expression::types::TimestampType; use databend_common_expression::types::UInt64Type; use databend_common_expression::utils::FromData; -use databend_common_expression::DataBlock; -use databend_common_expression::TableDataType; -use databend_common_expression::TableField; -use databend_common_expression::TableSchemaRefExt; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::principal::OwnershipObject; -use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_app::schema::CatalogInfo; use databend_common_meta_app::schema::CatalogNameIdent; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; +use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_app::tenant::Tenant; use databend_common_users::Object; use databend_common_users::UserApiProvider; diff --git a/src/query/storages/system/src/dictionaries_table.rs b/src/query/storages/system/src/dictionaries_table.rs index 39cd6644b82af..6196477742f0a 100644 --- a/src/query/storages/system/src/dictionaries_table.rs +++ b/src/query/storages/system/src/dictionaries_table.rs @@ -18,9 +18,6 @@ use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::DataType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::TimestampType; use databend_common_expression::ColumnBuilder; use databend_common_expression::DataBlock; use databend_common_expression::FromData; @@ -28,6 +25,9 @@ use databend_common_expression::ScalarRef; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::DataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::TimestampType; use databend_common_meta_app::schema::ListDictionaryReq; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; diff --git a/src/query/storages/system/src/engines_table.rs b/src/query/storages/system/src/engines_table.rs index 520430aeeeb0d..3c8da0cca9ebf 100644 --- a/src/query/storages/system/src/engines_table.rs +++ b/src/query/storages/system/src/engines_table.rs @@ -19,12 +19,12 @@ use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::StringType; -use databend_common_expression::utils::FromData; use databend_common_expression::DataBlock; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::StringType; +use databend_common_expression::utils::FromData; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; diff --git a/src/query/storages/system/src/functions_table.rs b/src/query/storages/system/src/functions_table.rs index 9c9417346c71f..6833a91ae68c8 100644 --- a/src/query/storages/system/src/functions_table.rs +++ b/src/query/storages/system/src/functions_table.rs @@ -18,15 +18,15 @@ use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::BooleanType; -use databend_common_expression::types::StringType; -use databend_common_expression::utils::FromData; use databend_common_expression::DataBlock; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; -use databend_common_functions::aggregates::AggregateFunctionFactory; +use databend_common_expression::types::BooleanType; +use databend_common_expression::types::StringType; +use databend_common_expression::utils::FromData; use databend_common_functions::BUILTIN_FUNCTIONS; +use databend_common_functions::aggregates::AggregateFunctionFactory; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; diff --git a/src/query/storages/system/src/indexes_table.rs b/src/query/storages/system/src/indexes_table.rs index 2d3ca7320dd13..28e53557119bd 100644 --- a/src/query/storages/system/src/indexes_table.rs +++ b/src/query/storages/system/src/indexes_table.rs @@ -18,13 +18,13 @@ use databend_common_catalog::catalog::CATALOG_DEFAULT; use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table::Table; use databend_common_exception::Result; -use databend_common_expression::types::StringType; -use databend_common_expression::types::TimestampType; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::StringType; +use databend_common_expression::types::TimestampType; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::schema::ListIndexesReq; use databend_common_meta_app::schema::TableIdent; diff --git a/src/query/storages/system/src/lib.rs b/src/query/storages/system/src/lib.rs index 7284248db1a0b..d1d0658895529 100644 --- a/src/query/storages/system/src/lib.rs +++ b/src/query/storages/system/src/lib.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(clippy::uninlined_format_args)] +#![allow(clippy::collapsible_if, clippy::uninlined_format_args)] #![allow(clippy::useless_asref)] #![feature(type_alias_impl_trait)] #![feature(impl_trait_in_assoc_type)] @@ -101,8 +101,8 @@ pub use malloc_stats_table::MallocStatsTable; pub use malloc_stats_totals_table::MallocStatsTotalsTable; pub use metrics_table::MetricsTable; pub use notification_history_table::NotificationHistoryTable; -pub use notifications_table::parse_notifications_to_datablock; pub use notifications_table::NotificationsTable; +pub use notifications_table::parse_notifications_to_datablock; pub use one_table::OneTable; pub use password_policies_table::PasswordPoliciesTable; pub use private_task_history_table::PrivateTaskHistoryTable; @@ -129,10 +129,10 @@ pub use tables_table::TablesTableWithHistory; pub use tables_table::TablesTableWithoutHistory; pub use tables_table::ViewsTableWithHistory; pub use tables_table::ViewsTableWithoutHistory; -pub use task_history_table::parse_task_runs_to_datablock; pub use task_history_table::TaskHistoryTable; -pub use tasks_table::parse_tasks_to_datablock; +pub use task_history_table::parse_task_runs_to_datablock; pub use tasks_table::TasksTable; +pub use tasks_table::parse_tasks_to_datablock; pub use temp_files_table::TempFilesTable; pub use user_functions_table::UserFunctionsTable; pub use users_table::UsersTable; diff --git a/src/query/storages/system/src/locks_table.rs b/src/query/storages/system/src/locks_table.rs index e4451041f5fcf..77201d09aaede 100644 --- a/src/query/storages/system/src/locks_table.rs +++ b/src/query/storages/system/src/locks_table.rs @@ -19,13 +19,6 @@ use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::filter_helper::FilterHelpers; -use databend_common_expression::type_check::check_number; -use databend_common_expression::types::number::UInt64Type; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::TimestampType; -use databend_common_expression::utils::FromData; use databend_common_expression::Constant; use databend_common_expression::DataBlock; use databend_common_expression::Expr; @@ -33,6 +26,13 @@ use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::filter_helper::FilterHelpers; +use databend_common_expression::type_check::check_number; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::TimestampType; +use databend_common_expression::types::number::UInt64Type; +use databend_common_expression::utils::FromData; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::schema::CatalogInfo; use databend_common_meta_app::schema::CatalogNameIdent; diff --git a/src/query/storages/system/src/log_queue.rs b/src/query/storages/system/src/log_queue.rs index 3c67ed80e08ad..9265dd1bb001f 100644 --- a/src/query/storages/system/src/log_queue.rs +++ b/src/query/storages/system/src/log_queue.rs @@ -28,10 +28,10 @@ use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::DataType; use databend_common_expression::ColumnBuilder; use databend_common_expression::DataBlock; use databend_common_expression::TableSchemaRef; +use databend_common_expression::types::DataType; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; diff --git a/src/query/storages/system/src/malloc_stats_table.rs b/src/query/storages/system/src/malloc_stats_table.rs index 5744ae74d7425..2406607b7b5e0 100644 --- a/src/query/storages/system/src/malloc_stats_table.rs +++ b/src/query/storages/system/src/malloc_stats_table.rs @@ -19,12 +19,12 @@ use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::VariantType; use databend_common_expression::BlockEntry; use databend_common_expression::DataBlock; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::VariantType; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; diff --git a/src/query/storages/system/src/malloc_stats_totals_table.rs b/src/query/storages/system/src/malloc_stats_totals_table.rs index 02424a6ce2972..18e6e15c888ba 100644 --- a/src/query/storages/system/src/malloc_stats_totals_table.rs +++ b/src/query/storages/system/src/malloc_stats_totals_table.rs @@ -20,16 +20,16 @@ use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::string::StringColumnBuilder; -use databend_common_expression::types::ArgType; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::NumberType; -use databend_common_expression::types::StringType; use databend_common_expression::Column; use databend_common_expression::DataBlock; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::ArgType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::NumberType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::string::StringColumnBuilder; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; diff --git a/src/query/storages/system/src/metrics_table.rs b/src/query/storages/system/src/metrics_table.rs index d325f31e82b19..8665fad814a91 100644 --- a/src/query/storages/system/src/metrics_table.rs +++ b/src/query/storages/system/src/metrics_table.rs @@ -15,21 +15,21 @@ use std::collections::HashMap; use std::sync::Arc; +use databend_common_base::runtime::GLOBAL_MEM_STAT; +use databend_common_base::runtime::metrics::GLOBAL_METRICS_REGISTRY; use databend_common_base::runtime::metrics::MetricSample; use databend_common_base::runtime::metrics::MetricValue; -use databend_common_base::runtime::metrics::GLOBAL_METRICS_REGISTRY; -use databend_common_base::runtime::GLOBAL_MEM_STAT; use databend_common_catalog::table::DistributionLevel; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::StringType; -use databend_common_expression::utils::FromData; use databend_common_expression::DataBlock; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::StringType; +use databend_common_expression::utils::FromData; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; diff --git a/src/query/storages/system/src/notification_history_table.rs b/src/query/storages/system/src/notification_history_table.rs index e4e0540205b82..a68d772bc8684 100644 --- a/src/query/storages/system/src/notification_history_table.rs +++ b/src/query/storages/system/src/notification_history_table.rs @@ -26,11 +26,11 @@ use databend_common_cloud_control::pb::NotificationHistory; use databend_common_config::GlobalConfig; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::DataBlock; +use databend_common_expression::FromData; use databend_common_expression::infer_table_schema; use databend_common_expression::types::StringType; use databend_common_expression::types::TimestampType; -use databend_common_expression::DataBlock; -use databend_common_expression::FromData; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; diff --git a/src/query/storages/system/src/notifications_table.rs b/src/query/storages/system/src/notifications_table.rs index 2e474353ff7ed..0e34985889f1b 100644 --- a/src/query/storages/system/src/notifications_table.rs +++ b/src/query/storages/system/src/notifications_table.rs @@ -26,14 +26,14 @@ use databend_common_cloud_control::pb::Notification; use databend_common_config::GlobalConfig; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::DataBlock; +use databend_common_expression::FromData; use databend_common_expression::infer_table_schema; use databend_common_expression::types::BooleanType; use databend_common_expression::types::StringType; use databend_common_expression::types::TimestampType; use databend_common_expression::types::UInt64Type; use databend_common_expression::types::VariantType; -use databend_common_expression::DataBlock; -use databend_common_expression::FromData; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; diff --git a/src/query/storages/system/src/one_table.rs b/src/query/storages/system/src/one_table.rs index 4456740641a2e..1df02d0dac780 100644 --- a/src/query/storages/system/src/one_table.rs +++ b/src/query/storages/system/src/one_table.rs @@ -21,20 +21,20 @@ use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::number::UInt8Type; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::utils::FromData; use databend_common_expression::DataBlock; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::number::UInt8Type; +use databend_common_expression::utils::FromData; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; -use crate::table::SystemTablePart; use crate::SyncOneBlockSystemTable; use crate::SyncSystemTable; +use crate::table::SystemTablePart; pub struct OneTable { table_info: TableInfo, diff --git a/src/query/storages/system/src/password_policies_table.rs b/src/query/storages/system/src/password_policies_table.rs index 2f2455ad7cc0a..7c76c7edf7cba 100644 --- a/src/query/storages/system/src/password_policies_table.rs +++ b/src/query/storages/system/src/password_policies_table.rs @@ -18,13 +18,13 @@ use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::StringType; -use databend_common_expression::types::TimestampType; -use databend_common_expression::utils::FromData; use databend_common_expression::DataBlock; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::StringType; +use databend_common_expression::types::TimestampType; +use databend_common_expression::utils::FromData; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; diff --git a/src/query/storages/system/src/private_task_history_table.rs b/src/query/storages/system/src/private_task_history_table.rs index a8f3e41f21b76..8f0e81f4e71bf 100644 --- a/src/query/storages/system/src/private_task_history_table.rs +++ b/src/query/storages/system/src/private_task_history_table.rs @@ -23,8 +23,8 @@ use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; use databend_common_meta_app::tenant::Tenant; -use databend_common_storages_basic::view_table::ViewTable; use databend_common_storages_basic::view_table::QUERY; +use databend_common_storages_basic::view_table::ViewTable; use crate::generate_catalog_meta; diff --git a/src/query/storages/system/src/private_tasks_table.rs b/src/query/storages/system/src/private_tasks_table.rs index 76ca8db802316..c54149b6b52f6 100644 --- a/src/query/storages/system/src/private_tasks_table.rs +++ b/src/query/storages/system/src/private_tasks_table.rs @@ -20,8 +20,8 @@ use databend_common_catalog::table_context::TableContext; use databend_common_cloud_control::pb; use databend_common_cloud_control::task_utils; use databend_common_exception::Result; -use databend_common_expression::infer_table_schema; use databend_common_expression::DataBlock; +use databend_common_expression::infer_table_schema; use databend_common_meta_app::principal::Status; use databend_common_meta_app::principal::Task; use databend_common_meta_app::schema::TableIdent; diff --git a/src/query/storages/system/src/procedures_table.rs b/src/query/storages/system/src/procedures_table.rs index d0255632db095..330a3d05d1bf3 100644 --- a/src/query/storages/system/src/procedures_table.rs +++ b/src/query/storages/system/src/procedures_table.rs @@ -18,15 +18,15 @@ use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; +use databend_common_expression::DataBlock; +use databend_common_expression::TableDataType; +use databend_common_expression::TableField; +use databend_common_expression::TableSchemaRefExt; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::StringType; use databend_common_expression::types::TimestampType; use databend_common_expression::types::UInt64Type; use databend_common_expression::utils::FromData; -use databend_common_expression::DataBlock; -use databend_common_expression::TableDataType; -use databend_common_expression::TableField; -use databend_common_expression::TableSchemaRefExt; use databend_common_meta_app::principal::ListProcedureReq; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; diff --git a/src/query/storages/system/src/processes_table.rs b/src/query/storages/system/src/processes_table.rs index d34f90d242368..16b6085225979 100644 --- a/src/query/storages/system/src/processes_table.rs +++ b/src/query/storages/system/src/processes_table.rs @@ -21,17 +21,17 @@ use databend_common_catalog::table::DistributionLevel; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::number::Int64Type; -use databend_common_expression::types::number::UInt32Type; -use databend_common_expression::types::number::UInt64Type; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::TimestampType; -use databend_common_expression::utils::FromData; use databend_common_expression::DataBlock; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::TimestampType; +use databend_common_expression::types::number::Int64Type; +use databend_common_expression::types::number::UInt32Type; +use databend_common_expression::types::number::UInt64Type; +use databend_common_expression::utils::FromData; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; diff --git a/src/query/storages/system/src/query_cache_table.rs b/src/query/storages/system/src/query_cache_table.rs index b5a9d2762707b..bfd80af0bcd7c 100644 --- a/src/query/storages/system/src/query_cache_table.rs +++ b/src/query/storages/system/src/query_cache_table.rs @@ -18,20 +18,20 @@ use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::BooleanType; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::UInt64Type; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::BooleanType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::UInt64Type; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; -use databend_common_storages_basic::gen_result_cache_prefix; use databend_common_storages_basic::ResultCacheMetaManager; +use databend_common_storages_basic::gen_result_cache_prefix; use databend_common_users::UserApiProvider; use itertools::Itertools; diff --git a/src/query/storages/system/src/query_execution_table.rs b/src/query/storages/system/src/query_execution_table.rs index 5b6f262ddc538..cd3fb63bc2fcf 100644 --- a/src/query/storages/system/src/query_execution_table.rs +++ b/src/query/storages/system/src/query_execution_table.rs @@ -21,10 +21,6 @@ use databend_common_catalog::table::DistributionLevel; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::TimestampType; -use databend_common_expression::types::UInt64Type; use databend_common_expression::ColumnBuilder; use databend_common_expression::DataBlock; use databend_common_expression::FromData; @@ -32,6 +28,10 @@ use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::TimestampType; +use databend_common_expression::types::UInt64Type; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; diff --git a/src/query/storages/system/src/roles_table.rs b/src/query/storages/system/src/roles_table.rs index 62b4099f7a090..eaa8f83fe036f 100644 --- a/src/query/storages/system/src/roles_table.rs +++ b/src/query/storages/system/src/roles_table.rs @@ -18,15 +18,15 @@ use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::number::UInt64Type; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::TimestampType; -use databend_common_expression::utils::FromData; use databend_common_expression::DataBlock; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::TimestampType; +use databend_common_expression::types::number::UInt64Type; +use databend_common_expression::utils::FromData; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; diff --git a/src/query/storages/system/src/settings_table.rs b/src/query/storages/system/src/settings_table.rs index 2defe591305cb..3e6783fefdb71 100644 --- a/src/query/storages/system/src/settings_table.rs +++ b/src/query/storages/system/src/settings_table.rs @@ -17,12 +17,12 @@ use std::sync::Arc; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::StringType; -use databend_common_expression::utils::FromData; use databend_common_expression::DataBlock; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::StringType; +use databend_common_expression::utils::FromData; use databend_common_meta_app::principal::UserSettingValue; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; diff --git a/src/query/storages/system/src/stages_table.rs b/src/query/storages/system/src/stages_table.rs index c394ec3bdc989..0d64c213b0d48 100644 --- a/src/query/storages/system/src/stages_table.rs +++ b/src/query/storages/system/src/stages_table.rs @@ -19,15 +19,15 @@ use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::number::UInt64Type; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::TimestampType; -use databend_common_expression::utils::FromData; use databend_common_expression::DataBlock; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::TimestampType; +use databend_common_expression::types::number::UInt64Type; +use databend_common_expression::utils::FromData; use databend_common_meta_app::principal::OwnershipObject; use databend_common_meta_app::principal::StageType; use databend_common_meta_app::schema::TableIdent; diff --git a/src/query/storages/system/src/statistics_table.rs b/src/query/storages/system/src/statistics_table.rs index 05f4bdad3e149..0bc02558692c3 100644 --- a/src/query/storages/system/src/statistics_table.rs +++ b/src/query/storages/system/src/statistics_table.rs @@ -20,15 +20,15 @@ use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table::DummyColumnStatisticsProvider; use databend_common_catalog::table::Table; use databend_common_exception::Result; -use databend_common_expression::infer_table_schema; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::UInt64Type; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::infer_table_schema; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::UInt64Type; use databend_common_meta_app::schema::CatalogInfo; use databend_common_meta_app::schema::CatalogNameIdent; use databend_common_meta_app::schema::TableIdent; diff --git a/src/query/storages/system/src/streams_table.rs b/src/query/storages/system/src/streams_table.rs index 6a9f471e75d17..38bbf67de9bfe 100644 --- a/src/query/storages/system/src/streams_table.rs +++ b/src/query/storages/system/src/streams_table.rs @@ -24,17 +24,17 @@ use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::number::UInt64Type; -use databend_common_expression::types::BooleanType; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::TimestampType; -use databend_common_expression::utils::FromData; use databend_common_expression::DataBlock; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::BooleanType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::TimestampType; +use databend_common_expression::types::number::UInt64Type; +use databend_common_expression::utils::FromData; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::principal::OwnershipObject; use databend_common_meta_app::schema::CatalogInfo; @@ -43,8 +43,8 @@ use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; use databend_common_meta_app::tenant::Tenant; -use databend_common_storages_fuse::operations::acquire_task_permit; use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::operations::acquire_task_permit; use databend_common_storages_stream::stream_table::StreamTable; use databend_common_users::Object; use databend_common_users::UserApiProvider; diff --git a/src/query/storages/system/src/table_functions_table.rs b/src/query/storages/system/src/table_functions_table.rs index 15c7ba1e0ceea..bb10cc4f3894e 100644 --- a/src/query/storages/system/src/table_functions_table.rs +++ b/src/query/storages/system/src/table_functions_table.rs @@ -17,13 +17,13 @@ use std::sync::Arc; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::StringType; -use databend_common_expression::utils::FromData; use databend_common_expression::DataBlock; use databend_common_expression::FunctionKind; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::StringType; +use databend_common_expression::utils::FromData; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; diff --git a/src/query/storages/system/src/tables_table.rs b/src/query/storages/system/src/tables_table.rs index dd4a5d2039472..eacedbf563dcb 100644 --- a/src/query/storages/system/src/tables_table.rs +++ b/src/query/storages/system/src/tables_table.rs @@ -25,15 +25,6 @@ use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::filter_helper::FilterHelpers; -use databend_common_expression::type_check::check_number; -use databend_common_expression::type_check::check_string; -use databend_common_expression::types::number::UInt64Type; -use databend_common_expression::types::BooleanType; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::TimestampType; -use databend_common_expression::utils::FromData; use databend_common_expression::Constant; use databend_common_expression::DataBlock; use databend_common_expression::Expr; @@ -42,19 +33,28 @@ use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::filter_helper::FilterHelpers; +use databend_common_expression::type_check::check_number; +use databend_common_expression::type_check::check_string; +use databend_common_expression::types::BooleanType; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::TimestampType; +use databend_common_expression::types::number::UInt64Type; +use databend_common_expression::utils::FromData; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_management::RoleApi; use databend_common_meta_app::principal::OwnershipObject; -use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_app::schema::CatalogInfo; use databend_common_meta_app::schema::CatalogNameIdent; use databend_common_meta_app::schema::CatalogType; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; +use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_app::tenant::Tenant; -use databend_common_storages_basic::view_table::QUERY; use databend_common_storages_basic::NullTable; +use databend_common_storages_basic::view_table::QUERY; use databend_common_storages_fuse::FuseTable; use databend_common_users::Object; use databend_common_users::UserApiProvider; @@ -116,7 +116,7 @@ macro_rules! impl_history_aware { CatalogType::Default => { return Err(ErrorCode::Internal( "Catalog type is Default, can not mock table".to_string(), - )) + )); } CatalogType::Iceberg => "Iceberg".to_string(), CatalogType::Hive => "Hive".to_string(), diff --git a/src/query/storages/system/src/task_history_table.rs b/src/query/storages/system/src/task_history_table.rs index ad3b2b9fa7730..aed315ae7f5f3 100644 --- a/src/query/storages/system/src/task_history_table.rs +++ b/src/query/storages/system/src/task_history_table.rs @@ -24,6 +24,9 @@ use databend_common_cloud_control::pb::TaskRun; use databend_common_config::GlobalConfig; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::DataBlock; +use databend_common_expression::FromData; +use databend_common_expression::Scalar; use databend_common_expression::date_helper::DateConverter; use databend_common_expression::infer_table_schema; use databend_common_expression::types::Int32Type; @@ -32,9 +35,6 @@ use databend_common_expression::types::StringType; use databend_common_expression::types::TimestampType; use databend_common_expression::types::UInt64Type; use databend_common_expression::types::VariantType; -use databend_common_expression::DataBlock; -use databend_common_expression::FromData; -use databend_common_expression::Scalar; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; diff --git a/src/query/storages/system/src/tasks_table.rs b/src/query/storages/system/src/tasks_table.rs index e9340254f5550..02c6bec4fd4b6 100644 --- a/src/query/storages/system/src/tasks_table.rs +++ b/src/query/storages/system/src/tasks_table.rs @@ -25,13 +25,13 @@ use databend_common_cloud_control::task_utils::Task; use databend_common_config::GlobalConfig; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::DataBlock; +use databend_common_expression::FromData; use databend_common_expression::infer_table_schema; use databend_common_expression::types::StringType; use databend_common_expression::types::TimestampType; use databend_common_expression::types::UInt64Type; use databend_common_expression::types::VariantType; -use databend_common_expression::DataBlock; -use databend_common_expression::FromData; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; diff --git a/src/query/storages/system/src/temp_files_table.rs b/src/query/storages/system/src/temp_files_table.rs index 70cbd5f2cd383..ff6f6f3726394 100644 --- a/src/query/storages/system/src/temp_files_table.rs +++ b/src/query/storages/system/src/temp_files_table.rs @@ -25,10 +25,6 @@ use databend_common_catalog::table::DistributionLevel; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::NumberType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::TimestampType; use databend_common_expression::BlockEntry; use databend_common_expression::DataBlock; use databend_common_expression::FromData; @@ -36,6 +32,10 @@ use databend_common_expression::SendableDataBlockStream; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::NumberType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::TimestampType; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; @@ -45,14 +45,14 @@ use databend_common_pipeline::core::ProcessorPtr; use databend_common_pipeline::sources::EmptySource; use databend_common_pipeline::sources::StreamSource; use databend_common_storage::DataOperator; +use futures::StreamExt; use futures::stream; use futures::stream::Chunks; use futures::stream::Take; -use futures::StreamExt; -use opendal::operator_futures::FutureLister; use opendal::Lister; use opendal::Metadata; use opendal::Operator; +use opendal::options::ListOptions; use crate::table::SystemTablePart; @@ -152,22 +152,35 @@ impl TempFilesTable { let limit = push_downs.as_ref().and_then(|x| x.limit); let operator = DataOperator::instance().spill_operator(); - let lister = operator.lister_with(&location_prefix).recursive(true); + let lister = { + let op = operator.clone(); + let path = location_prefix.clone(); + async move { + op.lister_options(&path, ListOptions { + recursive: true, + ..Default::default() + }) + .await + } + }; let stream = { let prefix = location_prefix.clone(); let mut counter = 0; let ctx = ctx.clone(); - let builder = ListerStreamSourceBuilder::with_lister_fut(operator, lister); - builder - .limit_opt(limit) - .chunk_size(MAX_BATCH_SIZE) - .build(move |entries| { + + stream_source_from_entry_lister_with_chunk_size( + operator.clone(), + lister, + limit, + MAX_BATCH_SIZE, + move |entries| { counter += entries.len(); let block = Self::block_from_entries(&prefix, entries)?; ctx.set_status_info(format!("{} entries processed", counter).as_str()); Ok(block) - })? + }, + )? }; StreamSource::create(ctx.get_scan_progress(), Some(stream), output) @@ -202,8 +215,11 @@ impl TempFilesTable { if metadata.is_file() { temp_files_name.push(path.trim_start_matches(location_prefix).to_string()); - temp_files_last_modified - .push(metadata.last_modified().map(|x| x.timestamp_micros())); + temp_files_last_modified.push( + metadata + .last_modified() + .map(|x| x.into_inner().as_microsecond()), + ); temp_files_content_length.push(metadata.content_length()); } } @@ -219,66 +235,18 @@ impl TempFilesTable { const MAX_BATCH_SIZE: usize = 1000; -pub struct ListerStreamSourceBuilder -where T: Future> + Send + 'static -{ - op: Operator, - lister_fut: FutureLister, - limit: Option, - chunk_size: usize, -} - -impl ListerStreamSourceBuilder -where T: Future> + Send + 'static -{ - pub fn with_lister_fut(op: Operator, lister_fut: FutureLister) -> Self { - Self { - op, - lister_fut, - limit: None, - chunk_size: MAX_BATCH_SIZE, - } - } - - pub fn limit_opt(mut self, limit: Option) -> Self { - self.limit = limit; - self - } - - pub fn chunk_size(mut self, chunk_size: usize) -> Self { - self.chunk_size = chunk_size; - self - } - - pub fn build( - self, - block_builder: (impl FnMut(Vec<(String, Metadata)>) -> Result - + Sync - + Send - + 'static), - ) -> Result { - stream_source_from_entry_lister_with_chunk_size( - self.op.clone(), - self.lister_fut, - self.limit, - self.chunk_size, - block_builder, - ) - } -} - fn stream_source_from_entry_lister_with_chunk_size( op: Operator, - lister_fut: FutureLister, + lister_fut: T, limit: Option, chunk_size: usize, - block_builder: (impl FnMut(Vec<(String, Metadata)>) -> Result + Sync + Send + 'static), + block_builder: impl FnMut(Vec<(String, Metadata)>) -> Result + Sync + Send + 'static, ) -> Result where T: Future> + Send + 'static, { enum ListerState> + Send + 'static> { - Uninitialized(FutureLister), + Uninitialized(U), Initialized(Chunks>), } diff --git a/src/query/storages/system/src/user_functions_table.rs b/src/query/storages/system/src/user_functions_table.rs index 56daf82f71214..425f8d950892b 100644 --- a/src/query/storages/system/src/user_functions_table.rs +++ b/src/query/storages/system/src/user_functions_table.rs @@ -21,15 +21,15 @@ use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; +use databend_common_expression::DataBlock; +use databend_common_expression::TableDataType; +use databend_common_expression::TableField; +use databend_common_expression::TableSchemaRefExt; use databend_common_expression::types::BooleanType; use databend_common_expression::types::StringType; use databend_common_expression::types::TimestampType; use databend_common_expression::types::VariantType; use databend_common_expression::utils::FromData; -use databend_common_expression::DataBlock; -use databend_common_expression::TableDataType; -use databend_common_expression::TableField; -use databend_common_expression::TableSchemaRefExt; use databend_common_meta_app::principal::UDFDefinition; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; diff --git a/src/query/storages/system/src/users_table.rs b/src/query/storages/system/src/users_table.rs index 62c9891976c8e..76dce0899e737 100644 --- a/src/query/storages/system/src/users_table.rs +++ b/src/query/storages/system/src/users_table.rs @@ -19,21 +19,21 @@ use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::BooleanType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::TimestampType; -use databend_common_expression::utils::FromData; use databend_common_expression::DataBlock; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::BooleanType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::TimestampType; +use databend_common_expression::utils::FromData; use databend_common_management::WorkloadApi; use databend_common_management::WorkloadMgr; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; -use databend_common_users::UserApiProvider; use databend_common_users::BUILTIN_ROLE_ACCOUNT_ADMIN; +use databend_common_users::UserApiProvider; use itertools::Itertools; use crate::table::AsyncOneBlockSystemTable; diff --git a/src/query/storages/system/src/util.rs b/src/query/storages/system/src/util.rs index d26838b9bbacd..b29f37ddeb910 100644 --- a/src/query/storages/system/src/util.rs +++ b/src/query/storages/system/src/util.rs @@ -14,11 +14,11 @@ use databend_common_catalog::catalog_kind::CATALOG_DEFAULT; use databend_common_exception::Result; +use databend_common_expression::FunctionContext; +use databend_common_expression::Scalar; use databend_common_expression::expr::*; use databend_common_expression::filter_helper::FilterHelpers; use databend_common_expression::type_check::check_string; -use databend_common_expression::FunctionContext; -use databend_common_expression::Scalar; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::schema::CatalogMeta; use databend_common_meta_app::schema::CatalogOption; @@ -52,9 +52,14 @@ pub fn find_gt_filter(expr: &Expr, visitor: &mut impl FnMut(&str, &Scala Expr::FunctionCall(FunctionCall { function, args, .. }) => { if function.signature.name == "gt" || function.signature.name == "gte" { match args.as_slice() { - [Expr::ColumnRef(ColumnRef { id, .. }), Expr::Constant(Constant { scalar, .. })] - | [Expr::Constant(Constant { scalar, .. }), Expr::ColumnRef(ColumnRef { id, .. })] => - { + [ + Expr::ColumnRef(ColumnRef { id, .. }), + Expr::Constant(Constant { scalar, .. }), + ] + | [ + Expr::Constant(Constant { scalar, .. }), + Expr::ColumnRef(ColumnRef { id, .. }), + ] => { visitor(id, scalar); } _ => {} @@ -83,9 +88,14 @@ pub fn find_lt_filter(expr: &Expr, visitor: &mut impl FnMut(&str, &Scala Expr::FunctionCall(FunctionCall { function, args, .. }) => { if function.signature.name == "lt" || function.signature.name == "lte" { match args.as_slice() { - [Expr::ColumnRef(ColumnRef { id, .. }), Expr::Constant(Constant { scalar, .. })] - | [Expr::Constant(Constant { scalar, .. }), Expr::ColumnRef(ColumnRef { id, .. })] => - { + [ + Expr::ColumnRef(ColumnRef { id, .. }), + Expr::Constant(Constant { scalar, .. }), + ] + | [ + Expr::Constant(Constant { scalar, .. }), + Expr::ColumnRef(ColumnRef { id, .. }), + ] => { visitor(id, scalar); } _ => {} diff --git a/src/query/storages/system/src/virtual_columns_table.rs b/src/query/storages/system/src/virtual_columns_table.rs index a0cff56204c6f..6100fd5169900 100644 --- a/src/query/storages/system/src/virtual_columns_table.rs +++ b/src/query/storages/system/src/virtual_columns_table.rs @@ -18,14 +18,14 @@ use databend_common_catalog::catalog::CatalogManager; use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table::Table; use databend_common_exception::Result; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::StringType; -use databend_common_expression::types::UInt32Type; use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::StringType; +use databend_common_expression::types::UInt32Type; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; diff --git a/src/query/storages/system/src/zero_table.rs b/src/query/storages/system/src/zero_table.rs index bfcb28653dcd6..c011893ff9551 100644 --- a/src/query/storages/system/src/zero_table.rs +++ b/src/query/storages/system/src/zero_table.rs @@ -21,14 +21,14 @@ use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table::Table; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; -use databend_common_expression::types::number::UInt8Type; -use databend_common_expression::types::NumberDataType; -use databend_common_expression::utils::FromData; use databend_common_expression::BlockEntry; use databend_common_expression::DataBlock; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::number::UInt8Type; +use databend_common_expression::utils::FromData; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; diff --git a/src/query/users/benches/visibility_checker.rs b/src/query/users/benches/visibility_checker.rs index 9fb70e1525f96..1112c9e5a9874 100644 --- a/src/query/users/benches/visibility_checker.rs +++ b/src/query/users/benches/visibility_checker.rs @@ -17,9 +17,9 @@ use databend_common_meta_app::principal::OwnershipObject; use databend_common_meta_app::principal::RoleInfo; use databend_common_meta_app::principal::UserInfo; use databend_common_users::GrantObjectVisibilityChecker; -use rand::rngs::StdRng; use rand::Rng; use rand::SeedableRng; +use rand::rngs::StdRng; fn main() { divan::main(); @@ -55,7 +55,7 @@ fn create_realistic_test_data( let mut tables_per_db = vec![0usize; num_databases]; for _ in 0..total_tables { // Use power-law distribution: top 20% DBs contain 80% tables - let db_idx = if rng.gen::() < 0.8 { + let db_idx = if rng.r#gen::() < 0.8 { // 80% of tables go to top 20% of DBs rng.gen_range(0..(num_databases / 5).max(1)) } else { diff --git a/src/query/users/src/lib.rs b/src/query/users/src/lib.rs index b9ab1360313c6..d119b49059880 100644 --- a/src/query/users/src/lib.rs +++ b/src/query/users/src/lib.rs @@ -12,8 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![feature(let_chains)] -#![allow(clippy::uninlined_format_args)] +#![allow( + clippy::cloned_ref_to_slice_refs, + clippy::collapsible_if, + clippy::iter_kv_map, + clippy::uninlined_format_args +)] extern crate core; diff --git a/src/query/users/src/role_cache_mgr.rs b/src/query/users/src/role_cache_mgr.rs index ce77721b3de9c..a2c8a61d6efd7 100644 --- a/src/query/users/src/role_cache_mgr.rs +++ b/src/query/users/src/role_cache_mgr.rs @@ -17,9 +17,9 @@ use std::sync::Arc; use std::time::Duration; use std::time::Instant; +use databend_common_base::base::GlobalInstance; use databend_common_base::base::tokio; use databend_common_base::base::tokio::task::JoinHandle; -use databend_common_base::base::GlobalInstance; use databend_common_exception::Result; use databend_common_meta_app::principal::OwnershipObject; use databend_common_meta_app::principal::RoleInfo; @@ -27,8 +27,8 @@ use databend_common_meta_app::tenant::Tenant; use log::warn; use parking_lot::RwLock; -use crate::role_util::find_all_related_roles; use crate::UserApiProvider; +use crate::role_util::find_all_related_roles; struct CachedRoles { roles: HashMap, diff --git a/src/query/users/src/role_mgr.rs b/src/query/users/src/role_mgr.rs index 43c8fe87cc8f9..bd65b41a59dc7 100644 --- a/src/query/users/src/role_mgr.rs +++ b/src/query/users/src/role_mgr.rs @@ -18,18 +18,18 @@ use databend_common_exception::ErrorCode; use databend_common_exception::ErrorCodeResultExt; use databend_common_exception::Result; use databend_common_management::RoleApi; +use databend_common_meta_app::principal::BUILTIN_ROLE_ACCOUNT_ADMIN; use databend_common_meta_app::principal::GrantObject; use databend_common_meta_app::principal::OwnershipInfo; use databend_common_meta_app::principal::OwnershipObject; use databend_common_meta_app::principal::RoleInfo; use databend_common_meta_app::principal::UserPrivilegeSet; -use databend_common_meta_app::principal::BUILTIN_ROLE_ACCOUNT_ADMIN; use databend_common_meta_app::schema::CreateOption; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_types::MatchSeq; -use crate::role_util::find_all_related_roles; use crate::UserApiProvider; +use crate::role_util::find_all_related_roles; impl UserApiProvider { // Get one role from by tenant. diff --git a/src/query/users/src/user_api.rs b/src/query/users/src/user_api.rs index 5c57a26c3edea..ff394ac4e74d5 100644 --- a/src/query/users/src/user_api.rs +++ b/src/query/users/src/user_api.rs @@ -16,15 +16,13 @@ use std::collections::HashMap; use std::ops::Deref; use std::sync::Arc; -use databend_common_base::base::tokio::sync::Mutex; use databend_common_base::base::GlobalInstance; +use databend_common_base::base::tokio::sync::Mutex; use databend_common_config::CacheConfig; use databend_common_config::GlobalConfig; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_grpc::RpcClientConf; -use databend_common_management::task::TaskMgr; -use databend_common_management::udf::UdfMgr; use databend_common_management::ClientSessionMgr; use databend_common_management::ConnectionMgr; use databend_common_management::FileFormatMgr; @@ -39,6 +37,8 @@ use databend_common_management::StageApi; use databend_common_management::StageMgr; use databend_common_management::UserApi; use databend_common_management::UserMgr; +use databend_common_management::task::TaskMgr; +use databend_common_management::udf::UdfMgr; use databend_common_meta_app::principal::AuthInfo; use databend_common_meta_app::principal::RoleInfo; use databend_common_meta_app::principal::UserDefinedFunction; @@ -53,8 +53,8 @@ use databend_common_meta_types::MatchSeq; use databend_common_meta_types::MetaError; use log::debug; -use crate::builtin::BuiltIn; use crate::BUILTIN_ROLE_PUBLIC; +use crate::builtin::BuiltIn; pub struct UserApiProvider { meta: MetaStore, diff --git a/src/query/users/src/user_mgr.rs b/src/query/users/src/user_mgr.rs index 3c4b70cfc2f35..a04463838df6e 100644 --- a/src/query/users/src/user_mgr.rs +++ b/src/query/users/src/user_mgr.rs @@ -30,8 +30,8 @@ use databend_common_meta_app::tenant::Tenant; use databend_common_meta_types::MatchSeq; use databend_common_meta_types::SeqV; -use crate::UserApiProvider; use crate::BUILTIN_ROLE_ACCOUNT_ADMIN; +use crate::UserApiProvider; impl UserApiProvider { // Get one user from by tenant. diff --git a/src/query/users/tests/it/jwt/authenticator.rs b/src/query/users/tests/it/jwt/authenticator.rs index 46cd9968f42b2..d4dfb2d0c0f52 100644 --- a/src/query/users/tests/it/jwt/authenticator.rs +++ b/src/query/users/tests/it/jwt/authenticator.rs @@ -23,11 +23,11 @@ use databend_common_users::EnsureUser; use databend_common_users::JwtAuthenticator; use databend_common_version::BUILD_INFO; use jwt_simple::prelude::*; -use wiremock::matchers::method; -use wiremock::matchers::path; use wiremock::Mock; use wiremock::MockServer; use wiremock::ResponseTemplate; +use wiremock::matchers::method; +use wiremock::matchers::path; #[derive(Serialize, Deserialize)] struct MyAdditionalData { diff --git a/src/query/users/tests/it/role_cache_mgr.rs b/src/query/users/tests/it/role_cache_mgr.rs index 83fa66a7c62f7..43adcf8cb0907 100644 --- a/src/query/users/tests/it/role_cache_mgr.rs +++ b/src/query/users/tests/it/role_cache_mgr.rs @@ -25,10 +25,10 @@ use databend_common_meta_app::principal::RoleInfo; use databend_common_meta_app::principal::UserPrivilegeSet; use databend_common_meta_app::schema::CreateOption; use databend_common_meta_app::tenant::Tenant; -use databend_common_users::role_util::find_all_related_roles; +use databend_common_users::BUILTIN_ROLE_PUBLIC; use databend_common_users::RoleCacheManager; use databend_common_users::UserApiProvider; -use databend_common_users::BUILTIN_ROLE_PUBLIC; +use databend_common_users::role_util::find_all_related_roles; use databend_common_version::BUILD_INFO; pub const CATALOG_DEFAULT: &str = "default"; diff --git a/src/query/users/tests/it/role_mgr.rs b/src/query/users/tests/it/role_mgr.rs index 269488d192cc4..08077d9de6d62 100644 --- a/src/query/users/tests/it/role_mgr.rs +++ b/src/query/users/tests/it/role_mgr.rs @@ -26,9 +26,9 @@ use databend_common_meta_app::schema::CreateOption::Create; use databend_common_meta_app::schema::CreateOption::CreateIfNotExists; use databend_common_meta_app::schema::CreateOption::CreateOrReplace; use databend_common_meta_app::tenant::Tenant; -use databend_common_users::UserApiProvider; use databend_common_users::BUILTIN_ROLE_ACCOUNT_ADMIN; use databend_common_users::BUILTIN_ROLE_PUBLIC; +use databend_common_users::UserApiProvider; use databend_common_version::BUILD_INFO; use pretty_assertions::assert_eq; @@ -106,9 +106,10 @@ async fn test_role_manager() -> Result<()> { ) .await?; let role = role_mgr.get_role(&tenant, role_name.clone()).await?; - assert!(role - .grants - .verify_privilege(&GrantObject::Global, UserPrivilegeType::Alter)); + assert!( + role.grants + .verify_privilege(&GrantObject::Global, UserPrivilegeType::Alter) + ); } // revoke privilege from role diff --git a/src/query/users/tests/it/role_util.rs b/src/query/users/tests/it/role_util.rs index 6c03fe46e86e8..1be9b510e73b3 100644 --- a/src/query/users/tests/it/role_util.rs +++ b/src/query/users/tests/it/role_util.rs @@ -17,8 +17,8 @@ use std::collections::HashMap; use databend_common_config::GlobalConfig; use databend_common_config::InnerConfig; use databend_common_meta_app::principal::RoleInfo; -use databend_common_users::role_util::find_all_related_roles; use databend_common_users::BUILTIN_ROLE_ACCOUNT_ADMIN; +use databend_common_users::role_util::find_all_related_roles; use databend_common_version::BUILD_INFO; #[test] diff --git a/src/query/users/tests/it/user_mgr.rs b/src/query/users/tests/it/user_mgr.rs index 1342e84c773b4..00347bb68d500 100644 --- a/src/query/users/tests/it/user_mgr.rs +++ b/src/query/users/tests/it/user_mgr.rs @@ -140,12 +140,16 @@ async fn test_user_manager() -> Result<()> { .grant_privileges_to_user(&tenant, user_info.identity(), GrantObject::Global, add_priv) .await?; let new_user = user_mgr.get_user(&tenant, user_info.identity()).await?; - assert!(new_user - .grants - .verify_privilege(&GrantObject::Global, UserPrivilegeType::Set)); - assert!(!new_user - .grants - .verify_privilege(&GrantObject::Global, UserPrivilegeType::Create)); + assert!( + new_user + .grants + .verify_privilege(&GrantObject::Global, UserPrivilegeType::Set) + ); + assert!( + !new_user + .grants + .verify_privilege(&GrantObject::Global, UserPrivilegeType::Create) + ); user_mgr .drop_user(&tenant, new_user.identity(), true) .await?; diff --git a/src/tests/sqlsmith/src/http_client.rs b/src/tests/sqlsmith/src/http_client.rs index 911623bccb5db..cd3422b496311 100644 --- a/src/tests/sqlsmith/src/http_client.rs +++ b/src/tests/sqlsmith/src/http_client.rs @@ -21,11 +21,11 @@ use std::time::Duration; use cookie::Cookie; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use reqwest::Client; +use reqwest::ClientBuilder; use reqwest::cookie::CookieStore; use reqwest::header::HeaderMap; use reqwest::header::HeaderValue; -use reqwest::Client; -use reqwest::ClientBuilder; use serde::Deserialize; use serde::Serialize; use url::Url; diff --git a/src/tests/sqlsmith/src/lib.rs b/src/tests/sqlsmith/src/lib.rs index ff87a3b175e45..bf5473a7a0894 100644 --- a/src/tests/sqlsmith/src/lib.rs +++ b/src/tests/sqlsmith/src/lib.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![allow(clippy::collapsible_if)] #![feature(box_patterns)] mod http_client; diff --git a/src/tests/sqlsmith/src/query_fuzzer.rs b/src/tests/sqlsmith/src/query_fuzzer.rs index 7ac9d8ee79b59..9fe653b074bae 100644 --- a/src/tests/sqlsmith/src/query_fuzzer.rs +++ b/src/tests/sqlsmith/src/query_fuzzer.rs @@ -14,6 +14,8 @@ use std::collections::HashMap; +use Expr::TimeSlice; +use databend_common_ast::Span; use databend_common_ast::ast::BinaryOperator; use databend_common_ast::ast::ColumnRef; use databend_common_ast::ast::Expr; @@ -33,16 +35,14 @@ use databend_common_ast::ast::WindowDesc; use databend_common_ast::ast::WindowFrame; use databend_common_ast::ast::WindowFrameBound; use databend_common_ast::ast::WindowFrameUnits; -use databend_common_ast::Span; use derive_visitor::Drive; use derive_visitor::DriveMut; use derive_visitor::Visitor; use derive_visitor::VisitorMut; use ethnum::I256; -use rand::rngs::SmallRng; use rand::Rng; use rand::SeedableRng; -use Expr::TimeSlice; +use rand::rngs::SmallRng; const BAD_INT64_VALUES: [i64; 31] = [ -2, diff --git a/src/tests/sqlsmith/src/runner.rs b/src/tests/sqlsmith/src/runner.rs index 3d6351238a239..67507fc23c33b 100644 --- a/src/tests/sqlsmith/src/runner.rs +++ b/src/tests/sqlsmith/src/runner.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::fs::create_dir_all; use std::fs::File; +use std::fs::create_dir_all; use std::future::Future; use std::io::BufWriter; use std::io::Write; @@ -26,21 +26,21 @@ use databend_common_ast::ast::DropTableStmt; use databend_common_ast::ast::DropViewStmt; use databend_common_ast::ast::Query; use databend_common_ast::ast::Statement; +use databend_common_ast::parser::Dialect; use databend_common_ast::parser::parse_sql; use databend_common_ast::parser::tokenize_sql; -use databend_common_ast::parser::Dialect; use databend_common_exception::Result; +use databend_common_expression::TableField; +use databend_common_expression::TableSchemaRefExt; use databend_common_expression::infer_schema_type; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; -use databend_common_expression::TableField; -use databend_common_expression::TableSchemaRefExt; use databend_common_sql::resolve_type_name; -use jiff::fmt::strtime; use jiff::Zoned; -use rand::rngs::SmallRng; +use jiff::fmt::strtime; use rand::Rng; use rand::SeedableRng; +use rand::rngs::SmallRng; use crate::http_client::HttpClient; use crate::query_fuzzer::QueryFuzzer; diff --git a/src/tests/sqlsmith/src/sql_gen/ddl.rs b/src/tests/sqlsmith/src/sql_gen/ddl.rs index 30e48c367f7ac..6a645fcbe8c0e 100644 --- a/src/tests/sqlsmith/src/sql_gen/ddl.rs +++ b/src/tests/sqlsmith/src/sql_gen/ddl.rs @@ -31,8 +31,8 @@ use databend_common_ast::ast::TableType; use databend_common_ast::ast::TypeName; use databend_common_exception::ErrorCode; use databend_common_expression::types::DataType; -use rand::distributions::Alphanumeric; use rand::Rng; +use rand::distributions::Alphanumeric; use crate::sql_gen::SqlGenerator; diff --git a/src/tests/sqlsmith/src/sql_gen/dml.rs b/src/tests/sqlsmith/src/sql_gen/dml.rs index 2fcd3656ead4f..7eac5da3a4029 100644 --- a/src/tests/sqlsmith/src/sql_gen/dml.rs +++ b/src/tests/sqlsmith/src/sql_gen/dml.rs @@ -15,6 +15,7 @@ use std::sync::Arc; use chrono_tz::Tz; +use databend_common_ast::Span; use databend_common_ast::ast::AddColumnOption; use databend_common_ast::ast::AlterTableAction; use databend_common_ast::ast::AlterTableStmt; @@ -37,13 +38,12 @@ use databend_common_ast::ast::Statement; use databend_common_ast::ast::TableReference; use databend_common_ast::ast::UnmatchedClause; use databend_common_ast::ast::UpdateStmt; -use databend_common_ast::Span; -use databend_common_expression::types::DataType; use databend_common_expression::Column; use databend_common_expression::ScalarRef; use databend_common_expression::TableField; -use databend_common_formats::field_encoder::FieldEncoderValues; +use databend_common_expression::types::DataType; use databend_common_formats::OutputCommonSettings; +use databend_common_formats::field_encoder::FieldEncoderValues; use databend_common_io::constants::FALSE_BYTES_LOWER; use databend_common_io::constants::INF_BYTES_LOWER; use databend_common_io::constants::NAN_BYTES_LOWER; diff --git a/src/tests/sqlsmith/src/sql_gen/expr.rs b/src/tests/sqlsmith/src/sql_gen/expr.rs index 96e12902e0f77..71a26ab293323 100644 --- a/src/tests/sqlsmith/src/sql_gen/expr.rs +++ b/src/tests/sqlsmith/src/sql_gen/expr.rs @@ -25,12 +25,12 @@ use databend_common_ast::ast::MapAccessor; use databend_common_ast::ast::SubqueryModifier; use databend_common_ast::ast::TrimWhere; use databend_common_ast::ast::UnaryOperator; -use databend_common_expression::types::convert_to_type_name; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::convert_to_type_name; use ethnum::I256; -use rand::distributions::Alphanumeric; use rand::Rng; +use rand::distributions::Alphanumeric; use crate::sql_gen::SqlGenerator; diff --git a/src/tests/sqlsmith/src/sql_gen/func.rs b/src/tests/sqlsmith/src/sql_gen/func.rs index 07dc5335eed31..bdf2f6f28f206 100644 --- a/src/tests/sqlsmith/src/sql_gen/func.rs +++ b/src/tests/sqlsmith/src/sql_gen/func.rs @@ -27,11 +27,11 @@ use databend_common_ast::ast::WindowFrameBound; use databend_common_ast::ast::WindowFrameUnits; use databend_common_ast::ast::WindowRef; use databend_common_ast::ast::WindowSpec; +use databend_common_expression::types::ALL_FLOAT_TYPES; +use databend_common_expression::types::ALL_INTEGER_TYPES; use databend_common_expression::types::DataType; use databend_common_expression::types::DecimalSize; use databend_common_expression::types::NumberDataType; -use databend_common_expression::types::ALL_FLOAT_TYPES; -use databend_common_expression::types::ALL_INTEGER_TYPES; use databend_common_functions::RANK_WINDOW_FUNCTIONS; use rand::Rng; diff --git a/src/tests/sqlsmith/src/sql_gen/query.rs b/src/tests/sqlsmith/src/sql_gen/query.rs index 694bf3a304023..acf9fb7cb5803 100644 --- a/src/tests/sqlsmith/src/sql_gen/query.rs +++ b/src/tests/sqlsmith/src/sql_gen/query.rs @@ -14,6 +14,7 @@ use std::mem; +use databend_common_ast::ast::CTE; use databend_common_ast::ast::Expr; use databend_common_ast::ast::FunctionCall; use databend_common_ast::ast::GroupBy; @@ -31,14 +32,13 @@ use databend_common_ast::ast::TableAlias; use databend_common_ast::ast::TableReference; use databend_common_ast::ast::WindowDefinition; use databend_common_ast::ast::With; -use databend_common_ast::ast::CTE; -use databend_common_expression::infer_schema_type; -use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDataType; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::infer_schema_type; +use databend_common_expression::types::DataType; +use databend_common_expression::types::NumberDataType; use rand::Rng; use crate::sql_gen::Column; diff --git a/src/tests/sqlsmith/src/sql_gen/sql_generator.rs b/src/tests/sqlsmith/src/sql_gen/sql_generator.rs index 66488c30b070a..5f9f48e063a71 100644 --- a/src/tests/sqlsmith/src/sql_gen/sql_generator.rs +++ b/src/tests/sqlsmith/src/sql_gen/sql_generator.rs @@ -14,9 +14,9 @@ use databend_common_ast::ast::GroupBy; use databend_common_ast::ast::Identifier; -use databend_common_expression::types::DataType; use databend_common_expression::FunctionSignature; use databend_common_expression::TableSchemaRef; +use databend_common_expression::types::DataType; use databend_common_functions::BUILTIN_FUNCTIONS; use rand::Rng; diff --git a/src/tests/sqlsmith/src/sql_gen/types.rs b/src/tests/sqlsmith/src/sql_gen/types.rs index 923e5f72dbfd0..0dd138d528565 100644 --- a/src/tests/sqlsmith/src/sql_gen/types.rs +++ b/src/tests/sqlsmith/src/sql_gen/types.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_expression::types::decimal::DecimalSize; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::decimal::DecimalSize; use rand::Rng; use crate::sql_gen::SqlGenerator; diff --git a/tests/sqllogictests/Cargo.toml b/tests/sqllogictests/Cargo.toml index 0b18c7eb2110c..8da56c70f5ffe 100644 --- a/tests/sqllogictests/Cargo.toml +++ b/tests/sqllogictests/Cargo.toml @@ -1,14 +1,15 @@ [package] -authors = ["Databend Authors "] categories = ["command-line-utilities", "development-tools::build-utils"] description = "Databend-Sqllogictests is a testing tool to verify the correctness of Databend." -edition = "2021" keywords = ["sqllogictest"] -license = "Apache-2.0" name = "databend-sqllogictests" + +version = { workspace = true } +authors = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +edition = { workspace = true } readme = "README.md" -repository = "https://github.com/datafuselabs/databend/" -version = "0.1.0" [[bin]] name = "databend-sqllogictests" diff --git a/tests/sqllogictests/src/client/http_client.rs b/tests/sqllogictests/src/client/http_client.rs index f88b39c04f4d4..515d815e15472 100644 --- a/tests/sqllogictests/src/client/http_client.rs +++ b/tests/sqllogictests/src/client/http_client.rs @@ -18,18 +18,18 @@ use std::time::Duration; use std::time::Instant; use regex::Regex; -use reqwest::header::HeaderMap; -use reqwest::header::HeaderValue; use reqwest::Client; use reqwest::ClientBuilder; +use reqwest::header::HeaderMap; +use reqwest::header::HeaderValue; use serde::Deserialize; use sqllogictest::DBOutput; use crate::client::global_cookie_store::GlobalCookieStore; use crate::error::Result; -use crate::util::parser_rows; use crate::util::ColumnType; use crate::util::HttpSessionConf; +use crate::util::parser_rows; pub struct HttpClient { pub client: Client, diff --git a/tests/sqllogictests/src/client/mod.rs b/tests/sqllogictests/src/client/mod.rs index fdb065f25ee81..154260847826f 100644 --- a/tests/sqllogictests/src/client/mod.rs +++ b/tests/sqllogictests/src/client/mod.rs @@ -22,8 +22,8 @@ use std::fmt; pub use http_client::HttpClient; pub use mysql_client::MySQLClient; -use rand::distributions::Alphanumeric; use rand::Rng; +use rand::distributions::Alphanumeric; use regex::Regex; use sqllogictest::DBOutput; pub use ttc_client::TTCClient; diff --git a/tests/sqllogictests/src/client/mysql_client.rs b/tests/sqllogictests/src/client/mysql_client.rs index 3a557942d86d1..594875f6e8b49 100644 --- a/tests/sqllogictests/src/client/mysql_client.rs +++ b/tests/sqllogictests/src/client/mysql_client.rs @@ -14,10 +14,10 @@ use std::time::Instant; -use mysql_async::prelude::Queryable; use mysql_async::Conn; use mysql_async::Pool; use mysql_async::Row; +use mysql_async::prelude::Queryable; use sqllogictest::DBOutput; use crate::error::Result; diff --git a/tests/sqllogictests/src/error.rs b/tests/sqllogictests/src/error.rs index acb626b56c5ff..3f305c056568e 100644 --- a/tests/sqllogictests/src/error.rs +++ b/tests/sqllogictests/src/error.rs @@ -12,14 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::fmt::Display; use std::io::Error as IOError; use databend_common_exception::ErrorCode; use mysql_async::Error as MysqlClientError; use reqwest::Error as HttpClientError; -use serde::Deserialize; -use serde::Serialize; use serde_json::Error as SerdeJsonError; use sqllogictest::TestError; use testcontainers::core::error::ClientError; @@ -28,26 +25,6 @@ use walkdir::Error as WalkDirError; pub type Result = std::result::Result; -#[derive(Debug, Serialize, Deserialize)] -struct Inner { - code: String, - message: String, -} -#[derive(Debug, Serialize, Deserialize, Error)] -pub struct TextError { - error: Inner, -} - -impl Display for TextError { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!( - f, - ". Code: {}, Text = {}.", - self.error.code, self.error.message, - ) - } -} - #[derive(Debug, Error)] pub enum DSqlLogicTestError { // Error from sqllogictest-rs diff --git a/tests/sqllogictests/src/main.rs b/tests/sqllogictests/src/main.rs index 2a1adbc85d2c3..6ea947c4334fa 100644 --- a/tests/sqllogictests/src/main.rs +++ b/tests/sqllogictests/src/main.rs @@ -19,18 +19,18 @@ use std::time::Instant; use clap::Parser; use client::TTCClient; -use futures_util::stream; use futures_util::StreamExt; +use futures_util::stream; use rand::Rng; -use sqllogictest::default_column_validator; -use sqllogictest::default_validator; -use sqllogictest::parse_file; use sqllogictest::DBOutput; use sqllogictest::Location; use sqllogictest::QueryExpect; use sqllogictest::Record; use sqllogictest::Runner; use sqllogictest::TestError; +use sqllogictest::default_column_validator; +use sqllogictest::default_validator; +use sqllogictest::parse_file; use testcontainers::ContainerAsync; use testcontainers::GenericImage; use testcontainers::Image; @@ -43,12 +43,12 @@ use crate::client::HttpClient; use crate::client::MySQLClient; use crate::error::DSqlLogicTestError; use crate::error::Result; +use crate::util::ColumnType; use crate::util::collect_lazy_dir; use crate::util::get_files; use crate::util::lazy_prepare_data; use crate::util::lazy_run_dictionary_containers; use crate::util::run_ttc_container; -use crate::util::ColumnType; mod arg; mod client; @@ -308,15 +308,15 @@ async fn run_suits(args: SqlLogicTestArgs, client_type: ClientType) -> Result<() if !file_name.ends_with(".test") { continue; } - if let Some(ref specific_file) = args.file { - if !specific_file.split(',').any(|f| f.eq(&file_name)) { - continue; - } + if let Some(ref specific_file) = args.file + && !specific_file.split(',').any(|f| f.eq(&file_name)) + { + continue; } - if let Some(ref skip_file) = args.skipped_file { - if skip_file.split(',').any(|f| f.eq(&file_name)) { - continue; - } + if let Some(ref skip_file) = args.skipped_file + && skip_file.split(',').any(|f| f.eq(&file_name)) + { + continue; } num_of_tests += parse_file::(suit_file.as_ref().unwrap().path()) .unwrap() diff --git a/tests/sqllogictests/src/util.rs b/tests/sqllogictests/src/util.rs index ad0991a2ccad0..826743af0d070 100644 --- a/tests/sqllogictests/src/util.rs +++ b/tests/sqllogictests/src/util.rs @@ -19,25 +19,25 @@ use std::path::PathBuf; use std::time::Duration; use std::time::Instant; +use bollard::Docker; use bollard::container::ListContainersOptions; use bollard::container::RemoveContainerOptions; -use bollard::Docker; use clap::Parser; use redis::Commands; use serde::Deserialize; use serde::Serialize; use serde_json::Value; -use testcontainers::core::client::docker_client_instance; -use testcontainers::core::logs::consumer::logging_consumer::LoggingConsumer; -use testcontainers::core::IntoContainerPort; -use testcontainers::core::WaitFor; -use testcontainers::runners::AsyncRunner; use testcontainers::ContainerAsync; use testcontainers::GenericImage; use testcontainers::ImageExt; +use testcontainers::core::IntoContainerPort; +use testcontainers::core::WaitFor; +use testcontainers::core::client::docker_client_instance; +use testcontainers::core::logs::consumer::logging_consumer::LoggingConsumer; +use testcontainers::runners::AsyncRunner; use testcontainers_modules::mysql::Mysql; -use testcontainers_modules::redis::Redis; use testcontainers_modules::redis::REDIS_PORT; +use testcontainers_modules::redis::Redis; use walkdir::DirEntry; use walkdir::WalkDir; @@ -50,12 +50,6 @@ const CONTAINER_RETRY_TIMES: usize = 3; const CONTAINER_STARTUP_TIMEOUT_SECONDS: u64 = 60; const CONTAINER_TIMEOUT_SECONDS: u64 = 300; -#[derive(Deserialize, Serialize, Debug, Clone, PartialEq, Eq)] -pub struct ServerInfo { - pub id: String, - pub start_time: String, -} - #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] pub struct HttpSessionConf { pub catalog: Option, @@ -262,8 +256,8 @@ pub async fn run_ttc_container( let image = images.next().unwrap(); let tag = images.next().unwrap_or("latest"); - use rand::distributions::Alphanumeric; use rand::Rng; + use rand::distributions::Alphanumeric; let rng = rand::thread_rng(); let x: String = rng .sample_iter(&Alphanumeric) @@ -476,16 +470,16 @@ async fn stop_container(docker: &Docker, container_name: &str) { ..Default::default() }); let containers = docker.list_containers(opts).await; - if let Ok(containers) = containers { - if !containers.is_empty() { - println!("==> list containers"); - for container in containers { - if let Some(names) = container.names { - println!( - " -> container name: {:?}, status: {:?}", - names, container.state - ); - } + if let Ok(containers) = containers + && !containers.is_empty() + { + println!("==> list containers"); + for container in containers { + if let Some(names) = container.names { + println!( + " -> container name: {:?}, status: {:?}", + names, container.state + ); } } }