chore(query): refactor common/arrow removes arrow/io crates (#16808)

* refactor native engine

* update

* update

* [ci skip] update

* [ci skip] update

* remove ffi

* remove ffi

* remove ffi

* fix init nested

* fix init nested

* fix stat

Author: sundy-li

Cargo.lock

@@ -27,7 +27,6 @@ members = [
     "src/common/storage",
     "src/common/vector",
     "src/common/license",
-    "src/common/parquet2",
     "src/query/ast",
     "src/query/codegen",
     "src/query/config",
@@ -140,7 +139,6 @@ databend-common-meta-store = { path = "src/meta/store" }
 databend-common-meta-types = { path = "src/meta/types" }
 databend-common-metrics = { path = "src/common/metrics" }
 databend-common-openai = { path = "src/common/openai" }
-databend-common-parquet2 = { path = "src/common/parquet2" }
 databend-common-pipeline-core = { path = "src/query/pipeline/core" }
 databend-common-pipeline-sinks = { path = "src/query/pipeline/sinks" }
 databend-common-pipeline-sources = { path = "src/query/pipeline/sources" }
@@ -196,12 +194,6 @@ databend-storages-common-session = { path = "src/query/storages/common/session"
 databend-storages-common-stage = { path = "src/query/storages/common/stage" }
 databend-storages-common-table-meta = { path = "src/query/storages/common/table_meta" }
 
-# Specific dependencies
-parquet2 = { package = "databend-common-parquet2", path = "src/common/parquet2", default-features = false, features = [
-    "serde_types",
-    "async",
-] }
-
 # Crates.io dependencies
 ahash = "0.8"
 aho-corasick = { version = "1.0.1" } #
@@ -214,7 +206,6 @@ arrow-buffer = { version = "53" }
 arrow-cast = { version = "53", features = ["prettyprint"] }
 arrow-data = { version = "53" }
 arrow-flight = { version = "53", features = ["flight-sql-experimental", "tls"] }
-arrow-format = { version = "0.8.1", features = ["flight-data", "flight-service", "ipc"] }
 arrow-ipc = { version = "53" }
 arrow-ord = { version = "53" }
 arrow-schema = { version = "53", features = ["serde"] }
@@ -612,7 +603,6 @@ overflow-checks = true
 rpath = false
 
 [patch.crates-io]
-arrow-format = { git = "https://github.com/Xuanwo/arrow-format", rev = "5502823a" }
 async-backtrace = { git = "https://github.com/datafuse-extras/async-backtrace.git", rev = "dea4553" }
 async-recursion = { git = "https://github.com/datafuse-extras/async-recursion.git", rev = "a353334" }
 backtrace = { git = "https://github.com/rust-lang/backtrace-rs.git", rev = "72265be", features = [

Cargo.toml

@@ -11,87 +11,33 @@ doctest = false
 test = true
 
 [features]
-default = ["arrow-default", "parquet-default"]
+default = ["arrow-default"]
 
 arrow = ["arrow-buffer", "arrow-schema", "arrow-data", "arrow-array"]
-io_flight = ["io_ipc", "arrow-format/flight-data"]
-io_ipc = []
-io_ipc_compression = []
-
-# base64 + io_ipc because arrow schemas are stored as base64-encoded ipc format.
-io_parquet = ["io_ipc", "base64", "streaming-iterator", "fallible-streaming-iterator"]
-io_parquet_async = ["futures", "io_parquet", "parquet2/async"]
-
-io_parquet_compression = [
-    "io_parquet_zstd",
-    "io_parquet_gzip",
-    "io_parquet_snappy",
-    "io_parquet_lz4",
-    "io_parquet_brotli",
-]
 
 # sample testing of generated arrow data
-io_parquet_sample_test = ["io_parquet_async"]
-
-# compression backends
-io_parquet_brotli = ["parquet2/brotli"]
-io_parquet_gzip = ["parquet2/gzip"]
-io_parquet_lz4 = ["parquet2/lz4"]
-io_parquet_snappy = ["parquet2/snappy"]
-io_parquet_zstd = ["parquet2/zstd"]
-
-# parquet bloom filter functions
-io_parquet_bloom_filter = ["parquet2/bloom_filter"]
 
 compute = [
-    "compute_aggregate",
-    "compute_cast",
     "compute_concatenate",
-    "compute_merge_sort",
-    "compute_sort",
-    "compute_take",
 ]
-compute_aggregate = []
-compute_cast = ["lexical-core", "compute_take"]
 compute_concatenate = []
-compute_merge_sort = ["itertools", "compute_sort"]
-compute_sort = ["compute_take"]
-compute_take = []
 
 serde_types = ["serde", "serde_derive"]
 simd = []
 
 arrow-default = [
     "arrow",
-    "io_ipc",
-    "io_ipc_compression",
-    "io_flight",
-    "io_parquet_async",
-    "io_parquet_compression",
-    "io_parquet",
     "compute",
     "serde_types",
     "simd",
 ]
 
-parquet-default = [
-    "parquet2/lz4",
-    "parquet2/zstd",
-    "parquet2/snappy",
-    # this feature can't be built in musl
-    # "parquet2/gzip_zlib_ng",
-    "parquet2/brotli",
-]
-
 [dependencies]
 ahash = { workspace = true }
 arrow-array = { workspace = true, optional = true }
 arrow-buffer = { workspace = true, optional = true }
 arrow-data = { workspace = true, optional = true }
-arrow-format = { workspace = true }
 arrow-schema = { workspace = true, optional = true }
-async-stream = { workspace = true, optional = true }
-base64 = { workspace = true, optional = true }
 bitpacking = { workspace = true }
 bytemuck = { workspace = true }
 byteorder = { workspace = true }
@@ -101,28 +47,23 @@ chrono-tz = { workspace = true, optional = true }
 dyn-clone = { workspace = true }
 either = { workspace = true }
 ethnum = { workspace = true }
-fallible-streaming-iterator = { workspace = true, optional = true }
 foreign_vec = { workspace = true }
-futures = { workspace = true, optional = true }
 hashbrown_v0_14 = { workspace = true }
 indexmap = { workspace = true }
-itertools = { workspace = true, optional = true }
-lexical-core = { workspace = true, optional = true }
 log = { workspace = true }
 lz4 = { workspace = true }
 num = { workspace = true, features = ["std"] }
 num-traits = { workspace = true }
 opendal = { workspace = true }
 ordered-float = { workspace = true }
-parquet2 = { workspace = true }
 rand = { workspace = true }
 ringbuffer = { workspace = true }
 roaring = { workspace = true }
 serde = { workspace = true, features = ["rc"], optional = true }
 serde_derive = { workspace = true, optional = true }
+serde_json = { workspace = true }
 simdutf8 = { workspace = true }
 snap = { workspace = true }
-streaming-iterator = { workspace = true, optional = true }
 zstd = { workspace = true }
 
 [dev-dependencies]