Skip to content

Commit 7cff135

Browse files
b41shb41sh
authored
feat(query): virtual column allow cast to other type (#16903)
* feat(query): virtual column allow cast to other type * add tests * fix typos * fix
1 parent 06c4ab6 commit 7cff135

File tree

30 files changed

+866
-454
lines changed

30 files changed

+866
-454
lines changed

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/meta/api/src/schema_api_test_suite.rs‎

Lines changed: 174 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6416,7 +6416,26 @@ impl SchemaApiTestSuite {
64166416
let req = CreateVirtualColumnReq {
64176417
create_option: CreateOption::Create,
64186418
name_ident: name_ident.clone(),
6419-
virtual_columns: vec!["variant:k1".to_string(), "variant[1]".to_string()],
6419+
virtual_columns: vec![
6420+
(
6421+
"variant:k1".to_string(),
6422+
TableDataType::Nullable(Box::new(TableDataType::Variant)),
6423+
),
6424+
(
6425+
"variant[1]".to_string(),
6426+
TableDataType::Nullable(Box::new(TableDataType::Variant)),
6427+
),
6428+
(
6429+
"variant:k1:k2".to_string(),
6430+
TableDataType::Nullable(Box::new(TableDataType::String)),
6431+
),
6432+
(
6433+
"variant:k1:k3".to_string(),
6434+
TableDataType::Nullable(Box::new(TableDataType::Number(
6435+
NumberDataType::UInt64,
6436+
))),
6437+
),
6438+
],
64206439
};
64216440

64226441
mt.create_virtual_column(req.clone()).await?;
@@ -6425,7 +6444,26 @@ impl SchemaApiTestSuite {
64256444
let req = CreateVirtualColumnReq {
64266445
create_option: CreateOption::Create,
64276446
name_ident: name_ident.clone(),
6428-
virtual_columns: vec!["variant:k1".to_string(), "variant[1]".to_string()],
6447+
virtual_columns: vec![
6448+
(
6449+
"variant:k1".to_string(),
6450+
TableDataType::Nullable(Box::new(TableDataType::Variant)),
6451+
),
6452+
(
6453+
"variant[1]".to_string(),
6454+
TableDataType::Nullable(Box::new(TableDataType::Variant)),
6455+
),
6456+
(
6457+
"variant:k1:k2".to_string(),
6458+
TableDataType::Nullable(Box::new(TableDataType::String)),
6459+
),
6460+
(
6461+
"variant:k1:k3".to_string(),
6462+
TableDataType::Nullable(Box::new(TableDataType::Number(
6463+
NumberDataType::UInt64,
6464+
))),
6465+
),
6466+
],
64296467
};
64306468

64316469
let res = mt.create_virtual_column(req).await;
@@ -6439,8 +6477,24 @@ impl SchemaApiTestSuite {
64396477
let res = mt.list_virtual_columns(req).await?;
64406478
assert_eq!(1, res.len());
64416479
assert_eq!(res[0].virtual_columns, vec![
6442-
"variant:k1".to_string(),
6443-
"variant[1]".to_string(),
6480+
(
6481+
"variant:k1".to_string(),
6482+
TableDataType::Nullable(Box::new(TableDataType::Variant))
6483+
),
6484+
(
6485+
"variant[1]".to_string(),
6486+
TableDataType::Nullable(Box::new(TableDataType::Variant))
6487+
),
6488+
(
6489+
"variant:k1:k2".to_string(),
6490+
TableDataType::Nullable(Box::new(TableDataType::String)),
6491+
),
6492+
(
6493+
"variant:k1:k3".to_string(),
6494+
TableDataType::Nullable(Box::new(TableDataType::Number(
6495+
NumberDataType::UInt64
6496+
))),
6497+
),
64446498
]);
64456499

64466500
let req = ListVirtualColumnsReq::new(&tenant, Some(u64::MAX));
@@ -6454,7 +6508,26 @@ impl SchemaApiTestSuite {
64546508
let req = UpdateVirtualColumnReq {
64556509
if_exists: false,
64566510
name_ident: name_ident.clone(),
6457-
virtual_columns: vec!["variant:k2".to_string(), "variant[2]".to_string()],
6511+
virtual_columns: vec![
6512+
(
6513+
"variant:k2".to_string(),
6514+
TableDataType::Nullable(Box::new(TableDataType::Variant)),
6515+
),
6516+
(
6517+
"variant[2]".to_string(),
6518+
TableDataType::Nullable(Box::new(TableDataType::Variant)),
6519+
),
6520+
(
6521+
"variant:k2:k3".to_string(),
6522+
TableDataType::Nullable(Box::new(TableDataType::String)),
6523+
),
6524+
(
6525+
"variant:k2:k4".to_string(),
6526+
TableDataType::Nullable(Box::new(TableDataType::Number(
6527+
NumberDataType::UInt64,
6528+
))),
6529+
),
6530+
],
64586531
};
64596532

64606533
mt.update_virtual_column(req).await?;
@@ -6467,8 +6540,24 @@ impl SchemaApiTestSuite {
64676540
let res = mt.list_virtual_columns(req).await?;
64686541
assert_eq!(1, res.len());
64696542
assert_eq!(res[0].virtual_columns, vec![
6470-
"variant:k2".to_string(),
6471-
"variant[2]".to_string(),
6543+
(
6544+
"variant:k2".to_string(),
6545+
TableDataType::Nullable(Box::new(TableDataType::Variant))
6546+
),
6547+
(
6548+
"variant[2]".to_string(),
6549+
TableDataType::Nullable(Box::new(TableDataType::Variant))
6550+
),
6551+
(
6552+
"variant:k2:k3".to_string(),
6553+
TableDataType::Nullable(Box::new(TableDataType::String)),
6554+
),
6555+
(
6556+
"variant:k2:k4".to_string(),
6557+
TableDataType::Nullable(Box::new(TableDataType::Number(
6558+
NumberDataType::UInt64
6559+
))),
6560+
),
64726561
]);
64736562
}
64746563

@@ -6495,7 +6584,26 @@ impl SchemaApiTestSuite {
64956584
let req = UpdateVirtualColumnReq {
64966585
if_exists: false,
64976586
name_ident: name_ident.clone(),
6498-
virtual_columns: vec!["variant:k3".to_string(), "variant[3]".to_string()],
6587+
virtual_columns: vec![
6588+
(
6589+
"variant:k3".to_string(),
6590+
TableDataType::Nullable(Box::new(TableDataType::Variant)),
6591+
),
6592+
(
6593+
"variant[3]".to_string(),
6594+
TableDataType::Nullable(Box::new(TableDataType::Variant)),
6595+
),
6596+
(
6597+
"variant:k3:k4".to_string(),
6598+
TableDataType::Nullable(Box::new(TableDataType::String)),
6599+
),
6600+
(
6601+
"variant:k3:k5".to_string(),
6602+
TableDataType::Nullable(Box::new(TableDataType::Number(
6603+
NumberDataType::UInt64,
6604+
))),
6605+
),
6606+
],
64996607
};
65006608

65016609
let res = mt.update_virtual_column(req).await;
@@ -6507,7 +6615,26 @@ impl SchemaApiTestSuite {
65076615
let req = CreateVirtualColumnReq {
65086616
create_option: CreateOption::Create,
65096617
name_ident: name_ident.clone(),
6510-
virtual_columns: vec!["variant:k1".to_string(), "variant[1]".to_string()],
6618+
virtual_columns: vec![
6619+
(
6620+
"variant:k1".to_string(),
6621+
TableDataType::Nullable(Box::new(TableDataType::Variant)),
6622+
),
6623+
(
6624+
"variant[1]".to_string(),
6625+
TableDataType::Nullable(Box::new(TableDataType::Variant)),
6626+
),
6627+
(
6628+
"variant:k1:k4".to_string(),
6629+
TableDataType::Nullable(Box::new(TableDataType::String)),
6630+
),
6631+
(
6632+
"variant:k1:k5".to_string(),
6633+
TableDataType::Nullable(Box::new(TableDataType::Number(
6634+
NumberDataType::UInt64,
6635+
))),
6636+
),
6637+
],
65116638
};
65126639

65136640
mt.create_virtual_column(req.clone()).await?;
@@ -6517,14 +6644,39 @@ impl SchemaApiTestSuite {
65176644
let res = mt.list_virtual_columns(req).await?;
65186645
assert_eq!(1, res.len());
65196646
assert_eq!(res[0].virtual_columns, vec![
6520-
"variant:k1".to_string(),
6521-
"variant[1]".to_string(),
6647+
(
6648+
"variant:k1".to_string(),
6649+
TableDataType::Nullable(Box::new(TableDataType::Variant))
6650+
),
6651+
(
6652+
"variant[1]".to_string(),
6653+
TableDataType::Nullable(Box::new(TableDataType::Variant))
6654+
),
6655+
(
6656+
"variant:k1:k4".to_string(),
6657+
TableDataType::Nullable(Box::new(TableDataType::String)),
6658+
),
6659+
(
6660+
"variant:k1:k5".to_string(),
6661+
TableDataType::Nullable(Box::new(TableDataType::Number(
6662+
NumberDataType::UInt64
6663+
))),
6664+
),
65226665
]);
65236666

65246667
let req = CreateVirtualColumnReq {
65256668
create_option: CreateOption::CreateOrReplace,
65266669
name_ident: name_ident.clone(),
6527-
virtual_columns: vec!["variant:k2".to_string()],
6670+
virtual_columns: vec![
6671+
(
6672+
"variant:k2".to_string(),
6673+
TableDataType::Nullable(Box::new(TableDataType::Variant)),
6674+
),
6675+
(
6676+
"variant:k3".to_string(),
6677+
TableDataType::Nullable(Box::new(TableDataType::String)),
6678+
),
6679+
],
65286680
};
65296681

65306682
mt.create_virtual_column(req.clone()).await?;
@@ -6533,7 +6685,16 @@ impl SchemaApiTestSuite {
65336685

65346686
let res = mt.list_virtual_columns(req).await?;
65356687
assert_eq!(1, res.len());
6536-
assert_eq!(res[0].virtual_columns, vec!["variant:k2".to_string(),]);
6688+
assert_eq!(res[0].virtual_columns, vec![
6689+
(
6690+
"variant:k2".to_string(),
6691+
TableDataType::Nullable(Box::new(TableDataType::Variant))
6692+
),
6693+
(
6694+
"variant:k3".to_string(),
6695+
TableDataType::Nullable(Box::new(TableDataType::String)),
6696+
)
6697+
]);
65376698
}
65386699

65396700
Ok(())

‎src/meta/app/src/schema/virtual_column.rs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ use std::fmt::Formatter;
1818

1919
use chrono::DateTime;
2020
use chrono::Utc;
21+
use databend_common_expression::TableDataType;
2122
use databend_common_meta_types::MetaId;
2223

2324
use super::CreateOption;
@@ -29,7 +30,7 @@ use crate::tenant::ToTenant;
2930
pub struct VirtualColumnMeta {
3031
pub table_id: MetaId,
3132

32-
pub virtual_columns: Vec<String>,
33+
pub virtual_columns: Vec<(String, TableDataType)>,
3334
pub created_on: DateTime<Utc>,
3435
pub updated_on: Option<DateTime<Utc>>,
3536
}
@@ -38,7 +39,7 @@ pub struct VirtualColumnMeta {
3839
pub struct CreateVirtualColumnReq {
3940
pub create_option: CreateOption,
4041
pub name_ident: VirtualColumnIdent,
41-
pub virtual_columns: Vec<String>,
42+
pub virtual_columns: Vec<(String, TableDataType)>,
4243
}
4344

4445
impl Display for CreateVirtualColumnReq {
@@ -56,7 +57,7 @@ impl Display for CreateVirtualColumnReq {
5657
pub struct UpdateVirtualColumnReq {
5758
pub if_exists: bool,
5859
pub name_ident: VirtualColumnIdent,
59-
pub virtual_columns: Vec<String>,
60+
pub virtual_columns: Vec<(String, TableDataType)>,
6061
}
6162

6263
impl Display for UpdateVirtualColumnReq {

‎src/meta/proto-conv/src/util.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,7 @@ const META_CHANGE_LOG: &[(u64, &str)] = &[
141141
(109, "2024-08-29: Refactor: ProcedureMeta add arg_names"),
142142
(110, "2024-09-18: Add: database.proto: DatabaseMeta.gc_in_progress"),
143143
(111, "2024-11-13: Add: Enable AWS Glue as an Apache Iceberg type when creating a catalog."),
144+
(112, "2024-11-28: Add: virtual_column add data_types field"),
144145
// Dear developer:
145146
// If you're gonna add a new metadata version, you'll have to add a test for it.
146147
// You could just copy an existing test file(e.g., `../tests/it/v024_table_meta.rs`)

‎src/meta/proto-conv/src/virtual_column_from_to_protobuf_impl.rs‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
1818
use chrono::DateTime;
1919
use chrono::Utc;
20+
use databend_common_expression::TableDataType;
2021
use databend_common_meta_app::schema as mt;
2122
use databend_common_protos::pb;
2223

@@ -36,10 +37,36 @@ impl FromToProto for mt::VirtualColumnMeta {
3637
fn from_pb(p: Self::PB) -> Result<Self, Incompatible>
3738
where Self: Sized {
3839
reader_check_msg(p.ver, p.min_reader_ver)?;
40+
let virtual_columns = if p.data_types.is_empty() {
41+
p.virtual_columns
42+
.iter()
43+
.map(|v| {
44+
(
45+
v.clone(),
46+
TableDataType::Nullable(Box::new(TableDataType::Variant)),
47+
)
48+
})
49+
.collect()
50+
} else {
51+
if p.virtual_columns.len() != p.data_types.len() {
52+
return Err(Incompatible {
53+
reason: format!(
54+
"Incompatible virtual columns length is {}, but data types length is {}",
55+
p.virtual_columns.len(),
56+
p.data_types.len()
57+
),
58+
});
59+
}
60+
let mut virtual_columns = Vec::new();
61+
for (v, ty) in p.virtual_columns.iter().zip(p.data_types.iter()) {
62+
virtual_columns.push((v.clone(), TableDataType::from_pb(ty.clone())?));
63+
}
64+
virtual_columns
65+
};
3966

4067
let v = Self {
4168
table_id: p.table_id,
42-
virtual_columns: p.virtual_columns,
69+
virtual_columns,
4370
created_on: DateTime::<Utc>::from_pb(p.created_on)?,
4471
updated_on: match p.updated_on {
4572
Some(updated_on) => Some(DateTime::<Utc>::from_pb(updated_on)?),
@@ -50,16 +77,23 @@ impl FromToProto for mt::VirtualColumnMeta {
5077
}
5178

5279
fn to_pb(&self) -> Result<Self::PB, Incompatible> {
80+
let mut data_types = Vec::new();
81+
let mut virtual_columns = Vec::new();
82+
for (v, ty) in self.virtual_columns.iter() {
83+
data_types.push(ty.to_pb()?);
84+
virtual_columns.push(v.clone());
85+
}
5386
let p = pb::VirtualColumnMeta {
5487
ver: VER,
5588
min_reader_ver: MIN_READER_VER,
5689
table_id: self.table_id,
57-
virtual_columns: self.virtual_columns.clone(),
90+
virtual_columns,
5891
created_on: self.created_on.to_pb()?,
5992
updated_on: match self.updated_on {
6093
Some(updated_on) => Some(updated_on.to_pb()?),
6194
None => None,
6295
},
96+
data_types,
6397
};
6498
Ok(p)
6599
}

‎src/meta/proto-conv/tests/it/main.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,3 +108,5 @@ mod v107_geography_datatype;
108108
mod v108_procedure;
109109
mod v109_procedure_with_args;
110110
mod v110_database_meta_gc_in_progress;
111+
mod v111_add_glue_as_iceberg_catalog_option;
112+
mod v112_virtual_column;

0 commit comments

Comments
 (0)