Skip to content

Commit 003b6d4

Browse files
committed
catalog: add pg_type.typsend and type typreceive as regproc
The Apache Arrow ADBC PostgreSQL driver runs a type-resolution query at connect time that references pg_catalog.pg_type.typsend. Materialize did not expose that column, so every adbc_driver_postgresql connection failed with `column "typsend" does not exist` before any user query ran. Carry the real PostgreSQL typsend OID per builtin type, verified against a live PostgreSQL 16 server for all 76 types that have a PG counterpart. Nine types get 0, matching PostgreSQL, which gives them no binary send function. _mz_aclitem is Materialize-invented and takes array_send by analogy with every other array type. typsend is appended to mz_internal.mz_type_pg_metadata, then projected through mz_internal.pg_type_all_databases and pg_catalog.pg_type. The new columns are appended rather than placed at PostgreSQL's ordinal positions, so existing column positions stay stable. pg_catalog.pg_type.typreceive also becomes regproc, which is what PostgreSQL declares and what lets a client's typreceive::TEXT comparison resolve a function name instead of digits. The underlying all-databases view keeps oid, because it backs a builtin index and resolving a regproc to a name reads current_database(), which is not materializable. Adds a test/adbc mzcompose suite covering the driver end to end. Connecting and streaming rows are hard assertions. Full Arrow type fidelity is not yet reachable, because Materialize encodes regproc over pgwire as a bare OID rather than as the function name, so the driver's name-keyed type map misses and every column resolves to Arrow binary. Those tests are marked expectedFailure so they report an unexpected success once that encoding is fixed.
1 parent 9346f8d commit 003b6d4

16 files changed

Lines changed: 439 additions & 8 deletions

File tree

‎ci/test/pipeline.template.yml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -760,6 +760,17 @@ steps:
760760
agents:
761761
queue: hetzner-aarch64-4cpu-8gb
762762

763+
- id: adbc
764+
label: Arrow ADBC driver
765+
depends_on: build-aarch64
766+
timeout_in_minutes: 20
767+
inputs: [test/adbc]
768+
plugins:
769+
- ./ci/plugins/mzcompose:
770+
composition: adbc
771+
agents:
772+
queue: hetzner-aarch64-4cpu-8gb
773+
763774
- id: chbench-demo
764775
topics: [debezium, kafka, mysql]
765776
label: chbench smoke

‎console/types/materialize.d.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4004,6 +4004,7 @@ export interface MzTypePgMetadata {
40044004
id: Generated<string>;
40054005
typinput: Generated<number>;
40064006
typreceive: Generated<number>;
4007+
typsend: Generated<number | null>;
40074008
}
40084009

40094010
export interface MzTypes {
@@ -4245,6 +4246,7 @@ export interface PgTypeAllDatabases {
42454246
typowner: number;
42464247
typreceive: number;
42474248
typrelid: number;
4249+
typsend: string;
42484250
typtype: string;
42494251
typtypmod: number;
42504252
}

‎src/adapter/src/catalog.rs‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2928,6 +2928,7 @@ mod tests {
29282928
array: u32,
29292929
input: u32,
29302930
receive: u32,
2931+
send: u32,
29312932
}
29322933

29332934
struct PgOper {
@@ -2967,7 +2968,7 @@ mod tests {
29672968
let pg_type: BTreeMap<_, _> = query(
29682969
&client,
29692970
sql!(
2970-
"SELECT oid, typname, typtype::text, typelem, typarray, typinput::oid, typreceive::oid as typreceive FROM pg_type"
2971+
"SELECT oid, typname, typtype::text, typelem, typarray, typinput::oid, typreceive::oid as typreceive, typsend::oid as typsend FROM pg_type"
29712972
),
29722973
&[],
29732974
)
@@ -2983,6 +2984,7 @@ mod tests {
29832984
array: row.get("typarray"),
29842985
input: row.get("typinput"),
29852986
receive: row.get("typreceive"),
2987+
send: row.get("typsend"),
29862988
};
29872989
(oid, pg_type)
29882990
})
@@ -3074,10 +3076,15 @@ mod tests {
30743076
ty.oid, pg_ty.name, ty.name,
30753077
);
30763078

3077-
let (typinput_oid, typreceive_oid) = match &ty.details.pg_metadata {
3078-
None => (0, 0),
3079-
Some(pgmeta) => (pgmeta.typinput_oid, pgmeta.typreceive_oid),
3080-
};
3079+
let (typinput_oid, typreceive_oid, typsend_oid) =
3080+
match &ty.details.pg_metadata {
3081+
None => (0, 0, 0),
3082+
Some(pgmeta) => (
3083+
pgmeta.typinput_oid,
3084+
pgmeta.typreceive_oid,
3085+
pgmeta.typsend_oid,
3086+
),
3087+
};
30813088
assert_eq!(
30823089
typinput_oid, pg_ty.input,
30833090
"type {} has typinput OID {:?} in mz but {:?} in pg",
@@ -3088,6 +3095,15 @@ mod tests {
30883095
"type {} has typreceive OID {:?} in mz but {:?} in pg",
30893096
ty.name, typreceive_oid, pg_ty.receive,
30903097
);
3098+
// Unlike typinput and typreceive below, typsend is not also
3099+
// checked against `func_oids`. Nothing resolves a typsend OID
3100+
// to a name, so the corresponding `*send` functions are
3101+
// deliberately not registered as builtins.
3102+
assert_eq!(
3103+
typsend_oid, pg_ty.send,
3104+
"type {} has typsend OID {:?} in mz but {:?} in pg",
3105+
ty.name, typsend_oid, pg_ty.send,
3106+
);
30913107
if typinput_oid != 0 {
30923108
assert!(
30933109
func_oids.contains(&typinput_oid),

‎src/adapter/src/catalog/builtin_table_updates.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1185,6 +1185,7 @@ impl CatalogState {
11851185
Datum::String(&id.to_string()),
11861186
Datum::UInt32(pg_metadata.typinput_oid),
11871187
Datum::UInt32(pg_metadata.typreceive_oid),
1188+
Datum::UInt32(pg_metadata.typsend_oid),
11881189
]),
11891190
diff,
11901191
));

‎src/adapter/src/catalog/open/builtin_schema_migration.rs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -301,6 +301,16 @@ static MIGRATIONS: LazyLock<Vec<MigrationStep>> = LazyLock::new(|| {
301301
MZ_CATALOG_SCHEMA,
302302
"mz_kafka_sources",
303303
),
304+
// `mz_type_pg_metadata` gained a trailing `typsend` column. Appending a
305+
// column is backward compatible, so the shard's schema can be evolved
306+
// in place. See the NOTE above: this version must stay at the
307+
// workspace's current dev version until the change ships.
308+
MigrationStep::evolution(
309+
"26.36.0-dev.0",
310+
CatalogItemType::Table,
311+
MZ_INTERNAL_SCHEMA,
312+
"mz_type_pg_metadata",
313+
),
304314
]
305315
});
306316

‎src/catalog/src/builtin/mz_catalog.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,7 @@ pub const TYPE_MZ_ACL_ITEM_ARRAY: BuiltinType<NameReference> = BuiltinType {
194194
pg_metadata: Some(CatalogTypePgMetadata {
195195
typinput_oid: 750,
196196
typreceive_oid: 2400,
197+
typsend_oid: 2401,
197198
}),
198199
},
199200
};

‎src/catalog/src/builtin/mz_internal.rs‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -586,6 +586,9 @@ pub static MZ_TYPE_PG_METADATA: LazyLock<BuiltinTable> = LazyLock::new(|| Builti
586586
.with_column("id", SqlScalarType::String.nullable(false))
587587
.with_column("typinput", SqlScalarType::Oid.nullable(false))
588588
.with_column("typreceive", SqlScalarType::Oid.nullable(false))
589+
// Always populated, but declared nullable because persist schema
590+
// evolution only accepts an appended field if it is nullable.
591+
.with_column("typsend", SqlScalarType::Oid.nullable(true))
589592
.finish(),
590593
column_comments: BTreeMap::new(),
591594
is_retained_metrics_object: false,
@@ -4372,6 +4375,9 @@ pub static PG_TYPE_ALL_DATABASES: LazyLock<BuiltinView> = LazyLock::new(|| {
43724375
.with_column("typcollation", SqlScalarType::Oid.nullable(false))
43734376
.with_column("typdefault", SqlScalarType::String.nullable(true))
43744377
.with_column("database_name", SqlScalarType::String.nullable(true))
4378+
// Appended rather than placed at PostgreSQL's ordinal position, to
4379+
// keep the existing columns' positions stable.
4380+
.with_column("typsend", SqlScalarType::RegProc.nullable(false))
43754381
.finish(),
43764382
column_comments: BTreeMap::new(),
43774383
sql: "
@@ -4440,7 +4446,8 @@ SELECT
44404446
-- MZ doesn't support COLLATE so typcollation is filled with 0
44414447
0::pg_catalog.oid AS typcollation,
44424448
NULL::pg_catalog.text AS typdefault,
4443-
d.name as database_name
4449+
d.name as database_name,
4450+
COALESCE(mz_internal.mz_type_pg_metadata.typsend, 0)::pg_catalog.regproc AS typsend
44444451
FROM
44454452
mz_catalog.mz_types
44464453
LEFT JOIN mz_internal.mz_type_pg_metadata ON mz_catalog.mz_types.id = mz_internal.mz_type_pg_metadata.id

0 commit comments

Comments
 (0)