From 003b6d4b14ec532b80947bbc00fa92010c032626 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 17:04:16 +0000 Subject: [PATCH 01/15] catalog: add pg_type.typsend and type typreceive as regproc The Apache Arrow ADBC PostgreSQL driver runs a type-resolution query at connect time that references pg_catalog.pg_type.typsend. Materialize did not expose that column, so every adbc_driver_postgresql connection failed with `column "typsend" does not exist` before any user query ran. Carry the real PostgreSQL typsend OID per builtin type, verified against a live PostgreSQL 16 server for all 76 types that have a PG counterpart. Nine types get 0, matching PostgreSQL, which gives them no binary send function. _mz_aclitem is Materialize-invented and takes array_send by analogy with every other array type. typsend is appended to mz_internal.mz_type_pg_metadata, then projected through mz_internal.pg_type_all_databases and pg_catalog.pg_type. The new columns are appended rather than placed at PostgreSQL's ordinal positions, so existing column positions stay stable. pg_catalog.pg_type.typreceive also becomes regproc, which is what PostgreSQL declares and what lets a client's typreceive::TEXT comparison resolve a function name instead of digits. The underlying all-databases view keeps oid, because it backs a builtin index and resolving a regproc to a name reads current_database(), which is not materializable. Adds a test/adbc mzcompose suite covering the driver end to end. Connecting and streaming rows are hard assertions. Full Arrow type fidelity is not yet reachable, because Materialize encodes regproc over pgwire as a bare OID rather than as the function name, so the driver's name-keyed type map misses and every column resolves to Arrow binary. Those tests are marked expectedFailure so they report an unexpected success once that encoding is fixed. --- ci/test/pipeline.template.yml | 11 + console/types/materialize.d.ts | 2 + src/adapter/src/catalog.rs | 26 ++- .../src/catalog/builtin_table_updates.rs | 1 + .../catalog/open/builtin_schema_migration.rs | 10 + src/catalog/src/builtin/mz_catalog.rs | 1 + src/catalog/src/builtin/mz_internal.rs | 9 +- src/catalog/src/builtin/pg_catalog.rs | 89 +++++++- src/sql/src/catalog.rs | 3 + test/adbc/mzcompose | 14 ++ test/adbc/mzcompose.py | 38 ++++ test/adbc/requirements.txt | 4 + test/adbc/smoketest.py | 212 ++++++++++++++++++ test/adbc/test.sh | 24 ++ .../mz_catalog_server_index_accounting.slt | 2 + .../system_catalog_identifiers.txt | 1 + 16 files changed, 439 insertions(+), 8 deletions(-) create mode 100755 test/adbc/mzcompose create mode 100644 test/adbc/mzcompose.py create mode 100644 test/adbc/requirements.txt create mode 100644 test/adbc/smoketest.py create mode 100755 test/adbc/test.sh diff --git a/ci/test/pipeline.template.yml b/ci/test/pipeline.template.yml index 9a013e7a8982d..233ad0051918e 100644 --- a/ci/test/pipeline.template.yml +++ b/ci/test/pipeline.template.yml @@ -760,6 +760,17 @@ steps: agents: queue: hetzner-aarch64-4cpu-8gb + - id: adbc + label: Arrow ADBC driver + depends_on: build-aarch64 + timeout_in_minutes: 20 + inputs: [test/adbc] + plugins: + - ./ci/plugins/mzcompose: + composition: adbc + agents: + queue: hetzner-aarch64-4cpu-8gb + - id: chbench-demo topics: [debezium, kafka, mysql] label: chbench smoke diff --git a/console/types/materialize.d.ts b/console/types/materialize.d.ts index 0d3ff49962a22..f9b5513e961f9 100644 --- a/console/types/materialize.d.ts +++ b/console/types/materialize.d.ts @@ -4004,6 +4004,7 @@ export interface MzTypePgMetadata { id: Generated; typinput: Generated; typreceive: Generated; + typsend: Generated; } export interface MzTypes { @@ -4245,6 +4246,7 @@ export interface PgTypeAllDatabases { typowner: number; typreceive: number; typrelid: number; + typsend: string; typtype: string; typtypmod: number; } diff --git a/src/adapter/src/catalog.rs b/src/adapter/src/catalog.rs index 8b3a1f919da38..23b4cd22f36e3 100644 --- a/src/adapter/src/catalog.rs +++ b/src/adapter/src/catalog.rs @@ -2928,6 +2928,7 @@ mod tests { array: u32, input: u32, receive: u32, + send: u32, } struct PgOper { @@ -2967,7 +2968,7 @@ mod tests { let pg_type: BTreeMap<_, _> = query( &client, sql!( - "SELECT oid, typname, typtype::text, typelem, typarray, typinput::oid, typreceive::oid as typreceive FROM pg_type" + "SELECT oid, typname, typtype::text, typelem, typarray, typinput::oid, typreceive::oid as typreceive, typsend::oid as typsend FROM pg_type" ), &[], ) @@ -2983,6 +2984,7 @@ mod tests { array: row.get("typarray"), input: row.get("typinput"), receive: row.get("typreceive"), + send: row.get("typsend"), }; (oid, pg_type) }) @@ -3074,10 +3076,15 @@ mod tests { ty.oid, pg_ty.name, ty.name, ); - let (typinput_oid, typreceive_oid) = match &ty.details.pg_metadata { - None => (0, 0), - Some(pgmeta) => (pgmeta.typinput_oid, pgmeta.typreceive_oid), - }; + let (typinput_oid, typreceive_oid, typsend_oid) = + match &ty.details.pg_metadata { + None => (0, 0, 0), + Some(pgmeta) => ( + pgmeta.typinput_oid, + pgmeta.typreceive_oid, + pgmeta.typsend_oid, + ), + }; assert_eq!( typinput_oid, pg_ty.input, "type {} has typinput OID {:?} in mz but {:?} in pg", @@ -3088,6 +3095,15 @@ mod tests { "type {} has typreceive OID {:?} in mz but {:?} in pg", ty.name, typreceive_oid, pg_ty.receive, ); + // Unlike typinput and typreceive below, typsend is not also + // checked against `func_oids`. Nothing resolves a typsend OID + // to a name, so the corresponding `*send` functions are + // deliberately not registered as builtins. + assert_eq!( + typsend_oid, pg_ty.send, + "type {} has typsend OID {:?} in mz but {:?} in pg", + ty.name, typsend_oid, pg_ty.send, + ); if typinput_oid != 0 { assert!( func_oids.contains(&typinput_oid), diff --git a/src/adapter/src/catalog/builtin_table_updates.rs b/src/adapter/src/catalog/builtin_table_updates.rs index ce1c40aa83108..598a31fd85fdb 100644 --- a/src/adapter/src/catalog/builtin_table_updates.rs +++ b/src/adapter/src/catalog/builtin_table_updates.rs @@ -1185,6 +1185,7 @@ impl CatalogState { Datum::String(&id.to_string()), Datum::UInt32(pg_metadata.typinput_oid), Datum::UInt32(pg_metadata.typreceive_oid), + Datum::UInt32(pg_metadata.typsend_oid), ]), diff, )); diff --git a/src/adapter/src/catalog/open/builtin_schema_migration.rs b/src/adapter/src/catalog/open/builtin_schema_migration.rs index bda72b0eafd1f..af570f530cb14 100644 --- a/src/adapter/src/catalog/open/builtin_schema_migration.rs +++ b/src/adapter/src/catalog/open/builtin_schema_migration.rs @@ -301,6 +301,16 @@ static MIGRATIONS: LazyLock> = LazyLock::new(|| { MZ_CATALOG_SCHEMA, "mz_kafka_sources", ), + // `mz_type_pg_metadata` gained a trailing `typsend` column. Appending a + // column is backward compatible, so the shard's schema can be evolved + // in place. See the NOTE above: this version must stay at the + // workspace's current dev version until the change ships. + MigrationStep::evolution( + "26.36.0-dev.0", + CatalogItemType::Table, + MZ_INTERNAL_SCHEMA, + "mz_type_pg_metadata", + ), ] }); diff --git a/src/catalog/src/builtin/mz_catalog.rs b/src/catalog/src/builtin/mz_catalog.rs index 3de16050b1f21..cc1fb9916a62b 100644 --- a/src/catalog/src/builtin/mz_catalog.rs +++ b/src/catalog/src/builtin/mz_catalog.rs @@ -194,6 +194,7 @@ pub const TYPE_MZ_ACL_ITEM_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; diff --git a/src/catalog/src/builtin/mz_internal.rs b/src/catalog/src/builtin/mz_internal.rs index fcf4cebc553d4..8746dce20bfd9 100644 --- a/src/catalog/src/builtin/mz_internal.rs +++ b/src/catalog/src/builtin/mz_internal.rs @@ -586,6 +586,9 @@ pub static MZ_TYPE_PG_METADATA: LazyLock = LazyLock::new(|| Builti .with_column("id", SqlScalarType::String.nullable(false)) .with_column("typinput", SqlScalarType::Oid.nullable(false)) .with_column("typreceive", SqlScalarType::Oid.nullable(false)) + // Always populated, but declared nullable because persist schema + // evolution only accepts an appended field if it is nullable. + .with_column("typsend", SqlScalarType::Oid.nullable(true)) .finish(), column_comments: BTreeMap::new(), is_retained_metrics_object: false, @@ -4372,6 +4375,9 @@ pub static PG_TYPE_ALL_DATABASES: LazyLock = LazyLock::new(|| { .with_column("typcollation", SqlScalarType::Oid.nullable(false)) .with_column("typdefault", SqlScalarType::String.nullable(true)) .with_column("database_name", SqlScalarType::String.nullable(true)) + // Appended rather than placed at PostgreSQL's ordinal position, to + // keep the existing columns' positions stable. + .with_column("typsend", SqlScalarType::RegProc.nullable(false)) .finish(), column_comments: BTreeMap::new(), sql: " @@ -4440,7 +4446,8 @@ SELECT -- MZ doesn't support COLLATE so typcollation is filled with 0 0::pg_catalog.oid AS typcollation, NULL::pg_catalog.text AS typdefault, - d.name as database_name + d.name as database_name, + COALESCE(mz_internal.mz_type_pg_metadata.typsend, 0)::pg_catalog.regproc AS typsend FROM mz_catalog.mz_types LEFT JOIN mz_internal.mz_type_pg_metadata ON mz_catalog.mz_types.id = mz_internal.mz_type_pg_metadata.id diff --git a/src/catalog/src/builtin/pg_catalog.rs b/src/catalog/src/builtin/pg_catalog.rs index ffc6f2484cf50..809c9344b1f77 100644 --- a/src/catalog/src/builtin/pg_catalog.rs +++ b/src/catalog/src/builtin/pg_catalog.rs @@ -52,6 +52,7 @@ pub const TYPE_BOOL: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 1242, typreceive_oid: 2436, + typsend_oid: 2437, }), }, }; @@ -66,6 +67,7 @@ pub const TYPE_BYTEA: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 1244, typreceive_oid: 2412, + typsend_oid: 2413, }), }, }; @@ -80,6 +82,7 @@ pub const TYPE_INT8: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 460, typreceive_oid: 2408, + typsend_oid: 2409, }), }, }; @@ -94,6 +97,7 @@ pub const TYPE_INT4: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 42, typreceive_oid: 2406, + typsend_oid: 2407, }), }, }; @@ -108,6 +112,7 @@ pub const TYPE_TEXT: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 46, typreceive_oid: 2414, + typsend_oid: 2415, }), }, }; @@ -122,6 +127,7 @@ pub const TYPE_OID: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 1798, typreceive_oid: 2418, + typsend_oid: 2419, }), }, }; @@ -136,6 +142,7 @@ pub const TYPE_FLOAT4: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 200, typreceive_oid: 2424, + typsend_oid: 2425, }), }, }; @@ -150,6 +157,7 @@ pub const TYPE_FLOAT8: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 214, typreceive_oid: 2426, + typsend_oid: 2427, }), }, }; @@ -166,6 +174,7 @@ pub const TYPE_BOOL_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -182,6 +191,7 @@ pub const TYPE_BYTEA_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -198,6 +208,7 @@ pub const TYPE_INT4_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -214,6 +225,7 @@ pub const TYPE_TEXT_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -230,6 +242,7 @@ pub const TYPE_INT8_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -246,6 +259,7 @@ pub const TYPE_FLOAT4_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -262,6 +276,7 @@ pub const TYPE_FLOAT8_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -278,6 +293,7 @@ pub const TYPE_OID_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -292,6 +308,7 @@ pub const TYPE_DATE: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 1084, typreceive_oid: 2468, + typsend_oid: 2469, }), }, }; @@ -306,6 +323,7 @@ pub const TYPE_TIME: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 1143, typreceive_oid: 2470, + typsend_oid: 2471, }), }, }; @@ -320,6 +338,7 @@ pub const TYPE_TIMESTAMP: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 1312, typreceive_oid: 2474, + typsend_oid: 2475, }), }, }; @@ -336,6 +355,7 @@ pub const TYPE_TIMESTAMP_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -352,6 +372,7 @@ pub const TYPE_DATE_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -368,6 +389,7 @@ pub const TYPE_TIME_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -382,6 +404,7 @@ pub const TYPE_TIMESTAMPTZ: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 1150, typreceive_oid: 2476, + typsend_oid: 2477, }), }, }; @@ -398,6 +421,7 @@ pub const TYPE_TIMESTAMPTZ_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -412,6 +436,7 @@ pub const TYPE_INTERVAL: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 1160, typreceive_oid: 2478, + typsend_oid: 2479, }), }, }; @@ -428,6 +453,7 @@ pub const TYPE_INTERVAL_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -442,6 +468,7 @@ pub const TYPE_NAME: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 34, typreceive_oid: 2422, + typsend_oid: 2423, }), }, }; @@ -458,6 +485,7 @@ pub const TYPE_NAME_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -472,6 +500,7 @@ pub const TYPE_NUMERIC: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 1701, typreceive_oid: 2460, + typsend_oid: 2461, }), }, }; @@ -488,6 +517,7 @@ pub const TYPE_NUMERIC_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -502,6 +532,7 @@ pub const TYPE_RECORD: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 2290, typreceive_oid: 2402, + typsend_oid: 2403, }), }, }; @@ -518,6 +549,7 @@ pub const TYPE_RECORD_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -532,6 +564,7 @@ pub const TYPE_UUID: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 2952, typreceive_oid: 2961, + typsend_oid: 2962, }), }, }; @@ -548,6 +581,7 @@ pub const TYPE_UUID_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -562,6 +596,7 @@ pub const TYPE_JSONB: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 3806, typreceive_oid: 3805, + typsend_oid: 3803, }), }, }; @@ -578,6 +613,7 @@ pub const TYPE_JSONB_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -592,6 +628,7 @@ pub const TYPE_ANY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 2294, typreceive_oid: 0, + typsend_oid: 0, }), }, }; @@ -606,6 +643,7 @@ pub const TYPE_ANYARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 2296, typreceive_oid: 2502, + typsend_oid: 2503, }), }, }; @@ -620,6 +658,7 @@ pub const TYPE_ANYELEMENT: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 2312, typreceive_oid: 0, + typsend_oid: 0, }), }, }; @@ -634,6 +673,7 @@ pub const TYPE_ANYNONARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 2777, typreceive_oid: 0, + typsend_oid: 0, }), }, }; @@ -648,6 +688,7 @@ pub const TYPE_ANYRANGE: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 3832, typreceive_oid: 0, + typsend_oid: 0, }), }, }; @@ -662,6 +703,7 @@ pub const TYPE_CHAR: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 1245, typreceive_oid: 2434, + typsend_oid: 2435, }), }, }; @@ -676,6 +718,7 @@ pub const TYPE_VARCHAR: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 1046, typreceive_oid: 2432, + typsend_oid: 2433, }), }, }; @@ -690,6 +733,7 @@ pub const TYPE_INT2: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 38, typreceive_oid: 2404, + typsend_oid: 2405, }), }, }; @@ -706,6 +750,7 @@ pub const TYPE_INT2_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -720,6 +765,7 @@ pub const TYPE_BPCHAR: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 1044, typreceive_oid: 2430, + typsend_oid: 2431, }), }, }; @@ -736,6 +782,7 @@ pub const TYPE_CHAR_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -752,6 +799,7 @@ pub const TYPE_VARCHAR_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -768,6 +816,7 @@ pub const TYPE_BPCHAR_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -782,6 +831,7 @@ pub const TYPE_REGPROC: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 44, typreceive_oid: 2444, + typsend_oid: 2445, }), }, }; @@ -798,6 +848,7 @@ pub const TYPE_REGPROC_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -812,6 +863,7 @@ pub const TYPE_REGTYPE: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 2220, typreceive_oid: 2454, + typsend_oid: 2455, }), }, }; @@ -828,6 +880,7 @@ pub const TYPE_REGTYPE_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -842,6 +895,7 @@ pub const TYPE_REGCLASS: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 2218, typreceive_oid: 2452, + typsend_oid: 2453, }), }, }; @@ -858,6 +912,7 @@ pub const TYPE_REGCLASS_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -872,6 +927,7 @@ pub const TYPE_INT2_VECTOR: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 40, typreceive_oid: 2410, + typsend_oid: 2411, }), }, }; @@ -888,6 +944,7 @@ pub const TYPE_INT2_VECTOR_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -902,6 +959,7 @@ pub const TYPE_ANYCOMPATIBLE: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 5086, typreceive_oid: 0, + typsend_oid: 0, }), }, }; @@ -916,6 +974,7 @@ pub const TYPE_ANYCOMPATIBLEARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 5088, typreceive_oid: 5090, + typsend_oid: 5091, }), }, }; @@ -930,6 +989,7 @@ pub const TYPE_ANYCOMPATIBLENONARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 5092, typreceive_oid: 0, + typsend_oid: 0, }), }, }; @@ -944,6 +1004,7 @@ pub const TYPE_ANYCOMPATIBLERANGE: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 5094, typreceive_oid: 0, + typsend_oid: 0, }), }, }; @@ -960,6 +1021,7 @@ pub const TYPE_INT4_RANGE: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 3834, typreceive_oid: 3836, + typsend_oid: 3837, }), }, }; @@ -976,6 +1038,7 @@ pub const TYPE_INT4_RANGE_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -992,6 +1055,7 @@ pub const TYPE_INT8_RANGE: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 3834, typreceive_oid: 3836, + typsend_oid: 3837, }), }, }; @@ -1008,6 +1072,7 @@ pub const TYPE_INT8_RANGE_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -1024,6 +1089,7 @@ pub const TYPE_DATE_RANGE: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 3834, typreceive_oid: 3836, + typsend_oid: 3837, }), }, }; @@ -1040,6 +1106,7 @@ pub const TYPE_DATE_RANGE_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -1056,6 +1123,7 @@ pub const TYPE_NUM_RANGE: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 3834, typreceive_oid: 3836, + typsend_oid: 3837, }), }, }; @@ -1072,6 +1140,7 @@ pub const TYPE_NUM_RANGE_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -1088,6 +1157,7 @@ pub const TYPE_TS_RANGE: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 3834, typreceive_oid: 3836, + typsend_oid: 3837, }), }, }; @@ -1104,6 +1174,7 @@ pub const TYPE_TS_RANGE_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -1120,6 +1191,7 @@ pub const TYPE_TSTZ_RANGE: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 3834, typreceive_oid: 3836, + typsend_oid: 3837, }), }, }; @@ -1136,6 +1208,7 @@ pub const TYPE_TSTZ_RANGE_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -1150,6 +1223,7 @@ pub const TYPE_ACL_ITEM: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 1031, typreceive_oid: 0, + typsend_oid: 0, }), }, }; @@ -1166,6 +1240,7 @@ pub const TYPE_ACL_ITEM_ARRAY: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 750, typreceive_oid: 2400, + typsend_oid: 2401, }), }, }; @@ -1180,6 +1255,7 @@ pub const TYPE_INTERNAL: BuiltinType = BuiltinType { pg_metadata: Some(CatalogTypePgMetadata { typinput_oid: 2304, typreceive_oid: 0, + typsend_oid: 0, }), }, }; @@ -1492,17 +1568,26 @@ pub static PG_TYPE: LazyLock = LazyLock::new(|| BuiltinView { .with_column("typelem", SqlScalarType::Oid.nullable(false)) .with_column("typarray", SqlScalarType::Oid.nullable(false)) .with_column("typinput", SqlScalarType::RegProc.nullable(true)) - .with_column("typreceive", SqlScalarType::Oid.nullable(false)) + // PostgreSQL declares `typreceive` and `typsend` as `regproc`, and clients + // compare the resolved function name against a known set. The underlying + // `mz_internal.pg_type_all_databases` keeps `typreceive` as `oid` because it + // backs a builtin index, and resolving a `regproc` to its name reads + // `current_database()`, which is unmaterializable. + .with_column("typreceive", SqlScalarType::RegProc.nullable(false)) .with_column("typnotnull", SqlScalarType::Bool.nullable(false)) .with_column("typbasetype", SqlScalarType::Oid.nullable(false)) .with_column("typtypmod", SqlScalarType::Int32.nullable(false)) .with_column("typcollation", SqlScalarType::Oid.nullable(false)) .with_column("typdefault", SqlScalarType::String.nullable(true)) + // Appended rather than placed at PostgreSQL's ordinal position, to keep the + // existing columns' positions stable. + .with_column("typsend", SqlScalarType::RegProc.nullable(false)) .finish(), column_comments: BTreeMap::new(), sql: "SELECT oid, typname, typnamespace, typowner, typlen, typtype, typcategory, typdelim, typrelid, typelem, - typarray, typinput, typreceive, typnotnull, typbasetype, typtypmod, typcollation, typdefault + typarray, typinput, typreceive::pg_catalog.regproc AS typreceive, typnotnull, typbasetype, + typtypmod, typcollation, typdefault, typsend FROM mz_internal.pg_type_all_databases WHERE database_name IS NULL OR database_name = pg_catalog.current_database();", access: vec![PUBLIC_SELECT], diff --git a/src/sql/src/catalog.rs b/src/sql/src/catalog.rs index 9806b73ccc2fc..e47987bd9d0d5 100644 --- a/src/sql/src/catalog.rs +++ b/src/sql/src/catalog.rs @@ -1068,6 +1068,9 @@ pub struct CatalogTypePgMetadata { pub typinput_oid: u32, /// The OID of the `typreceive` function in PostgreSQL. pub typreceive_oid: u32, + /// The OID of the `typsend` function in PostgreSQL, or 0 for the types that + /// PostgreSQL gives no binary send function. + pub typsend_oid: u32, } /// Represents a reference to type in the catalog diff --git a/test/adbc/mzcompose b/test/adbc/mzcompose new file mode 100755 index 0000000000000..1f866645dabc8 --- /dev/null +++ b/test/adbc/mzcompose @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. +# +# mzcompose — runs Docker Compose with Materialize customizations. + +exec "$(dirname "$0")"/../../bin/pyactivate -m materialize.cli.mzcompose "$@" diff --git a/test/adbc/mzcompose.py b/test/adbc/mzcompose.py new file mode 100644 index 0000000000000..aa223549c797b --- /dev/null +++ b/test/adbc/mzcompose.py @@ -0,0 +1,38 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +""" +Tests the Apache Arrow ADBC PostgreSQL driver against Materialize, and hands +the resulting Arrow tables to DuckDB. +""" + +from materialize.mzcompose.composition import Composition +from materialize.mzcompose.service import Service +from materialize.mzcompose.services.materialized import Materialized + +SERVICES = [ + Materialized(), + Service( + name="adbc", + config={ + "image": "python:3.14.1-trixie", + "volumes": [ + "../../:/workdir", + ], + "environment": [ + "PGHOST=materialized", + ], + }, + ), +] + + +def workflow_default(c: Composition) -> None: + c.up("materialized") + c.run("adbc", "/workdir/test/adbc/test.sh") diff --git a/test/adbc/requirements.txt b/test/adbc/requirements.txt new file mode 100644 index 0000000000000..660e0cc247979 --- /dev/null +++ b/test/adbc/requirements.txt @@ -0,0 +1,4 @@ +adbc-driver-manager==1.11.0 +adbc-driver-postgresql==1.11.0 +duckdb==1.5.5 +pyarrow==25.0.0 diff --git a/test/adbc/smoketest.py b/test/adbc/smoketest.py new file mode 100644 index 0000000000000..062c1ec0c2b20 --- /dev/null +++ b/test/adbc/smoketest.py @@ -0,0 +1,212 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +"""Apache Arrow ADBC PostgreSQL driver tests. + +The driver keys its Arrow type mapping off the *name* of each type's binary +receive function, which it reads from `pg_catalog.pg_type.typreceive`. +Materialize encodes a `regproc` over pgwire as a bare OID rather than as the +function name, so every lookup misses and the driver falls back to +`arrow.opaque` over binary. Connecting and streaming rows work. Per-column type +fidelity does not, so the tests that require it are marked `expectedFailure` +rather than deleted. They report an unexpected success once `regproc` renders as +a name. +""" + +import unittest + +import adbc_driver_postgresql.dbapi +import duckdb +import pyarrow as pa + +MATERIALIZED_URL = "postgresql://materialize@materialized:6875/materialize" + +# One non-null value per scalar type the driver has to resolve. Every value is +# non-null so that a broken binary decoder shows up as a wrong value rather +# than as a column of nulls. +SCALAR_QUERY = """ +SELECT + true::bool AS c_bool, + 1::int2 AS c_int2, + 2::int4 AS c_int4, + 3::int8 AS c_int8, + 1.5::float4 AS c_float4, + 2.5::float8 AS c_float8, + 123.45::numeric AS c_numeric, + 'abc'::text AS c_text, + 'def'::varchar AS c_varchar, + '\\x0102'::bytea AS c_bytea, + '2024-01-01'::date AS c_date, + '12:34:56'::time AS c_time, + '2024-01-01 12:34:56'::timestamp AS c_timestamp, + '2024-01-01 12:34:56+00'::timestamptz AS c_timestamptz, + '1 day'::interval AS c_interval, + '00000000-0000-0000-0000-000000000001'::uuid AS c_uuid, + '{"a": 1}'::jsonb AS c_jsonb +""" + +# The Arrow type the driver must produce for each column of SCALAR_QUERY. +# `numeric` and `uuid` have no native Arrow type the driver maps them onto, so +# it wraps them in an `arrow.opaque` extension type over string or binary +# storage. +EXPECTED_SCALAR_TYPES = { + "c_bool": pa.bool_(), + "c_int2": pa.int16(), + "c_int4": pa.int32(), + "c_int8": pa.int64(), + "c_float4": pa.float32(), + "c_float8": pa.float64(), + "c_numeric": pa.opaque(pa.string(), "numeric", "PostgreSQL"), + "c_text": pa.string(), + "c_varchar": pa.string(), + "c_bytea": pa.binary(), + "c_date": pa.date32(), + "c_time": pa.time64("us"), + "c_timestamp": pa.timestamp("us"), + "c_timestamptz": pa.timestamp("us", tz="UTC"), + "c_interval": pa.month_day_nano_interval(), + "c_uuid": pa.opaque(pa.binary(), "uuid", "PostgreSQL"), + "c_jsonb": pa.string(), +} + + +def query_arrow(sql: str) -> pa.Table: + with adbc_driver_postgresql.dbapi.connect(MATERIALIZED_URL) as conn: + with conn.cursor() as cur: + cur.execute(sql) + return cur.fetch_arrow_table() + + +class SmokeTest(unittest.TestCase): + def test_connect(self) -> None: + """Connecting runs the driver's type resolver, which reads the binary + send and receive functions of every type out of `pg_catalog.pg_type`. A + column missing there fails every connection before any user query, so + connecting at all covers that catalog contract. This deliberately makes + no claim about the values, to stay a test of the catalog rather than of + Arrow type fidelity.""" + with adbc_driver_postgresql.dbapi.connect(MATERIALIZED_URL) as conn: + with conn.cursor() as cur: + cur.execute("SELECT 1") + table = cur.fetch_arrow_table() + self.assertEqual(table.num_rows, 1) + self.assertEqual(table.num_columns, 1) + + # Materialize encodes `regproc` as an OID rather than a function name, so + # the driver cannot key its type map and every column resolves to binary. + @unittest.expectedFailure + def test_scalar_types_resolve(self) -> None: + """Each scalar type must arrive as its own Arrow type. A type the + driver cannot resolve degrades to opaque binary instead of failing, so + asserting the exact Arrow type is what catches it.""" + table = query_arrow(SCALAR_QUERY) + self.assertEqual(table.num_rows, 1) + self.assertEqual( + sorted(table.schema.names), sorted(EXPECTED_SCALAR_TYPES.keys()) + ) + + for name, expected in EXPECTED_SCALAR_TYPES.items(): + actual = table.schema.field(name).type + self.assertEqual( + actual, + expected, + f"column {name} resolved to Arrow type {actual}, expected {expected}", + ) + + # `bytea` is the only column that is legitimately Arrow binary. Any + # other column landing there means the driver fell back to passing + # bytes through untyped. + for field in table.schema: + if field.type == pa.binary(): + self.assertEqual( + field.name, + "c_bytea", + f"column {field.name} fell back to opaque binary", + ) + + row = table.to_pylist()[0] + self.assertEqual(row["c_bool"], True) + self.assertEqual(row["c_int4"], 2) + self.assertEqual(row["c_text"], "abc") + self.assertEqual(row["c_bytea"], b"\x01\x02") + + # Materialize encodes `regproc` as an OID rather than a function name, so + # the driver cannot key its type map and the element type resolves to + # binary. + @unittest.expectedFailure + def test_array_type(self) -> None: + """An array must arrive as an Arrow list, not as an opaque blob.""" + table = query_arrow("SELECT ARRAY[1, 2, 3]::int4[] AS a") + actual = table.schema.field("a").type + self.assertEqual( + actual, + pa.list_(pa.int32()), + f"column a resolved to Arrow type {actual}, expected a list of int32", + ) + self.assertEqual(table.to_pylist(), [{"a": [1, 2, 3]}]) + + # Materialize encodes `regproc` as an OID rather than a function name, so + # the driver cannot key its type map and every column resolves to binary. + @unittest.expectedFailure + def test_null_handling(self) -> None: + """Nulls round-trip while keeping the column's resolved Arrow type.""" + table = query_arrow(""" + SELECT + NULL::int4 AS c_int4, + NULL::text AS c_text, + NULL::timestamptz AS c_timestamptz + """) + self.assertEqual(table.schema.field("c_int4").type, pa.int32()) + self.assertEqual(table.schema.field("c_text").type, pa.string()) + self.assertEqual( + table.schema.field("c_timestamptz").type, pa.timestamp("us", tz="UTC") + ) + self.assertEqual( + table.to_pylist(), + [{"c_int4": None, "c_text": None, "c_timestamptz": None}], + ) + + # Materialize encodes `regproc` as an OID rather than a function name, so + # the driver cannot key its type map and DuckDB is handed blobs it + # cannot aggregate. + @unittest.expectedFailure + def test_duckdb_handoff(self) -> None: + """DuckDB reads the Arrow table Materialize produced directly, with no + intermediate file or object store hop.""" + table = query_arrow(""" + SELECT n, n * 2 AS doubled, 'row' || n::text AS label + FROM generate_series(1, 100) AS g(n) + """) + + con = duckdb.connect() + try: + con.register("t", table) + self.assertEqual( + con.execute("SELECT count(*), sum(n), sum(doubled) FROM t").fetchall(), + [(100, 5050, 10100)], + ) + self.assertEqual( + con.execute("SELECT sum(doubled) FROM t WHERE n > 50").fetchall(), + [(7550,)], + ) + self.assertEqual( + con.execute("SELECT label FROM t WHERE n = 7").fetchall(), + [("row7",)], + ) + finally: + con.close() + + def test_fetch_many_rows(self) -> None: + """The driver fetches results as a binary COPY stream. This covers a + result large enough to span multiple reads of that stream. Row and + column counts hold regardless of how each column's type resolves, so + this stays a hard assertion.""" + table = query_arrow("SELECT n FROM generate_series(1, 5000) AS g(n)") + self.assertEqual(table.num_rows, 5000) + self.assertEqual(table.num_columns, 1) diff --git a/test/adbc/test.sh b/test/adbc/test.sh new file mode 100755 index 0000000000000..8ca39aa74dcf8 --- /dev/null +++ b/test/adbc/test.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. +# +# test.sh — run Apache Arrow ADBC driver tests. + +set -euo pipefail + +cd "$(dirname "$0")/../.." + +. misc/shlib/shlib.bash + +cd test/adbc + +pip install -r requirements.txt + +python -m unittest smoketest diff --git a/test/sqllogictest/mz_catalog_server_index_accounting.slt b/test/sqllogictest/mz_catalog_server_index_accounting.slt index 0259167bc76bf..7904f778cd29f 100644 --- a/test/sqllogictest/mz_catalog_server_index_accounting.slt +++ b/test/sqllogictest/mz_catalog_server_index_accounting.slt @@ -873,6 +873,7 @@ mz_tables source_id mz_type_pg_metadata id mz_type_pg_metadata typinput mz_type_pg_metadata typreceive +mz_type_pg_metadata typsend mz_types category mz_types create_sql mz_types id @@ -986,5 +987,6 @@ pg_type_all_databases typnotnull pg_type_all_databases typowner pg_type_all_databases typreceive pg_type_all_databases typrelid +pg_type_all_databases typsend pg_type_all_databases typtype pg_type_all_databases typtypmod diff --git a/test/workload-replay/system_catalog_identifiers.txt b/test/workload-replay/system_catalog_identifiers.txt index 730a57364f729..20c0f28badac6 100644 --- a/test/workload-replay/system_catalog_identifiers.txt +++ b/test/workload-replay/system_catalog_identifiers.txt @@ -1562,6 +1562,7 @@ typnotnull typowner typreceive typrelid +typsend typtype typtypmod _uint2 From a5af32266012b57d0999b037a06f6a1bcfd87480 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 17:17:13 +0000 Subject: [PATCH 02/15] test/adbc: silence pyright reportMissingImports on container-only deps `ci/test/lint-main/checks/check-python-files.sh` runs pyright with `--warnings`, and `pyproject.toml` maps `reportMissingImports` to a warning, so any import pyright cannot resolve fails the lint-and-rustfmt job. `adbc-driver-postgresql` and `duckdb` are pinned in `test/adbc/requirements.txt` and installed only inside the composition's test container, never in the repo-wide virtualenv pyright resolves against, so both imports reported as unresolved. Suppress the two imports with targeted `# pyright: ignore` comments rather than pulling the drivers into `misc/python/requirements.txt`, which would install heavy native wheels into every developer's virtualenv purely to satisfy a lint. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01K8HcpvmmntWK9QuJWvKVbg --- test/adbc/smoketest.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/adbc/smoketest.py b/test/adbc/smoketest.py index 062c1ec0c2b20..8ecad8f68f44b 100644 --- a/test/adbc/smoketest.py +++ b/test/adbc/smoketest.py @@ -21,8 +21,11 @@ import unittest -import adbc_driver_postgresql.dbapi -import duckdb +# `adbc-driver-postgresql` and `duckdb` are pinned in `test/adbc/requirements.txt` +# and installed only inside this composition's test container, so they are absent +# from the repo-wide Python virtualenv that pyright resolves against. +import adbc_driver_postgresql.dbapi # pyright: ignore[reportMissingImports] +import duckdb # pyright: ignore[reportMissingImports] import pyarrow as pa MATERIALIZED_URL = "postgresql://materialize@materialized:6875/materialize" From 908d4b3519a1cb9e6be693593677195624e06203 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 17:43:48 +0000 Subject: [PATCH 03/15] pgrepr: render regproc as the resolved function name in text format PostgreSQL renders a regproc through regprocout, which resolves the OID to the function's name. Materialize emitted the bare OID, so a client that keys on the text of pg_catalog.pg_type.typreceive, as the Apache Arrow ADBC PostgreSQL driver does, missed every lookup and degraded every column to opaque binary. Value::from_datum collapsed Oid, RegClass, RegProc and RegType into one Value::Oid, erasing the reg flavor before encode_text ran. RegProc gets its own Value variant, so the text encoding can resolve the name while the binary encoding stays a bare 4-byte OID, matching regprocsend. Resolution needs no catalog access. Materialize has no CREATE FUNCTION, so all 626 functions are builtins and the OID to name mapping is fixed when the binary is built. That matters because mz-pgrepr sits below the catalog crates and two of its encode paths, the Kafka FORMAT TEXT sink and COPY TO with an S3 target, run in clusterd where no catalog exists. Resolving from a static table therefore fixes every text rendering uniformly, at no per-statement cost and with no signature changes, so mz-pgwire, mz-pgcopy, mz-storage-operators, mz-interchange and mz-adapter are untouched. The table lives in mz-pgrepr-consts, which already holds the OID constants and has no dependencies. mz-pgrepr cannot read mz_sql::func directly, because mz-sql depends on mz-pgrepr, so the entries are checked in as data and mz_catalog::builtin's test_regproc_names_match_builtin_functions recomputes the whole table from the builtin registry, failing with the corrected contents on drift. Renderings follow regprocout, verified against PostgreSQL 16. OID 0 prints `-`, an OID naming no function prints in decimal, a uniquely named function prints bare, and an overloaded name prints schema-qualified, so one of the six abs impls prints pg_catalog.abs. That rule reproduces Materialize's own regproc::text cast for all 626 function OIDs, so the two paths agree except on OID 0, where the cast prints 0 and PostgreSQL prints `-`. decode_text accepts all of those renderings back, which keeps COPY TO output loadable by COPY FROM and matches regprocin, including its refusal of an ambiguous name. regclass and regtype keep rendering as digits. They can name objects a user created, which no static table can know. The JSON encoder keeps emitting a number, because build_row_schema_json in the same module declares these columns as 4-byte unsigned integers and the Avro encoder writes them that way. The HTTP SQL API shares that encoder. typreceive now resolves for all 67 builtin types that have one, which is the column the ADBC driver reads, so the driver's type fidelity tests become hard assertions rather than expected failures. typsend still renders in decimal, because the builtin registry defines receive functions but no send functions, leaving those OIDs out of the table. --- src/catalog/src/builtin.rs | 58 +++ src/interchange/src/json.rs | 7 + src/pgrepr-consts/src/lib.rs | 1 + src/pgrepr-consts/src/regproc.rs | 708 +++++++++++++++++++++++++++++++ src/pgrepr/src/lib.rs | 1 + src/pgrepr/src/regproc.rs | 61 +++ src/pgrepr/src/value.rs | 154 ++++++- test/adbc/smoketest.py | 31 +- test/pgtest-mz/datums.pt | 2 +- test/sqllogictest/regproc.slt | 30 ++ 10 files changed, 1019 insertions(+), 34 deletions(-) create mode 100644 src/pgrepr-consts/src/regproc.rs create mode 100644 src/pgrepr/src/regproc.rs diff --git a/src/catalog/src/builtin.rs b/src/catalog/src/builtin.rs index 931aac1423a17..fa304cc73dd0c 100644 --- a/src/catalog/src/builtin.rs +++ b/src/catalog/src/builtin.rs @@ -1659,6 +1659,64 @@ mod tests { use super::*; + /// `mz_pgrepr::regproc::NAMES` is a table of every builtin function OID and + /// the text a `regproc` renders it as. It has to be checked in as data + /// because `mz-pgrepr` sits below this crate in the dependency graph and so + /// cannot read the registry itself. This test is what keeps it honest: it + /// recomputes the whole table from the registry and, on drift, prints the + /// corrected table to paste into `src/pgrepr-consts/src/regproc.rs`. + #[mz_ore::test] + #[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux` + fn test_regproc_names_match_builtin_functions() { + // `effective_search_path` unconditionally prepends these two, so a + // uniquely named function in either resolves from its bare name. + const IMPLICITLY_SEARCHED: &[&str] = &[MZ_CATALOG_SCHEMA, PG_CATALOG_SCHEMA]; + + // A bare name only identifies one OID when exactly one impl anywhere in + // the registry carries it, so count impls across schemas. + let mut impls_per_name: BTreeMap<&str, usize> = BTreeMap::new(); + for func in BUILTINS::funcs() { + *impls_per_name.entry(func.name).or_default() += func.inner.func_impls().len(); + } + + let mut expected: BTreeMap = BTreeMap::new(); + for func in BUILTINS::funcs() { + // The rule `regprocout` applies: print the bare name when it + // resolves back to this OID on its own, otherwise qualify it. + let rendered = + if impls_per_name[func.name] == 1 && IMPLICITLY_SEARCHED.contains(&func.schema) { + func.name.to_string() + } else { + format!("{}.{}", func.schema, func.name) + }; + for details in func.inner.func_impls() { + let previous = expected.insert(details.oid, rendered.clone()); + assert_eq!( + previous, None, + "two builtin functions share OID {}", + details.oid + ); + } + } + + let actual: BTreeMap = mz_pgrepr::regproc::NAMES + .iter() + .map(|(oid, name)| (*oid, name.to_string())) + .collect(); + + if actual != expected { + let table: String = expected + .iter() + .map(|(oid, name)| format!(" ({}, \"{}\"),\n", oid, name)) + .collect(); + panic!( + "mz_pgrepr::regproc::NAMES has drifted from the builtin function registry. \ + Replace the entries of NAMES in src/pgrepr-consts/src/regproc.rs with:\n{}", + table + ); + } + } + #[mz_ore::test] #[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux` fn test_builtin_type_schema() { diff --git a/src/interchange/src/json.rs b/src/interchange/src/json.rs index 34a9c1affd438..35ef15ff7ccf6 100644 --- a/src/interchange/src/json.rs +++ b/src/interchange/src/json.rs @@ -117,6 +117,13 @@ impl ToJson for TypedDatum<'_> { SqlScalarType::Int32 => json!(datum.unwrap_int32()), SqlScalarType::Int64 => json!(datum.unwrap_int64()), SqlScalarType::UInt16 => json!(datum.unwrap_uint16()), + // NOTE: `regproc` stays a number here even though the pgwire text + // encoding resolves it to a function name. `build_row_schema_json` + // below declares these columns as 4-byte unsigned integers and + // `mz_interchange::avro` encodes them that way, so emitting a name + // would contradict the schema this module publishes for the same + // column. The HTTP SQL API shares this encoder and so also reports + // a number. SqlScalarType::UInt32 | SqlScalarType::Oid | SqlScalarType::RegClass diff --git a/src/pgrepr-consts/src/lib.rs b/src/pgrepr-consts/src/lib.rs index a9fa3f684635c..b69d78d247031 100644 --- a/src/pgrepr-consts/src/lib.rs +++ b/src/pgrepr-consts/src/lib.rs @@ -10,3 +10,4 @@ //! Constants used in the representation of and serialization for PostgreSQL datums. pub mod oid; +pub mod regproc; diff --git a/src/pgrepr-consts/src/regproc.rs b/src/pgrepr-consts/src/regproc.rs new file mode 100644 index 0000000000000..bc0940c9ebaa9 --- /dev/null +++ b/src/pgrepr-consts/src/regproc.rs @@ -0,0 +1,708 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +//! Text renderings of `regproc` values. +//! +//! PostgreSQL renders a `regproc` in text format through `regprocout`, which +//! resolves the OID to the function's name. Materialize has no +//! `CREATE FUNCTION`, so every function is a builtin and the OID to name +//! mapping is fixed when the binary is built. That is what makes a static +//! table viable: the text encoder has no catalog access and cannot gain any, +//! because the catalog crates depend on `mz-pgrepr` rather than the reverse, +//! and some encode paths run in `clusterd`, which has no catalog at all. +//! +//! [`NAMES`] is derived from the builtin function registry in `mz_sql::func`, +//! which is the authoritative definition of every function OID. +//! `mz_catalog::builtin`'s `test_regproc_names_match_builtin_functions` +//! recomputes the whole table from that registry and fails with the corrected +//! contents when the two have drifted, so do not hand-edit entries here. Add +//! the function to the registry and paste in the table the test prints. + +/// Every builtin function OID paired with the text `regproc` renders it as, +/// sorted by OID. +/// +/// The name is the full rendering, not the bare function name. It is +/// schema-qualified whenever the bare name would not resolve back to this OID, +/// which is the rule `regprocout` applies: a name shared by several overloads, +/// or a name in a schema that is not searched implicitly, gets qualified. +/// +/// NOTE: The rendering assumes the default search path, which is all a static +/// table can do. A session that puts `mz_internal` or `mz_unsafe` on its +/// `search_path` makes those bare names resolvable, so `regprocout` would print +/// them bare and this table still qualifies them. The SQL cast from `regproc` to +/// `text` reads the session's search path and does render them bare. +pub const NAMES: &[(u32, &str)] = &[ + (34, "namein"), + (38, "int2in"), + (40, "int2vectorin"), + (42, "int4in"), + (44, "regprocin"), + (46, "textin"), + (89, "version"), + (200, "float4in"), + (214, "float8in"), + (376, "pg_catalog.string_to_array"), + (383, "array_cat"), + (384, "pg_catalog.array_to_string"), + (394, "pg_catalog.string_to_array"), + (395, "pg_catalog.array_to_string"), + (460, "int8in"), + (720, "pg_catalog.octet_length"), + (721, "get_byte"), + (723, "get_bit"), + (745, "current_user"), + (746, "session_user"), + (750, "array_in"), + (849, "position"), + (861, "current_database"), + (868, "strpos"), + (870, "pg_catalog.lower"), + (871, "pg_catalog.upper"), + (872, "initcap"), + (873, "pg_catalog.lpad"), + (875, "pg_catalog.ltrim"), + (876, "pg_catalog.rtrim"), + (877, "pg_catalog.substr"), + (878, "translate"), + (879, "pg_catalog.lpad"), + (881, "pg_catalog.ltrim"), + (882, "pg_catalog.rtrim"), + (883, "pg_catalog.substr"), + (884, "pg_catalog.btrim"), + (885, "pg_catalog.btrim"), + (936, "pg_catalog.substring"), + (937, "pg_catalog.substring"), + (938, "pg_catalog.generate_series"), + (939, "pg_catalog.generate_series"), + (940, "pg_catalog.mod"), + (941, "pg_catalog.mod"), + (947, "pg_catalog.mod"), + (1026, "pg_catalog.timezone"), + (1031, "aclitemin"), + (1044, "bpcharin"), + (1046, "varcharin"), + (1066, "pg_catalog.generate_series"), + (1067, "pg_catalog.generate_series"), + (1068, "pg_catalog.generate_series"), + (1069, "pg_catalog.generate_series"), + (1081, "format_type"), + (1084, "date_in"), + (1143, "time_in"), + (1150, "timestamptz_in"), + (1158, "to_timestamp"), + (1159, "pg_catalog.timezone"), + (1160, "interval_in"), + (1171, "pg_catalog.date_part"), + (1172, "pg_catalog.date_part"), + (1175, "justify_hours"), + (1178, "pg_catalog.date"), + (1192, "generate_subscripts"), + (1193, "pg_catalog.array_fill"), + (1194, "pg_catalog.log10"), + (1199, "pg_catalog.age"), + (1215, "obj_description"), + (1216, "col_description"), + (1217, "pg_catalog.date_trunc"), + (1218, "pg_catalog.date_trunc"), + (1242, "boolin"), + (1244, "byteain"), + (1245, "charin"), + (1268, "pg_catalog.parse_ident"), + (1269, "pg_column_size"), + (1282, "quote_ident"), + (1286, "pg_catalog.array_fill"), + (1295, "justify_days"), + (1299, "now"), + (1312, "timestamp_in"), + (1317, "pg_catalog.length"), + (1340, "pg_catalog.log"), + (1341, "pg_catalog.ln"), + (1342, "pg_catalog.round"), + (1343, "pg_catalog.trunc"), + (1344, "pg_catalog.sqrt"), + (1345, "cbrt"), + (1346, "pow"), + (1347, "pg_catalog.exp"), + (1365, "makeaclitem"), + (1368, "pg_catalog.power"), + (1374, "pg_catalog.octet_length"), + (1375, "pg_catalog.octet_length"), + (1381, "char_length"), + (1385, "pg_catalog.date_part"), + (1387, "pg_catalog.pg_get_constraintdef"), + (1394, "pg_catalog.abs"), + (1395, "pg_catalog.abs"), + (1396, "pg_catalog.abs"), + (1397, "pg_catalog.abs"), + (1398, "pg_catalog.abs"), + (1402, "current_schema"), + (1403, "current_schemas"), + (1481, "pg_catalog.log10"), + (1573, "pg_catalog.pg_get_ruledef"), + (1597, "pg_encoding_to_char"), + (1600, "asin"), + (1601, "acos"), + (1602, "atan"), + (1604, "sin"), + (1605, "cos"), + (1606, "tan"), + (1607, "cot"), + (1608, "degrees"), + (1609, "radians"), + (1619, "pg_typeof"), + (1620, "ascii"), + (1621, "chr"), + (1622, "repeat"), + (1637, "like_escape"), + (1640, "pg_catalog.pg_get_viewdef"), + (1641, "pg_catalog.pg_get_viewdef"), + (1642, "pg_get_userbyid"), + (1643, "pg_catalog.pg_get_indexdef"), + (1689, "aclexplode"), + (1701, "numeric_in"), + (1705, "pg_catalog.abs"), + (1707, "pg_catalog.round"), + (1708, "pg_catalog.round"), + (1710, "pg_catalog.trunc"), + (1711, "pg_catalog.ceil"), + (1712, "pg_catalog.floor"), + (1713, "pg_catalog.length"), + (1714, "convert_from"), + (1716, "pg_catalog.pg_get_expr"), + (1728, "pg_catalog.mod"), + (1730, "pg_catalog.sqrt"), + (1732, "pg_catalog.exp"), + (1734, "pg_catalog.ln"), + (1736, "pg_catalog.log"), + (1741, "pg_catalog.log"), + (1770, "pg_catalog.to_char"), + (1798, "oidin"), + (1810, "pg_catalog.bit_length"), + (1811, "pg_catalog.bit_length"), + (1922, "pg_catalog.has_table_privilege"), + (1923, "pg_catalog.has_table_privilege"), + (1924, "pg_catalog.has_table_privilege"), + (1925, "pg_catalog.has_table_privilege"), + (1926, "pg_catalog.has_table_privilege"), + (1927, "pg_catalog.has_table_privilege"), + (1928, "pg_stat_get_numscans"), + (1946, "encode"), + (1947, "decode"), + (2010, "pg_catalog.length"), + (2020, "pg_catalog.date_trunc"), + (2021, "pg_catalog.date_part"), + (2026, "pg_backend_pid"), + (2029, "pg_catalog.date"), + (2037, "pg_catalog.timezone"), + (2038, "pg_catalog.timezone"), + (2049, "pg_catalog.to_char"), + (2058, "pg_catalog.age"), + (2069, "pg_catalog.timezone"), + (2070, "pg_catalog.timezone"), + (2077, "pg_catalog.current_setting"), + (2079, "pg_table_is_visible"), + (2080, "pg_type_is_visible"), + (2081, "pg_function_is_visible"), + (2087, "replace"), + (2088, "split_part"), + (2091, "array_lower"), + (2092, "array_upper"), + (2100, "pg_catalog.avg"), + (2101, "pg_catalog.avg"), + (2102, "pg_catalog.avg"), + (2104, "pg_catalog.avg"), + (2105, "pg_catalog.avg"), + (2106, "pg_catalog.avg"), + (2107, "pg_catalog.sum"), + (2108, "pg_catalog.sum"), + (2109, "pg_catalog.sum"), + (2110, "pg_catalog.sum"), + (2111, "pg_catalog.sum"), + (2113, "pg_catalog.sum"), + (2114, "pg_catalog.sum"), + (2115, "pg_catalog.max"), + (2116, "pg_catalog.max"), + (2117, "pg_catalog.max"), + (2119, "pg_catalog.max"), + (2120, "pg_catalog.max"), + (2122, "pg_catalog.max"), + (2123, "pg_catalog.max"), + (2126, "pg_catalog.max"), + (2127, "pg_catalog.max"), + (2128, "pg_catalog.max"), + (2129, "pg_catalog.max"), + (2131, "pg_catalog.min"), + (2132, "pg_catalog.min"), + (2133, "pg_catalog.min"), + (2135, "pg_catalog.min"), + (2136, "pg_catalog.min"), + (2138, "pg_catalog.min"), + (2139, "pg_catalog.min"), + (2142, "pg_catalog.min"), + (2143, "pg_catalog.min"), + (2144, "pg_catalog.min"), + (2145, "pg_catalog.min"), + (2147, "pg_catalog.count"), + (2148, "pg_catalog.variance"), + (2149, "pg_catalog.variance"), + (2150, "pg_catalog.variance"), + (2151, "pg_catalog.variance"), + (2152, "pg_catalog.variance"), + (2154, "pg_catalog.stddev"), + (2155, "pg_catalog.stddev"), + (2156, "pg_catalog.stddev"), + (2157, "pg_catalog.stddev"), + (2158, "pg_catalog.stddev"), + (2167, "pg_catalog.ceiling"), + (2169, "pg_catalog.power"), + (2171, "pg_cancel_backend"), + (2176, "array_length"), + (2218, "regclassin"), + (2220, "regtypein"), + (2244, "pg_catalog.max"), + (2245, "pg_catalog.min"), + (2250, "pg_catalog.has_database_privilege"), + (2251, "pg_catalog.has_database_privilege"), + (2252, "pg_catalog.has_database_privilege"), + (2253, "pg_catalog.has_database_privilege"), + (2254, "pg_catalog.has_database_privilege"), + (2255, "pg_catalog.has_database_privilege"), + (2268, "pg_catalog.has_schema_privilege"), + (2269, "pg_catalog.has_schema_privilege"), + (2270, "pg_catalog.has_schema_privilege"), + (2271, "pg_catalog.has_schema_privilege"), + (2272, "pg_catalog.has_schema_privilege"), + (2273, "pg_catalog.has_schema_privilege"), + (2284, "pg_catalog.regexp_replace"), + (2285, "pg_catalog.regexp_replace"), + (2290, "record_in"), + (2294, "any_in"), + (2296, "anyarray_in"), + (2304, "internal_in"), + (2308, "pg_catalog.ceil"), + (2309, "pg_catalog.floor"), + (2311, "pg_catalog.md5"), + (2312, "anyelement_in"), + (2320, "pg_catalog.ceiling"), + (2321, "pg_catalog.md5"), + (2325, "pg_catalog.pg_relation_size"), + (2331, "mz_catalog.unnest"), + (2332, "pg_catalog.pg_relation_size"), + (2335, "pg_catalog.array_agg"), + (2400, "array_recv"), + (2402, "record_recv"), + (2404, "int2recv"), + (2406, "int4recv"), + (2408, "int8recv"), + (2410, "int2vectorrecv"), + (2412, "bytearecv"), + (2414, "textrecv"), + (2418, "oidrecv"), + (2422, "namerecv"), + (2424, "float4recv"), + (2426, "float8recv"), + (2430, "bpcharrecv"), + (2432, "varcharrecv"), + (2434, "charrecv"), + (2436, "boolrecv"), + (2444, "regprocrecv"), + (2452, "regclassrecv"), + (2454, "regtyperecv"), + (2460, "numeric_recv"), + (2462, "sinh"), + (2463, "cosh"), + (2464, "tanh"), + (2465, "asinh"), + (2466, "acosh"), + (2467, "atanh"), + (2468, "date_recv"), + (2470, "time_recv"), + (2474, "timestamp_recv"), + (2476, "timestamptz_recv"), + (2478, "interval_recv"), + (2502, "anyarray_recv"), + (2504, "pg_catalog.pg_get_ruledef"), + (2505, "pg_catalog.pg_get_viewdef"), + (2506, "pg_catalog.pg_get_viewdef"), + (2507, "pg_catalog.pg_get_indexdef"), + (2508, "pg_catalog.pg_get_constraintdef"), + (2509, "pg_catalog.pg_get_expr"), + (2517, "bool_and"), + (2518, "bool_or"), + (2560, "pg_postmaster_start_time"), + (2641, "pg_catalog.var_samp"), + (2642, "pg_catalog.var_samp"), + (2643, "pg_catalog.var_samp"), + (2644, "pg_catalog.var_samp"), + (2645, "pg_catalog.var_samp"), + (2705, "pg_catalog.pg_has_role"), + (2706, "pg_catalog.pg_has_role"), + (2707, "pg_catalog.pg_has_role"), + (2708, "pg_catalog.pg_has_role"), + (2709, "pg_catalog.pg_has_role"), + (2710, "pg_catalog.pg_has_role"), + (2711, "justify_interval"), + (2712, "pg_catalog.stddev_samp"), + (2713, "pg_catalog.stddev_samp"), + (2714, "pg_catalog.stddev_samp"), + (2715, "pg_catalog.stddev_samp"), + (2716, "pg_catalog.stddev_samp"), + (2718, "pg_catalog.var_pop"), + (2719, "pg_catalog.var_pop"), + (2720, "pg_catalog.var_pop"), + (2721, "pg_catalog.var_pop"), + (2722, "pg_catalog.var_pop"), + (2724, "pg_catalog.stddev_pop"), + (2725, "pg_catalog.stddev_pop"), + (2726, "pg_catalog.stddev_pop"), + (2727, "pg_catalog.stddev_pop"), + (2728, "pg_catalog.stddev_pop"), + (2763, "pg_catalog.regexp_matches"), + (2764, "pg_catalog.regexp_matches"), + (2765, "pg_catalog.regexp_split_to_table"), + (2766, "pg_catalog.regexp_split_to_table"), + (2767, "pg_catalog.regexp_split_to_array"), + (2768, "pg_catalog.regexp_split_to_array"), + (2777, "anynonarray_in"), + (2803, "pg_catalog.count"), + (2952, "uuid_in"), + (2961, "uuid_recv"), + (3058, "concat"), + (3059, "concat_ws"), + (3060, "left"), + (3061, "right"), + (3062, "reverse"), + (3100, "row_number"), + (3101, "rank"), + (3102, "dense_rank"), + (3106, "pg_catalog.lag"), + (3107, "pg_catalog.lag"), + (3108, "pg_catalog.lag"), + (3109, "pg_catalog.lead"), + (3110, "pg_catalog.lead"), + (3111, "pg_catalog.lead"), + (3112, "first_value"), + (3113, "last_value"), + (3138, "mz_catalog.has_type_privilege"), + (3139, "mz_catalog.has_type_privilege"), + (3140, "mz_catalog.has_type_privilege"), + (3141, "mz_catalog.has_type_privilege"), + (3142, "mz_catalog.has_type_privilege"), + (3143, "mz_catalog.has_type_privilege"), + (3159, "pg_catalog.pg_get_viewdef"), + (3166, "pg_size_pretty"), + (3167, "array_remove"), + (3207, "jsonb_array_length"), + (3208, "jsonb_each"), + (3210, "jsonb_typeof"), + (3219, "jsonb_array_elements"), + (3262, "jsonb_strip_nulls"), + (3267, "jsonb_agg"), + (3270, "jsonb_object_agg"), + (3271, "pg_catalog.jsonb_build_array"), + (3272, "pg_catalog.jsonb_build_array"), + (3273, "pg_catalog.jsonb_build_object"), + (3274, "pg_catalog.jsonb_build_object"), + (3277, "pg_catalog.array_position"), + (3278, "pg_catalog.array_position"), + (3294, "pg_catalog.current_setting"), + (3306, "jsonb_pretty"), + (3396, "pg_catalog.regexp_match"), + (3397, "pg_catalog.regexp_match"), + (3419, "sha224"), + (3420, "sha256"), + (3421, "sha384"), + (3422, "sha512"), + (3461, "make_timestamp"), + (3465, "jsonb_array_elements_text"), + (3538, "pg_catalog.string_agg"), + (3545, "pg_catalog.string_agg"), + (3696, "starts_with"), + (3778, "pg_tablespace_location"), + (3787, "to_jsonb"), + (3805, "jsonb_recv"), + (3806, "jsonb_in"), + (3810, "pg_is_in_recovery"), + (3832, "anyrange_in"), + (3834, "range_in"), + (3836, "range_recv"), + (3840, "pg_catalog.int4range"), + (3841, "pg_catalog.int4range"), + (3844, "pg_catalog.numrange"), + (3845, "pg_catalog.numrange"), + (3848, "pg_catalog.lower"), + (3849, "pg_catalog.upper"), + (3850, "isempty"), + (3851, "lower_inc"), + (3852, "upper_inc"), + (3853, "lower_inf"), + (3854, "upper_inf"), + (3931, "jsonb_object_keys"), + (3932, "jsonb_each_text"), + (3933, "pg_catalog.tsrange"), + (3934, "pg_catalog.tsrange"), + (3937, "pg_catalog.tstzrange"), + (3938, "pg_catalog.tstzrange"), + (3941, "pg_catalog.daterange"), + (3942, "pg_catalog.daterange"), + (3945, "pg_catalog.int8range"), + (3946, "pg_catalog.int8range"), + (4053, "pg_catalog.array_agg"), + (4350, "normalize"), + (5086, "anycompatible_in"), + (5088, "anycompatiblearray_in"), + (5090, "anycompatiblearray_recv"), + (5092, "anycompatiblenonarray_in"), + (5094, "anycompatiblerange_in"), + (6163, "bit_count"), + (6177, "pg_catalog.date_bin"), + (6178, "pg_catalog.date_bin"), + (6199, "pg_catalog.extract"), + (6200, "pg_catalog.extract"), + (6202, "pg_catalog.extract"), + (6203, "pg_catalog.extract"), + (6204, "pg_catalog.extract"), + (12000, "information_schema._pg_expandarray"), + (12001, "pg_catalog.digest"), + (12002, "pg_catalog.digest"), + (12003, "pg_catalog.hmac"), + (12004, "pg_catalog.hmac"), + (16386, "pg_catalog.ceil"), + (16387, "concat_agg"), + (16388, "csv_extract"), + (16389, "current_timestamp"), + (16390, "pg_catalog.floor"), + (16392, "list_append"), + (16393, "list_cat"), + (16394, "list_length_max"), + (16395, "list_length"), + (16396, "list_n_layers"), + (16397, "list_prepend"), + (16398, "pg_catalog.max"), + (16399, "pg_catalog.min"), + (16400, "mz_unsafe.mz_all"), + (16401, "mz_unsafe.mz_any"), + (16403, "mz_unsafe.mz_avg_promotion_internal_v1"), + (16404, "mz_unsafe.mz_avg_promotion_internal_v1"), + (16405, "mz_unsafe.mz_avg_promotion_internal_v1"), + (16407, "mz_environment_id"), + (16409, "mz_logical_timestamp"), + (16410, "mz_internal.mz_render_typmod"), + (16411, "mz_version"), + (16412, "regexp_extract"), + (16413, "repeat_row"), + (16414, "pg_catalog.round"), + (16416, "mz_catalog.unnest"), + (16434, "mz_unsafe.mz_sleep"), + (16435, "mz_internal.mz_session_id"), + (16436, "mz_uptime"), + (16440, "mz_row_size"), + (16441, "pg_catalog.max"), + (16442, "pg_catalog.min"), + (16443, "mz_unsafe.mz_avg_promotion_internal_v1"), + (16444, "list_agg"), + (16445, "mz_unsafe.mz_error_if_null"), + (16446, "pg_catalog.date_bin"), + (16447, "pg_catalog.date_bin"), + (16448, "list_remove"), + (16449, "pg_catalog.date_bin_hopping"), + (16450, "pg_catalog.date_bin_hopping"), + (16451, "pg_catalog.date_bin_hopping"), + (16452, "pg_catalog.date_bin_hopping"), + (16453, "mz_internal.mz_type_name"), + (16456, "map_length"), + (16457, "mz_unsafe.mz_panic"), + (16458, "mz_version_num"), + (16459, "pg_catalog.trunc"), + (16496, "pg_catalog.max"), + (16497, "pg_catalog.max"), + (16498, "pg_catalog.max"), + (16499, "pg_catalog.min"), + (16500, "pg_catalog.min"), + (16501, "pg_catalog.min"), + (16502, "pg_catalog.sum"), + (16503, "pg_catalog.sum"), + (16504, "pg_catalog.sum"), + (16505, "pg_catalog.avg"), + (16506, "pg_catalog.avg"), + (16507, "pg_catalog.avg"), + (16508, "pg_catalog.mod"), + (16509, "pg_catalog.mod"), + (16510, "pg_catalog.mod"), + (16511, "pg_catalog.stddev"), + (16512, "pg_catalog.stddev"), + (16513, "pg_catalog.stddev"), + (16514, "pg_catalog.stddev_pop"), + (16515, "pg_catalog.stddev_pop"), + (16516, "pg_catalog.stddev_pop"), + (16517, "pg_catalog.stddev_samp"), + (16518, "pg_catalog.stddev_samp"), + (16519, "pg_catalog.stddev_samp"), + (16520, "pg_catalog.variance"), + (16521, "pg_catalog.variance"), + (16522, "pg_catalog.variance"), + (16523, "pg_catalog.var_pop"), + (16524, "pg_catalog.var_pop"), + (16525, "pg_catalog.var_pop"), + (16526, "pg_catalog.var_samp"), + (16527, "pg_catalog.var_samp"), + (16528, "pg_catalog.var_samp"), + (16550, "mz_unsafe.mz_avg_promotion_internal_v1"), + (16551, "mz_unsafe.mz_avg_promotion_internal_v1"), + (16560, "mz_now"), + (16561, "pg_catalog.max"), + (16562, "pg_catalog.min"), + (16563, "pg_catalog.date"), + (16564, "pg_catalog.ceiling"), + (16565, "uuid_generate_v5"), + (16570, "mz_internal.make_mz_aclitem"), + (16571, "mz_internal.mz_aclitem_grantor"), + (16572, "mz_internal.mz_aclitem_grantee"), + (16573, "mz_internal.mz_aclitem_privileges"), + (16574, "mz_internal.is_rbac_enabled"), + (16575, "mz_internal.mz_acl_item_contains_privilege"), + (16576, "mz_internal.mz_validate_privileges"), + (16577, "mz_internal.mz_role_oid"), + (16578, "mz_internal.mz_minimal_name_qualification"), + (16581, "mz_internal.mz_resolve_object_name"), + (16582, "mz_internal.mz_global_id_to_name"), + (16584, "pg_catalog.parse_ident"), + (16585, "mz_internal.mz_database_oid"), + (16586, "mz_internal.mz_schema_oid"), + (16587, "mz_catalog.has_cluster_privilege"), + (16588, "mz_catalog.has_cluster_privilege"), + (16589, "mz_catalog.has_cluster_privilege"), + (16591, "mz_catalog.has_connection_privilege"), + (16592, "mz_catalog.has_connection_privilege"), + (16593, "mz_catalog.has_connection_privilege"), + (16594, "mz_catalog.has_connection_privilege"), + (16595, "mz_catalog.has_connection_privilege"), + (16596, "mz_catalog.has_connection_privilege"), + (16597, "mz_catalog.has_secret_privilege"), + (16598, "mz_catalog.has_secret_privilege"), + (16599, "mz_catalog.has_secret_privilege"), + (16600, "mz_catalog.has_secret_privilege"), + (16601, "mz_catalog.has_secret_privilege"), + (16602, "mz_catalog.has_secret_privilege"), + (16603, "mz_internal.mz_normalize_object_name"), + (16604, "mz_catalog.datediff"), + (16605, "mz_catalog.datediff"), + (16606, "mz_catalog.datediff"), + (16607, "mz_catalog.datediff"), + (16608, "mz_catalog.has_system_privilege"), + (16609, "mz_catalog.has_system_privilege"), + (16610, "mz_catalog.has_system_privilege"), + (16611, "try_parse_monotonic_iso8601_timestamp"), + (16612, "mz_internal.mz_role_oid_memberships"), + (16613, "mz_internal.mz_validate_role_privilege"), + (16614, "mz_is_superuser"), + (16615, "mz_internal.mz_format_privileges"), + (16616, "mz_internal.aclitem_grantor"), + (16617, "mz_internal.aclitem_grantee"), + (16618, "mz_internal.aclitem_privileges"), + (16619, "mz_internal.mz_aclexplode"), + (16620, "mz_catalog.has_role"), + (16621, "mz_catalog.has_role"), + (16622, "mz_catalog.has_role"), + (16623, "mz_catalog.has_role"), + (16624, "mz_catalog.has_role"), + (16625, "mz_catalog.has_role"), + (16626, "mz_unsafe.mz_avg_promotion"), + (16627, "mz_unsafe.mz_avg_promotion"), + (16628, "mz_unsafe.mz_avg_promotion"), + (16629, "mz_unsafe.mz_avg_promotion"), + (16630, "mz_unsafe.mz_avg_promotion"), + (16631, "mz_unsafe.mz_avg_promotion"), + (16632, "mz_unsafe.mz_avg_promotion"), + (16633, "mz_unsafe.mz_avg_promotion"), + (16634, "mz_unsafe.mz_avg_promotion"), + (16635, "mz_catalog.avg_internal_v1"), + (16636, "mz_catalog.avg_internal_v1"), + (16637, "mz_catalog.avg_internal_v1"), + (16638, "mz_catalog.avg_internal_v1"), + (16639, "mz_catalog.avg_internal_v1"), + (16640, "mz_catalog.avg_internal_v1"), + (16641, "mz_catalog.avg_internal_v1"), + (16642, "mz_catalog.avg_internal_v1"), + (16643, "mz_catalog.avg_internal_v1"), + (16644, "mz_catalog.constant_time_eq"), + (16645, "mz_catalog.constant_time_eq"), + (16646, "timezone_offset"), + (16647, "mz_catalog.pretty_sql"), + (16648, "mz_catalog.pretty_sql"), + (16649, "mz_internal.mz_name_rank"), + (16650, "mz_internal.mz_connection_oid"), + (16651, "mz_internal.mz_secret_oid"), + (16652, "map_build"), + (16653, "map_agg"), + (16654, "mz_catalog.unnest"), + (16655, "mz_internal.mz_normalize_schema_name"), + (16957, "current_catalog"), + (16958, "current_role"), + (16959, "user"), + (16987, "mz_catalog.crc32"), + (16988, "mz_catalog.crc32"), + (16989, "mz_catalog.kafka_murmur2"), + (16990, "mz_catalog.kafka_murmur2"), + (16991, "mz_catalog.seahash"), + (16992, "mz_catalog.seahash"), + (17069, "mz_internal.parse_catalog_id"), + (17070, "mz_internal.parse_catalog_privileges"), + (17073, "mz_internal.parse_catalog_create_sql"), + (17074, "mz_internal.redact_sql"), + (17075, "repeat_row_non_negative"), + (17086, "mz_internal.mz_session_role_memberships"), + (17090, "mz_internal.parse_catalog_acl_mode"), + (17093, "mz_unsafe.generate_series_unoptimized"), + (17094, "mz_unsafe.generate_series_unoptimized"), + (17100, "mz_internal.parse_catalog_audit_log_details"), + (17105, "mz_internal.parse_postgres_source_details"), + (17106, "mz_internal.parse_kafka_source_details"), +]; + +/// Returns the text that `regproc` OID `oid` renders as, or `None` when no +/// builtin function has that OID. +/// +/// PostgreSQL renders an unmatched OID in decimal and OID 0 as `-`. Neither is +/// in [`NAMES`], so callers handle both. +pub fn name(oid: u32) -> Option<&'static str> { + NAMES + .binary_search_by_key(&oid, |(entry_oid, _)| *entry_oid) + .ok() + .map(|i| NAMES[i].1) +} + +/// Returns the OID whose [`NAMES`] rendering is `name`. +/// +/// `Err` distinguishes no match from an ambiguous one. A rendering is ambiguous +/// when several overloads share it, in which case PostgreSQL's `regprocin` +/// likewise refuses to resolve it. The scan is linear because reads of this +/// direction only happen while decoding text-format input, which is rare +/// compared to encoding. +pub fn oid(name: &str) -> Result { + let mut found = None; + for (entry_oid, entry_name) in NAMES { + if *entry_name == name { + if found.is_some() { + return Err(NameLookupError::Ambiguous); + } + found = Some(*entry_oid); + } + } + found.ok_or(NameLookupError::NotFound) +} + +/// Why [`oid`] could not resolve a name. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NameLookupError { + /// No builtin function renders as the given name. + NotFound, + /// Several builtin functions render as the given name. + Ambiguous, +} diff --git a/src/pgrepr/src/lib.rs b/src/pgrepr/src/lib.rs index 12244010b54fc..d429f7a5898ae 100644 --- a/src/pgrepr/src/lib.rs +++ b/src/pgrepr/src/lib.rs @@ -23,6 +23,7 @@ mod types; mod value; pub mod oid; +pub mod regproc; pub use types::{ ANYCOMPATIBLELIST, ANYCOMPATIBLEMAP, LIST, MAP, Type, TypeConversionError, TypeFromOidError, diff --git a/src/pgrepr/src/regproc.rs b/src/pgrepr/src/regproc.rs new file mode 100644 index 0000000000000..63165b9216256 --- /dev/null +++ b/src/pgrepr/src/regproc.rs @@ -0,0 +1,61 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +#![allow(missing_docs)] + +pub use mz_pgrepr_consts::regproc::*; + +#[cfg(test)] +mod tests { + use super::*; + + /// [`name`] binary searches [`NAMES`], so the table must be sorted by OID + /// and free of duplicate OIDs. + #[mz_ore::test] + fn names_is_sorted_by_oid() { + for window in NAMES.windows(2) { + assert!( + window[0].0 < window[1].0, + "NAMES is not sorted by OID: {} precedes {}", + window[0].0, + window[1].0, + ); + } + } + + #[mz_ore::test] + fn lookups_round_trip() { + for (entry_oid, entry_name) in NAMES { + assert_eq!(name(*entry_oid), Some(*entry_name)); + // Overloads sharing a rendering can only resolve back to one OID, + // so ambiguity is the expected outcome for those. + match oid(entry_name) { + Ok(resolved) => assert_eq!(resolved, *entry_oid), + Err(err) => assert_eq!(err, NameLookupError::Ambiguous), + } + } + assert_eq!(name(0), None); + assert_eq!(oid("no_such_function"), Err(NameLookupError::NotFound)); + } + + /// The renderings that matter to clients are the `pg_type` columns declared + /// `regproc`, which is what motivates resolving names at all. Spot-check a + /// few so a regeneration that dropped them fails loudly. + #[mz_ore::test] + fn type_io_functions_resolve() { + for (func_oid, expected) in [ + (1242, "boolin"), + (2436, "boolrecv"), + (2400, "array_recv"), + (2414, "textrecv"), + ] { + assert_eq!(name(func_oid), Some(expected), "regproc {func_oid}"); + } + } +} diff --git a/src/pgrepr/src/value.rs b/src/pgrepr/src/value.rs index 0a5374eedee34..4e19dcafcc5b0 100644 --- a/src/pgrepr/src/value.rs +++ b/src/pgrepr/src/value.rs @@ -34,7 +34,7 @@ use uuid::Uuid; use crate::types::{NumericConstraints, UINT2, UINT4, UINT8}; use crate::value::error::{IntoDatumError, NulCharacterError}; -use crate::{Interval, Jsonb, Numeric, Type, UInt2, UInt4, UInt8}; +use crate::{Interval, Jsonb, Numeric, Type, UInt2, UInt4, UInt8, regproc}; pub mod error; pub mod interval; @@ -91,6 +91,12 @@ pub enum Value { Numeric(Numeric), /// An object identifier. Oid(u32), + /// An object identifier naming a function. + /// + /// Kept distinct from [`Value::Oid`] because the text encoding resolves the + /// OID to the function's name, matching PostgreSQL's `regprocout`. The + /// binary encoding is identical to an `oid`, as it is in PostgreSQL. + RegProc(u32), /// A sequence of heterogeneous values. Record(Vec>), /// A time. @@ -140,8 +146,15 @@ impl Value { (Datum::UInt8(c), SqlScalarType::PgLegacyChar) => Some(Value::Char(c)), (Datum::UInt16(u), SqlScalarType::UInt16) => Some(Value::UInt2(UInt2(u))), (Datum::UInt32(oid), SqlScalarType::Oid) => Some(Value::Oid(oid)), + (Datum::UInt32(oid), SqlScalarType::RegProc) => Some(Value::RegProc(oid)), + // NOTE: `regclass` and `regtype` collapse into `Value::Oid`, so they + // render as digits where PostgreSQL renders a name. Unlike + // `regproc`, they can name objects a user created, which no static + // table can know and which the encoder cannot look up: the catalog + // crates depend on `mz-pgrepr`, and the Kafka sink and `COPY TO` + // encoders run in `clusterd`, where there is no catalog at all. + // Resolution for those two lives in the SQL cast to `text` instead. (Datum::UInt32(oid), SqlScalarType::RegClass) => Some(Value::Oid(oid)), - (Datum::UInt32(oid), SqlScalarType::RegProc) => Some(Value::Oid(oid)), (Datum::UInt32(oid), SqlScalarType::RegType) => Some(Value::Oid(oid)), (Datum::UInt32(u), SqlScalarType::UInt32) => Some(Value::UInt4(UInt4(u))), (Datum::UInt64(u), SqlScalarType::UInt64) => Some(Value::UInt8(UInt8(u))), @@ -304,7 +317,7 @@ impl Value { }) })? } - Value::Oid(oid) => Datum::UInt32(oid), + Value::Oid(oid) | Value::RegProc(oid) => Datum::UInt32(oid), Value::Record(_) => { // This situation is handled gracefully by Value::decode; if we // wind up here it's a programming error. @@ -408,6 +421,20 @@ impl Value { }) .expect("provided closure never fails"), Value::Oid(oid) => strconv::format_uint32(buf, *oid), + // Mirrors PostgreSQL's `regprocout`: OID 0 renders as `-`, an OID + // that names a function renders as that name, and any other OID + // renders in decimal. + // + // NOTE: The SQL cast from `regproc` to `text` renders OID 0 as `0` + // rather than `-`, so the two disagree on that one value. PostgreSQL + // routes both through `regprocout` and prints `-` for both. + Value::RegProc(oid) => match *oid { + 0 => strconv::format_string(buf, REGPROC_NULL), + oid => match regproc::name(oid) { + Some(name) => strconv::format_string(buf, name), + None => strconv::format_uint32(buf, oid), + }, + }, Value::Record(elems) => strconv::format_record(buf, elems, |buf, elem| match elem { None => Ok::<_, ()>(buf.write_null()), Some(elem) => Ok(elem.encode_text(buf.nonnull_buffer())), @@ -530,7 +557,9 @@ impl Value { Err("binary encoding of map types is not implemented".into()) } Value::Name(s) => s.to_sql(&PgType::NAME, buf), - Value::Oid(i) => i.to_sql(&PgType::OID, buf), + // PostgreSQL's `regprocsend` is `int4send`, so a `regproc` is + // indistinguishable from an `oid` on the wire in binary format. + Value::Oid(i) | Value::RegProc(i) => i.to_sql(&PgType::OID, buf), Value::Record(fields) => { let nfields = pg_len("record field length", fields.len())?; buf.put_i32(nfields); @@ -720,9 +749,8 @@ impl Value { strconv::parse_numeric(s)?, constraints.as_ref(), )?)), - Type::Oid | Type::RegClass | Type::RegProc | Type::RegType => { - Value::Oid(strconv::parse_oid(s)?) - } + Type::Oid | Type::RegClass | Type::RegType => Value::Oid(strconv::parse_oid(s)?), + Type::RegProc => Value::RegProc(parse_regproc(s)?), Type::Record(_) => { return Err("input of anonymous composite types is not implemented".into()); } @@ -830,9 +858,10 @@ impl Value { strconv::parse_numeric(s)?, constraints.as_ref(), )?)), - Type::Oid | Type::RegClass | Type::RegProc | Type::RegType => { + Type::Oid | Type::RegClass | Type::RegType => { packer.push(Datum::UInt32(strconv::parse_oid(s)?)) } + Type::RegProc => packer.push(Datum::UInt32(parse_regproc(s)?)), Type::Record(_) => { return Err("input of anonymous composite types is not implemented".into()); } @@ -909,9 +938,10 @@ impl Value { constraints.as_ref(), )?))) } - Type::Oid | Type::RegClass | Type::RegProc | Type::RegType => { + Type::Oid | Type::RegClass | Type::RegType => { u32::from_sql(ty.inner(), raw).map(Value::Oid) } + Type::RegProc => u32::from_sql(ty.inner(), raw).map(Value::RegProc), Type::Record(_) => Err("input of anonymous composite types is not implemented".into()), Type::Text => decode_binary_string(ty, raw).map(Value::Text), Type::BpChar { .. } => decode_binary_string(ty, raw).map(Value::BpChar), @@ -962,6 +992,45 @@ impl Value { } } +/// The text PostgreSQL's `regprocout` emits for OID 0, and that `regprocin` +/// reads back as OID 0. +const REGPROC_NULL: &str = "-"; + +/// Parses the text format of a `regproc`, matching PostgreSQL's `regprocin`. +/// +/// A function name resolves to that function's OID, `-` is OID 0, and anything +/// else is read as a decimal OID. Accepting names is what keeps a `COPY ... TO` +/// rendering loadable by `COPY ... FROM`, since the text encoding emits names. +fn parse_regproc(s: &str) -> Result> { + if s == REGPROC_NULL { + return Ok(0); + } + match regproc::oid(s) { + Ok(oid) => Ok(oid), + // PostgreSQL refuses the same input, because a name shared by several + // overloads does not identify one of them. + Err(regproc::NameLookupError::Ambiguous) => { + Err(format!("more than one function named \"{}\"", s).into()) + } + Err(regproc::NameLookupError::NotFound) => match strconv::parse_oid(s) { + Ok(oid) => Ok(oid), + // Number-shaped input keeps its numeric diagnosis, which is what + // reports an out of range OID. Anything else was meant as a name, + // so report a missing function, matching both PostgreSQL and the + // SQL cast from `text` to `regproc`. + Err(err) if is_number_shaped(s) => Err(err.into()), + Err(_) => Err(format!("function \"{}\" does not exist", s).into()), + }, + } +} + +/// Whether `s` is shaped like a decimal number, ignoring whether it is in range. +fn is_number_shaped(s: &str) -> bool { + let digits = s.trim(); + let digits = digits.strip_prefix(['+', '-']).unwrap_or(digits); + !digits.is_empty() && digits.bytes().all(|b| b.is_ascii_digit()) +} + /// Returns an error if `s` contains a NUL character, which PostgreSQL rejects /// in text values. fn reject_nul(s: &str) -> Result<(), Box> { @@ -1067,6 +1136,73 @@ mod tests { }); } + /// A `regproc` renders as the function's name in text format and as a bare + /// OID in binary, which is what PostgreSQL's `regprocout` and + /// `regprocsend` do. Every text rendering must load back through + /// `decode_text`. + #[mz_ore::test] + fn regproc_text_encoding_resolves_names() { + // 1242 is `boolin`, uniquely named. 1398 is one of six `abs` + // overloads, so PostgreSQL qualifies it. 99999 names no function. + for (oid, expected) in [ + (0, "-"), + (1242, "boolin"), + (1398, "pg_catalog.abs"), + (99999, "99999"), + (u32::MAX, "4294967295"), + ] { + let mut buf = BytesMut::new(); + Value::from_datum(Datum::UInt32(oid), &SqlScalarType::RegProc) + .expect("a non-null datum encodes") + .encode_text(&mut buf); + assert_eq!(str::from_utf8(&buf).unwrap(), expected, "regproc {oid}"); + + let Value::RegProc(decoded) = Value::decode_text(&Type::RegProc, &buf).unwrap() else { + panic!("decoding a regproc must yield Value::RegProc"); + }; + assert_eq!(decoded, oid, "text rendering {expected} did not round trip"); + + // Binary stays a 4-byte OID, so the name resolution is invisible + // there. + let mut binary = BytesMut::new(); + Value::RegProc(oid) + .encode_binary(&Type::RegProc, &mut binary) + .expect("regproc has a binary encoding"); + assert_eq!(&binary[..], &oid.to_be_bytes()[..], "regproc {oid} binary"); + } + } + + /// A name shared by several overloads does not identify one of them, so + /// reading it back is an error rather than an arbitrary choice. PostgreSQL + /// rejects the same input. + #[mz_ore::test] + fn regproc_text_decoding_rejects_ambiguous_names() { + assert_eq!( + Value::decode_text(&Type::RegProc, b"pg_catalog.abs") + .map(|_| ()) + .map_err(|e| e.to_string()) + .unwrap_err(), + "more than one function named \"pg_catalog.abs\"", + ); + assert_eq!( + Value::decode_text(&Type::RegProc, b"no_such_function") + .map(|_| ()) + .map_err(|e| e.to_string()) + .unwrap_err(), + "function \"no_such_function\" does not exist", + ); + // Number-shaped input keeps its numeric diagnosis rather than being + // reported as a missing function. + let out_of_range = Value::decode_text(&Type::RegProc, b"99999999999999") + .map(|_| ()) + .map_err(|e| e.to_string()) + .unwrap_err(); + assert!( + out_of_range.starts_with("invalid input syntax for type oid"), + "unexpected error for an out of range OID: {out_of_range}", + ); + } + /// Verifies that we correctly print the chain of parsing errors, all the way through the stack. #[mz_ore::test] fn decode_text_error_smoke_test() { diff --git a/test/adbc/smoketest.py b/test/adbc/smoketest.py index 8ecad8f68f44b..60a638bc4e16f 100644 --- a/test/adbc/smoketest.py +++ b/test/adbc/smoketest.py @@ -10,13 +10,12 @@ """Apache Arrow ADBC PostgreSQL driver tests. The driver keys its Arrow type mapping off the *name* of each type's binary -receive function, which it reads from `pg_catalog.pg_type.typreceive`. -Materialize encodes a `regproc` over pgwire as a bare OID rather than as the -function name, so every lookup misses and the driver falls back to -`arrow.opaque` over binary. Connecting and streaming rows work. Per-column type -fidelity does not, so the tests that require it are marked `expectedFailure` -rather than deleted. They report an unexpected success once `regproc` renders as -a name. +receive function, which it reads from `pg_catalog.pg_type.typreceive`. That +column is declared `regproc`, and Materialize resolves a `regproc` to its +function name in the pgwire text encoding, so the driver's lookups hit and each +column arrives as its own Arrow type. A type the driver cannot resolve degrades +to `arrow.opaque` over binary rather than erroring, so asserting the exact Arrow +type per column is what catches a regression here. """ import unittest @@ -101,9 +100,6 @@ def test_connect(self) -> None: self.assertEqual(table.num_rows, 1) self.assertEqual(table.num_columns, 1) - # Materialize encodes `regproc` as an OID rather than a function name, so - # the driver cannot key its type map and every column resolves to binary. - @unittest.expectedFailure def test_scalar_types_resolve(self) -> None: """Each scalar type must arrive as its own Arrow type. A type the driver cannot resolve degrades to opaque binary instead of failing, so @@ -139,10 +135,6 @@ def test_scalar_types_resolve(self) -> None: self.assertEqual(row["c_text"], "abc") self.assertEqual(row["c_bytea"], b"\x01\x02") - # Materialize encodes `regproc` as an OID rather than a function name, so - # the driver cannot key its type map and the element type resolves to - # binary. - @unittest.expectedFailure def test_array_type(self) -> None: """An array must arrive as an Arrow list, not as an opaque blob.""" table = query_arrow("SELECT ARRAY[1, 2, 3]::int4[] AS a") @@ -154,9 +146,6 @@ def test_array_type(self) -> None: ) self.assertEqual(table.to_pylist(), [{"a": [1, 2, 3]}]) - # Materialize encodes `regproc` as an OID rather than a function name, so - # the driver cannot key its type map and every column resolves to binary. - @unittest.expectedFailure def test_null_handling(self) -> None: """Nulls round-trip while keeping the column's resolved Arrow type.""" table = query_arrow(""" @@ -175,10 +164,6 @@ def test_null_handling(self) -> None: [{"c_int4": None, "c_text": None, "c_timestamptz": None}], ) - # Materialize encodes `regproc` as an OID rather than a function name, so - # the driver cannot key its type map and DuckDB is handed blobs it - # cannot aggregate. - @unittest.expectedFailure def test_duckdb_handoff(self) -> None: """DuckDB reads the Arrow table Materialize produced directly, with no intermediate file or object store hop.""" @@ -207,9 +192,7 @@ def test_duckdb_handoff(self) -> None: def test_fetch_many_rows(self) -> None: """The driver fetches results as a binary COPY stream. This covers a - result large enough to span multiple reads of that stream. Row and - column counts hold regardless of how each column's type resolves, so - this stays a hard assertion.""" + result large enough to span multiple reads of that stream.""" table = query_arrow("SELECT n FROM generate_series(1, 5000) AS g(n)") self.assertEqual(table.num_rows, 5000) self.assertEqual(table.num_columns, 1) diff --git a/test/pgtest-mz/datums.pt b/test/pgtest-mz/datums.pt index 5b6eef6f1f89e..3b924bc164155 100644 --- a/test/pgtest-mz/datums.pt +++ b/test/pgtest-mz/datums.pt @@ -23,7 +23,7 @@ ReadyForQuery {"status":"I"} CommandComplete {"tag":"CREATE TABLE"} ReadyForQuery {"status":"I"} RowDescription {"fields":[{"name":"rowid"},{"name":"_bool"},{"name":"_int16"},{"name":"_int32"},{"name":"_int64"},{"name":"_uint16"},{"name":"_uint32"},{"name":"_uint64"},{"name":"_float32"},{"name":"_float64"},{"name":"_numeric"},{"name":"_date"},{"name":"_time"},{"name":"_timestamp"},{"name":"_timestamp_"},{"name":"_timestamp__"},{"name":"_timestamptz"},{"name":"_timestamptz_"},{"name":"_timestamptz__"},{"name":"_interval"},{"name":"_pglegacychar"},{"name":"_bytes"},{"name":"_string"},{"name":"_char"},{"name":"_varchar"},{"name":"_jsonb"},{"name":"_uuid"},{"name":"_oid"},{"name":"_regproc"},{"name":"_regtype"},{"name":"_regclass"},{"name":"_int2vector"},{"name":"_mztimestamp"},{"name":"_mzaclitem"}]} -DataRow {"fields":["1","t","0","0","0","0","0","0","0","0","0","2000-01-01","00:00:00","1970-01-01 00:00:00","1970-01-01 00:00:00","1970-01-01 00:00:00","1970-01-01 00:00:00+00","1970-01-01 00:00:00+00","1970-01-01 00:00:00+00","00:00:00","\u0000","\\x",""," ","","true","00000000-0000-0000-0000-000000000000","0","0","0","0","NULL","0","=/p"]} +DataRow {"fields":["1","t","0","0","0","0","0","0","0","0","0","2000-01-01","00:00:00","1970-01-01 00:00:00","1970-01-01 00:00:00","1970-01-01 00:00:00","1970-01-01 00:00:00+00","1970-01-01 00:00:00+00","1970-01-01 00:00:00+00","00:00:00","\u0000","\\x",""," ","","true","00000000-0000-0000-0000-000000000000","0","-","0","0","NULL","0","=/p"]} DataRow {"fields":["2","f","1","1","1","1","1","1","1","1","1","4714-11-24 BC","23:59:59.999999","4714-12-31 00:00:00 BC","4714-12-31 00:00:00 BC","4714-12-31 00:00:00 BC","4714-12-31 00:00:00+00 BC","4714-12-31 00:00:00+00 BC","4714-12-31 00:00:00+00 BC","1 mon 1 day 00:00:00.000001","[255]","\\x00"," ","'"," ","false","ffffffff-ffff-ffff-ffff-ffffffffffff","4294967295","4294967295","4294967295","4294967295","NULL","18446744073709551615","=arwdUCRBNP/p"]} DataRow {"fields":["3","NULL","-1","-1","-1","65535","4294967295","18446744073709551615","-1","-1","-1","262142-12-31","NULL","262142-12-31 23:59:59","262142-12-31 23:59:59","262142-12-31 23:59:59","262142-12-31 23:59:59+00","262142-12-31 23:59:59+00","262142-12-31 23:59:59+00","-1 mons -1 days -00:00:00.000001","NULL","\\xff","'","\"","'","null","NULL","NULL","NULL","NULL","NULL","NULL","NULL","u42=/p"]} DataRow {"fields":["4","NULL","-32768","-2147483648","-9223372036854775808","255","32767","2147483647","-3.4028235e+38","-1.7976931348623157e+308","-Infinity","NULL","NULL","1970-01-01 00:00:00.123457","1970-01-01 00:00:00.123457","1970-01-01 00:00:00.123457","1970-01-01 00:00:00.123457+00","1970-01-01 00:00:00.123457+00","1970-01-01 00:00:00.123457+00","1 mon","NULL","NULL","\"",".","\"","\"\"","NULL","NULL","NULL","NULL","NULL","NULL","NULL","u42=arwdUCRBNP/p"]} diff --git a/test/sqllogictest/regproc.slt b/test/sqllogictest/regproc.slt index c9bc16ff5eb64..a55a01608a599 100644 --- a/test/sqllogictest/regproc.slt +++ b/test/sqllogictest/regproc.slt @@ -192,6 +192,36 @@ true true true +# The pgwire text encoding resolves a regproc to its function name, matching +# PostgreSQL's regprocout. These have to be `simple` records: `query` records +# read results in the binary format, where a regproc is a bare OID, so the text +# rendering is invisible to them. + +# 1242 is boolin and is uniquely named. 1398 is one of six abs overloads, so the +# bare name would not identify it and it is qualified. 99999 names no function +# and renders in decimal. OID 0 renders as `-`. +simple +SELECT 1242::regproc, 1398::regproc, 99999::regproc, 0::regproc +---- +boolin,pg_catalog.abs,99999,- +COMPLETE 1 + +# The motivating case: clients resolve a type's I/O function names to decide how +# to decode its values. +simple +SELECT typinput, typreceive FROM pg_type WHERE typname = 'bool' +---- +boolin,boolrecv +COMPLETE 1 + +# regclass and regtype keep rendering as digits. They can name objects a user +# created, which the wire encoder cannot resolve. +simple +SELECT 1082::regtype, 1242::regclass +---- +1082,1242 +COMPLETE 1 + # ensure that catalog functions can be resolved if the active database is invalid statement OK From 208d36b2672b5615e4792cc4fd50561d6f51eadd Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 17:51:18 +0000 Subject: [PATCH 04/15] pgrepr: do not assert an ambiguous regproc rendering round trips A rendering that is schema-qualified because its name is overloaded still names every one of that name's impls, so decode_text cannot pick one and returns the same error PostgreSQL's regprocin does. The encoding test asserted pg_catalog.abs round tripped, contradicting the decoding test right below it, and parse_regproc's doc comment claimed round tripping without that qualification. Also skip the reverse-lookup coverage test under miri. Covering every entry costs a table scan per entry, which is milliseconds natively and far too slow interpreted. --- src/pgrepr/src/regproc.rs | 3 +++ src/pgrepr/src/value.rs | 43 ++++++++++++++++++++++++--------------- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/src/pgrepr/src/regproc.rs b/src/pgrepr/src/regproc.rs index 63165b9216256..9de5faba5dc50 100644 --- a/src/pgrepr/src/regproc.rs +++ b/src/pgrepr/src/regproc.rs @@ -30,6 +30,9 @@ mod tests { } #[mz_ore::test] + // `oid` scans the table, so covering every entry is quadratic in its 626 + // rows. That is milliseconds natively and far too slow interpreted. + #[cfg_attr(miri, ignore)] fn lookups_round_trip() { for (entry_oid, entry_name) in NAMES { assert_eq!(name(*entry_oid), Some(*entry_name)); diff --git a/src/pgrepr/src/value.rs b/src/pgrepr/src/value.rs index 4e19dcafcc5b0..3dd56d024d694 100644 --- a/src/pgrepr/src/value.rs +++ b/src/pgrepr/src/value.rs @@ -999,8 +999,13 @@ const REGPROC_NULL: &str = "-"; /// Parses the text format of a `regproc`, matching PostgreSQL's `regprocin`. /// /// A function name resolves to that function's OID, `-` is OID 0, and anything -/// else is read as a decimal OID. Accepting names is what keeps a `COPY ... TO` -/// rendering loadable by `COPY ... FROM`, since the text encoding emits names. +/// else is read as a decimal OID. +/// +/// Accepting names is what keeps a `COPY ... TO` rendering loadable by +/// `COPY ... FROM`, since the text encoding emits names. That holds for every +/// rendering that names exactly one function. An overloaded name renders +/// schema-qualified and still names every one of its impls, so it does not round +/// trip, which is true of PostgreSQL as well. fn parse_regproc(s: &str) -> Result> { if s == REGPROC_NULL { return Ok(0); @@ -1138,18 +1143,21 @@ mod tests { /// A `regproc` renders as the function's name in text format and as a bare /// OID in binary, which is what PostgreSQL's `regprocout` and - /// `regprocsend` do. Every text rendering must load back through - /// `decode_text`. + /// `regprocsend` do. #[mz_ore::test] fn regproc_text_encoding_resolves_names() { - // 1242 is `boolin`, uniquely named. 1398 is one of six `abs` - // overloads, so PostgreSQL qualifies it. 99999 names no function. - for (oid, expected) in [ - (0, "-"), - (1242, "boolin"), - (1398, "pg_catalog.abs"), - (99999, "99999"), - (u32::MAX, "4294967295"), + // 1242 is `boolin`, uniquely named. 1398 is one of six `abs` overloads, + // so PostgreSQL qualifies it. 99999 names no function. + // + // `round_trips` is false for the qualified overload: that rendering + // names all six impls, so reading it back cannot pick one. PostgreSQL + // does not round trip it either. + for (oid, expected, round_trips) in [ + (0, "-", true), + (1242, "boolin", true), + (1398, "pg_catalog.abs", false), + (99999, "99999", true), + (u32::MAX, "4294967295", true), ] { let mut buf = BytesMut::new(); Value::from_datum(Datum::UInt32(oid), &SqlScalarType::RegProc) @@ -1157,10 +1165,13 @@ mod tests { .encode_text(&mut buf); assert_eq!(str::from_utf8(&buf).unwrap(), expected, "regproc {oid}"); - let Value::RegProc(decoded) = Value::decode_text(&Type::RegProc, &buf).unwrap() else { - panic!("decoding a regproc must yield Value::RegProc"); - }; - assert_eq!(decoded, oid, "text rendering {expected} did not round trip"); + if round_trips { + let Value::RegProc(decoded) = Value::decode_text(&Type::RegProc, &buf).unwrap() + else { + panic!("decoding a regproc must yield Value::RegProc"); + }; + assert_eq!(decoded, oid, "text rendering {expected} did not round trip"); + } // Binary stays a 4-byte OID, so the name resolution is invisible // there. From ec484146f6455405a8cfa33642a59283f0389883 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 17:51:43 +0000 Subject: [PATCH 05/15] catalog: migrate mz_type_pg_metadata by replacement, not evolution The `typsend` addition to `mz_internal.mz_type_pg_metadata` was registered as a `MigrationStep::evolution`. `Evolution` is the wrong mechanism for a builtin table. Under `Evolution` the read-only side of a 0dt upgrade never registers the new schema. `migrate_evolve_one` only checks `backward_compatible` and returns, leaving the shard advertising the old three-column schema while the new binary's catalog describes four. Every dataflow the read-only environment must hydrate before it reports ready, here the `pg_type_all_databases_ind` builtin index, then has to read the old parts through persist's part-migration path. No builtin table has ever taken a column addition, so that combination is unexercised. `Replacement` is what the rest of the list uses. It is also what populates `MigrationResult::replaced_items`, which flows into `migrated_storage_collections_0dt` and switches on the read-only back-fill in `Coordinator::bootstrap`. The new shard is written directly and the dependent dataflow hydrates from it. Replacement discards the old shard contents, which costs nothing here. `Coordinator::bootstrap_tables` retracts every system table's full persist snapshot on each boot and re-derives the rows from the catalog. The only two builtin tables whose contents must survive, `mz_storage_usage_by_shard` and `mz_object_arrangement_size_history`, are excluded there and rejected outright by `validate_migration_steps`. `mz_type_pg_metadata` is in neither set. Also corrects the justification for `typsend` being `nullable(true)`. It claimed persist schema evolution requires an appended field to be nullable. That rule exists in `mz_persist_types::schema::backward_compatible`, but it is unreachable for row columns because `RowColumnarEncoder::finish` hardcodes Arrow nullability to `true` for every column. The real situation is that the declaration is simply wider than `pack_type_update` needs. --- .../catalog/open/builtin_schema_migration.rs | 19 +++++++++++++++---- src/catalog/src/builtin/mz_internal.rs | 9 +++++++-- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/adapter/src/catalog/open/builtin_schema_migration.rs b/src/adapter/src/catalog/open/builtin_schema_migration.rs index af570f530cb14..6a9b3eb4a1226 100644 --- a/src/adapter/src/catalog/open/builtin_schema_migration.rs +++ b/src/adapter/src/catalog/open/builtin_schema_migration.rs @@ -302,10 +302,21 @@ static MIGRATIONS: LazyLock> = LazyLock::new(|| { "mz_kafka_sources", ), // `mz_type_pg_metadata` gained a trailing `typsend` column. Appending a - // column is backward compatible, so the shard's schema can be evolved - // in place. See the NOTE above: this version must stay at the - // workspace's current dev version until the change ships. - MigrationStep::evolution( + // column would be backward compatible enough for `Evolution`, but + // `Replacement` is the right mechanism for a builtin table anyway. + // `Coordinator::bootstrap_tables` retracts every system table's persist + // snapshot and re-derives the rows from the catalog on each boot, so a + // fresh shard loses nothing. More importantly, only `Replacement` + // reports the item in `MigrationResult::replaced_items`, which is what + // makes the read-only coordinator back-fill the new shard so dependent + // dataflows such as `pg_type_all_databases_ind` can hydrate. Under + // `Evolution` a read-only replica would instead have to read the old + // parts through the schema-migration path, which no builtin table has + // ever exercised. + // + // See the NOTE above: this version must stay at the workspace's current + // dev version until the change ships. + MigrationStep::replacement( "26.36.0-dev.0", CatalogItemType::Table, MZ_INTERNAL_SCHEMA, diff --git a/src/catalog/src/builtin/mz_internal.rs b/src/catalog/src/builtin/mz_internal.rs index 8746dce20bfd9..ccb26716859ac 100644 --- a/src/catalog/src/builtin/mz_internal.rs +++ b/src/catalog/src/builtin/mz_internal.rs @@ -586,8 +586,13 @@ pub static MZ_TYPE_PG_METADATA: LazyLock = LazyLock::new(|| Builti .with_column("id", SqlScalarType::String.nullable(false)) .with_column("typinput", SqlScalarType::Oid.nullable(false)) .with_column("typreceive", SqlScalarType::Oid.nullable(false)) - // Always populated, but declared nullable because persist schema - // evolution only accepts an appended field if it is nullable. + // `pack_type_update` always writes an OID here, so this declaration is + // wider than the data requires. Downstream `COALESCE`s on this column + // are there for the `LEFT JOIN` against this table, not for this. + // + // TODO: Tighten to `nullable(false)` to match `typinput` and + // `typreceive`. That also changes the descs of the views projecting the + // column. .with_column("typsend", SqlScalarType::Oid.nullable(true)) .finish(), column_comments: BTreeMap::new(), From 033eca370c8c0dbd31333924515213f0ec8cb466 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 18:17:16 +0000 Subject: [PATCH 06/15] pgrepr: document the regproc reserved-word quoting divergence `regprocout` quotes a rendering that is not a bare identifier, printing `"left"` and `pg_catalog."substring"`. The static table leaves those unquoted to match Materialize's `regproc` to `text` cast, so the wire format and the in-product cast agree. Record that divergence and why the unquoted form is still loadable. Measured against PostgreSQL 16.14. Qualification itself is not a divergence: `regprocout` qualifies whenever the bare name would not be found uniquely by `regprocin`, so a merely overloaded name such as `abs` renders `pg_catalog.abs` there too, independent of `search_path`. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01K8HcpvmmntWK9QuJWvKVbg --- src/pgrepr-consts/src/regproc.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/pgrepr-consts/src/regproc.rs b/src/pgrepr-consts/src/regproc.rs index bc0940c9ebaa9..cb7012840309f 100644 --- a/src/pgrepr-consts/src/regproc.rs +++ b/src/pgrepr-consts/src/regproc.rs @@ -37,6 +37,13 @@ /// `search_path` makes those bare names resolvable, so `regprocout` would print /// them bare and this table still qualifies them. The SQL cast from `regproc` to /// `text` reads the session's search path and does render them bare. +/// +/// NOTE: Names that are not bare identifiers are left unquoted here, so `left` +/// renders as `left` and OID 936 as `pg_catalog.substring`. PostgreSQL's +/// `regprocout` quotes those, rendering `"left"` and `pg_catalog."substring"`. +/// This table matches the SQL cast from `regproc` to `text` instead, so the two +/// Materialize paths agree with each other. The unquoted form still resolves, +/// because `regprocin` accepts a reserved word with or without quotes. pub const NAMES: &[(u32, &str)] = &[ (34, "namein"), (38, "int2in"), From 4f47bff49a7a10080a3351c3dba49aeae7cc9d88 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 20:09:15 +0000 Subject: [PATCH 07/15] sqllogictest: regenerate catalog_server_explain.slt for pg_type typsend Adding typsend to pg_type_all_databases widens the view from 19 to 20 columns, which shifts column references throughout the recorded EXPLAIN plans and adds an oidtoregproc(coalesce(#12{typsend}, 0)) projection alongside the existing oidtoregproc(#10{typinput}). Regenerated from the rewrite-results diff emitted by the slt-4 job of Buildkite build #129572, which reported the two expected output failures in this file (lines 4203 and 19091) and no others. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01K8HcpvmmntWK9QuJWvKVbg --- test/sqllogictest/catalog_server_explain.slt | 95 ++++++++++---------- 1 file changed, 48 insertions(+), 47 deletions(-) diff --git a/test/sqllogictest/catalog_server_explain.slt b/test/sqllogictest/catalog_server_explain.slt index 39ca0049f5634..eb491bc64e69b 100644 --- a/test/sqllogictest/catalog_server_explain.slt +++ b/test/sqllogictest/catalog_server_explain.slt @@ -4223,8 +4223,8 @@ mz_internal.pg_type_all_databases: →Arrange (#0{id}) (#2{schema_id}) →Union →Map/Filter/Project - Project: #0..=#11 - Map: null, null, null + Project: #0..=#12 + Map: null, null, null, null →Consolidating Union →Negate Diffs →Differential Join %1[#0] » %0:l0[#0{id}] @@ -4238,7 +4238,7 @@ mz_internal.pg_type_all_databases: Project: #1, #2, #0, #3..=#8 →Arranged mz_catalog.mz_types →Fused with Child Map/Filter/Project - Project: #0..=#8, #0, #9, #10 + Project: #0..=#8, #0, #9..=#11 →Read l1 →Arrange (#0{id}) →Fused with Parent Map/Filter/Project @@ -4258,10 +4258,10 @@ mz_internal.pg_type_all_databases: →Read mz_catalog.mz_map_types →Read mz_catalog.mz_pseudo_types cte l3 = - →Differential Join %1:mz_databases[#0] » %0:l2[#13{database_id}] - →Arrange (#13{database_id}) + →Differential Join %1:mz_databases[#0] » %0:l2[#14{database_id}] + →Arrange (#14{database_id}) →Fused with Child Map/Filter/Project - Filter: (#13{database_id}) IS NOT NULL + Filter: (#14{database_id}) IS NOT NULL →Read l2 →Arrange (#0) →Fused with Parent Map/Filter/Project @@ -4272,18 +4272,18 @@ mz_internal.pg_type_all_databases: →Arrange (#5{owner_id}) →Union →Map/Filter/Project - Project: #0..=#24 + Project: #0..=#25 Map: null, null, null, null, null →Consolidating Union →Negate Diffs →Fused with Child Map/Filter/Project - Project: #0..=#11, #2, #12..=#16, #0, #17 + Project: #0..=#12, #2, #13..=#17, #0, #18 →Read l3 →Fused with Child Map/Filter/Project - Project: #0..=#11, #2, #12..=#16, #0, #17 + Project: #0..=#12, #2, #13..=#17, #0, #18 →Read l2 →Fused with Child Map/Filter/Project - Project: #0..=#11, #2, #12..=#16, #0, #17, #13, #18..=#21 + Project: #0..=#12, #2, #13..=#17, #0, #18, #14, #19..=#22 →Read l3 →Arranged mz_catalog.mz_roles cte l5 = @@ -4294,7 +4294,7 @@ mz_internal.pg_type_all_databases: →Fused with Child Map/Filter/Project Project: #0 →Arranged l5 - Key: (#0..=#29) + Key: (#0..=#30) cte l7 = →Arranged l6 cte l8 = @@ -4361,73 +4361,73 @@ mz_internal.pg_type_all_databases: Project: #0 →Read l15 →Return - →Delta Join [%0:l4[#0..=#29] » %1:l5[#0..=#29] » %2[#0] » %3[#0] » %4[#0]] [%1:l5[#0] » %0:l4[#0..=#29] » %2[#0] » %3[#0] » %4[#0]] [%2[#0] » %1:l5[#0] » %0:l4[#0..=#29] » %3[#0] » %4[#0]] [%3[#0] » %1:l5[#0] » %0:l4[#0..=#29] » %2[#0] » %4[#0]] [%4[#0] » %1:l5[#0] » %0:l4[#0..=#29] » %2[#0] » %3[#0]] + →Delta Join [%0:l4[#0..=#30] » %1:l5[#0..=#30] » %2[#0] » %3[#0] » %4[#0]] [%1:l5[#0] » %0:l4[#0..=#30] » %2[#0] » %3[#0] » %4[#0]] [%2[#0] » %1:l5[#0] » %0:l4[#0..=#30] » %3[#0] » %4[#0]] [%3[#0] » %1:l5[#0] » %0:l4[#0..=#30] » %2[#0] » %4[#0]] [%4[#0] » %1:l5[#0] » %0:l4[#0..=#30] » %2[#0] » %3[#0]] path %0: after %3: - Project: #0..=#5, #7..=#10, #12 - Map: coalesce(#6, #11, 0) + Project: #0..=#5, #7..=#11, #13 + Map: coalesce(#6, #12, 0) after %4: - Project: #1..=#10, #12 - Map: coalesce(#11, 0) + Project: #1..=#11, #13 + Map: coalesce(#12, 0) Final closure: - Project: #0..=#2, #4, #11, #5, #6, #12, #13, #9, #10, #7, #8, #14, #13, #15, #13, #16, #3 + Project: #0..=#2, #4, #12, #5, #6, #13, #14, #10, #11, #7, #8, #15, #14, #16, #14, #17, #3, #9 Map: null, 44, 0, false, -1, null path %1: after %0: - Project: #1, #3, #13, #22, #25, #0, #30..=#33 - Map: text_to_"char"(case when (#19{mztype} = "a") then "b" else #19{mztype} end), text_to_"char"(case when (#4{category} = "array") then "A" else case when (#4{category} = "bit-string") then "V" else case when (#4{category} = "boolean") then "B" else case when (#4{category} = "composite") then "C" else case when (#4{category} = "date-time") then "D" else case when (#4{category} = "enum") then "E" else case when (#4{category} = "geometric") then "G" else case when (#4{category} = "list") then "U" else case when (#4{category} = "network-address") then "I" else case when (#4{category} = "numeric") then "N" else case when (#4{category} = "pseudo") then "P" else case when (#4{category} = "string") then "S" else case when (#4{category} = "timespan") then "T" else case when (#4{category} = "user-defined") then "U" else case when (#4{category} = "unknown") then "X" else null end end end end end end end end end end end end end end end), oidtoregproc(#10{typinput}), coalesce(#11{typreceive}, 0) + Project: #1, #3, #14, #23, #26, #0, #31..=#35 + Map: text_to_"char"(case when (#20{mztype} = "a") then "b" else #20{mztype} end), text_to_"char"(case when (#4{category} = "array") then "A" else case when (#4{category} = "bit-string") then "V" else case when (#4{category} = "boolean") then "B" else case when (#4{category} = "composite") then "C" else case when (#4{category} = "date-time") then "D" else case when (#4{category} = "enum") then "E" else case when (#4{category} = "geometric") then "G" else case when (#4{category} = "list") then "U" else case when (#4{category} = "network-address") then "I" else case when (#4{category} = "numeric") then "N" else case when (#4{category} = "pseudo") then "P" else case when (#4{category} = "string") then "S" else case when (#4{category} = "timespan") then "T" else case when (#4{category} = "user-defined") then "U" else case when (#4{category} = "unknown") then "X" else null end end end end end end end end end end end end end end end), oidtoregproc(#10{typinput}), coalesce(#11{typreceive}, 0), oidtoregproc(coalesce(#12{typsend}, 0)) after %3: - Project: #1..=#5, #0, #7..=#10, #12 - Map: coalesce(#6, #11, 0) + Project: #1..=#5, #0, #7..=#11, #13 + Map: coalesce(#6, #12, 0) after %4: - Project: #1..=#10, #12 - Map: coalesce(#11, 0) + Project: #1..=#11, #13 + Map: coalesce(#12, 0) Final closure: - Project: #0..=#2, #4, #11, #5, #6, #12, #13, #9, #10, #7, #8, #14, #13, #15, #13, #16, #3 + Project: #0..=#2, #4, #12, #5, #6, #13, #14, #10, #11, #7, #8, #15, #14, #16, #14, #17, #3, #9 Map: null, 44, 0, false, -1, null path %2: after %0: - Project: #1, #3, #13, #22, #25, #0, #30..=#34 - Map: text_to_"char"(case when (#19{mztype} = "a") then "b" else #19{mztype} end), text_to_"char"(case when (#4{category} = "array") then "A" else case when (#4{category} = "bit-string") then "V" else case when (#4{category} = "boolean") then "B" else case when (#4{category} = "composite") then "C" else case when (#4{category} = "date-time") then "D" else case when (#4{category} = "enum") then "E" else case when (#4{category} = "geometric") then "G" else case when (#4{category} = "list") then "U" else case when (#4{category} = "network-address") then "I" else case when (#4{category} = "numeric") then "N" else case when (#4{category} = "pseudo") then "P" else case when (#4{category} = "string") then "S" else case when (#4{category} = "timespan") then "T" else case when (#4{category} = "user-defined") then "U" else case when (#4{category} = "unknown") then "X" else null end end end end end end end end end end end end end end end), oidtoregproc(#10{typinput}), coalesce(#11{typreceive}, 0) + Project: #1, #3, #14, #23, #26, #0, #31..=#36 + Map: text_to_"char"(case when (#20{mztype} = "a") then "b" else #20{mztype} end), text_to_"char"(case when (#4{category} = "array") then "A" else case when (#4{category} = "bit-string") then "V" else case when (#4{category} = "boolean") then "B" else case when (#4{category} = "composite") then "C" else case when (#4{category} = "date-time") then "D" else case when (#4{category} = "enum") then "E" else case when (#4{category} = "geometric") then "G" else case when (#4{category} = "list") then "U" else case when (#4{category} = "network-address") then "I" else case when (#4{category} = "numeric") then "N" else case when (#4{category} = "pseudo") then "P" else case when (#4{category} = "string") then "S" else case when (#4{category} = "timespan") then "T" else case when (#4{category} = "user-defined") then "U" else case when (#4{category} = "unknown") then "X" else null end end end end end end end end end end end end end end end), oidtoregproc(#10{typinput}), coalesce(#11{typreceive}, 0), oidtoregproc(coalesce(#12{typsend}, 0)) after %3: - Project: #1..=#5, #0, #7..=#10, #12 - Map: coalesce(#6, #11, 0) + Project: #1..=#5, #0, #7..=#11, #13 + Map: coalesce(#6, #12, 0) after %4: - Project: #1..=#10, #12 - Map: coalesce(#11, 0) + Project: #1..=#11, #13 + Map: coalesce(#12, 0) Final closure: - Project: #0..=#2, #4, #11, #5, #6, #12, #13, #9, #10, #7, #8, #14, #13, #15, #13, #16, #3 + Project: #0..=#2, #4, #12, #5, #6, #13, #14, #10, #11, #7, #8, #15, #14, #16, #14, #17, #3, #9 Map: null, 44, 0, false, -1, null path %3: after %0: - Project: #1, #3, #13, #22, #25, #0, #30..=#34 - Map: text_to_"char"(case when (#19{mztype} = "a") then "b" else #19{mztype} end), text_to_"char"(case when (#4{category} = "array") then "A" else case when (#4{category} = "bit-string") then "V" else case when (#4{category} = "boolean") then "B" else case when (#4{category} = "composite") then "C" else case when (#4{category} = "date-time") then "D" else case when (#4{category} = "enum") then "E" else case when (#4{category} = "geometric") then "G" else case when (#4{category} = "list") then "U" else case when (#4{category} = "network-address") then "I" else case when (#4{category} = "numeric") then "N" else case when (#4{category} = "pseudo") then "P" else case when (#4{category} = "string") then "S" else case when (#4{category} = "timespan") then "T" else case when (#4{category} = "user-defined") then "U" else case when (#4{category} = "unknown") then "X" else null end end end end end end end end end end end end end end end), oidtoregproc(#10{typinput}), coalesce(#11{typreceive}, 0) + Project: #1, #3, #14, #23, #26, #0, #31..=#36 + Map: text_to_"char"(case when (#20{mztype} = "a") then "b" else #20{mztype} end), text_to_"char"(case when (#4{category} = "array") then "A" else case when (#4{category} = "bit-string") then "V" else case when (#4{category} = "boolean") then "B" else case when (#4{category} = "composite") then "C" else case when (#4{category} = "date-time") then "D" else case when (#4{category} = "enum") then "E" else case when (#4{category} = "geometric") then "G" else case when (#4{category} = "list") then "U" else case when (#4{category} = "network-address") then "I" else case when (#4{category} = "numeric") then "N" else case when (#4{category} = "pseudo") then "P" else case when (#4{category} = "string") then "S" else case when (#4{category} = "timespan") then "T" else case when (#4{category} = "user-defined") then "U" else case when (#4{category} = "unknown") then "X" else null end end end end end end end end end end end end end end end), oidtoregproc(#10{typinput}), coalesce(#11{typreceive}, 0), oidtoregproc(coalesce(#12{typsend}, 0)) after %2: - Project: #1..=#5, #0, #7..=#10, #12 - Map: coalesce(#11, #6, 0) + Project: #1..=#5, #0, #7..=#11, #13 + Map: coalesce(#12, #6, 0) after %4: - Project: #1..=#10, #12 - Map: coalesce(#11, 0) + Project: #1..=#11, #13 + Map: coalesce(#12, 0) Final closure: - Project: #0..=#2, #4, #11, #5, #6, #12, #13, #9, #10, #7, #8, #14, #13, #15, #13, #16, #3 + Project: #0..=#2, #4, #12, #5, #6, #13, #14, #10, #11, #7, #8, #15, #14, #16, #14, #17, #3, #9 Map: null, 44, 0, false, -1, null path %4: after %0: - Project: #1, #3, #13, #22, #25, #0, #30..=#34 - Map: text_to_"char"(case when (#19{mztype} = "a") then "b" else #19{mztype} end), text_to_"char"(case when (#4{category} = "array") then "A" else case when (#4{category} = "bit-string") then "V" else case when (#4{category} = "boolean") then "B" else case when (#4{category} = "composite") then "C" else case when (#4{category} = "date-time") then "D" else case when (#4{category} = "enum") then "E" else case when (#4{category} = "geometric") then "G" else case when (#4{category} = "list") then "U" else case when (#4{category} = "network-address") then "I" else case when (#4{category} = "numeric") then "N" else case when (#4{category} = "pseudo") then "P" else case when (#4{category} = "string") then "S" else case when (#4{category} = "timespan") then "T" else case when (#4{category} = "user-defined") then "U" else case when (#4{category} = "unknown") then "X" else null end end end end end end end end end end end end end end end), oidtoregproc(#10{typinput}), coalesce(#11{typreceive}, 0) + Project: #1, #3, #14, #23, #26, #0, #31..=#36 + Map: text_to_"char"(case when (#20{mztype} = "a") then "b" else #20{mztype} end), text_to_"char"(case when (#4{category} = "array") then "A" else case when (#4{category} = "bit-string") then "V" else case when (#4{category} = "boolean") then "B" else case when (#4{category} = "composite") then "C" else case when (#4{category} = "date-time") then "D" else case when (#4{category} = "enum") then "E" else case when (#4{category} = "geometric") then "G" else case when (#4{category} = "list") then "U" else case when (#4{category} = "network-address") then "I" else case when (#4{category} = "numeric") then "N" else case when (#4{category} = "pseudo") then "P" else case when (#4{category} = "string") then "S" else case when (#4{category} = "timespan") then "T" else case when (#4{category} = "user-defined") then "U" else case when (#4{category} = "unknown") then "X" else null end end end end end end end end end end end end end end end), oidtoregproc(#10{typinput}), coalesce(#11{typreceive}, 0), oidtoregproc(coalesce(#12{typsend}, 0)) after %3: - Project: #1..=#5, #7..=#11, #13 - Map: coalesce(#6, #12, 0) + Project: #1..=#5, #7..=#12, #14 + Map: coalesce(#6, #13, 0) Final closure: - Project: #0..=#2, #4, #11, #6, #7, #12, #13, #10, #5, #8, #9, #14, #13, #15, #13, #16, #3 + Project: #0..=#2, #4, #12, #6, #7, #13, #14, #11, #5, #8, #9, #15, #14, #16, #14, #17, #3, #10 Map: null, 44, 0, false, -1, null - →Arrange (#0..=#29) + →Arrange (#0..=#30) →Stream l4 - →Arrange (#0) (#0..=#29) + →Arrange (#0) (#0..=#30) →Fused with Child Map/Filter/Project Filter: (#5 = #5) →Arranged l5 - Key: (#0..=#29) + Key: (#0..=#30) →Arrange (#0) →Union →Stream l11 @@ -19092,8 +19092,9 @@ EXPLAIN SELECT * FROM "pg_catalog"."pg_type"; ---- Explained Query (fast path): →Map/Filter/Project - Project: #0..=#17 + Project: #0..=#11, #20, #13..=#17, #19 Filter: ((#18{database_name}) IS NULL OR (#18{database_name} = "materialize")) + Map: oidtoregproc(#12{typreceive}) →Indexed mz_internal.pg_type_all_databases (using mz_internal.pg_type_all_databases_ind) Used Indexes: From 8075a5ecf5aea7359f2cc122855020f548b85f5d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 20:21:14 +0000 Subject: [PATCH 08/15] catalog: tighten mz_type_pg_metadata.typsend and document OID provenance `pack_type_update` always writes an OID for `typsend`, so declare the column `nullable(false)` to match `typinput` and `typreceive`. The `COALESCE`s in `pg_type_all_databases` stay, because they guard the `LEFT JOIN` against this table rather than the column's declaration. `typreceive` is the precedent: it is already `nullable(false)` and still needs its `COALESCE`, and `typinput` shows the other side, non-nullable in the base table but `nullable(true)` in the view because nothing coalesces it. Update the console's generated `MzTypePgMetadata.typsend` to match. The two views' `typsend` descs already read `RegProc.nullable(false)` and are unchanged, since `COALESCE(x, 0)` is non-nullable either way. Document where `regproc::NAMES` comes from: the builtin function registry reached through `BUILTINS::funcs()`, kept honest by `test_regproc_names_match_builtin_functions`, with PostgreSQL-assigned OIDs for the compatible subset and `crate::oid` ranges for the rest. Correct the `MigrationStep::replacement` justification. `bootstrap_tables` does not re-derive *every* system table, it holds back `mz_storage_usage_by_shard` and `mz_object_arrangement_size_history` through `is_retained_across_restarts`. Name them, and state that `mz_type_pg_metadata` is not among them. Also trace `replaced_items` to `new_shards` and `migrated_storage_collections_0dt` instead of asserting the back-fill, and replace the unfalsifiable "no builtin table has ever exercised" with the checkable fact that the list's only `Evolution` step targets a source. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01K8HcpvmmntWK9QuJWvKVbg --- console/types/materialize.d.ts | 2 +- .../catalog/open/builtin_schema_migration.rs | 30 ++++++++++++------- src/catalog/src/builtin/mz_internal.rs | 13 ++++---- src/pgrepr-consts/src/regproc.rs | 25 ++++++++++++++-- 4 files changed, 49 insertions(+), 21 deletions(-) diff --git a/console/types/materialize.d.ts b/console/types/materialize.d.ts index f9b5513e961f9..1c4d3cb32f79f 100644 --- a/console/types/materialize.d.ts +++ b/console/types/materialize.d.ts @@ -4004,7 +4004,7 @@ export interface MzTypePgMetadata { id: Generated; typinput: Generated; typreceive: Generated; - typsend: Generated; + typsend: Generated; } export interface MzTypes { diff --git a/src/adapter/src/catalog/open/builtin_schema_migration.rs b/src/adapter/src/catalog/open/builtin_schema_migration.rs index 6a9b3eb4a1226..193e716b529e1 100644 --- a/src/adapter/src/catalog/open/builtin_schema_migration.rs +++ b/src/adapter/src/catalog/open/builtin_schema_migration.rs @@ -303,16 +303,26 @@ static MIGRATIONS: LazyLock> = LazyLock::new(|| { ), // `mz_type_pg_metadata` gained a trailing `typsend` column. Appending a // column would be backward compatible enough for `Evolution`, but - // `Replacement` is the right mechanism for a builtin table anyway. - // `Coordinator::bootstrap_tables` retracts every system table's persist - // snapshot and re-derives the rows from the catalog on each boot, so a - // fresh shard loses nothing. More importantly, only `Replacement` - // reports the item in `MigrationResult::replaced_items`, which is what - // makes the read-only coordinator back-fill the new shard so dependent - // dataflows such as `pg_type_all_databases_ind` can hydrate. Under - // `Evolution` a read-only replica would instead have to read the old - // parts through the schema-migration path, which no builtin table has - // ever exercised. + // `Replacement` costs nothing here and buys the read-only back-fill. + // `Coordinator::bootstrap_tables` retracts each system table's persist + // snapshot and re-derives its rows from the catalog on every boot, + // except for the two it holds back through + // `is_retained_across_restarts`, `mz_storage_usage_by_shard` and + // `mz_object_arrangement_size_history`, whose contents live nowhere + // else. `validate_migration_steps` refuses to migrate those two for the + // same reason. `mz_type_pg_metadata` is not one of them, so a fresh + // shard for it loses nothing. + // + // Only `Replacement` puts the item in `MigrationResult::replaced_items`, + // because that set is derived from `MigrationRunResult::new_shards`, + // which only `migrate_replace` populates. `replaced_items` reaches the + // coordinator as `migrated_storage_collections_0dt`, which is what makes + // a read-only coordinator back-fill the new shard so dependent dataflows + // such as `pg_type_all_databases_ind` can hydrate. Under `Evolution` the + // read-only path only checks backward compatibility and leaves the new + // schema unregistered, so readers would take the old parts through + // persist's schema migration instead. The only `Evolution` step in this + // list targets a source, so that path is untried for tables. // // See the NOTE above: this version must stay at the workspace's current // dev version until the change ships. diff --git a/src/catalog/src/builtin/mz_internal.rs b/src/catalog/src/builtin/mz_internal.rs index ccb26716859ac..73a269c8e0d2f 100644 --- a/src/catalog/src/builtin/mz_internal.rs +++ b/src/catalog/src/builtin/mz_internal.rs @@ -586,14 +586,11 @@ pub static MZ_TYPE_PG_METADATA: LazyLock = LazyLock::new(|| Builti .with_column("id", SqlScalarType::String.nullable(false)) .with_column("typinput", SqlScalarType::Oid.nullable(false)) .with_column("typreceive", SqlScalarType::Oid.nullable(false)) - // `pack_type_update` always writes an OID here, so this declaration is - // wider than the data requires. Downstream `COALESCE`s on this column - // are there for the `LEFT JOIN` against this table, not for this. - // - // TODO: Tighten to `nullable(false)` to match `typinput` and - // `typreceive`. That also changes the descs of the views projecting the - // column. - .with_column("typsend", SqlScalarType::Oid.nullable(true)) + // NOTE: `pg_type_all_databases` still wraps `typreceive` and `typsend` + // in `COALESCE`. That guards its `LEFT JOIN` against this table, which + // yields NULLs for types without PostgreSQL metadata, so it is required + // even though neither column is nullable here. + .with_column("typsend", SqlScalarType::Oid.nullable(false)) .finish(), column_comments: BTreeMap::new(), is_retained_metrics_object: false, diff --git a/src/pgrepr-consts/src/regproc.rs b/src/pgrepr-consts/src/regproc.rs index cb7012840309f..cfa0925244d0d 100644 --- a/src/pgrepr-consts/src/regproc.rs +++ b/src/pgrepr-consts/src/regproc.rs @@ -17,12 +17,33 @@ //! because the catalog crates depend on `mz-pgrepr` rather than the reverse, //! and some encode paths run in `clusterd`, which has no catalog at all. //! -//! [`NAMES`] is derived from the builtin function registry in `mz_sql::func`, -//! which is the authoritative definition of every function OID. +//! [`NAMES`] is derived from Materialize's builtin function registry, reached +//! through `mz_catalog::builtin::BUILTINS::funcs()`, which flattens the +//! per-schema maps in `mz_sql::func` (`PG_CATALOG_BUILTINS`, +//! `MZ_CATALOG_BUILTINS`, `MZ_INTERNAL_BUILTINS`, and so on). That registry is +//! the authoritative definition of every function OID. The same registry backs +//! `mz_catalog.mz_functions`, and `pg_catalog.pg_proc` is a view over that +//! table, so this table cannot disagree with the catalog without also +//! disagreeing with itself. +//! //! `mz_catalog::builtin`'s `test_regproc_names_match_builtin_functions` //! recomputes the whole table from that registry and fails with the corrected //! contents when the two have drifted, so do not hand-edit entries here. Add //! the function to the registry and paste in the table the test prints. +//! +//! The OIDs themselves are not invented by Materialize wherever PostgreSQL +//! already assigns one. `mz_sql::func`'s `PG_CATALOG_BUILTINS` records that its +//! literal OIDs were read out of a PostgreSQL 13 `pg_proc`, and points at +//! +//! for the same values. Everything else draws from [`crate::oid`], which +//! carves out two ranges: `FIRST_UNPINNED_OID` upward for functions PostgreSQL +//! ships but does not pin an OID for, and `FIRST_MATERIALIZE_OID` upward for +//! functions that exist only in Materialize. So for the PostgreSQL-compatible +//! subset a client resolving an OID here sees the name PostgreSQL would give it. +//! +//! There is no PostgreSQL document that lists this mapping. `pg_proc.dat` is +//! the closest thing to a canonical source, and it covers only the OIDs +//! PostgreSQL itself defines. /// Every builtin function OID paired with the text `regproc` renders it as, /// sorted by OID. From 8596cf05c4f88907f5f7320a5ee09bd41b998b26 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 20:37:31 +0000 Subject: [PATCH 09/15] all: trim comments that do not earn their keep Cut restatement, motivation, and duplicated reasoning from the comments this branch adds. What stays records a non-obvious constraint, a measured PostgreSQL behaviour, or a reason a tempting simplification is wrong. The `mz_type_pg_metadata` migration step no longer defends its use of `MigrationStep::replacement`, which is what every other step in the list uses. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01K8HcpvmmntWK9QuJWvKVbg --- .../catalog/open/builtin_schema_migration.rs | 28 +-------- src/catalog/src/builtin.rs | 14 ++--- src/catalog/src/builtin/mz_internal.rs | 9 +-- src/catalog/src/builtin/pg_catalog.rs | 10 +--- src/interchange/src/json.rs | 7 +-- src/pgrepr-consts/src/regproc.rs | 60 +++++++------------ src/pgrepr/src/regproc.rs | 9 ++- src/pgrepr/src/value.rs | 55 ++++++----------- src/sql/src/catalog.rs | 3 +- test/adbc/smoketest.py | 39 +++++------- test/sqllogictest/regproc.slt | 20 +++---- 11 files changed, 84 insertions(+), 170 deletions(-) diff --git a/src/adapter/src/catalog/open/builtin_schema_migration.rs b/src/adapter/src/catalog/open/builtin_schema_migration.rs index 193e716b529e1..50cb4b23a05a2 100644 --- a/src/adapter/src/catalog/open/builtin_schema_migration.rs +++ b/src/adapter/src/catalog/open/builtin_schema_migration.rs @@ -301,31 +301,9 @@ static MIGRATIONS: LazyLock> = LazyLock::new(|| { MZ_CATALOG_SCHEMA, "mz_kafka_sources", ), - // `mz_type_pg_metadata` gained a trailing `typsend` column. Appending a - // column would be backward compatible enough for `Evolution`, but - // `Replacement` costs nothing here and buys the read-only back-fill. - // `Coordinator::bootstrap_tables` retracts each system table's persist - // snapshot and re-derives its rows from the catalog on every boot, - // except for the two it holds back through - // `is_retained_across_restarts`, `mz_storage_usage_by_shard` and - // `mz_object_arrangement_size_history`, whose contents live nowhere - // else. `validate_migration_steps` refuses to migrate those two for the - // same reason. `mz_type_pg_metadata` is not one of them, so a fresh - // shard for it loses nothing. - // - // Only `Replacement` puts the item in `MigrationResult::replaced_items`, - // because that set is derived from `MigrationRunResult::new_shards`, - // which only `migrate_replace` populates. `replaced_items` reaches the - // coordinator as `migrated_storage_collections_0dt`, which is what makes - // a read-only coordinator back-fill the new shard so dependent dataflows - // such as `pg_type_all_databases_ind` can hydrate. Under `Evolution` the - // read-only path only checks backward compatibility and leaves the new - // schema unregistered, so readers would take the old parts through - // persist's schema migration instead. The only `Evolution` step in this - // list targets a source, so that path is untried for tables. - // - // See the NOTE above: this version must stay at the workspace's current - // dev version until the change ships. + // `mz_type_pg_metadata` gained a trailing `typsend` column. See the NOTE + // above: this version must stay at the workspace's current dev version + // until the change ships. MigrationStep::replacement( "26.36.0-dev.0", CatalogItemType::Table, diff --git a/src/catalog/src/builtin.rs b/src/catalog/src/builtin.rs index fa304cc73dd0c..df2c3b295723d 100644 --- a/src/catalog/src/builtin.rs +++ b/src/catalog/src/builtin.rs @@ -1659,12 +1659,10 @@ mod tests { use super::*; - /// `mz_pgrepr::regproc::NAMES` is a table of every builtin function OID and - /// the text a `regproc` renders it as. It has to be checked in as data - /// because `mz-pgrepr` sits below this crate in the dependency graph and so - /// cannot read the registry itself. This test is what keeps it honest: it - /// recomputes the whole table from the registry and, on drift, prints the - /// corrected table to paste into `src/pgrepr-consts/src/regproc.rs`. + /// Recomputes `mz_pgrepr::regproc::NAMES` from the builtin function registry + /// and fails with the corrected table when the checked-in copy has drifted. + /// It is checked in as data because `mz-pgrepr` sits below this crate in the + /// dependency graph and so cannot read the registry itself. #[mz_ore::test] #[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux` fn test_regproc_names_match_builtin_functions() { @@ -1681,8 +1679,8 @@ mod tests { let mut expected: BTreeMap = BTreeMap::new(); for func in BUILTINS::funcs() { - // The rule `regprocout` applies: print the bare name when it - // resolves back to this OID on its own, otherwise qualify it. + // Mirrors PostgreSQL's `regprocout`, which qualifies a name that + // would not resolve back to this OID on its own. let rendered = if impls_per_name[func.name] == 1 && IMPLICITLY_SEARCHED.contains(&func.schema) { func.name.to_string() diff --git a/src/catalog/src/builtin/mz_internal.rs b/src/catalog/src/builtin/mz_internal.rs index 73a269c8e0d2f..12dbff60271f5 100644 --- a/src/catalog/src/builtin/mz_internal.rs +++ b/src/catalog/src/builtin/mz_internal.rs @@ -586,10 +586,9 @@ pub static MZ_TYPE_PG_METADATA: LazyLock = LazyLock::new(|| Builti .with_column("id", SqlScalarType::String.nullable(false)) .with_column("typinput", SqlScalarType::Oid.nullable(false)) .with_column("typreceive", SqlScalarType::Oid.nullable(false)) - // NOTE: `pg_type_all_databases` still wraps `typreceive` and `typsend` - // in `COALESCE`. That guards its `LEFT JOIN` against this table, which - // yields NULLs for types without PostgreSQL metadata, so it is required - // even though neither column is nullable here. + // NOTE: `pg_type_all_databases` still needs `COALESCE` on this column, + // because its `LEFT JOIN` against this table yields NULLs for types with + // no PostgreSQL metadata. .with_column("typsend", SqlScalarType::Oid.nullable(false)) .finish(), column_comments: BTreeMap::new(), @@ -4377,8 +4376,6 @@ pub static PG_TYPE_ALL_DATABASES: LazyLock = LazyLock::new(|| { .with_column("typcollation", SqlScalarType::Oid.nullable(false)) .with_column("typdefault", SqlScalarType::String.nullable(true)) .with_column("database_name", SqlScalarType::String.nullable(true)) - // Appended rather than placed at PostgreSQL's ordinal position, to - // keep the existing columns' positions stable. .with_column("typsend", SqlScalarType::RegProc.nullable(false)) .finish(), column_comments: BTreeMap::new(), diff --git a/src/catalog/src/builtin/pg_catalog.rs b/src/catalog/src/builtin/pg_catalog.rs index 809c9344b1f77..78d914d54a86e 100644 --- a/src/catalog/src/builtin/pg_catalog.rs +++ b/src/catalog/src/builtin/pg_catalog.rs @@ -1568,19 +1568,15 @@ pub static PG_TYPE: LazyLock = LazyLock::new(|| BuiltinView { .with_column("typelem", SqlScalarType::Oid.nullable(false)) .with_column("typarray", SqlScalarType::Oid.nullable(false)) .with_column("typinput", SqlScalarType::RegProc.nullable(true)) - // PostgreSQL declares `typreceive` and `typsend` as `regproc`, and clients - // compare the resolved function name against a known set. The underlying - // `mz_internal.pg_type_all_databases` keeps `typreceive` as `oid` because it - // backs a builtin index, and resolving a `regproc` to its name reads - // `current_database()`, which is unmaterializable. + // The underlying `mz_internal.pg_type_all_databases` keeps `typreceive` + // as `oid` because it backs a builtin index, and resolving a `regproc` to + // a name reads `current_database()`, which is unmaterializable. .with_column("typreceive", SqlScalarType::RegProc.nullable(false)) .with_column("typnotnull", SqlScalarType::Bool.nullable(false)) .with_column("typbasetype", SqlScalarType::Oid.nullable(false)) .with_column("typtypmod", SqlScalarType::Int32.nullable(false)) .with_column("typcollation", SqlScalarType::Oid.nullable(false)) .with_column("typdefault", SqlScalarType::String.nullable(true)) - // Appended rather than placed at PostgreSQL's ordinal position, to keep the - // existing columns' positions stable. .with_column("typsend", SqlScalarType::RegProc.nullable(false)) .finish(), column_comments: BTreeMap::new(), diff --git a/src/interchange/src/json.rs b/src/interchange/src/json.rs index 35ef15ff7ccf6..36470fc7b02dc 100644 --- a/src/interchange/src/json.rs +++ b/src/interchange/src/json.rs @@ -119,11 +119,8 @@ impl ToJson for TypedDatum<'_> { SqlScalarType::UInt16 => json!(datum.unwrap_uint16()), // NOTE: `regproc` stays a number here even though the pgwire text // encoding resolves it to a function name. `build_row_schema_json` - // below declares these columns as 4-byte unsigned integers and - // `mz_interchange::avro` encodes them that way, so emitting a name - // would contradict the schema this module publishes for the same - // column. The HTTP SQL API shares this encoder and so also reports - // a number. + // below declares these columns as 4-byte unsigned integers, so a + // name would contradict the schema this module publishes for them. SqlScalarType::UInt32 | SqlScalarType::Oid | SqlScalarType::RegClass diff --git a/src/pgrepr-consts/src/regproc.rs b/src/pgrepr-consts/src/regproc.rs index cfa0925244d0d..8b7ac281c41f3 100644 --- a/src/pgrepr-consts/src/regproc.rs +++ b/src/pgrepr-consts/src/regproc.rs @@ -18,53 +18,37 @@ //! and some encode paths run in `clusterd`, which has no catalog at all. //! //! [`NAMES`] is derived from Materialize's builtin function registry, reached -//! through `mz_catalog::builtin::BUILTINS::funcs()`, which flattens the -//! per-schema maps in `mz_sql::func` (`PG_CATALOG_BUILTINS`, -//! `MZ_CATALOG_BUILTINS`, `MZ_INTERNAL_BUILTINS`, and so on). That registry is -//! the authoritative definition of every function OID. The same registry backs -//! `mz_catalog.mz_functions`, and `pg_catalog.pg_proc` is a view over that -//! table, so this table cannot disagree with the catalog without also -//! disagreeing with itself. +//! through `mz_catalog::builtin::BUILTINS::funcs()`, which is the authoritative +//! definition of every function OID. `mz_catalog::builtin`'s +//! `test_regproc_names_match_builtin_functions` recomputes the whole table from +//! that registry and fails with the corrected contents when the two have +//! drifted, so do not hand-edit entries here. Add the function to the registry +//! and paste in the table the test prints. //! -//! `mz_catalog::builtin`'s `test_regproc_names_match_builtin_functions` -//! recomputes the whole table from that registry and fails with the corrected -//! contents when the two have drifted, so do not hand-edit entries here. Add -//! the function to the registry and paste in the table the test prints. -//! -//! The OIDs themselves are not invented by Materialize wherever PostgreSQL -//! already assigns one. `mz_sql::func`'s `PG_CATALOG_BUILTINS` records that its -//! literal OIDs were read out of a PostgreSQL 13 `pg_proc`, and points at +//! The OIDs are not invented by Materialize wherever PostgreSQL already assigns +//! one. `mz_sql::func`'s `PG_CATALOG_BUILTINS` records that its literal OIDs +//! were read out of a PostgreSQL 13 `pg_proc`, and points at //! -//! for the same values. Everything else draws from [`crate::oid`], which -//! carves out two ranges: `FIRST_UNPINNED_OID` upward for functions PostgreSQL -//! ships but does not pin an OID for, and `FIRST_MATERIALIZE_OID` upward for -//! functions that exist only in Materialize. So for the PostgreSQL-compatible -//! subset a client resolving an OID here sees the name PostgreSQL would give it. -//! -//! There is no PostgreSQL document that lists this mapping. `pg_proc.dat` is -//! the closest thing to a canonical source, and it covers only the OIDs -//! PostgreSQL itself defines. +//! for the same values. Everything else draws from [`crate::oid`], which carves +//! out `FIRST_UNPINNED_OID` upward for functions PostgreSQL ships without a +//! pinned OID, and `FIRST_MATERIALIZE_OID` upward for functions that exist only +//! in Materialize. /// Every builtin function OID paired with the text `regproc` renders it as, /// sorted by OID. /// -/// The name is the full rendering, not the bare function name. It is -/// schema-qualified whenever the bare name would not resolve back to this OID, -/// which is the rule `regprocout` applies: a name shared by several overloads, -/// or a name in a schema that is not searched implicitly, gets qualified. +/// The name is schema-qualified whenever the bare name would not resolve back to +/// this OID, which is the rule `regprocout` applies. /// /// NOTE: The rendering assumes the default search path, which is all a static -/// table can do. A session that puts `mz_internal` or `mz_unsafe` on its -/// `search_path` makes those bare names resolvable, so `regprocout` would print -/// them bare and this table still qualifies them. The SQL cast from `regproc` to -/// `text` reads the session's search path and does render them bare. +/// table can do. A session with `mz_internal` or `mz_unsafe` on its +/// `search_path` makes those bare names resolvable, so the SQL cast from +/// `regproc` to `text` renders them bare where this table still qualifies them. /// -/// NOTE: Names that are not bare identifiers are left unquoted here, so `left` -/// renders as `left` and OID 936 as `pg_catalog.substring`. PostgreSQL's -/// `regprocout` quotes those, rendering `"left"` and `pg_catalog."substring"`. -/// This table matches the SQL cast from `regproc` to `text` instead, so the two -/// Materialize paths agree with each other. The unquoted form still resolves, -/// because `regprocin` accepts a reserved word with or without quotes. +/// NOTE: Reserved words are left unquoted here, so `left` renders as `left` +/// where PostgreSQL's `regprocout` renders `"left"`. This matches the SQL cast +/// from `regproc` to `text`, and the unquoted form still resolves because +/// `regprocin` accepts a reserved word either way. pub const NAMES: &[(u32, &str)] = &[ (34, "namein"), (38, "int2in"), diff --git a/src/pgrepr/src/regproc.rs b/src/pgrepr/src/regproc.rs index 9de5faba5dc50..438dd72ddaa6e 100644 --- a/src/pgrepr/src/regproc.rs +++ b/src/pgrepr/src/regproc.rs @@ -30,8 +30,8 @@ mod tests { } #[mz_ore::test] - // `oid` scans the table, so covering every entry is quadratic in its 626 - // rows. That is milliseconds natively and far too slow interpreted. + // `oid` scans the table, so covering every entry is quadratic and far too + // slow interpreted. #[cfg_attr(miri, ignore)] fn lookups_round_trip() { for (entry_oid, entry_name) in NAMES { @@ -47,9 +47,8 @@ mod tests { assert_eq!(oid("no_such_function"), Err(NameLookupError::NotFound)); } - /// The renderings that matter to clients are the `pg_type` columns declared - /// `regproc`, which is what motivates resolving names at all. Spot-check a - /// few so a regeneration that dropped them fails loudly. + /// Spot-checks the `pg_type` I/O function renderings, so a regeneration that + /// dropped them fails loudly. #[mz_ore::test] fn type_io_functions_resolve() { for (func_oid, expected) in [ diff --git a/src/pgrepr/src/value.rs b/src/pgrepr/src/value.rs index 3dd56d024d694..95ca4283a00b9 100644 --- a/src/pgrepr/src/value.rs +++ b/src/pgrepr/src/value.rs @@ -148,12 +148,9 @@ impl Value { (Datum::UInt32(oid), SqlScalarType::Oid) => Some(Value::Oid(oid)), (Datum::UInt32(oid), SqlScalarType::RegProc) => Some(Value::RegProc(oid)), // NOTE: `regclass` and `regtype` collapse into `Value::Oid`, so they - // render as digits where PostgreSQL renders a name. Unlike - // `regproc`, they can name objects a user created, which no static - // table can know and which the encoder cannot look up: the catalog - // crates depend on `mz-pgrepr`, and the Kafka sink and `COPY TO` - // encoders run in `clusterd`, where there is no catalog at all. - // Resolution for those two lives in the SQL cast to `text` instead. + // render as digits where PostgreSQL renders a name. Unlike `regproc` + // they can name objects a user created, which no static table can + // know. Resolution for those two lives in the SQL cast to `text`. (Datum::UInt32(oid), SqlScalarType::RegClass) => Some(Value::Oid(oid)), (Datum::UInt32(oid), SqlScalarType::RegType) => Some(Value::Oid(oid)), (Datum::UInt32(u), SqlScalarType::UInt32) => Some(Value::UInt4(UInt4(u))), @@ -421,13 +418,11 @@ impl Value { }) .expect("provided closure never fails"), Value::Oid(oid) => strconv::format_uint32(buf, *oid), - // Mirrors PostgreSQL's `regprocout`: OID 0 renders as `-`, an OID - // that names a function renders as that name, and any other OID - // renders in decimal. + // Mirrors PostgreSQL's `regprocout`. // // NOTE: The SQL cast from `regproc` to `text` renders OID 0 as `0` - // rather than `-`, so the two disagree on that one value. PostgreSQL - // routes both through `regprocout` and prints `-` for both. + // rather than `-`, so the two Materialize paths disagree on that one + // value. PostgreSQL prints `-` for both. Value::RegProc(oid) => match *oid { 0 => strconv::format_string(buf, REGPROC_NULL), oid => match regproc::name(oid) { @@ -557,8 +552,8 @@ impl Value { Err("binary encoding of map types is not implemented".into()) } Value::Name(s) => s.to_sql(&PgType::NAME, buf), - // PostgreSQL's `regprocsend` is `int4send`, so a `regproc` is - // indistinguishable from an `oid` on the wire in binary format. + // PostgreSQL's `regprocsend` is `int4send`, so binary is identical to + // an `oid`. Value::Oid(i) | Value::RegProc(i) => i.to_sql(&PgType::OID, buf), Value::Record(fields) => { let nfields = pg_len("record field length", fields.len())?; @@ -998,31 +993,25 @@ const REGPROC_NULL: &str = "-"; /// Parses the text format of a `regproc`, matching PostgreSQL's `regprocin`. /// -/// A function name resolves to that function's OID, `-` is OID 0, and anything -/// else is read as a decimal OID. -/// /// Accepting names is what keeps a `COPY ... TO` rendering loadable by -/// `COPY ... FROM`, since the text encoding emits names. That holds for every -/// rendering that names exactly one function. An overloaded name renders -/// schema-qualified and still names every one of its impls, so it does not round -/// trip, which is true of PostgreSQL as well. +/// `COPY ... FROM`, since the text encoding emits names. An overloaded name +/// renders schema-qualified and names every one of its impls, so it does not +/// round trip, which is true of PostgreSQL as well. fn parse_regproc(s: &str) -> Result> { if s == REGPROC_NULL { return Ok(0); } match regproc::oid(s) { Ok(oid) => Ok(oid), - // PostgreSQL refuses the same input, because a name shared by several - // overloads does not identify one of them. + // PostgreSQL refuses the same input rather than picking an overload. Err(regproc::NameLookupError::Ambiguous) => { Err(format!("more than one function named \"{}\"", s).into()) } Err(regproc::NameLookupError::NotFound) => match strconv::parse_oid(s) { Ok(oid) => Ok(oid), // Number-shaped input keeps its numeric diagnosis, which is what - // reports an out of range OID. Anything else was meant as a name, - // so report a missing function, matching both PostgreSQL and the - // SQL cast from `text` to `regproc`. + // reports an out of range OID. Anything else was meant as a name, so + // report a missing function, matching PostgreSQL. Err(err) if is_number_shaped(s) => Err(err.into()), Err(_) => Err(format!("function \"{}\" does not exist", s).into()), }, @@ -1147,11 +1136,8 @@ mod tests { #[mz_ore::test] fn regproc_text_encoding_resolves_names() { // 1242 is `boolin`, uniquely named. 1398 is one of six `abs` overloads, - // so PostgreSQL qualifies it. 99999 names no function. - // - // `round_trips` is false for the qualified overload: that rendering - // names all six impls, so reading it back cannot pick one. PostgreSQL - // does not round trip it either. + // so PostgreSQL qualifies it. 99999 names no function. The qualified + // overload does not round trip, because its rendering names all six. for (oid, expected, round_trips) in [ (0, "-", true), (1242, "boolin", true), @@ -1173,8 +1159,6 @@ mod tests { assert_eq!(decoded, oid, "text rendering {expected} did not round trip"); } - // Binary stays a 4-byte OID, so the name resolution is invisible - // there. let mut binary = BytesMut::new(); Value::RegProc(oid) .encode_binary(&Type::RegProc, &mut binary) @@ -1183,9 +1167,8 @@ mod tests { } } - /// A name shared by several overloads does not identify one of them, so - /// reading it back is an error rather than an arbitrary choice. PostgreSQL - /// rejects the same input. + /// An overloaded name is an error rather than an arbitrary choice, which is + /// what PostgreSQL does too. #[mz_ore::test] fn regproc_text_decoding_rejects_ambiguous_names() { assert_eq!( @@ -1202,8 +1185,6 @@ mod tests { .unwrap_err(), "function \"no_such_function\" does not exist", ); - // Number-shaped input keeps its numeric diagnosis rather than being - // reported as a missing function. let out_of_range = Value::decode_text(&Type::RegProc, b"99999999999999") .map(|_| ()) .map_err(|e| e.to_string()) diff --git a/src/sql/src/catalog.rs b/src/sql/src/catalog.rs index e47987bd9d0d5..cb7d738a0c2a9 100644 --- a/src/sql/src/catalog.rs +++ b/src/sql/src/catalog.rs @@ -1068,8 +1068,7 @@ pub struct CatalogTypePgMetadata { pub typinput_oid: u32, /// The OID of the `typreceive` function in PostgreSQL. pub typreceive_oid: u32, - /// The OID of the `typsend` function in PostgreSQL, or 0 for the types that - /// PostgreSQL gives no binary send function. + /// The OID of the `typsend` function in PostgreSQL. pub typsend_oid: u32, } diff --git a/test/adbc/smoketest.py b/test/adbc/smoketest.py index 60a638bc4e16f..b79726c422366 100644 --- a/test/adbc/smoketest.py +++ b/test/adbc/smoketest.py @@ -10,28 +10,24 @@ """Apache Arrow ADBC PostgreSQL driver tests. The driver keys its Arrow type mapping off the *name* of each type's binary -receive function, which it reads from `pg_catalog.pg_type.typreceive`. That -column is declared `regproc`, and Materialize resolves a `regproc` to its -function name in the pgwire text encoding, so the driver's lookups hit and each -column arrives as its own Arrow type. A type the driver cannot resolve degrades -to `arrow.opaque` over binary rather than erroring, so asserting the exact Arrow -type per column is what catches a regression here. +receive function, which it reads from `pg_catalog.pg_type.typreceive`. A type it +cannot resolve degrades to `arrow.opaque` over binary rather than erroring, so +asserting the exact Arrow type per column is what catches a regression here. """ import unittest -# `adbc-driver-postgresql` and `duckdb` are pinned in `test/adbc/requirements.txt` -# and installed only inside this composition's test container, so they are absent -# from the repo-wide Python virtualenv that pyright resolves against. +# Pinned in `test/adbc/requirements.txt` and installed only inside this +# composition's test container, so they are absent from the repo-wide virtualenv +# that pyright resolves against. import adbc_driver_postgresql.dbapi # pyright: ignore[reportMissingImports] import duckdb # pyright: ignore[reportMissingImports] import pyarrow as pa MATERIALIZED_URL = "postgresql://materialize@materialized:6875/materialize" -# One non-null value per scalar type the driver has to resolve. Every value is -# non-null so that a broken binary decoder shows up as a wrong value rather -# than as a column of nulls. +# Every value is non-null so that a broken binary decoder shows up as a wrong +# value rather than as a column of nulls. SCALAR_QUERY = """ SELECT true::bool AS c_bool, @@ -53,10 +49,8 @@ '{"a": 1}'::jsonb AS c_jsonb """ -# The Arrow type the driver must produce for each column of SCALAR_QUERY. -# `numeric` and `uuid` have no native Arrow type the driver maps them onto, so -# it wraps them in an `arrow.opaque` extension type over string or binary -# storage. +# `numeric` and `uuid` have no native Arrow type the driver maps them onto, so it +# wraps them in an `arrow.opaque` extension type over string or binary storage. EXPECTED_SCALAR_TYPES = { "c_bool": pa.bool_(), "c_int2": pa.int16(), @@ -90,9 +84,7 @@ def test_connect(self) -> None: """Connecting runs the driver's type resolver, which reads the binary send and receive functions of every type out of `pg_catalog.pg_type`. A column missing there fails every connection before any user query, so - connecting at all covers that catalog contract. This deliberately makes - no claim about the values, to stay a test of the catalog rather than of - Arrow type fidelity.""" + connecting at all covers that catalog contract.""" with adbc_driver_postgresql.dbapi.connect(MATERIALIZED_URL) as conn: with conn.cursor() as cur: cur.execute("SELECT 1") @@ -101,9 +93,7 @@ def test_connect(self) -> None: self.assertEqual(table.num_columns, 1) def test_scalar_types_resolve(self) -> None: - """Each scalar type must arrive as its own Arrow type. A type the - driver cannot resolve degrades to opaque binary instead of failing, so - asserting the exact Arrow type is what catches it.""" + """Each scalar type must arrive as its own Arrow type.""" table = query_arrow(SCALAR_QUERY) self.assertEqual(table.num_rows, 1) self.assertEqual( @@ -118,9 +108,8 @@ def test_scalar_types_resolve(self) -> None: f"column {name} resolved to Arrow type {actual}, expected {expected}", ) - # `bytea` is the only column that is legitimately Arrow binary. Any - # other column landing there means the driver fell back to passing - # bytes through untyped. + # `bytea` is the only column that is legitimately Arrow binary. Any other + # column landing there fell back to untyped bytes. for field in table.schema: if field.type == pa.binary(): self.assertEqual( diff --git a/test/sqllogictest/regproc.slt b/test/sqllogictest/regproc.slt index a55a01608a599..ab8a8a8ae154f 100644 --- a/test/sqllogictest/regproc.slt +++ b/test/sqllogictest/regproc.slt @@ -192,30 +192,26 @@ true true true -# The pgwire text encoding resolves a regproc to its function name, matching -# PostgreSQL's regprocout. These have to be `simple` records: `query` records -# read results in the binary format, where a regproc is a bare OID, so the text -# rendering is invisible to them. - -# 1242 is boolin and is uniquely named. 1398 is one of six abs overloads, so the -# bare name would not identify it and it is qualified. 99999 names no function -# and renders in decimal. OID 0 renders as `-`. +# The pgwire text encoding resolves a regproc to its function name. These must be +# `simple` records: `query` records read results in the binary format, where a +# regproc is a bare OID. + +# 1398 is one of six abs overloads, so it is qualified. 99999 names no function. simple SELECT 1242::regproc, 1398::regproc, 99999::regproc, 0::regproc ---- boolin,pg_catalog.abs,99999,- COMPLETE 1 -# The motivating case: clients resolve a type's I/O function names to decide how -# to decode its values. +# Type I/O function names, which is what clients read to decode values simple SELECT typinput, typreceive FROM pg_type WHERE typname = 'bool' ---- boolin,boolrecv COMPLETE 1 -# regclass and regtype keep rendering as digits. They can name objects a user -# created, which the wire encoder cannot resolve. +# regclass and regtype keep rendering as digits, since they can name objects a +# user created simple SELECT 1082::regtype, 1242::regclass ---- From 2d7aff09fe170542a318d6e13a75faeb215ac13b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 20:53:29 +0000 Subject: [PATCH 10/15] sql: render reg-type OID 0 as `-` in text casts The SQL casts from `regproc`, `regclass` and `regtype` to `text` rendered OID 0 as `0`. OID 0 misses the `mz_objects` join, so the cast fell through to stringifying the OID. PostgreSQL's `regprocout`, `regclassout` and `regtypeout` all render OID 0 as `-`, and so does our own pgwire TEXT output for `regproc`. The SQL cast therefore disagreed with both PostgreSQL and our wire path. Add an OID 0 case to the shared cast template so all three reg types render `-`. A nonzero OID that names nothing still renders as its digits, matching PostgreSQL. NULL still casts to NULL. Measured against PostgreSQL 16.13: SELECT 0::regproc::text, 0::regclass::text, 0::regtype::text; - | - | - SELECT 99999::regproc::text, 99999::regclass::text, 99999::regtype::text; 99999 | 99999 | 99999 Fixing all three is safe for builtin plans. Every reg-to-text cast is registered `Explicit`, so no implicit or assignment coercion can reach the template, and no builtin view or index SQL casts a reg type to text. No builtin plan moves, so `catalog_server_explain.slt` is unaffected. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01K8HcpvmmntWK9QuJWvKVbg --- src/pgrepr/src/value.rs | 4 ---- src/sql/src/plan/typeconv.rs | 9 ++++++++- test/sqllogictest/regclass.slt | 12 ++++++++++++ test/sqllogictest/regproc.slt | 12 ++++++++++++ test/sqllogictest/regtype.slt | 12 ++++++++++++ 5 files changed, 44 insertions(+), 5 deletions(-) diff --git a/src/pgrepr/src/value.rs b/src/pgrepr/src/value.rs index 95ca4283a00b9..590590791b910 100644 --- a/src/pgrepr/src/value.rs +++ b/src/pgrepr/src/value.rs @@ -419,10 +419,6 @@ impl Value { .expect("provided closure never fails"), Value::Oid(oid) => strconv::format_uint32(buf, *oid), // Mirrors PostgreSQL's `regprocout`. - // - // NOTE: The SQL cast from `regproc` to `text` renders OID 0 as `0` - // rather than `-`, so the two Materialize paths disagree on that one - // value. PostgreSQL prints `-` for both. Value::RegProc(oid) => match *oid { 0 => strconv::format_string(buf, REGPROC_NULL), oid => match regproc::name(oid) { diff --git a/src/sql/src/plan/typeconv.rs b/src/sql/src/plan/typeconv.rs index 6b69af3b8f78d..523e3018011f7 100644 --- a/src/sql/src/plan/typeconv.rs +++ b/src/sql/src/plan/typeconv.rs @@ -203,9 +203,16 @@ static STRING_TO_REGTYPE: LazyLock = LazyLock::new(|| { .to_string() }); +// OID 0 renders as `-` to match PostgreSQL's `regprocout`, `regclassout` and +// `regtypeout`, which all spell the absent reference that way. A nonzero OID +// that names nothing still renders as its digits, also as PostgreSQL does. const REG_STRING_CAST_TEMPLATE: &str = "( SELECT - COALESCE(mz_internal.mz_global_id_to_name(o.id), CAST($1 AS pg_catalog.oid)::pg_catalog.text) + CASE + WHEN CAST($1 AS pg_catalog.oid) = 0::pg_catalog.oid THEN '-' + ELSE + COALESCE(mz_internal.mz_global_id_to_name(o.id), CAST($1 AS pg_catalog.oid)::pg_catalog.text) + END AS text FROM ( diff --git a/test/sqllogictest/regclass.slt b/test/sqllogictest/regclass.slt index d3816146ae8ea..309fc2c899980 100644 --- a/test/sqllogictest/regclass.slt +++ b/test/sqllogictest/regclass.slt @@ -274,6 +274,18 @@ SELECT NULL::regclass::text ---- NULL +# OID 0 is the absent reference, rendered as `-` like PostgreSQL's `regclassout`. +# A nonzero OID that names no relation keeps its digits. +query T +SELECT 0::regclass::text +---- +- + +query T +SELECT 99999::regclass::text +---- +99999 + # ensure that all existing types can be cast to their respective names statement OK select oid, oid::regclass::text from (select oid from pg_catalog.pg_class) diff --git a/test/sqllogictest/regproc.slt b/test/sqllogictest/regproc.slt index ab8a8a8ae154f..d25268a12d55b 100644 --- a/test/sqllogictest/regproc.slt +++ b/test/sqllogictest/regproc.slt @@ -85,6 +85,18 @@ SELECT NULL::regproc::text ---- NULL +# OID 0 is the absent reference, rendered as `-` like PostgreSQL's `regprocout`. +# A nonzero OID that names no function keeps its digits. +query T +SELECT 0::regproc::text +---- +- + +query T +SELECT 99999::regproc::text +---- +99999 + query error more than one function named "max" SELECT 'max'::regproc diff --git a/test/sqllogictest/regtype.slt b/test/sqllogictest/regtype.slt index 8ba6b5b88e8f3..f8b4ccd34472e 100644 --- a/test/sqllogictest/regtype.slt +++ b/test/sqllogictest/regtype.slt @@ -119,6 +119,18 @@ SELECT NULL::regtype::text ---- NULL +# OID 0 is the absent reference, rendered as `-` like PostgreSQL's `regtypeout`. +# A nonzero OID that names no type keeps its digits. +query T +SELECT 0::regtype::text +---- +- + +query T +SELECT 99999::regtype::text +---- +99999 + # ensure that all existing types can be cast to their respective names statement OK select oid, oid::regtype::text from (select oid from mz_catalog.mz_types) From 7ddce8f87b20602dad31c43e692f33dae88b76bb Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 27 Aug 2026 18:02:51 +0000 Subject: [PATCH 11/15] test/adbc: bump ADBC to 1.12.0, pyarrow to 25.0.1, python image to 3.14.7 The ADBC 1.12.0 bump is not a no-op for this test. apache/arrow-adbc#4415 makes the postgres driver return JSON and JSONB as the `arrow.json` Arrow extension type instead of plain utf8, so the `c_jsonb` expectation in `smoketest.py` has to move from `pa.string()` to `pa.json_(pa.string())` or the type assertion fails. pyarrow 25.0.1 is a patch release with no changes near this path. `test/lang/python/mzcompose.py` moves along with `test/adbc/mzcompose.py` because it pinned the identical `python:3.14.1-trixie`; the two pins are deliberately kept in sync, and bumping only one would leave the repo with two different python image tags for no reason. --- test/adbc/mzcompose.py | 2 +- test/adbc/requirements.txt | 6 +++--- test/adbc/smoketest.py | 2 +- test/lang/python/mzcompose.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/adbc/mzcompose.py b/test/adbc/mzcompose.py index aa223549c797b..4a63167ef5255 100644 --- a/test/adbc/mzcompose.py +++ b/test/adbc/mzcompose.py @@ -21,7 +21,7 @@ Service( name="adbc", config={ - "image": "python:3.14.1-trixie", + "image": "python:3.14.7-trixie", "volumes": [ "../../:/workdir", ], diff --git a/test/adbc/requirements.txt b/test/adbc/requirements.txt index 660e0cc247979..0d25792f3e17d 100644 --- a/test/adbc/requirements.txt +++ b/test/adbc/requirements.txt @@ -1,4 +1,4 @@ -adbc-driver-manager==1.11.0 -adbc-driver-postgresql==1.11.0 +adbc-driver-manager==1.12.0 +adbc-driver-postgresql==1.12.0 duckdb==1.5.5 -pyarrow==25.0.0 +pyarrow==25.0.1 diff --git a/test/adbc/smoketest.py b/test/adbc/smoketest.py index b79726c422366..2b0e5d22da46d 100644 --- a/test/adbc/smoketest.py +++ b/test/adbc/smoketest.py @@ -68,7 +68,7 @@ "c_timestamptz": pa.timestamp("us", tz="UTC"), "c_interval": pa.month_day_nano_interval(), "c_uuid": pa.opaque(pa.binary(), "uuid", "PostgreSQL"), - "c_jsonb": pa.string(), + "c_jsonb": pa.json_(pa.string()), } diff --git a/test/lang/python/mzcompose.py b/test/lang/python/mzcompose.py index a65335fdf37fd..4a4b130079db8 100644 --- a/test/lang/python/mzcompose.py +++ b/test/lang/python/mzcompose.py @@ -20,7 +20,7 @@ Service( name="python", config={ - "image": "python:3.14.1-trixie", + "image": "python:3.14.7-trixie", "volumes": [ "../../../:/workdir", ], From a6e03928a3131a333ff2ae7844f330c54978d5bd Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 27 Aug 2026 18:09:13 +0000 Subject: [PATCH 12/15] sql: accept `-` as OID 0 when casting a string to a reg* type Rendering OID 0 as `-` in the reg* text output was only half of the round trip: `STRING_REG_CAST_TEMPLATE` had no branch for `-`, so Materialize could not parse back its own rendering. `parse_ident` rejected it with `string is not a valid identifier: "-"`, which broke `0::regproc::text::regproc` and `0::regtype::text::regtype`. PostgreSQL's `regprocin`, `regclassin` and `regtypein` all go through `parseDashOrOid`, so `-` means OID 0 for every reg* type. Accept it in all cases, including text to regclass: `-` is not number-shaped, so the restriction that disables the OID-like branch there does not apply. Adds round-trip coverage to regproc.slt, regtype.slt and regclass.slt. --- src/sql/src/plan/typeconv.rs | 5 +++++ test/sqllogictest/regclass.slt | 12 ++++++++++++ test/sqllogictest/regproc.slt | 11 +++++++++++ test/sqllogictest/regtype.slt | 11 +++++++++++ 4 files changed, 39 insertions(+) diff --git a/src/sql/src/plan/typeconv.rs b/src/sql/src/plan/typeconv.rs index 523e3018011f7..6f09ee1c5cec7 100644 --- a/src/sql/src/plan/typeconv.rs +++ b/src/sql/src/plan/typeconv.rs @@ -123,6 +123,8 @@ impl From<[UnaryFunc; N]> for CastTemplate { /// A reg* type represents a specific type of object by oid. /// /// Casting from a string to a reg*: +/// - Accepts `-`, the absent reference, as OID 0 in all cases, matching +/// PostgreSQL's `parseDashOrOid`. /// - Accepts a string that looks like an OID and converts the value to the /// specified reg* type. This is available in all cases except explicitly /// casting text values to regclass (e.g. `SELECT '2'::text::regclass`) @@ -138,6 +140,9 @@ const STRING_REG_CAST_TEMPLATE: &str = " (SELECT CASE WHEN $1 IS NULL THEN NULL +-- Handle the absent reference, which the reg* text output spells `-`. PostgreSQL's +-- `parseDashOrOid` accepts it for every reg* type, including text to regclass. + WHEN $1 = '-' THEN 0::pg_catalog.oid::pg_catalog.{0} -- Handle OID-like input, if available via {2} WHEN {2} AND pg_catalog.substring($1, 1, 1) BETWEEN '0' AND '9' THEN $1::pg_catalog.oid::pg_catalog.{0} diff --git a/test/sqllogictest/regclass.slt b/test/sqllogictest/regclass.slt index 8f76c5ad98f4f..71844235ea1b8 100644 --- a/test/sqllogictest/regclass.slt +++ b/test/sqllogictest/regclass.slt @@ -286,6 +286,18 @@ SELECT 99999::regclass::text ---- 99999 +# `-` is the absent reference on input as well, so the text rendering round trips. +# Unlike a numeric string, `-` is accepted even casting from text to regclass. +query T +SELECT 0::regclass::text::regclass::text +---- +- + +query T +SELECT '-'::regclass::oid +---- +0 + # ensure that all existing types can be cast to their respective names statement OK select oid, oid::regclass::text from (select oid from pg_catalog.pg_class) diff --git a/test/sqllogictest/regproc.slt b/test/sqllogictest/regproc.slt index d25268a12d55b..d5c519a9c0946 100644 --- a/test/sqllogictest/regproc.slt +++ b/test/sqllogictest/regproc.slt @@ -97,6 +97,17 @@ SELECT 99999::regproc::text ---- 99999 +# `-` is the absent reference on input as well, so the text rendering round trips. +query T +SELECT 0::regproc::text::regproc::text +---- +- + +query T +SELECT '-'::regproc::oid +---- +0 + query error more than one function named "max" SELECT 'max'::regproc diff --git a/test/sqllogictest/regtype.slt b/test/sqllogictest/regtype.slt index d7b655830ea6c..69fd3282ab11d 100644 --- a/test/sqllogictest/regtype.slt +++ b/test/sqllogictest/regtype.slt @@ -131,6 +131,17 @@ SELECT 99999::regtype::text ---- 99999 +# `-` is the absent reference on input as well, so the text rendering round trips. +query T +SELECT 0::regtype::text::regtype::text +---- +- + +query T +SELECT '-'::regtype::oid +---- +0 + # ensure that all existing types can be cast to their respective names statement OK select oid, oid::regtype::text from (select oid from mz_catalog.mz_types) From ff48413d12f25201eb05ee2172fc97171c42bf4a Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 27 Aug 2026 18:21:01 +0000 Subject: [PATCH 13/15] pgrepr: regenerate the regproc name table with REWRITE=1 The drift test recomputed `regproc::NAMES` from the builtin function registry and printed the corrected table in its panic message, leaving the reader to paste hundreds of lines back into the source. It now splices the recomputed table into `src/pgrepr-consts/src/regproc.rs` itself when `REWRITE` is set, matching the idiom in `src/compute-types/tests/lir_schema.rs`, and names that command when it fails without it. `// BEGIN GENERATED` / `// END GENERATED` mark the region the splice replaces. Both markers must appear exactly once before anything is written, since a splice anchored elsewhere in the file would clobber the lookup functions below the table. The table stays static: `mz-pgrepr-consts` sits below the catalog crates and the text encoder also runs in `clusterd`, which links no catalog. --- src/catalog/src/builtin.rs | 70 +++++++++++++++++++++++++++----- src/pgrepr-consts/src/regproc.rs | 11 +++-- 2 files changed, 68 insertions(+), 13 deletions(-) diff --git a/src/catalog/src/builtin.rs b/src/catalog/src/builtin.rs index 5fb7b00902e9a..6fb54b7867f51 100644 --- a/src/catalog/src/builtin.rs +++ b/src/catalog/src/builtin.rs @@ -1656,9 +1656,12 @@ mod tests { use super::*; /// Recomputes `mz_pgrepr::regproc::NAMES` from the builtin function registry - /// and fails with the corrected table when the checked-in copy has drifted. - /// It is checked in as data because `mz-pgrepr` sits below this crate in the - /// dependency graph and so cannot read the registry itself. + /// and fails when the checked-in copy has drifted. It is checked in as data + /// because `mz-pgrepr` sits below this crate in the dependency graph and so + /// cannot read the registry itself. + /// + /// Run with `REWRITE=1` to splice the recomputed table back into + /// `src/pgrepr-consts/src/regproc.rs`. #[mz_ore::test] #[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux` fn test_regproc_names_match_builtin_functions() { @@ -1693,24 +1696,71 @@ mod tests { } } + let table: String = expected + .iter() + .map(|(oid, name)| format!(" ({}, \"{}\"),\n", oid, name)) + .collect(); + + if std::env::var_os("REWRITE").is_some() { + rewrite_regproc_names(&table); + return; + } + let actual: BTreeMap = mz_pgrepr::regproc::NAMES .iter() .map(|(oid, name)| (*oid, name.to_string())) .collect(); if actual != expected { - let table: String = expected - .iter() - .map(|(oid, name)| format!(" ({}, \"{}\"),\n", oid, name)) - .collect(); panic!( - "mz_pgrepr::regproc::NAMES has drifted from the builtin function registry. \ - Replace the entries of NAMES in src/pgrepr-consts/src/regproc.rs with:\n{}", - table + "mz_pgrepr::regproc::NAMES has drifted from the builtin function \ + registry. Regenerate it with:\n\n \ + REWRITE=1 cargo test -p mz-catalog \ + test_regproc_names_match_builtin_functions\n" ); } } + /// Replaces the generated region of `mz_pgrepr::regproc::NAMES` with + /// `table`, leaving every other byte of the file alone. + /// + /// The path is relative to this crate's directory, which is the working + /// directory `cargo test` runs in. + /// + /// A splice anchored anywhere but the table would clobber the lookup + /// functions below it, so both markers have to appear exactly once. + fn rewrite_regproc_names(table: &str) { + const PATH: &str = "../pgrepr-consts/src/regproc.rs"; + const BEGIN: &str = " // BEGIN GENERATED\n"; + const END: &str = " // END GENERATED\n"; + + let contents = + std::fs::read_to_string(PATH).unwrap_or_else(|e| panic!("reading '{PATH}': {e}")); + for marker in [BEGIN, END] { + let count = contents.matches(marker).count(); + assert_eq!( + count, + 1, + "'{}' appears {} times in '{}', expected exactly once", + marker.trim(), + count, + PATH + ); + } + let begin = contents.find(BEGIN).expect("checked above") + BEGIN.len(); + let end = contents.find(END).expect("checked above"); + assert!( + begin <= end, + "'{}' precedes '{}' in '{}'", + END.trim(), + BEGIN.trim(), + PATH + ); + + let rewritten = format!("{}{}{}", &contents[..begin], table, &contents[end..]); + std::fs::write(PATH, rewritten).unwrap_or_else(|e| panic!("writing '{PATH}': {e}")); + } + #[mz_ore::test] #[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux` fn test_builtin_type_schema() { diff --git a/src/pgrepr-consts/src/regproc.rs b/src/pgrepr-consts/src/regproc.rs index 2c1a0146153dd..d3ad9c9c76674 100644 --- a/src/pgrepr-consts/src/regproc.rs +++ b/src/pgrepr-consts/src/regproc.rs @@ -21,9 +21,12 @@ //! through `mz_catalog::builtin::BUILTINS::funcs()`, which is the authoritative //! definition of every function OID. `mz_catalog::builtin`'s //! `test_regproc_names_match_builtin_functions` recomputes the whole table from -//! that registry and fails with the corrected contents when the two have -//! drifted, so do not hand-edit entries here. Add the function to the registry -//! and paste in the table the test prints. +//! that registry and fails when the two have drifted, so do not hand-edit +//! entries here. Add the function to the registry, then regenerate the table: +//! +//! ```shell +//! REWRITE=1 cargo test -p mz-catalog test_regproc_names_match_builtin_functions +//! ``` //! //! The OIDs are not invented by Materialize wherever PostgreSQL already assigns //! one. `mz_sql::func`'s `PG_CATALOG_BUILTINS` records that its literal OIDs @@ -50,6 +53,7 @@ /// from `regproc` to `text`, and the unquoted form still resolves because /// `regprocin` accepts a reserved word either way. pub const NAMES: &[(u32, &str)] = &[ + // BEGIN GENERATED (34, "namein"), (38, "int2in"), (40, "int2vectorin"), @@ -681,6 +685,7 @@ pub const NAMES: &[(u32, &str)] = &[ (17113, "mz_aws_account_id"), (17114, "mz_aws_external_id_prefix"), (17115, "mz_aws_connection_role_arn"), + // END GENERATED ]; /// Returns the text that `regproc` OID `oid` renders as, or `None` when no From 8822b54137803c2584077cd3f99d7217efc06c94 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 27 Aug 2026 18:54:35 +0000 Subject: [PATCH 14/15] sql: spell the reg* `-` round-trip tests with a literal input The round-trip cases fed a cast-derived value into the string to reg* cast: `0::reg*::text::reg*::text`. A non-constant input cannot fold away the `If` branches of `STRING_REG_CAST_TEMPLATE`, so MFP fusion eagerly evaluates the name-resolution `ELSE` and its `mz_error_if_null` raises even though the `-` branch matched. That is the pre-existing evaluation order limitation akin to STG-54 already asserted as a `query error` in regtype.slt and behind the neighbouring commented-out cases, not something these tests should be exercising. Assert the same behaviour from a literal input instead, matching the shape of the passing neighbours. regclass additionally covers `'-'::text`, which routes through the explicit text to regclass template that rejects OID-like input and so is the interesting context for the ungated `-` branch. --- test/sqllogictest/regclass.slt | 14 +++++++++++--- test/sqllogictest/regproc.slt | 6 ++++-- test/sqllogictest/regtype.slt | 6 ++++-- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/test/sqllogictest/regclass.slt b/test/sqllogictest/regclass.slt index 71844235ea1b8..7c961fb25171c 100644 --- a/test/sqllogictest/regclass.slt +++ b/test/sqllogictest/regclass.slt @@ -286,10 +286,11 @@ SELECT 99999::regclass::text ---- 99999 -# `-` is the absent reference on input as well, so the text rendering round trips. -# Unlike a numeric string, `-` is accepted even casting from text to regclass. +# `-` round trips on input too. Spelled from a literal because feeding a +# cast-derived value into the string to reg* cast hits the evaluation order +# problem behind the cases commented out above. query T -SELECT 0::regclass::text::regclass::text +SELECT '-'::regclass::text ---- - @@ -298,6 +299,13 @@ SELECT '-'::regclass::oid ---- 0 +# Unlike a numeric string, `-` is accepted even by the explicit text to regclass +# cast. +query T +SELECT '-'::text::regclass::oid +---- +0 + # ensure that all existing types can be cast to their respective names statement OK select oid, oid::regclass::text from (select oid from pg_catalog.pg_class) diff --git a/test/sqllogictest/regproc.slt b/test/sqllogictest/regproc.slt index d5c519a9c0946..6f1a31d6198b1 100644 --- a/test/sqllogictest/regproc.slt +++ b/test/sqllogictest/regproc.slt @@ -97,9 +97,11 @@ SELECT 99999::regproc::text ---- 99999 -# `-` is the absent reference on input as well, so the text rendering round trips. +# `-` round trips on input too. Spelled from a literal because feeding a +# cast-derived value into the string to reg* cast hits the evaluation order +# problem behind the cases commented out above. query T -SELECT 0::regproc::text::regproc::text +SELECT '-'::regproc::text ---- - diff --git a/test/sqllogictest/regtype.slt b/test/sqllogictest/regtype.slt index 69fd3282ab11d..817cb6225b7ee 100644 --- a/test/sqllogictest/regtype.slt +++ b/test/sqllogictest/regtype.slt @@ -131,9 +131,11 @@ SELECT 99999::regtype::text ---- 99999 -# `-` is the absent reference on input as well, so the text rendering round trips. +# `-` round trips on input too. Spelled from a literal because feeding a +# cast-derived value into the string to reg* cast hits the evaluation order +# problem behind the cases commented out above. query T -SELECT 0::regtype::text::regtype::text +SELECT '-'::regtype::text ---- - From 6e7030d5468ec5bceecf4f108a8e18ee41c7ab07 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 27 Aug 2026 19:01:09 +0000 Subject: [PATCH 15/15] ci: move ADBC test suite from the test pipeline to nightly The Arrow ADBC driver suite ran on every PR, which costs CI time for a driver-integration test that rarely regresses. Move it to the nightly pipeline, where the other client-library suites live. The step drops its `inputs: [test/adbc]` gating, which only has meaning in the per-PR pipeline, and takes the 30 minute timeout used by the rest of the Language group. Composition, agent queue, and build dependency are unchanged. --- ci/nightly/pipeline.template.yml | 10 ++++++++++ ci/test/pipeline.template.yml | 11 ----------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/ci/nightly/pipeline.template.yml b/ci/nightly/pipeline.template.yml index f25c31d5ef852..3e3fec1668155 100644 --- a/ci/nightly/pipeline.template.yml +++ b/ci/nightly/pipeline.template.yml @@ -3026,6 +3026,16 @@ steps: - group: Language key: language-tests steps: + - id: adbc + label: Arrow ADBC driver + depends_on: build-aarch64 + timeout_in_minutes: 30 + plugins: + - ./ci/plugins/mzcompose: + composition: adbc + agents: + queue: hetzner-aarch64-4cpu-8gb + - id: lang-csharp label: ":csharp:" depends_on: build-aarch64 diff --git a/ci/test/pipeline.template.yml b/ci/test/pipeline.template.yml index 5bef529d08af5..6d915c47c1db5 100644 --- a/ci/test/pipeline.template.yml +++ b/ci/test/pipeline.template.yml @@ -679,17 +679,6 @@ steps: agents: queue: hetzner-aarch64-4cpu-8gb - - id: adbc - label: Arrow ADBC driver - depends_on: build-aarch64 - timeout_in_minutes: 20 - inputs: [test/adbc] - plugins: - - ./ci/plugins/mzcompose: - composition: adbc - agents: - queue: hetzner-aarch64-4cpu-8gb - - group: Docs key: docs-tests steps: