From 1dea044124b62520145afc41be8d9b871b8c5ef9 Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Mon, 8 Jan 2024 13:44:25 +0200 Subject: [PATCH 1/7] Enable introspection of electrified enum types in the proxy --- .../electric/postgres/proxy/prisma/query.ex | 19 +++-- .../postgres/proxy/prisma/query_test.exs | 78 ++++++++++++++++++- .../test/support/prisma/002_query_test.sql | 7 ++ .../test/support/prisma/003_query_test.sql | 2 +- 4 files changed, 96 insertions(+), 10 deletions(-) diff --git a/components/electric/lib/electric/postgres/proxy/prisma/query.ex b/components/electric/lib/electric/postgres/proxy/prisma/query.ex index 2a5046c06f..4d10f9b638 100644 --- a/components/electric/lib/electric/postgres/proxy/prisma/query.ex +++ b/components/electric/lib/electric/postgres/proxy/prisma/query.ex @@ -489,9 +489,12 @@ defmodule Electric.Postgres.Proxy.Prisma.Query.TypeV4_8 do ] end - # we don't support custom types - def data_rows([_nspname], _schema_version, _config) do - [] + def data_rows([nspname_array], schema_version, _config) do + namespaces = Electric.Postgres.Proxy.Prisma.Query.parse_name_array(nspname_array) + + for %{name: name} = enum <- schema_version.schema.enums, name.schema in namespaces do + [name.name, enum.values, name.schema] + end end end @@ -530,9 +533,13 @@ defmodule Electric.Postgres.Proxy.Prisma.Query.TypeV5_2 do ] end - # we don't support custom types - def data_rows([_nspname], _schema_version, _config) do - [] + def data_rows([nspname_array], schema_version, config) do + Electric.Postgres.Proxy.Prisma.Query.TypeV4_8.data_rows( + [nspname_array], + schema_version, + config + ) + |> Enum.map(fn [name, value, namespace] -> [name, value, namespace, nil] end) end end diff --git a/components/electric/test/electric/postgres/proxy/prisma/query_test.exs b/components/electric/test/electric/postgres/proxy/prisma/query_test.exs index 51bb311100..c1fcf77a02 100644 --- a/components/electric/test/electric/postgres/proxy/prisma/query_test.exs +++ b/components/electric/test/electric/postgres/proxy/prisma/query_test.exs @@ -37,6 +37,7 @@ defmodule Electric.Postgres.Proxy.Prisma.QueryTest do ["with_constraint", "public", <<0>>, <<0>>, <<0>>, nil, nil], ["checked", "public", <<0>>, <<0>>, <<0>>, nil, nil], ["interesting", "public", <<0>>, <<0>>, <<0>>, nil, nil], + ["manuals", "public", <<0>>, <<0>>, <<0>>, nil, nil], ["pointy", "public", <<0>>, <<0>>, <<0>>, nil, nil], ["pointy2", "public", <<0>>, <<0>>, <<0>>, nil, nil] ]) @@ -82,7 +83,8 @@ defmodule Electric.Postgres.Proxy.Prisma.QueryTest do end test "TypeV5_2", cxt do - [] = Query.TypeV5_2.data_rows([@public], cxt.schema, config()) + [["oses", ["linux", "macos", "windows"], "public", nil]] = + Query.TypeV5_2.data_rows([@public], cxt.schema, config()) end test "ColumnV5_2", cxt do @@ -312,6 +314,60 @@ defmodule Electric.Postgres.Proxy.Prisma.QueryTest do nil, nil ], + [ + "public", + "manuals", + "id", + "text", + nil, + nil, + nil, + nil, + "text", + "pg_catalog", + "text", + nil, + "NO", + "NO", + nil, + nil + ], + [ + "public", + "manuals", + "manual_url", + "text", + nil, + nil, + nil, + nil, + "text", + "pg_catalog", + "text", + nil, + "NO", + "NO", + nil, + nil + ], + [ + "public", + "manuals", + "os", + "oses", + nil, + nil, + nil, + nil, + "oses", + "pg_catalog", + "oses", + nil, + "NO", + "NO", + nil, + nil + ], [ "public", "with_constraint", @@ -498,7 +554,7 @@ defmodule Electric.Postgres.Proxy.Prisma.QueryTest do test "ForeignKeyV5_2", cxt do data_rows = Query.ForeignKeyV5_2.data_rows([@public], cxt.schema, config()) - # can't do an equality check as the actual oids + # can't do an equality check as the actual oids assert Enum.sort(data_rows) == Enum.sort([ [ @@ -555,7 +611,7 @@ defmodule Electric.Postgres.Proxy.Prisma.QueryTest do test "IndexV5_2", cxt do data_rows = Query.IndexV5_2.data_rows([@public], cxt.schema, config()) - # can't do an equality check as the actual oids + # can't do an equality check as the actual oids assert Enum.sort(data_rows) == Enum.sort([ [ @@ -654,6 +710,22 @@ defmodule Electric.Postgres.Proxy.Prisma.QueryTest do <<0>>, <<0>> ], + [ + "public", + "manuals_pkey", + "manuals", + "id", + <<1>>, + <<1>>, + <<0, 0, 0, 0>>, + "text_ops", + <<1>>, + "btree", + "ASC", + <<0>>, + <<0>>, + <<0>> + ], [ "public", "pointy_pkey", diff --git a/components/electric/test/support/prisma/002_query_test.sql b/components/electric/test/support/prisma/002_query_test.sql index 20c94ed9db..49cef2a77e 100644 --- a/components/electric/test/support/prisma/002_query_test.sql +++ b/components/electric/test/support/prisma/002_query_test.sql @@ -41,3 +41,10 @@ CREATE TABLE public.pointy2 ( ); +CREATE TYPE oses AS ENUM ('linux', 'macos', 'windows'); + +CREATE TABLE public.manuals ( + id text PRIMARY KEY, + os oses NOT NULL, + manual_url text NOT NULL +); diff --git a/components/electric/test/support/prisma/003_query_test.sql b/components/electric/test/support/prisma/003_query_test.sql index be14fcd8ed..7427bce93d 100644 --- a/components/electric/test/support/prisma/003_query_test.sql +++ b/components/electric/test/support/prisma/003_query_test.sql @@ -7,4 +7,4 @@ ALTER TABLE public.checked ENABLE ELECTRIC; ALTER TABLE public.interesting ENABLE ELECTRIC; ALTER TABLE public.pointy ENABLE ELECTRIC; ALTER TABLE public.pointy2 ENABLE ELECTRIC; - +ALTER TABLE public.oses ENABLE ELECTRIC; From e6ab855fbaf701eba1217ec7eb98d68a5873e7bd Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Mon, 8 Jan 2024 13:35:11 +0200 Subject: [PATCH 2/7] Remove outdated comment --- components/electric/test/support/prisma/003_query_test.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/components/electric/test/support/prisma/003_query_test.sql b/components/electric/test/support/prisma/003_query_test.sql index 7427bce93d..030b85dc12 100644 --- a/components/electric/test/support/prisma/003_query_test.sql +++ b/components/electric/test/support/prisma/003_query_test.sql @@ -2,8 +2,6 @@ ALTER TABLE public.with_constraint ENABLE ELECTRIC; ALTER TABLE public.checked ENABLE ELECTRIC; --- NOTE: `interesting` can't currently be electrified as it contains columns of --- types we currently (as of 09/2023) don't support ALTER TABLE public.interesting ENABLE ELECTRIC; ALTER TABLE public.pointy ENABLE ELECTRIC; ALTER TABLE public.pointy2 ENABLE ELECTRIC; From d44cb3a4c1204944a7d89995770b87b4e3e91520 Mon Sep 17 00:00:00 2001 From: Kevin Date: Tue, 28 Nov 2023 10:57:02 +0100 Subject: [PATCH 3/7] feat (DAL): support for enumerations (#604) This PR adds support for enumerations by: - modifying the generator to store enumeration types as "TEXT" type in the generated client - modifying satellite to serialise/deserialise enumerations as text - since enumeration types are unknown to Satellite it serialises/deserialises them as text --- .changeset/moody-carpets-allow.md | 6 +++ clients/typescript/src/satellite/client.ts | 4 +- .../test/satellite/serialization.test.ts | 52 +++++++++++++++---- .../03.20_node_satellite_can_sync_enums.lux | 2 +- .../writeTableSchemas.ts | 1 + 5 files changed, 52 insertions(+), 13 deletions(-) create mode 100644 .changeset/moody-carpets-allow.md diff --git a/.changeset/moody-carpets-allow.md b/.changeset/moody-carpets-allow.md new file mode 100644 index 0000000000..773acf612d --- /dev/null +++ b/.changeset/moody-carpets-allow.md @@ -0,0 +1,6 @@ +--- +"electric-sql": patch +"@electric-sql/prisma-generator": patch +--- + +Adds client-side support for enumerations. diff --git a/clients/typescript/src/satellite/client.ts b/clients/typescript/src/satellite/client.ts index f27c7e7e8f..5512449dfc 100644 --- a/clients/typescript/src/satellite/client.ts +++ b/clients/typescript/src/satellite/client.ts @@ -739,8 +739,7 @@ export class SatelliteClient implements Client { 'error', new SatelliteError( SatelliteErrorCode.UNEXPECTED_STATE, - `unexpected state ${ - ReplicationStatus[this.inbound.isReplicating] + `unexpected state ${ReplicationStatus[this.inbound.isReplicating] } handling 'relation' message` ) ) @@ -1130,6 +1129,7 @@ function deserializeColumnData( case PgDateType.PG_TIMETZ: return typeDecoder.timetz(column) default: + // also covers user-defined enumeration types return typeDecoder.text(column) } } diff --git a/clients/typescript/test/satellite/serialization.test.ts b/clients/typescript/test/satellite/serialization.test.ts index 0620a7b059..e8c26449e5 100644 --- a/clients/typescript/test/satellite/serialization.test.ts +++ b/clients/typescript/test/satellite/serialization.test.ts @@ -28,6 +28,9 @@ test('serialize/deserialize row data', async (t) => { { name: 'bool1', type: 'BOOL', isNullable: true }, { name: 'bool2', type: 'BOOL', isNullable: true }, { name: 'bool3', type: 'BOOL', isNullable: true }, + // bundled migrations contain type 'TEXT' for enums + { name: 'enum1', type: 'TEXT', isNullable: true }, + { name: 'enum2', type: 'TEXT', isNullable: true }, ], } @@ -46,6 +49,9 @@ test('serialize/deserialize row data', async (t) => { ['bool1', PgBasicType.PG_BOOL], ['bool2', PgBasicType.PG_BOOL], ['bool3', PgBasicType.PG_BOOL], + // enum types are transformed to text type by our generator + ['enum1', PgBasicType.PG_TEXT], + ['enum2', PgBasicType.PG_TEXT], ]), relations: [], } as unknown as TableSchema< @@ -76,12 +82,28 @@ test('serialize/deserialize row data', async (t) => { bool1: 1, bool2: 0, bool3: null, + enum1: 'red', + enum2: null, } const s_row = serializeRow(record, rel, dbDescription) t.deepEqual( s_row.values.map((bytes) => new TextDecoder().decode(bytes)), - ['Hello', 'World!', '', '1', '-30', '1', '-30.3', '5e+234', 't', 'f', ''] + [ + 'Hello', + 'World!', + '', + '1', + '-30', + '1', + '-30.3', + '5e+234', + 't', + 'f', + '', + 'red', + '', + ] ) const d_row = deserializeRow(s_row, rel, dbDescription) @@ -100,6 +122,8 @@ test('serialize/deserialize row data', async (t) => { bool1: null, bool2: null, bool3: null, + enum1: 'red', + enum2: null, } const s_row2 = serializeRow(record2, rel, dbDescription) @@ -117,6 +141,8 @@ test('serialize/deserialize row data', async (t) => { '', '', '', + 'red', + '', ] ) @@ -282,17 +308,23 @@ test('Use incoming Relation types if not found in the schema', async (t) => { schema: 'schema', table: 'new_table', tableType: SatRelation_RelationType.TABLE, - columns: [{ name: 'value', type: 'INTEGER', isNullable: true }], + columns: [ + { name: 'value', type: 'INTEGER', isNullable: true }, + { name: 'color', type: 'COLOR', isNullable: true }, // at runtime, incoming SatRelation messages contain the name of the enum type + ], } - const satOpRow = serializeRow( - { value: 6 }, - newTableRelation, - testDbDescription - ) + const row = { + value: 6, + color: 'red', + } + + const satOpRow = serializeRow(row, newTableRelation, testDbDescription) - // Encoded values ["6"] - t.deepEqual(satOpRow.values, [new Uint8Array(['6'.charCodeAt(0)])]) + t.deepEqual( + satOpRow.values.map((bytes) => new TextDecoder().decode(bytes)), + ['6', 'red'] + ) const deserializedRow = deserializeRow( satOpRow, @@ -300,5 +332,5 @@ test('Use incoming Relation types if not found in the schema', async (t) => { testDbDescription ) - t.deepEqual(deserializedRow, { value: 6 }) + t.deepEqual(deserializedRow, row) }) diff --git a/e2e/tests/03.20_node_satellite_can_sync_enums.lux b/e2e/tests/03.20_node_satellite_can_sync_enums.lux index 8dec753f1d..99b9c7ad90 100644 --- a/e2e/tests/03.20_node_satellite_can_sync_enums.lux +++ b/e2e/tests/03.20_node_satellite_can_sync_enums.lux @@ -39,7 +39,7 @@ [invoke node_get_enum "row3" null] [shell pg_1] - [invoke wait-for "SELECT * FROM public.enums;" "row1" 10 $psql] + [invoke wait-for "SELECT * FROM public.enums;" "row2" 10 $psql] !SELECT * FROM public.enums; ??row1 | RED diff --git a/generator/src/functions/tableDescriptionWriters/writeTableSchemas.ts b/generator/src/functions/tableDescriptionWriters/writeTableSchemas.ts index e92042bc6b..530f8c84ff 100644 --- a/generator/src/functions/tableDescriptionWriters/writeTableSchemas.ts +++ b/generator/src/functions/tableDescriptionWriters/writeTableSchemas.ts @@ -128,6 +128,7 @@ function pgType(field: ExtendedDMMFField, modelName: string): string { case 'Json': return jsonToPg(attributes) default: + if (field.kind === 'enum') return 'TEXT' // treat enums as TEXT such that the ts-client correctly serializes/deserialises them as text return 'UNRECOGNIZED PRISMA TYPE' } } From 8245630a5b8c0dc0afe6474bd3f577289eb8d9d0 Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Mon, 8 Jan 2024 15:24:04 +0200 Subject: [PATCH 4/7] Fix TypeScript code formatting --- clients/typescript/src/satellite/client.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clients/typescript/src/satellite/client.ts b/clients/typescript/src/satellite/client.ts index 5512449dfc..8a2c19f7b7 100644 --- a/clients/typescript/src/satellite/client.ts +++ b/clients/typescript/src/satellite/client.ts @@ -739,7 +739,8 @@ export class SatelliteClient implements Client { 'error', new SatelliteError( SatelliteErrorCode.UNEXPECTED_STATE, - `unexpected state ${ReplicationStatus[this.inbound.isReplicating] + `unexpected state ${ + ReplicationStatus[this.inbound.isReplicating] } handling 'relation' message` ) ) From 68b3e8ccf1e8fa56f959ff34cdd572f3cefc21ec Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Mon, 8 Jan 2024 16:23:14 +0200 Subject: [PATCH 5/7] Add server changeset --- .changeset/dirty-hairs-mate.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/dirty-hairs-mate.md diff --git a/.changeset/dirty-hairs-mate.md b/.changeset/dirty-hairs-mate.md new file mode 100644 index 0000000000..bad8b0f632 --- /dev/null +++ b/.changeset/dirty-hairs-mate.md @@ -0,0 +1,5 @@ +--- +"@core/electric": patch +--- + +[VAX-1040] [VAX-1041] [VAX-1042] Add support for user-defined enum types in electrified tables. From 02e021188ecf3a63a936f591b5444c488fb38c96 Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Tue, 9 Jan 2024 13:20:38 +0200 Subject: [PATCH 6/7] fixup! Enable introspection of electrified enum types in the proxy --- .../electric/lib/electric/postgres/proxy/prisma/query.ex | 6 ++++-- .../test/electric/postgres/proxy/prisma/query_test.exs | 7 +++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/components/electric/lib/electric/postgres/proxy/prisma/query.ex b/components/electric/lib/electric/postgres/proxy/prisma/query.ex index 4d10f9b638..6daa0b9b25 100644 --- a/components/electric/lib/electric/postgres/proxy/prisma/query.ex +++ b/components/electric/lib/electric/postgres/proxy/prisma/query.ex @@ -492,8 +492,10 @@ defmodule Electric.Postgres.Proxy.Prisma.Query.TypeV4_8 do def data_rows([nspname_array], schema_version, _config) do namespaces = Electric.Postgres.Proxy.Prisma.Query.parse_name_array(nspname_array) - for %{name: name} = enum <- schema_version.schema.enums, name.schema in namespaces do - [name.name, enum.values, name.schema] + for %{name: %{name: name, schema: schema}} = enum <- schema_version.schema.enums, + schema in namespaces, + value <- enum.values do + [name, value, schema] end end end diff --git a/components/electric/test/electric/postgres/proxy/prisma/query_test.exs b/components/electric/test/electric/postgres/proxy/prisma/query_test.exs index c1fcf77a02..1e84a191fa 100644 --- a/components/electric/test/electric/postgres/proxy/prisma/query_test.exs +++ b/components/electric/test/electric/postgres/proxy/prisma/query_test.exs @@ -83,8 +83,11 @@ defmodule Electric.Postgres.Proxy.Prisma.QueryTest do end test "TypeV5_2", cxt do - [["oses", ["linux", "macos", "windows"], "public", nil]] = - Query.TypeV5_2.data_rows([@public], cxt.schema, config()) + assert [ + ["oses", "linux", "public", nil], + ["oses", "macos", "public", nil], + ["oses", "windows", "public", nil] + ] == Query.TypeV5_2.data_rows([@public], cxt.schema, config()) end test "ColumnV5_2", cxt do From 59fdb47f6a928a041e37a44fe34ddee7c0a6414e Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Tue, 9 Jan 2024 13:20:54 +0200 Subject: [PATCH 7/7] Fix trailing whitespace issues in prisma/query.ex --- .../lib/electric/postgres/proxy/prisma/query.ex | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/components/electric/lib/electric/postgres/proxy/prisma/query.ex b/components/electric/lib/electric/postgres/proxy/prisma/query.ex index 6daa0b9b25..816780bce5 100644 --- a/components/electric/lib/electric/postgres/proxy/prisma/query.ex +++ b/components/electric/lib/electric/postgres/proxy/prisma/query.ex @@ -116,7 +116,7 @@ end defmodule Electric.Postgres.Proxy.Prisma.Query.NamespaceVersionV5_2 do @moduledoc """ - SELECT + SELECT EXISTS(SELECT 1 FROM pg_namespace WHERE nspname = $1), version(), current_setting('server_version_num')::integer as numeric_version; @@ -312,7 +312,7 @@ defmodule Electric.Postgres.Proxy.Prisma.Query.ConstraintV5_2 do AND contype NOT IN ('p', 'u', 'f') ORDER BY namespace, table_name, constr.contype, constraint_name; - Lists: + Lists: - check constraints - constraint trigger @@ -569,11 +569,11 @@ defmodule Electric.Postgres.Proxy.Prisma.Query.ColumnV4_8 do FROM pg_class JOIN pg_namespace on pg_namespace.oid = pg_class.relnamespace AND pg_namespace.nspname = ANY ( $1 ) - ) as oid on oid.oid = att.attrelid + ) as oid on oid.oid = att.attrelid AND relname = info.table_name AND namespace = info.table_schema LEFT OUTER JOIN pg_attrdef attdef ON attdef.adrelid = att.attrelid AND attdef.adnum = att.attnum AND table_schema = namespace - WHERE table_schema = ANY ( $1 ) + WHERE table_schema = ANY ( $1 ) ORDER BY namespace, table_name, ordinal_position; """ @behaviour Electric.Postgres.Proxy.Prisma.Query @@ -1048,9 +1048,9 @@ defmodule Electric.Postgres.Proxy.Prisma.Query.ForeignKeyV4_8 do conname AS constraint_name, child, parent, - table_name, + table_name, namespace - FROM (SELECT + FROM (SELECT ns.nspname AS \"namespace\", unnest(con1.conkey) AS \"parent\", unnest(con1.confkey) AS \"child\", @@ -1349,7 +1349,7 @@ defmodule Electric.Postgres.Proxy.Prisma.Query.IndexV4_8 do @moduledoc """ WITH rawindex AS ( SELECT - indrelid, + indrelid, indexrelid, indisunique, indisprimary,