feat(electric): Enable introspection of electrified enum types in the proxy

.changeset/dirty-hairs-mate.md

@@ -0,0 +1,5 @@
+---
+"@core/electric": patch
+---
+
+[VAX-1040] [VAX-1041] [VAX-1042] Add support for user-defined enum types in electrified tables.

.changeset/moody-carpets-allow.md

@@ -0,0 +1,6 @@
+---
+"electric-sql": patch
+"@electric-sql/prisma-generator": patch
+---
+
+Adds client-side support for enumerations.

clients/typescript/src/satellite/client.ts

@@ -1130,6 +1130,7 @@ function deserializeColumnData(
     case PgDateType.PG_TIMETZ:
       return typeDecoder.timetz(column)
     default:
+      // also covers user-defined enumeration types
       return typeDecoder.text(column)
   }
 }

clients/typescript/test/satellite/serialization.test.ts

@@ -28,6 +28,9 @@ test('serialize/deserialize row data', async (t) => {
       { name: 'bool1', type: 'BOOL', isNullable: true },
       { name: 'bool2', type: 'BOOL', isNullable: true },
       { name: 'bool3', type: 'BOOL', isNullable: true },
+      // bundled migrations contain type 'TEXT' for enums
+      { name: 'enum1', type: 'TEXT', isNullable: true },
+      { name: 'enum2', type: 'TEXT', isNullable: true },
     ],
   }
 
@@ -46,6 +49,9 @@ test('serialize/deserialize row data', async (t) => {
           ['bool1', PgBasicType.PG_BOOL],
           ['bool2', PgBasicType.PG_BOOL],
           ['bool3', PgBasicType.PG_BOOL],
+          // enum types are transformed to text type by our generator
+          ['enum1', PgBasicType.PG_TEXT],
+          ['enum2', PgBasicType.PG_TEXT],
         ]),
         relations: [],
       } as unknown as TableSchema<
@@ -76,12 +82,28 @@ test('serialize/deserialize row data', async (t) => {
     bool1: 1,
     bool2: 0,
     bool3: null,
+    enum1: 'red',
+    enum2: null,
   }
 
   const s_row = serializeRow(record, rel, dbDescription)
   t.deepEqual(
     s_row.values.map((bytes) => new TextDecoder().decode(bytes)),
-    ['Hello', 'World!', '', '1', '-30', '1', '-30.3', '5e+234', 't', 'f', '']
+    [
+      'Hello',
+      'World!',
+      '',
+      '1',
+      '-30',
+      '1',
+      '-30.3',
+      '5e+234',
+      't',
+      'f',
+      '',
+      'red',
+      '',
+    ]
   )
 
   const d_row = deserializeRow(s_row, rel, dbDescription)
@@ -100,6 +122,8 @@ test('serialize/deserialize row data', async (t) => {
     bool1: null,
     bool2: null,
     bool3: null,
+    enum1: 'red',
+    enum2: null,
   }
 
   const s_row2 = serializeRow(record2, rel, dbDescription)
@@ -117,6 +141,8 @@ test('serialize/deserialize row data', async (t) => {
       '',
       '',
       '',
+      'red',
+      '',
     ]
   )
 
@@ -282,23 +308,29 @@ test('Use incoming Relation types if not found in the schema', async (t) => {
     schema: 'schema',
     table: 'new_table',
     tableType: SatRelation_RelationType.TABLE,
-    columns: [{ name: 'value', type: 'INTEGER', isNullable: true }],
+    columns: [
+      { name: 'value', type: 'INTEGER', isNullable: true },
+      { name: 'color', type: 'COLOR', isNullable: true }, // at runtime, incoming SatRelation messages contain the name of the enum type
+    ],
   }
 
-  const satOpRow = serializeRow(
-    { value: 6 },
-    newTableRelation,
-    testDbDescription
-  )
+  const row = {
+    value: 6,
+    color: 'red',
+  }
+
+  const satOpRow = serializeRow(row, newTableRelation, testDbDescription)
 
-  // Encoded values ["6"]
-  t.deepEqual(satOpRow.values, [new Uint8Array(['6'.charCodeAt(0)])])
+  t.deepEqual(
+    satOpRow.values.map((bytes) => new TextDecoder().decode(bytes)),
+    ['6', 'red']
+  )
 
   const deserializedRow = deserializeRow(
     satOpRow,
     newTableRelation,
     testDbDescription
   )
 
-  t.deepEqual(deserializedRow, { value: 6 })
+  t.deepEqual(deserializedRow, row)
 })

components/electric/lib/electric/postgres/proxy/prisma/query.ex

@@ -116,7 +116,7 @@ end
 
 defmodule Electric.Postgres.Proxy.Prisma.Query.NamespaceVersionV5_2 do
   @moduledoc """
-  SELECT 
+  SELECT
     EXISTS(SELECT 1 FROM pg_namespace WHERE nspname = $1),
     version(),
     current_setting('server_version_num')::integer as numeric_version;
@@ -312,7 +312,7 @@ defmodule Electric.Postgres.Proxy.Prisma.Query.ConstraintV5_2 do
     AND contype NOT IN ('p', 'u', 'f')
   ORDER BY namespace, table_name, constr.contype, constraint_name;
 
-  Lists: 
+  Lists:
 
   - check constraints
   - constraint trigger
@@ -489,9 +489,14 @@ defmodule Electric.Postgres.Proxy.Prisma.Query.TypeV4_8 do
     ]
   end
 
-  # we don't support custom types
-  def data_rows([_nspname], _schema_version, _config) do
-    []
+  def data_rows([nspname_array], schema_version, _config) do
+    namespaces = Electric.Postgres.Proxy.Prisma.Query.parse_name_array(nspname_array)
+
+    for %{name: %{name: name, schema: schema}} = enum <- schema_version.schema.enums,
+        schema in namespaces,
+        value <- enum.values do
+      [name, value, schema]
+    end
   end
 end
 
@@ -530,9 +535,13 @@ defmodule Electric.Postgres.Proxy.Prisma.Query.TypeV5_2 do
     ]
   end
 
-  # we don't support custom types
-  def data_rows([_nspname], _schema_version, _config) do
-    []
+  def data_rows([nspname_array], schema_version, config) do
+    Electric.Postgres.Proxy.Prisma.Query.TypeV4_8.data_rows(
+      [nspname_array],
+      schema_version,
+      config
+    )
+    |> Enum.map(fn [name, value, namespace] -> [name, value, namespace, nil] end)
   end
 end
 
@@ -560,11 +569,11 @@ defmodule Electric.Postgres.Proxy.Prisma.Query.ColumnV4_8 do
        FROM pg_class
        JOIN pg_namespace on pg_namespace.oid = pg_class.relnamespace
        AND pg_namespace.nspname = ANY ( $1 )
-      ) as oid on oid.oid = att.attrelid 
+      ) as oid on oid.oid = att.attrelid
         AND relname = info.table_name
         AND namespace = info.table_schema
   LEFT OUTER JOIN pg_attrdef attdef ON attdef.adrelid = att.attrelid AND attdef.adnum = att.attnum AND table_schema = namespace
-  WHERE table_schema = ANY ( $1 ) 
+  WHERE table_schema = ANY ( $1 )
   ORDER BY namespace, table_name, ordinal_position;
   """
   @behaviour Electric.Postgres.Proxy.Prisma.Query
@@ -1039,9 +1048,9 @@ defmodule Electric.Postgres.Proxy.Prisma.Query.ForeignKeyV4_8 do
       conname         AS constraint_name,
       child,
       parent,
-      table_name, 
+      table_name,
       namespace
-  FROM (SELECT 
+  FROM (SELECT
               ns.nspname AS \"namespace\",
               unnest(con1.conkey)                AS \"parent\",
               unnest(con1.confkey)                AS \"child\",
@@ -1340,7 +1349,7 @@ defmodule Electric.Postgres.Proxy.Prisma.Query.IndexV4_8 do
   @moduledoc """
   WITH rawindex AS (
       SELECT
-          indrelid, 
+          indrelid,
           indexrelid,
           indisunique,
           indisprimary,

components/electric/test/electric/postgres/proxy/prisma/query_test.exs

@@ -37,6 +37,7 @@ defmodule Electric.Postgres.Proxy.Prisma.QueryTest do
                ["with_constraint", "public", <<0>>, <<0>>, <<0>>, nil, nil],
                ["checked", "public", <<0>>, <<0>>, <<0>>, nil, nil],
                ["interesting", "public", <<0>>, <<0>>, <<0>>, nil, nil],
+               ["manuals", "public", <<0>>, <<0>>, <<0>>, nil, nil],
                ["pointy", "public", <<0>>, <<0>>, <<0>>, nil, nil],
                ["pointy2", "public", <<0>>, <<0>>, <<0>>, nil, nil]
              ])
@@ -82,7 +83,11 @@ defmodule Electric.Postgres.Proxy.Prisma.QueryTest do
   end
 
   test "TypeV5_2", cxt do
-    [] = Query.TypeV5_2.data_rows([@public], cxt.schema, config())
+    assert [
+             ["oses", "linux", "public", nil],
+             ["oses", "macos", "public", nil],
+             ["oses", "windows", "public", nil]
+           ] == Query.TypeV5_2.data_rows([@public], cxt.schema, config())
   end
 
   test "ColumnV5_2", cxt do
@@ -312,6 +317,60 @@ defmodule Electric.Postgres.Proxy.Prisma.QueryTest do
                  nil,
                  nil
                ],
+               [
+                 "public",
+                 "manuals",
+                 "id",
+                 "text",
+                 nil,
+                 nil,
+                 nil,
+                 nil,
+                 "text",
+                 "pg_catalog",
+                 "text",
+                 nil,
+                 "NO",
+                 "NO",
+                 nil,
+                 nil
+               ],
+               [
+                 "public",
+                 "manuals",
+                 "manual_url",
+                 "text",
+                 nil,
+                 nil,
+                 nil,
+                 nil,
+                 "text",
+                 "pg_catalog",
+                 "text",
+                 nil,
+                 "NO",
+                 "NO",
+                 nil,
+                 nil
+               ],
+               [
+                 "public",
+                 "manuals",
+                 "os",
+                 "oses",
+                 nil,
+                 nil,
+                 nil,
+                 nil,
+                 "oses",
+                 "pg_catalog",
+                 "oses",
+                 nil,
+                 "NO",
+                 "NO",
+                 nil,
+                 nil
+               ],
                [
                  "public",
                  "with_constraint",
@@ -498,7 +557,7 @@ defmodule Electric.Postgres.Proxy.Prisma.QueryTest do
   test "ForeignKeyV5_2", cxt do
     data_rows = Query.ForeignKeyV5_2.data_rows([@public], cxt.schema, config())
 
-    # can't do an equality check as the actual oids 
+    # can't do an equality check as the actual oids
     assert Enum.sort(data_rows) ==
              Enum.sort([
                [
@@ -555,7 +614,7 @@ defmodule Electric.Postgres.Proxy.Prisma.QueryTest do
   test "IndexV5_2", cxt do
     data_rows = Query.IndexV5_2.data_rows([@public], cxt.schema, config())
 
-    # can't do an equality check as the actual oids 
+    # can't do an equality check as the actual oids
     assert Enum.sort(data_rows) ==
              Enum.sort([
                [
@@ -654,6 +713,22 @@ defmodule Electric.Postgres.Proxy.Prisma.QueryTest do
                  <<0>>,
                  <<0>>
                ],
+               [
+                 "public",
+                 "manuals_pkey",
+                 "manuals",
+                 "id",
+                 <<1>>,
+                 <<1>>,
+                 <<0, 0, 0, 0>>,
+                 "text_ops",
+                 <<1>>,
+                 "btree",
+                 "ASC",
+                 <<0>>,
+                 <<0>>,
+                 <<0>>
+               ],
                [
                  "public",
                  "pointy_pkey",

components/electric/test/support/prisma/002_query_test.sql

@@ -41,3 +41,10 @@ CREATE TABLE public.pointy2 (
 );
 
 
+CREATE TYPE oses AS ENUM ('linux', 'macos', 'windows');
+
+CREATE TABLE public.manuals (
+  id text PRIMARY KEY,
+  os oses NOT NULL,
+  manual_url text NOT NULL
+);

components/electric/test/support/prisma/003_query_test.sql

@@ -2,9 +2,7 @@
 
 ALTER TABLE public.with_constraint ENABLE ELECTRIC;
 ALTER TABLE public.checked ENABLE ELECTRIC;
--- NOTE: `interesting` can't currently be electrified as it contains columns of
--- types we currently (as of 09/2023) don't support
 ALTER TABLE public.interesting ENABLE ELECTRIC;
 ALTER TABLE public.pointy ENABLE ELECTRIC;
 ALTER TABLE public.pointy2 ENABLE ELECTRIC;
-
+ALTER TABLE public.oses ENABLE ELECTRIC;

e2e/tests/03.20_node_satellite_can_sync_enums.lux

@@ -39,7 +39,7 @@
     [invoke node_get_enum "row3" null]
 
 [shell pg_1]
-    [invoke wait-for "SELECT * FROM public.enums;" "row1" 10 $psql]
+    [invoke wait-for "SELECT * FROM public.enums;" "row2" 10 $psql]
 
     !SELECT * FROM public.enums;
     ??row1 | RED

generator/src/functions/tableDescriptionWriters/writeTableSchemas.ts

@@ -128,6 +128,7 @@ function pgType(field: ExtendedDMMFField, modelName: string): string {
     case 'Json':
       return jsonToPg(attributes)
     default:
+      if (field.kind === 'enum') return 'TEXT' // treat enums as TEXT such that the ts-client correctly serializes/deserialises them as text
       return 'UNRECOGNIZED PRISMA TYPE'
   }
 }