feat(DAL): client-side support for JSON type

.changeset/nine-bulldogs-learn.md

@@ -0,0 +1,6 @@
+---
+"electric-sql": patch
+"@electric-sql/prisma-generator": patch
+---
+
+Adds client-side support for JSON type.

clients/typescript/src/cli/migrations/migrate.ts

@@ -8,6 +8,7 @@ import decompress from 'decompress'
 import { buildMigrations, getMigrationNames } from './builder'
 import { exec } from 'child_process'
 import { dedent } from 'ts-dedent'
+import { findAndReplaceInFile } from '../util'
 
 const appRoot = path.resolve() // path where the user ran `npx electric migrate`
 
@@ -182,6 +183,12 @@ async function _generate(opts: Omit<GeneratorOptions, 'watch'>) {
     console.log('Generating Electric client...')
     await generateElectricClient(prismaSchema)
     const relativePath = path.relative(appRoot, opts.out)
+    // Modify the type of JSON input values in the generated Prisma client
+    // because we deviate from Prisma's typing for JSON values
+    const outDir = opts.out
+    await extendJsonType(outDir)
+    // Delete all files generated for the Prisma client, except the typings
+    await keepOnlyPrismaTypings(outDir)
     console.log(`Successfully generated Electric client at: ./${relativePath}`)
 
     // Build the migrations
@@ -214,16 +221,17 @@ async function createPrismaSchema(
   )
   const output = path.resolve(out)
   const schema = dedent`
-    generator client {
-      provider = "prisma-client-js"
-    }
-
     generator electric {
       provider      = "${provider}"
       output        = "${output}"
       relationModel = "false"
     }
 
+    generator client {
+      provider = "prisma-client-js"
+      output   = "${output}"
+    }
+
     datasource db {
       provider = "postgresql"
       url      = "${proxy}"
@@ -561,3 +569,29 @@ function parseAttributes(attributes: string): Array<Attribute> {
     }
   })
 }
+
+/*
+ * Modifies Prisma's `InputJsonValue` type to include `null`
+ */
+function extendJsonType(prismaDir: string): Promise<void> {
+  const prismaTypings = path.join(prismaDir, 'index.d.ts')
+  const inputJsonValueRegex = /^\s*export\s*type\s*InputJsonValue\s*(=)\s*/gm
+  const replacement = 'export type InputJsonValue = null | '
+  return findAndReplaceInFile(inputJsonValueRegex, replacement, prismaTypings)
+}
+
+async function keepOnlyPrismaTypings(prismaDir: string): Promise<void> {
+  const contents = await fs.readdir(prismaDir)
+  // Delete all files except the generated Electric client and the Prisma typings
+  const proms = contents.map(async (fileOrDir) => {
+    const filePath = path.join(prismaDir, fileOrDir)
+    if (fileOrDir === 'index.d.ts') {
+      // rename this file to `prismaClient.d.ts`
+      return fs.rename(filePath, path.join(prismaDir, 'prismaClient.d.ts'))
+    } else if (fileOrDir !== 'index.ts') {
+      // delete the file or folder
+      return fs.rm(filePath, { recursive: true })
+    }
+  })
+  await Promise.all(proms)
+}

clients/typescript/src/cli/util/index.ts

@@ -0,0 +1 @@
+export * from './io'

clients/typescript/src/cli/util/io.ts

@@ -0,0 +1,15 @@
+import { readFile, writeFile } from 'fs/promises'
+
+/*
+ * Replaces the first occurence of `find` by `replace` in the file `file`.
+ * If `find` is a regular expression that sets the `g` flag, then it replaces all occurences.
+ */
+export async function findAndReplaceInFile(
+  find: string | RegExp,
+  replace: string,
+  file: string
+) {
+  const content = await readFile(file, 'utf8')
+  const replacedContent = content.replace(find, replace)
+  await writeFile(file, replacedContent)
+}

clients/typescript/src/client/conversions/datatypes/json.ts

@@ -0,0 +1,27 @@
+// not the most precise JSON type
+// but good enough for serialising/deserialising
+type JSON = string | number | boolean | Array<any> | Record<string, any>
+
+export function serialiseJSON(v: JSON): string {
+  if (isJsonNull(v)) {
+    // user provided the special `JsonNull` value
+    // to indicate a JSON null value rather than a DB NULL
+    return JSON.stringify(null)
+  }
+  return JSON.stringify(v)
+}
+
+export function deserialiseJSON(v: string): JSON {
+  if (v === JSON.stringify(null)) return { __is_electric_json_null__: true }
+  return JSON.parse(v)
+}
+
+function isJsonNull(v: JSON): boolean {
+  return (
+    typeof v === 'object' &&
+    !Array.isArray(v) &&
+    v !== null &&
+    Object.hasOwn(v, '__is_electric_json_null__') &&
+    v['__is_electric_json_null__']
+  )
+}

clients/typescript/src/client/conversions/sqlite.ts

@@ -1,6 +1,7 @@
 import { InvalidArgumentError } from '../validation/errors/invalidArgumentError'
 import { deserialiseBoolean, serialiseBoolean } from './datatypes/boolean'
 import { deserialiseDate, serialiseDate } from './datatypes/date'
+import { deserialiseJSON, serialiseJSON } from './datatypes/json'
 import { PgBasicType, PgDateType, PgType } from './types'
 
 /**
@@ -29,6 +30,11 @@ export function toSqlite(v: any, pgType: PgType): any {
     // and deserialise it back to `NaN` when reading from the DB.
     // cf. https://github.com/WiseLibs/better-sqlite3/issues/1088
     return 'NaN'
+  } else if (
+    pgType === PgBasicType.PG_JSON ||
+    pgType === PgBasicType.PG_JSONB
+  ) {
+    return serialiseJSON(v)
   } else {
     return v
   }
@@ -50,6 +56,12 @@ export function fromSqlite(v: any, pgType: PgType): any {
   ) {
     // it's a serialised NaN
     return NaN
+  } else if (
+    pgType === PgBasicType.PG_JSON ||
+    pgType === PgBasicType.PG_JSONB
+  ) {
+    // it's serialised JSON
+    return deserialiseJSON(v)
   } else {
     return v
   }

clients/typescript/src/client/conversions/types.ts

@@ -12,6 +12,8 @@ export enum PgBasicType {
   PG_VARCHAR = 'VARCHAR',
   PG_CHAR = 'CHAR',
   PG_UUID = 'UUID',
+  PG_JSON = 'JSON',
+  PG_JSONB = 'JSONB',
 }
 
 /**

clients/typescript/src/client/model/builder.ts

@@ -280,6 +280,7 @@ function makeFilter(
     // an object containing filters is provided
     // e.g. users.findMany({ where: { id: { in: [1, 2, 3] } } })
     const fs = {
+      equals: z.any(),
       in: z.any().array().optional(),
       not: z.any().optional(),
       notIn: z.any().optional(),
@@ -293,6 +294,7 @@ function makeFilter(
     }
 
     const fsHandlers = {
+      equals: makeEqualsFilter.bind(null),
       in: makeInFilter.bind(null),
       not: makeNotFilter.bind(null),
       notIn: makeNotInFilter.bind(null),
@@ -380,6 +382,13 @@ function makeBooleanFilter(
   }
 }
 
+function makeEqualsFilter(
+  fieldName: string,
+  value: unknown | undefined
+): { sql: string; args?: unknown[] } {
+  return { sql: `${fieldName} = ?`, args: [value] }
+}
+
 function makeInFilter(
   fieldName: string,
   values: unknown[] | undefined

clients/typescript/src/satellite/client.ts

@@ -1128,15 +1128,6 @@ function deserializeColumnData(
   columnType: PgType
 ): string | number {
   switch (columnType) {
-    case PgBasicType.PG_CHAR:
-    case PgDateType.PG_DATE:
-    case PgBasicType.PG_TEXT:
-    case PgDateType.PG_TIME:
-    case PgDateType.PG_TIMESTAMP:
-    case PgDateType.PG_TIMESTAMPTZ:
-    case PgBasicType.PG_UUID:
-    case PgBasicType.PG_VARCHAR:
-      return typeDecoder.text(column)
     case PgBasicType.PG_BOOL:
       return typeDecoder.bool(column)
     case PgBasicType.PG_INT:
@@ -1152,17 +1143,13 @@ function deserializeColumnData(
     case PgDateType.PG_TIMETZ:
       return typeDecoder.timetz(column)
     default:
-      // should not occur
-      throw new SatelliteError(
-        SatelliteErrorCode.UNKNOWN_DATA_TYPE,
-        `can't deserialize ${columnType}`
-      )
+      return typeDecoder.text(column)
   }
 }
 
 // All values serialized as textual representation
 function serializeColumnData(
-  columnValue: string | number,
+  columnValue: string | number | object,
   columnType: PgType
 ): Uint8Array {
   switch (columnType) {

clients/typescript/test/client/conversions/input.test.ts

@@ -8,7 +8,7 @@ import {
   _NOT_UNIQUE_,
   _RECORD_NOT_FOUND_,
 } from '../../../src/client/validation/errors/messages'
-import { schema } from '../generated'
+import { JsonNull, schema } from '../generated'
 import { DataTypes, Dummy } from '../generated/client'
 
 const db = new Database(':memory:')
@@ -31,7 +31,7 @@ await tbl.sync()
 function setupDB() {
   db.exec('DROP TABLE IF EXISTS DataTypes')
   db.exec(
-    "CREATE TABLE DataTypes('id' int PRIMARY KEY, 'date' varchar, 'time' varchar, 'timetz' varchar, 'timestamp' varchar, 'timestamptz' varchar, 'bool' int, 'uuid' varchar, 'int2' int2, 'int4' int4, 'float8' real, 'relatedId' int);"
+    "CREATE TABLE DataTypes('id' int PRIMARY KEY, 'date' varchar, 'time' varchar, 'timetz' varchar, 'timestamp' varchar, 'timestamptz' varchar, 'bool' int, 'uuid' varchar, 'int2' int2, 'int4' int4, 'float8' real, 'json' varchar, 'relatedId' int);"
   )
 
   db.exec('DROP TABLE IF EXISTS Dummy')
@@ -91,6 +91,26 @@ test.serial('findFirst transforms booleans to integer in SQLite', async (t) => {
   t.is(res?.bool, true)
 })
 
+test.serial(
+  'findFirst transforms json values to strings in SQLite',
+  async (t) => {
+    await electric.adapter.run({
+      sql: `INSERT INTO DataTypes('id', 'json') VALUES (1, NULL), (2, '{ "a": 5 }'), (3, 'null')`,
+    })
+
+    const res = await tbl.findFirst({
+      where: {
+        json: {
+          equals: JsonNull,
+        },
+      },
+    })
+
+    t.is(res?.id, 3)
+    t.deepEqual(res?.json, JsonNull)
+  }
+)
+
 test.serial(
   'findFirst transforms JS objects in equals filter to SQLite',
   async (t) => {
@@ -234,6 +254,7 @@ const dateNulls = {
   int4: null,
   float8: null,
   uuid: null,
+  json: null,
 }
 
 const nulls = {

clients/typescript/test/client/conversions/sqlite.test.ts

@@ -8,7 +8,7 @@ import {
   _NOT_UNIQUE_,
   _RECORD_NOT_FOUND_,
 } from '../../../src/client/validation/errors/messages'
-import { schema } from '../generated'
+import { schema, JsonNull } from '../generated'
 
 const db = new Database(':memory:')
 const electric = await electrify(
@@ -30,7 +30,7 @@ await tbl.sync()
 function setupDB() {
   db.exec('DROP TABLE IF EXISTS DataTypes')
   db.exec(
-    "CREATE TABLE DataTypes('id' int PRIMARY KEY, 'date' varchar, 'time' varchar, 'timetz' varchar, 'timestamp' varchar, 'timestamptz' varchar, 'bool' int, 'uuid' varchar, 'int2' int2, 'int4' int4, 'float8' real, 'relatedId' int);"
+    "CREATE TABLE DataTypes('id' int PRIMARY KEY, 'date' varchar, 'time' varchar, 'timetz' varchar, 'timestamp' varchar, 'timestamptz' varchar, 'bool' int, 'uuid' varchar, 'int2' int2, 'int4' int4, 'float8' real, 'json' varchar, 'relatedId' int);"
   )
 }
 
@@ -211,3 +211,79 @@ test.serial('floats are converted correctly to SQLite', async (t) => {
     { id: 4, float8: -Infinity },
   ])
 })
+
+test.serial('json is converted correctly to SQLite', async (t) => {
+  const json = { a: 1, b: true, c: { d: 'nested' }, e: [1, 2, 3], f: null }
+  await tbl.create({
+    data: {
+      id: 1,
+      json,
+    },
+  })
+
+  const rawRes = await electric.db.raw({
+    sql: 'SELECT json FROM DataTypes WHERE id = ?',
+    args: [1],
+  })
+  t.is(rawRes[0].json, JSON.stringify(json))
+
+  // Also test null values
+  // this null value is not a JSON null
+  // but a DB NULL that indicates absence of a value
+  await tbl.create({
+    data: {
+      id: 2,
+      json: null,
+    },
+  })
+
+  const rawRes2 = await electric.db.raw({
+    sql: 'SELECT json FROM DataTypes WHERE id = ?',
+    args: [2],
+  })
+  t.is(rawRes2[0].json, null)
+
+  // Also test JSON null value
+  await tbl.create({
+    data: {
+      id: 3,
+      json: JsonNull,
+    },
+  })
+
+  const rawRes3 = await electric.db.raw({
+    sql: 'SELECT json FROM DataTypes WHERE id = ?',
+    args: [3],
+  })
+  t.is(rawRes3[0].json, JSON.stringify(null))
+
+  // also test regular values
+  await tbl.create({
+    data: {
+      id: 4,
+      json: 'foo',
+    },
+  })
+
+  const rawRes4 = await electric.db.raw({
+    sql: 'SELECT json FROM DataTypes WHERE id = ?',
+    args: [4],
+  })
+
+  t.is(rawRes4[0].json, JSON.stringify('foo'))
+
+  // also test arrays
+  await tbl.create({
+    data: {
+      id: 5,
+      json: [1, 2, 3],
+    },
+  })
+
+  const rawRes5 = await electric.db.raw({
+    sql: 'SELECT json FROM DataTypes WHERE id = ?',
+    args: [5],
+  })
+
+  t.is(rawRes5[0].json, JSON.stringify([1, 2, 3]))
+})