feat(DAL): client-side support for JSON type (#633)

This PR adds client-side support for the JSON type.
The DAL accepts JS values representing JSON and serialises them to a
string that is stored in SQLite.
When reading a JSON value from the database the string is deserialised
back into a JS value.
A corner case arises when having an optional column of type JSON because
we need to differentiate between the database NULL value and the JSON
null value. We treat the regular JS null value as a database NULL (to be
consistent with how null values are interpreted for other column types)
and require users to pass a special JsonNull object in order to store a
top-level JSON null value.

Still need to add an E2E test for json values.

---------

Co-authored-by: Oleksii Sholik <[email protected]>

.changeset/nine-bulldogs-learn.md

@@ -0,0 +1,6 @@
+---
+"electric-sql": patch
+"@electric-sql/prisma-generator": patch
+---
+
+Adds client-side support for JSON type.

clients/typescript/src/cli/migrations/migrate.ts

@@ -8,6 +8,7 @@ import decompress from 'decompress'
 import { buildMigrations, getMigrationNames } from './builder'
 import { exec } from 'child_process'
 import { dedent } from 'ts-dedent'
+import { findAndReplaceInFile } from '../util'
 
 const appRoot = path.resolve() // path where the user ran `npx electric migrate`
 
@@ -182,6 +183,12 @@ async function _generate(opts: Omit<GeneratorOptions, 'watch'>) {
     console.log('Generating Electric client...')
     await generateElectricClient(prismaSchema)
     const relativePath = path.relative(appRoot, opts.out)
+    // Modify the type of JSON input values in the generated Prisma client
+    // because we deviate from Prisma's typing for JSON values
+    const outDir = opts.out
+    await extendJsonType(outDir)
+    // Delete all files generated for the Prisma client, except the typings
+    await keepOnlyPrismaTypings(outDir)
     console.log(`Successfully generated Electric client at: ./${relativePath}`)
 
     // Build the migrations
@@ -227,16 +234,17 @@ async function createPrismaSchema(
   )
   const output = path.resolve(out)
   const schema = dedent`
-    generator client {
-      provider = "prisma-client-js"
-    }
-
     generator electric {
       provider      = "${escapePathForString(provider)}"
       output        = "${escapePathForString(output)}"
       relationModel = "false"
     }
 
+    generator client {
+      provider = "prisma-client-js"
+      output   = "${output}"
+    }
+
     datasource db {
       provider = "postgresql"
       url      = "${proxy}"
@@ -568,3 +576,29 @@ function parseAttributes(attributes: string): Array<Attribute> {
     }
   })
 }
+
+/*
+ * Modifies Prisma's `InputJsonValue` type to include `null`
+ */
+function extendJsonType(prismaDir: string): Promise<void> {
+  const prismaTypings = path.join(prismaDir, 'index.d.ts')
+  const inputJsonValueRegex = /^\s*export\s*type\s*InputJsonValue\s*(=)\s*/gm
+  const replacement = 'export type InputJsonValue = null | '
+  return findAndReplaceInFile(inputJsonValueRegex, replacement, prismaTypings)
+}
+
+async function keepOnlyPrismaTypings(prismaDir: string): Promise<void> {
+  const contents = await fs.readdir(prismaDir)
+  // Delete all files except the generated Electric client and the Prisma typings
+  const proms = contents.map(async (fileOrDir) => {
+    const filePath = path.join(prismaDir, fileOrDir)
+    if (fileOrDir === 'index.d.ts') {
+      // rename this file to `prismaClient.d.ts`
+      return fs.rename(filePath, path.join(prismaDir, 'prismaClient.d.ts'))
+    } else if (fileOrDir !== 'index.ts') {
+      // delete the file or folder
+      return fs.rm(filePath, { recursive: true })
+    }
+  })
+  await Promise.all(proms)
+}

clients/typescript/src/cli/util/index.ts

@@ -0,0 +1 @@
+export * from './io'

clients/typescript/src/cli/util/io.ts

@@ -0,0 +1,15 @@
+import { readFile, writeFile } from 'fs/promises'
+
+/*
+ * Replaces the first occurence of `find` by `replace` in the file `file`.
+ * If `find` is a regular expression that sets the `g` flag, then it replaces all occurences.
+ */
+export async function findAndReplaceInFile(
+  find: string | RegExp,
+  replace: string,
+  file: string
+) {
+  const content = await readFile(file, 'utf8')
+  const replacedContent = content.replace(find, replace)
+  await writeFile(file, replacedContent)
+}

clients/typescript/src/client/conversions/datatypes/json.ts

@@ -0,0 +1,27 @@
+// not the most precise JSON type
+// but good enough for serialising/deserialising
+type JSON = string | number | boolean | Array<any> | Record<string, any>
+
+export function serialiseJSON(v: JSON): string {
+  if (isJsonNull(v)) {
+    // user provided the special `JsonNull` value
+    // to indicate a JSON null value rather than a DB NULL
+    return JSON.stringify(null)
+  }
+  return JSON.stringify(v)
+}
+
+export function deserialiseJSON(v: string): JSON {
+  if (v === JSON.stringify(null)) return { __is_electric_json_null__: true }
+  return JSON.parse(v)
+}
+
+function isJsonNull(v: JSON): boolean {
+  return (
+    typeof v === 'object' &&
+    !Array.isArray(v) &&
+    v !== null &&
+    Object.hasOwn(v, '__is_electric_json_null__') &&
+    v['__is_electric_json_null__']
+  )
+}

clients/typescript/src/client/conversions/sqlite.ts

@@ -1,6 +1,7 @@
 import { InvalidArgumentError } from '../validation/errors/invalidArgumentError'
 import { deserialiseBoolean, serialiseBoolean } from './datatypes/boolean'
 import { deserialiseDate, serialiseDate } from './datatypes/date'
+import { deserialiseJSON, serialiseJSON } from './datatypes/json'
 import { PgBasicType, PgDateType, PgType } from './types'
 
 /**
@@ -34,6 +35,11 @@ export function toSqlite(v: any, pgType: PgType): any {
     pgType === PgBasicType.PG_REAL
   ) {
     return Math.fround(v)
+  } else if (
+    pgType === PgBasicType.PG_JSON ||
+    pgType === PgBasicType.PG_JSONB
+  ) {
+    return serialiseJSON(v)
   } else {
     return v
   }
@@ -68,6 +74,12 @@ export function fromSqlite(v: any, pgType: PgType): any {
     // because some drivers (e.g. wa-sqlite) return a regular JS number if the value fits into a JS number
     // but we know that it should be a BigInt based on the column type
     return BigInt(v)
+  } else if (
+    pgType === PgBasicType.PG_JSON ||
+    pgType === PgBasicType.PG_JSONB
+  ) {
+    // it's serialised JSON
+    return deserialiseJSON(v)
   } else {
     return v
   }

clients/typescript/src/client/conversions/types.ts

@@ -12,6 +12,8 @@ export enum PgBasicType {
   PG_VARCHAR = 'VARCHAR',
   PG_CHAR = 'CHAR',
   PG_UUID = 'UUID',
+  PG_JSON = 'JSON',
+  PG_JSONB = 'JSONB',
 }
 
 /**

clients/typescript/src/client/model/builder.ts

@@ -321,6 +321,7 @@ function makeFilter(
     // an object containing filters is provided
     // e.g. users.findMany({ where: { id: { in: [1, 2, 3] } } })
     const fs = {
+      equals: z.any(),
       in: z.any().array().optional(),
       not: z.any().optional(),
       notIn: z.any().optional(),
@@ -334,6 +335,7 @@ function makeFilter(
     }
 
     const fsHandlers = {
+      equals: makeEqualsFilter.bind(null),
       in: makeInFilter.bind(null),
       not: makeNotFilter.bind(null),
       notIn: makeNotInFilter.bind(null),
@@ -421,6 +423,13 @@ function makeBooleanFilter(
   }
 }
 
+function makeEqualsFilter(
+  fieldName: string,
+  value: unknown | undefined
+): { sql: string; args?: unknown[] } {
+  return { sql: `${fieldName} = ?`, args: [value] }
+}
+
 function makeInFilter(
   fieldName: string,
   values: unknown[] | undefined

clients/typescript/src/satellite/client.ts

@@ -1137,16 +1137,6 @@ function deserializeColumnData(
   columnType: PgType
 ): string | number {
   switch (columnType) {
-    case PgBasicType.PG_CHAR:
-    case PgDateType.PG_DATE:
-    case PgBasicType.PG_INT8:
-    case PgBasicType.PG_TEXT:
-    case PgDateType.PG_TIME:
-    case PgDateType.PG_TIMESTAMP:
-    case PgDateType.PG_TIMESTAMPTZ:
-    case PgBasicType.PG_UUID:
-    case PgBasicType.PG_VARCHAR:
-      return typeDecoder.text(column)
     case PgBasicType.PG_BOOL:
       return typeDecoder.bool(column)
     case PgBasicType.PG_INT:
@@ -1161,17 +1151,13 @@ function deserializeColumnData(
     case PgDateType.PG_TIMETZ:
       return typeDecoder.timetz(column)
     default:
-      // should not occur
-      throw new SatelliteError(
-        SatelliteErrorCode.UNKNOWN_DATA_TYPE,
-        `can't deserialize ${columnType}`
-      )
+      return typeDecoder.text(column)
   }
 }
 
 // All values serialized as textual representation
 function serializeColumnData(
-  columnValue: string | number,
+  columnValue: string | number | object,
   columnType: PgType
 ): Uint8Array {
   switch (columnType) {

clients/typescript/test/client/conversions/input.test.ts

@@ -8,7 +8,7 @@ import {
   _NOT_UNIQUE_,
   _RECORD_NOT_FOUND_,
 } from '../../../src/client/validation/errors/messages'
-import { schema } from '../generated'
+import { JsonNull, schema } from '../generated'
 import { DataTypes, Dummy } from '../generated/client'
 
 const db = new Database(':memory:')
@@ -31,7 +31,7 @@ await tbl.sync()
 function setupDB() {
   db.exec('DROP TABLE IF EXISTS DataTypes')
   db.exec(
-    "CREATE TABLE DataTypes('id' int PRIMARY KEY, 'date' varchar, 'time' varchar, 'timetz' varchar, 'timestamp' varchar, 'timestamptz' varchar, 'bool' int, 'uuid' varchar, 'int2' int2, 'int4' int4, 'int8' int8, 'float4' real, 'float8' real, 'relatedId' int);"
+    "CREATE TABLE DataTypes('id' int PRIMARY KEY, 'date' varchar, 'time' varchar, 'timetz' varchar, 'timestamp' varchar, 'timestamptz' varchar, 'bool' int, 'uuid' varchar, 'int2' int2, 'int4' int4, 'int8' int8, 'float4' real, 'float8' real, 'json' varchar, 'relatedId' int);"
   )
 
   db.exec('DROP TABLE IF EXISTS Dummy')
@@ -91,6 +91,26 @@ test.serial('findFirst transforms booleans to integer in SQLite', async (t) => {
   t.is(res?.bool, true)
 })
 
+test.serial(
+  'findFirst transforms json values to strings in SQLite',
+  async (t) => {
+    await electric.adapter.run({
+      sql: `INSERT INTO DataTypes('id', 'json') VALUES (1, NULL), (2, '{ "a": 5 }'), (3, 'null')`,
+    })
+
+    const res = await tbl.findFirst({
+      where: {
+        json: {
+          equals: JsonNull,
+        },
+      },
+    })
+
+    t.is(res?.id, 3)
+    t.deepEqual(res?.json, JsonNull)
+  }
+)
+
 test.serial(
   'findFirst transforms JS objects in equals filter to SQLite',
   async (t) => {
@@ -236,6 +256,7 @@ const dateNulls = {
   float4: null,
   float8: null,
   uuid: null,
+  json: null,
 }
 
 const nulls = {

clients/typescript/test/client/conversions/sqlite.test.ts

@@ -8,7 +8,7 @@ import {
   _NOT_UNIQUE_,
   _RECORD_NOT_FOUND_,
 } from '../../../src/client/validation/errors/messages'
-import { schema } from '../generated'
+import { schema, JsonNull } from '../generated'
 
 const db = new Database(':memory:')
 const electric = await electrify(
@@ -30,7 +30,7 @@ await tbl.sync()
 function setupDB() {
   db.exec('DROP TABLE IF EXISTS DataTypes')
   db.exec(
-    "CREATE TABLE DataTypes('id' int PRIMARY KEY, 'date' varchar, 'time' varchar, 'timetz' varchar, 'timestamp' varchar, 'timestamptz' varchar, 'bool' int, 'uuid' varchar, 'int2' int2, 'int4' int4, 'int8' int8, 'float4' real, 'float8' real, 'relatedId' int);"
+    "CREATE TABLE DataTypes('id' int PRIMARY KEY, 'date' varchar, 'time' varchar, 'timetz' varchar, 'timestamp' varchar, 'timestamptz' varchar, 'bool' int, 'uuid' varchar, 'int2' int2, 'int4' int4, 'int8' int8, 'float4' real, 'float8' real, 'json' varchar, 'relatedId' int);"
   )
 }
 
@@ -243,3 +243,79 @@ test.serial('BigInts are converted correctly to SQLite', async (t) => {
   t.deepEqual(rawRes, [{ id: 1, int8: bigInt.toString() }])
   //db.defaultSafeIntegers(false) // disables BigInt support
 })
+
+test.serial('json is converted correctly to SQLite', async (t) => {
+  const json = { a: 1, b: true, c: { d: 'nested' }, e: [1, 2, 3], f: null }
+  await tbl.create({
+    data: {
+      id: 1,
+      json,
+    },
+  })
+
+  const rawRes = await electric.db.raw({
+    sql: 'SELECT json FROM DataTypes WHERE id = ?',
+    args: [1],
+  })
+  t.is(rawRes[0].json, JSON.stringify(json))
+
+  // Also test null values
+  // this null value is not a JSON null
+  // but a DB NULL that indicates absence of a value
+  await tbl.create({
+    data: {
+      id: 2,
+      json: null,
+    },
+  })
+
+  const rawRes2 = await electric.db.raw({
+    sql: 'SELECT json FROM DataTypes WHERE id = ?',
+    args: [2],
+  })
+  t.is(rawRes2[0].json, null)
+
+  // Also test JSON null value
+  await tbl.create({
+    data: {
+      id: 3,
+      json: JsonNull,
+    },
+  })
+
+  const rawRes3 = await electric.db.raw({
+    sql: 'SELECT json FROM DataTypes WHERE id = ?',
+    args: [3],
+  })
+  t.is(rawRes3[0].json, JSON.stringify(null))
+
+  // also test regular values
+  await tbl.create({
+    data: {
+      id: 4,
+      json: 'foo',
+    },
+  })
+
+  const rawRes4 = await electric.db.raw({
+    sql: 'SELECT json FROM DataTypes WHERE id = ?',
+    args: [4],
+  })
+
+  t.is(rawRes4[0].json, JSON.stringify('foo'))
+
+  // also test arrays
+  await tbl.create({
+    data: {
+      id: 5,
+      json: [1, 2, 3],
+    },
+  })
+
+  const rawRes5 = await electric.db.raw({
+    sql: 'SELECT json FROM DataTypes WHERE id = ?',
+    args: [5],
+  })
+
+  t.is(rawRes5[0].json, JSON.stringify([1, 2, 3]))
+})