feat (client): client-side support for int8 (#665)

This PR adds client-side support for int8 aka bigint.
Supporting BigInts is tricky because of inconsistencies in the sqlite
drivers.
For example, better-sqlite3 requires BigInts to be explicitly enabled
but then *all* integers are returned as BigInts because SQLite does not
distinguish between them (they are all just 64bit integers). On the
other hand, wa-sqlite returns a regular JS number if the integer fits,
otherwise it returns a BigInt.

To avoid these problems, we always cast BigInt to text when reading from
the DB and then let the DAL turn it into a BigInt;
i.e., when the Postgres type of a column "foo" is bigint, we read as
follows: `SELECT cast(foo as TEXT) AS foo FROM table`. Similarly, we
cast returned values: `INSERT INTO table VALUES (9223372036854775807)
RETURNING cast(foo as TEXT) AS foo`.

A similar problem occurs in the oplog triggers. These triggers insert an
oplog entry for every insert/update/delete. This oplog entry contains
the old and new rows as JSON objects. However, JSON does not support
BigInts:
```sql
sqlite> SELECT json_valid('{"a": 123n}');
0
```
We also can't write it as a regular number because when parsing the JSON
it is read as a number and rounding errors occur:
```js
js> JSON.parse('{"a": 9223372036854775807}')
{ a: 9223372036854776000 }
```

Hence, the triggers also need to cast BigInts to text and the
deserialisation function for oplog entries (`deserialiseRow`) needs to
correctly deserialise them as BigInts (when the column type is
int8/bigint).

---------

Co-authored-by: David Martos <[email protected]>

clients/typescript/src/client/conversions/input.ts

@@ -331,8 +331,7 @@ function isObject(v: any): boolean {
 }
 
 function isFilterObject(value: any): boolean {
-  // if it is an object it can only be a timestamp or a filter object
-  // because those are the only objects we support in where clauses
+  // if it is an object it can only be a data object or a filter object
   return isObject(value) && !isDataObject(value)
 }
 

clients/typescript/src/client/conversions/sqlite.ts

@@ -50,6 +50,11 @@ export function fromSqlite(v: any, pgType: PgType): any {
   ) {
     // it's a serialised NaN
     return NaN
+  } else if (pgType === PgBasicType.PG_INT8) {
+    // always return BigInts for PG_INT8 values
+    // because some drivers (e.g. wa-sqlite) return a regular JS number if the value fits into a JS number
+    // but we know that it should be a BigInt based on the column type
+    return BigInt(v)
   } else {
     return v
   }

clients/typescript/src/client/model/builder.ts

@@ -13,16 +13,34 @@ import { InvalidArgumentError } from '../validation/errors/invalidArgumentError'
 import * as z from 'zod'
 import { IShapeManager } from './shapes'
 import Log from 'loglevel'
+import { ExtendedTableSchema } from './schema'
+import { PgBasicType } from '../conversions/types'
+import { HKT } from '../util/hkt'
 
 const squelPostgres = squel.useFlavour('postgres')
+squelPostgres.registerValueHandler('bigint', function (bigint) {
+  return bigint.toString()
+})
 
 type AnyFindInput = FindInput<any, any, any, any, any>
 
 export class Builder {
   constructor(
     private _tableName: string,
     private _fields: string[],
-    private shapeManager: IShapeManager
+    private shapeManager: IShapeManager,
+    private _tableDescription: ExtendedTableSchema<
+      any,
+      any,
+      any,
+      any,
+      any,
+      any,
+      any,
+      any,
+      any,
+      HKT
+    >
   ) {}
 
   create(i: CreateInput<any, any, any>): QueryBuilder {
@@ -196,10 +214,26 @@ export class Builder {
     const fields = identificationFields
       .filter((f) => this._fields.includes(f))
       .concat(selectedFields)
+      .map((f) => this.castBigIntToText(f))
 
     return q.fields(fields)
   }
 
+  /**
+   * Casts a field to TEXT if it is of type BigInt
+   * because not all adapters deal well with BigInts
+   * (e.g. better-sqlite3 requires BigInt support to be enabled
+   *       but then all integers are returned as BigInt...)
+   * The DAL will convert the string into a BigInt in the `fromSqlite` function from `../conversions/sqlite.ts`.
+   */
+  private castBigIntToText(field: string) {
+    const pgType = this._tableDescription.fields.get(field)
+    if (pgType === PgBasicType.PG_INT8) {
+      return `cast(${field} as TEXT) AS ${field}`
+    }
+    return field
+  }
+
   private addOrderBy(i: AnyFindInput, q: PostgresSelect): PostgresSelect {
     if (typeof i.orderBy === 'undefined') return q
     const orderByArray = Array.isArray(i.orderBy) ? i.orderBy : [i.orderBy]
@@ -243,6 +277,13 @@ export class Builder {
     query: T
   ): T {
     return this._fields.reduce((query, field) => {
+      // if field is of type BigInt cast the result to TEXT
+      // because not all adapters deal well with BigInts
+      // the DAL will convert the string into a BigInt in the `fromSqlite` function from `../conversions/sqlite.ts`.
+      const pgType = this._tableDescription.fields.get(field)
+      if (pgType === PgBasicType.PG_INT8) {
+        return query.returning(`cast(${field} as TEXT) AS ${field}`)
+      }
       return query.returning(field)
     }, query)
   }

clients/typescript/src/client/model/table.ts

@@ -107,12 +107,17 @@ export class Table<
   ) {
     this._fields = this._dbDescription.getFields(tableName)
     const fieldNames = this._dbDescription.getFieldNames(tableName)
-    this._builder = new Builder(tableName, fieldNames, shapeManager)
+    const tableDescription = this._dbDescription.getTableDescription(tableName)
+    this._builder = new Builder(
+      tableName,
+      fieldNames,
+      shapeManager,
+      tableDescription
+    )
     this._executor = new Executor(adapter, notifier, this._fields)
     this._shapeManager = shapeManager
     this._qualifiedTableName = new QualifiedTablename('main', tableName)
     this._tables = new Map()
-    const tableDescription = this._dbDescription.getTableDescription(tableName)
     this._schema = tableDescription.modelSchema
     this.createSchema = //transformCreateSchema(
       omitCountFromSelectAndIncludeSchema(tableDescription.createSchema) //, fields)

clients/typescript/src/migrators/triggers.ts

@@ -9,7 +9,12 @@ type ForeignKey = {
 
 type ColumnName = string
 type SQLiteType = string
-type ColumnTypes = Record<ColumnName, SQLiteType>
+type PgType = string
+type ColumnType = {
+  sqliteType: SQLiteType
+  pgType: PgType
+}
+type ColumnTypes = Record<ColumnName, ColumnType>
 
 export type Table = {
   tableName: string
@@ -220,6 +225,7 @@ export function generateTriggers(tables: Tables): Statement[] {
  * Joins the column names and values into a string of pairs of the form `'col1', val1, 'col2', val2, ...`
  * that can be used to build a JSON object in a SQLite `json_object` function call.
  * Values of type REAL are cast to text to avoid a bug in SQLite's `json_object` function (see below).
+ * Similarly, values of type INT8 (i.e. BigInts) are cast to text because JSON does not support BigInts.
  *
  * NOTE: There is a bug with SQLite's `json_object` function up to version 3.41.2
  *       that causes it to return an invalid JSON object if some value is +Infinity or -Infinity.
@@ -268,7 +274,10 @@ function joinColsForJSON(
   // casts the value to TEXT if it is of type REAL
   // to work around the bug in SQLite's `json_object` function
   const castIfNeeded = (col: string, targettedCol: string) => {
-    if (colTypes[col] === 'REAL') {
+    const tpes = colTypes[col]
+    const sqliteType = tpes.sqliteType
+    const pgType = tpes.pgType
+    if (sqliteType === 'REAL' || pgType === 'INT8' || pgType === 'BIGINT') {
       return `cast(${targettedCol} as TEXT)`
     } else {
       return targettedCol

clients/typescript/src/satellite/client.ts

@@ -1130,6 +1130,7 @@ function deserializeColumnData(
   switch (columnType) {
     case PgBasicType.PG_CHAR:
     case PgDateType.PG_DATE:
+    case PgBasicType.PG_INT8:
     case PgBasicType.PG_TEXT:
     case PgDateType.PG_TIME:
     case PgDateType.PG_TIMESTAMP:
@@ -1142,7 +1143,6 @@ function deserializeColumnData(
     case PgBasicType.PG_INT:
     case PgBasicType.PG_INT2:
     case PgBasicType.PG_INT4:
-    case PgBasicType.PG_INT8:
     case PgBasicType.PG_INTEGER:
       return Number(typeDecoder.text(column))
     case PgBasicType.PG_FLOAT4:

clients/typescript/src/satellite/oplog.ts

@@ -338,6 +338,9 @@ function deserialiseRow(str: string, rel: Pick<Relation, 'columns'>): Rec {
         return Number(value)
       }
     }
+    if (columnType === 'INT8' || columnType === 'BIGINT') {
+      return BigInt(value)
+    }
     return value
   })
 }

clients/typescript/src/satellite/process.ts

@@ -1471,7 +1471,13 @@ export function generateTriggersForTable(tbl: MigrationTable): Statement[] {
       }
     }),
     columnTypes: Object.fromEntries(
-      tbl.columns.map((col) => [col.name, col.sqliteType.toUpperCase()])
+      tbl.columns.map((col) => [
+        col.name,
+        {
+          sqliteType: col.sqliteType.toUpperCase(),
+          pgType: col.pgType!.name.toUpperCase(),
+        },
+      ])
     ),
   }
   const fullTableName = table.namespace + '.' + table.tableName

clients/typescript/src/util/types.ts

@@ -1,6 +1,7 @@
 import type Long from 'long'
 import {
   SatOpMigrate_Column,
+  SatOpMigrate_PgColumnType,
   SatOpMigrate_Table,
   SatOpMigrate_Type,
   SatRelation_RelationType,
@@ -130,9 +131,12 @@ export type DataChange = {
   tags: Tag[]
 }
 
-// The properties are omitted from columns because they are not currently used.
+export type SatOpMigrate_Col = Omit<SatOpMigrate_Column, '$type' | 'pgType'> & {
+  pgType: Omit<SatOpMigrate_PgColumnType, '$type'> | undefined
+}
+
 export type MigrationTable = Omit<SatOpMigrate_Table, '$type' | 'columns'> & {
-  columns: Omit<SatOpMigrate_Column, '$type' | 'pgType'>[]
+  columns: SatOpMigrate_Col[]
 }
 
 export type SchemaChange = {

clients/typescript/test/client/conversions/input.test.ts

@@ -31,7 +31,7 @@ await tbl.sync()
 function setupDB() {
   db.exec('DROP TABLE IF EXISTS DataTypes')
   db.exec(
-    "CREATE TABLE DataTypes('id' int PRIMARY KEY, 'date' varchar, 'time' varchar, 'timetz' varchar, 'timestamp' varchar, 'timestamptz' varchar, 'bool' int, 'uuid' varchar, 'int2' int2, 'int4' int4, 'float8' real, 'relatedId' int);"
+    "CREATE TABLE DataTypes('id' int PRIMARY KEY, 'date' varchar, 'time' varchar, 'timetz' varchar, 'timestamp' varchar, 'timestamptz' varchar, 'bool' int, 'uuid' varchar, 'int2' int2, 'int4' int4, 'int8' int8, 'float8' real, 'relatedId' int);"
   )
 
   db.exec('DROP TABLE IF EXISTS Dummy')
@@ -232,6 +232,7 @@ const dateNulls = {
   bool: null,
   int2: null,
   int4: null,
+  int8: null,
   float8: null,
   uuid: null,
 }

clients/typescript/test/client/conversions/sqlite.test.ts

@@ -30,7 +30,7 @@ await tbl.sync()
 function setupDB() {
   db.exec('DROP TABLE IF EXISTS DataTypes')
   db.exec(
-    "CREATE TABLE DataTypes('id' int PRIMARY KEY, 'date' varchar, 'time' varchar, 'timetz' varchar, 'timestamp' varchar, 'timestamptz' varchar, 'bool' int, 'uuid' varchar, 'int2' int2, 'int4' int4, 'float8' real, 'relatedId' int);"
+    "CREATE TABLE DataTypes('id' int PRIMARY KEY, 'date' varchar, 'time' varchar, 'timetz' varchar, 'timestamp' varchar, 'timestamptz' varchar, 'bool' int, 'uuid' varchar, 'int2' int2, 'int4' int4, 'int8' int8, 'float8' real, 'relatedId' int);"
   )
 }
 
@@ -211,3 +211,24 @@ test.serial('floats are converted correctly to SQLite', async (t) => {
     { id: 4, float8: -Infinity },
   ])
 })
+
+test.serial('BigInts are converted correctly to SQLite', async (t) => {
+  //db.defaultSafeIntegers(true) // enables BigInt support
+  const bigInt = 9_223_372_036_854_775_807n
+  await tbl.create({
+    data: {
+      id: 1,
+      int8: bigInt,
+    },
+  })
+
+  const rawRes = await electric.db.raw({
+    sql: 'SELECT id, cast(int8 as TEXT) AS int8 FROM DataTypes WHERE id = ?',
+    args: [1],
+  })
+  // because we are executing a raw query,
+  // the returned BigInt for the `id`
+  // is not converted into a regular number
+  t.deepEqual(rawRes, [{ id: 1, int8: bigInt.toString() }])
+  //db.defaultSafeIntegers(false) // disables BigInt support
+})