expose converter API, squashes all commits

.tool-versions

@@ -1,6 +1,6 @@
 elixir 1.17.0-otp-27
 erlang 27.0
-nodejs 20.2.0
+nodejs 21.4.0
 pnpm 9.4.0
 # keep synchronised with version pinned in .github/workflows/satellite_proto.yml
 protoc 26.1

clients/typescript/src/client/conversions/converter.ts

@@ -1,3 +1,5 @@
+import { Row } from '../../util'
+import { AnyTableSchema } from '../model'
 import { PgType } from './types'
 
 export interface Converter {
@@ -7,12 +9,48 @@ export interface Converter {
    * @param pgType The Postgres type of the column in which to store the value.
    */
   encode(v: any, pgType: PgType): any
+  /**
+   * Encodes the provided row for storing in the database.
+   * @param row The row to encode
+   * @param tableSchema The schema of the table for this row.
+   */
+  encodeRow(
+    row: Record<string, unknown>,
+    tableSchema: Pick<AnyTableSchema, 'fields'>
+  ): Row
+  /**
+   * Encodes the provided rows for storing in the database.
+   * @param rows The rows to encode
+   * @param tableSchema The schema of the table for these rows.
+   */
+  encodeRows(
+    rows: Array<Record<string, unknown>>,
+    tableSchema: Pick<AnyTableSchema, 'fields'>
+  ): Array<Row>
   /**
    * Decodes the provided value from the database.
    * @param v The value to decode.
    * @param pgType The Postgres type of the column from which to decode the value.
    */
   decode(v: any, pgType: PgType): any
+  /**
+   * Decodes the provided row from the database.
+   * @param row The row to decode
+   * @param tableSchema The schema of the table for this row.
+   */
+  decodeRow<T extends Record<string, any> = Record<string, any>>(
+    row: Record<string, unknown>,
+    tableSchema: Pick<AnyTableSchema, 'fields'>
+  ): T
+  /**
+   * Decodes the provided rows from the database.
+   * @param rows The rows to decode
+   * @param tableSchema The schema of the table for these rows.
+   */
+  decodeRows<T extends Record<string, any> = Record<string, any>>(
+    rows: Array<Record<string, unknown>>,
+    tableSchema: Pick<AnyTableSchema, 'fields'>
+  ): Array<T>
 }
 
 /**
@@ -26,3 +64,34 @@ export interface Converter {
 export function isDataObject(v: unknown): boolean {
   return v instanceof Date || typeof v === 'bigint' || ArrayBuffer.isView(v)
 }
+
+export function mapRow<
+  T extends Record<string, unknown> = Record<string, unknown>
+>(
+  row: Record<string, unknown>,
+  tableSchema: Pick<AnyTableSchema, 'fields'>,
+  f: (v: any, pgType: PgType) => any
+): T {
+  const mappedRow = {} as T
+
+  for (const [key, value] of Object.entries(row)) {
+    const pgType = tableSchema.fields[key]
+    const mappedValue =
+      pgType === undefined
+        ? value // it's an unknown column, leave it as is
+        : f(value, pgType)
+    mappedRow[key as keyof T] = mappedValue
+  }
+
+  return mappedRow
+}
+
+export function mapRows<
+  T extends Record<string, unknown> = Record<string, unknown>
+>(
+  rows: Array<Record<string, unknown>>,
+  tableSchema: Pick<AnyTableSchema, 'fields'>,
+  f: (v: any, pgType: PgType) => any
+): T[] {
+  return rows.map((row) => mapRow<T>(row, tableSchema, f))
+}

clients/typescript/src/client/conversions/index.ts

@@ -1,3 +1,4 @@
-export { postgresConverter } from './postgres'
+export type { Converter } from './converter'
 export { sqliteConverter } from './sqlite'
+export { postgresConverter } from './postgres'
 export { PgBasicType } from './types'

clients/typescript/src/client/conversions/postgres.ts

@@ -1,8 +1,9 @@
 import { InvalidArgumentError } from '../validation/errors/invalidArgumentError'
-import { Converter } from './converter'
+import { Converter, mapRow, mapRows } from './converter'
 import { deserialiseDate, serialiseDate } from './datatypes/date'
 import { isJsonNull } from './datatypes/json'
 import { PgBasicType, PgDateType, PgType } from './types'
+import { AnyTableSchema } from '../model/schema'
 
 /**
  * This module takes care of converting TypeScript values to a Postgres storeable value and back.
@@ -11,7 +12,7 @@ import { PgBasicType, PgDateType, PgType } from './types'
  * Currently, no conversions are needed for the data types we support.
  */
 
-function toPostgres(v: any, pgType: PgType): any {
+export function toPostgres(v: any, pgType: PgType): any {
   if (v === null) {
     // don't transform null values
     return v
@@ -52,7 +53,7 @@ function toPostgres(v: any, pgType: PgType): any {
   }
 }
 
-function fromPostgres(v: any, pgType: PgType): any {
+export function fromPostgres(v: any, pgType: PgType): any {
   if (v === null) {
     // don't transform null values
     return v
@@ -97,5 +98,21 @@ function fromPostgres(v: any, pgType: PgType): any {
 
 export const postgresConverter: Converter = {
   encode: toPostgres,
+  encodeRow: <T extends Record<string, unknown> = Record<string, unknown>>(
+    row: Record<string, unknown>,
+    tableSchema: AnyTableSchema
+  ) => mapRow<T>(row, tableSchema, toPostgres),
+  encodeRows: <T extends Record<string, unknown> = Record<string, unknown>>(
+    rows: Array<Record<string, unknown>>,
+    tableSchema: AnyTableSchema
+  ) => mapRows<T>(rows, tableSchema, toPostgres),
   decode: fromPostgres,
+  decodeRow: <T extends Record<string, any> = Record<string, any>>(
+    row: Record<string, unknown>,
+    tableSchema: AnyTableSchema
+  ) => mapRow<T>(row, tableSchema, fromPostgres),
+  decodeRows: <T extends Record<string, any> = Record<string, any>>(
+    rows: Array<Record<string, unknown>>,
+    tableSchema: AnyTableSchema
+  ) => mapRows<T>(rows, tableSchema, fromPostgres),
 }

clients/typescript/src/client/conversions/sqlite.ts

@@ -1,10 +1,11 @@
 import { InvalidArgumentError } from '../validation/errors/invalidArgumentError'
-import { Converter } from './converter'
+import { Converter, mapRow, mapRows } from './converter'
 import { deserialiseBoolean, serialiseBoolean } from './datatypes/boolean'
 import { deserialiseBlob, serialiseBlob } from './datatypes/blob'
 import { deserialiseDate, serialiseDate } from './datatypes/date'
 import { deserialiseJSON, serialiseJSON } from './datatypes/json'
 import { PgBasicType, PgDateType, PgType, isPgDateType } from './types'
+import { AnyTableSchema } from '../model/schema'
 
 /**
  * This module takes care of converting TypeScript values for Postgres-specific types to a SQLite storeable value and back.
@@ -14,7 +15,7 @@ import { PgBasicType, PgDateType, PgType, isPgDateType } from './types'
  * When reading from the SQLite database, the string can be parsed back into a `Date` object.
  */
 
-function toSqlite(v: any, pgType: PgType): any {
+export function toSqlite(v: any, pgType: PgType): any {
   if (v === null) {
     // don't transform null values
     return v
@@ -49,7 +50,7 @@ function toSqlite(v: any, pgType: PgType): any {
   }
 }
 
-function fromSqlite(v: any, pgType: PgType): any {
+export function fromSqlite(v: any, pgType: PgType): any {
   if (v === null) {
     // don't transform null values
     return v
@@ -93,5 +94,21 @@ function fromSqlite(v: any, pgType: PgType): any {
 
 export const sqliteConverter: Converter = {
   encode: toSqlite,
+  encodeRow: <T extends Record<string, unknown> = Record<string, unknown>>(
+    row: Record<string, unknown>,
+    tableSchema: AnyTableSchema
+  ) => mapRow<T>(row, tableSchema, toSqlite),
+  encodeRows: <T extends Record<string, unknown> = Record<string, unknown>>(
+    rows: Array<Record<string, unknown>>,
+    tableSchema: AnyTableSchema
+  ) => mapRows<T>(rows, tableSchema, toSqlite),
   decode: fromSqlite,
+  decodeRow: <T extends Record<string, any> = Record<string, any>>(
+    row: Record<string, unknown>,
+    tableSchema: AnyTableSchema
+  ) => mapRow<T>(row, tableSchema, fromSqlite),
+  decodeRows: <T extends Record<string, unknown> = Record<string, unknown>>(
+    rows: Array<Record<string, unknown>>,
+    tableSchema: AnyTableSchema
+  ) => mapRows<T>(rows, tableSchema, fromSqlite),
 }

clients/typescript/src/client/execution/nonTransactionalDB.ts

@@ -3,7 +3,6 @@ import { QueryBuilder } from 'squel'
 import { DB } from './db'
 import * as z from 'zod'
 import { Row, Statement } from '../../util'
-import { Transformation, transformFields } from '../conversions/input'
 import { Fields } from '../model/schema'
 import { Converter } from '../conversions/converter'
 
@@ -59,12 +58,9 @@ export class NonTransactionalDB implements DB {
             // convert SQLite/PG values back to JS values
             // and then parse the transformed object
             // with the Zod schema to validate it
-            const transformedRow = transformFields(
-              row,
-              this._fields,
-              this._converter,
-              Transformation.Decode
-            )
+            const transformedRow = this._converter.decodeRow(row, {
+              fields: this._fields,
+            })
             return schema.parse(transformedRow)
           })
           successCallback(this, objects)

clients/typescript/src/client/execution/transactionalDB.ts

@@ -4,7 +4,6 @@ import { DB } from './db'
 import * as z from 'zod'
 import { Row, Statement } from '../../util'
 import { Fields } from '../model/schema'
-import { Transformation, transformFields } from '../conversions/input'
 import { Converter } from '../conversions/converter'
 
 export class TransactionalDB implements DB {
@@ -52,12 +51,9 @@ export class TransactionalDB implements DB {
             // convert SQLite/PG values back to JS values
             // and then parse the transformed object
             // with the Zod schema to validate it
-            const transformedRow = transformFields(
-              row,
-              this._fields,
-              this._converter,
-              Transformation.Decode
-            )
+            const transformedRow = this._converter.decodeRow(row, {
+              fields: this._fields,
+            })
             return schema.parse(transformedRow)
           })
           successCallback(

clients/typescript/src/client/model/client.ts

@@ -153,9 +153,7 @@ export class ElectricClient<
     }
   }
 
-  setReplicationTransform<
-    T extends Record<string, unknown> = Record<string, unknown>
-  >(
+  setReplicationTransform<T extends Row = Row>(
     qualifiedTableName: QualifiedTablename,
     i: ReplicatedRowTransformer<T>
   ): void {