feat (client): support for PG on the client

.changeset/breezy-plums-dream.md

@@ -0,0 +1,7 @@
+---
+"@core/electric": patch
+"electric-sql": patch
+"@electric-sql/prisma-generator": patch
+---
+
+Support for a local Postgres database on the client. Also introduces drivers for node Postgres and PGlite.

.github/workflows/clients_typescript_tests.yml

@@ -70,7 +70,7 @@ jobs:
       - name: Build
         run: make build
       - name: Run tests
-        run: make tests
+        run: make tests-CI
   maybe_publish:
     runs-on: ubuntu-latest
     needs: [test, check_types, verify_formatting]

.github/workflows/e2e.yml

@@ -8,6 +8,7 @@ on:
       # Root files
       - "*"
       - "!pnpm-lock.yaml"
+      - "!clients/typescript/**" # Satellite tests run in a separate workflow
       # CI files not related to GH actions
       - ".buildkite/**"
       - "**/README.md"
@@ -80,7 +81,7 @@ jobs:
 
       - run: make lux
       - run: make deps pull
-      - run: make test_only
+      - run: make test-no-satellite
         id: tests
         env:
           ELECTRIC_IMAGE_NAME: electric-sql-ci/electric
@@ -92,7 +93,7 @@ jobs:
         if: ${{ failure() && steps.tests.outcome == 'failure'  }}
         with:
           name: lux_logs
-          path: e2e/lux_logs/run_*
+          path: e2e/**/lux_logs/run_*
       - name: Upload test results to Buildkite analytics
         if: ${{ !cancelled() && steps.tests.outcome != 'skipped' && env.BUILDKITE_ANALYTICS_TOKEN != '' }}
         working-directory: e2e/lux_logs/latest_run
@@ -110,3 +111,92 @@ jobs:
             -F "run_env[commit_sha]=$GITHUB_SHA" \
             -F "run_env[url]=https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \
             https://analytics-api.buildkite.com/v1/uploads
+
+  e2e_satellite_tests:
+    name: E2E Satellite tests
+    runs-on: electric-e2e-8-32
+    strategy:
+      matrix:
+        dialect: [SQLite, Postgres]
+    defaults:
+      run:
+        working-directory: e2e
+    env:
+      BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_TEST_ANALYTICS_E2E }}
+      DIALECT: ${{ matrix.dialect }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+
+      - name: Inject slug/short variables
+        uses: rlespinasse/github-slug-action@v4
+      - name: Inject variables for `docker buildx` github actions caching
+        uses: crazy-max/ghaction-github-runtime@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Log in to the Container registry
+        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - uses: erlef/setup-beam@v1
+        with:
+          otp-version: ${{ env.OTP_VERSION }}
+          elixir-version: ${{ env.ELIXIR_VERSION }}
+
+      - run: |
+          echo "ELECTRIC_VERSION=$(make --silent print_version_from_git)" >> $GITHUB_ENV
+        working-directory: components/electric
+      - run: make docker-build-ci
+        env:
+          ELECTRIC_IMAGE_NAME: electric-sql-ci/electric
+        working-directory: components/electric
+      - run: make docker-build-ws-client
+        env:
+          ELECTRIC_CLIENT_IMAGE_NAME: electric-sql-ci/electric-ws-client
+        working-directory: components/electric
+
+      - name: Cache built lux
+        uses: actions/cache@v3
+        with:
+          path: |
+            e2e/lux/bin
+            e2e/lux/ebin
+            e2e/lux/priv
+          key: ${{ runner.os }}-luxbuilt-${{ env.OTP_VERSION }}-${{ env.ELIXIR_VERSION }}
+
+      - run: make lux
+      - run: make deps pull
+      - run: make test-satellite-only
+        id: tests
+        env:
+          ELECTRIC_IMAGE_NAME: electric-sql-ci/electric
+          ELECTRIC_CLIENT_IMAGE_NAME: electric-sql-ci/electric-ws-client
+          ELECTRIC_IMAGE_TAG: ${{ env.ELECTRIC_VERSION }}
+
+      - name: Upload lux logs
+        uses: actions/upload-artifact@v3
+        if: ${{ failure() && steps.tests.outcome == 'failure'  }}
+        with:
+          name: lux_logs
+          path: e2e/**/lux_logs/run_*
+      - name: Upload test results to Buildkite analytics
+        if: ${{ !cancelled() && steps.tests.outcome != 'skipped' && env.BUILDKITE_ANALYTICS_TOKEN != '' }}
+        working-directory: e2e/lux_logs/latest_run
+        run: |
+          curl \
+            -X POST \
+            --fail-with-body \
+            -H "Authorization: Token token=\"$BUILDKITE_ANALYTICS_TOKEN\"" \
+            -F "data=@lux_junit.xml" \
+            -F "format=junit" \
+            -F "run_env[CI]=github_actions" \
+            -F "run_env[key]=$GITHUB_ACTION-$GITHUB_RUN_NUMBER-$GITHUB_RUN_ATTEMPT" \
+            -F "run_env[number]=$GITHUB_RUN_NUMBER" \
+            -F "run_env[branch]=$GITHUB_REF" \
+            -F "run_env[commit_sha]=$GITHUB_SHA" \
+            -F "run_env[url]=https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \
+            https://analytics-api.buildkite.com/v1/uploads

clients/typescript/Makefile

@@ -17,6 +17,9 @@ build-dev: node_modules
 tests:
 	pnpm run test
 
+tests-CI:
+	pnpm run test-CI
+
 style:
 	prettier --check --loglevel warn . && eslint src --quiet
 

clients/typescript/ava.config.js

@@ -1,5 +1,18 @@
 const [major, minor, _patch] = process.versions.node.split('.').map(Number)
 
+// Developers can provide a `TEST_ONLY_DIALECT` value of `postgres`, `pglite`, or `sqlite`
+// to run the unit tests only for that dialect.
+// Developers can also provide a `DISABLE_DIALECT` value of `postgres`, `pglite`, or `sqlite`
+// to disable the unit tests for that dialect but run all others.
+const testOnlyDialect = process.env.TEST_ONLY_DIALECT
+const disableDialect = process.env.DISABLE_DIALECT
+
+if (testOnlyDialect && disableDialect) {
+  throw new Error(
+    'Cannot set both TEST_ONLY_DIALECT and DISABLE_DIALECT environment variables.'
+  )
+}
+
 let loaderArg
 if (
   major > 20 ||
@@ -11,9 +24,38 @@ if (
   loaderArg = '--loader=tsx'
 }
 
+const files = ['test/**/*.test.ts', 'test/**/*.test.tsx']
+const ignorePostgres = ['!test/**/postgres/**']
+const ignorePglite = ['!test/**/pglite/**']
+const ignoreSqlite = ['!test/**/sqlite/**']
+
+switch (testOnlyDialect) {
+  case 'postgres':
+    files.push(...ignorePglite, ...ignoreSqlite)
+    break
+  case 'pglite':
+    files.push(...ignorePostgres, ...ignoreSqlite)
+    break
+  case 'sqlite':
+    files.push(...ignorePostgres, ...ignorePglite)
+    break
+}
+
+switch (disableDialect) {
+  case 'postgres':
+    files.push(...ignorePostgres)
+    break
+  case 'pglite':
+    files.push(...ignorePglite)
+    break
+  case 'sqlite':
+    files.push(...ignoreSqlite)
+    break
+}
+
 export default {
   timeout: '10m',
-  files: ['test/**/*.test.ts', 'test/**/*.test.tsx'],
+  files,
   extensions: {
     ts: 'module',
     tsx: 'module',

clients/typescript/package.json

@@ -55,7 +55,10 @@
     "./op-sqlite": "./dist/drivers/op-sqlite/index.js",
     "./generic": "./dist/drivers/generic/index.js",
     "./node": "./dist/drivers/better-sqlite3/index.js",
+    "./node-postgres": "./dist/drivers/node-postgres/index.js",
+    "./pglite": "./dist/drivers/pglite/index.js",
     "./react": "./dist/frameworks/react/index.js",
+    "./tauri-postgres": "./dist/drivers/tauri-postgres/index.js",
     "./vuejs": "./dist/frameworks/vuejs/index.js",
     "./wa-sqlite": "./dist/drivers/wa-sqlite/index.js",
     "./tauri": "./dist/drivers/tauri-sqlite/index.js",
@@ -90,9 +93,18 @@
       "node": [
         "./dist/drivers/better-sqlite3/index.d.ts"
       ],
+      "node-postgres": [
+        "./dist/drivers/node-postgres/index.d.ts"
+      ],
+      "pglite": [
+        "./dist/drivers/pglite/index.d.ts"
+      ],
       "react": [
         "./dist/frameworks/react/index.d.ts"
       ],
+      "tauri-postgres": [
+        "./dist/drivers/tauri-postgres/index.d.ts"
+      ],
       "vuejs": [
         "./dist/frameworks/vuejs/index.d.ts"
       ],
@@ -152,6 +164,7 @@
     "build": "shx rm -rf dist && npm run build:copy-docker && concurrently \"tsup\" \"tsc -p tsconfig.build.json\" && node scripts/fix-imports.js",
     "build:copy-docker": "shx mkdir -p ./dist/cli/docker-commands/docker && shx cp -r ./src/cli/docker-commands/docker ./dist/cli/docker-commands",
     "test": "ava",
+    "test-CI": "DISABLE_DIALECT=postgres npm run test",
     "generate-test-client": "npx tsx ./test/client/generateTestClient.ts",
     "typecheck": "tsc -p tsconfig.json",
     "posttest": "npm run typecheck",
@@ -202,6 +215,7 @@
     "zod": "3.21.1"
   },
   "devDependencies": {
+    "@electric-sql/pglite": "^0.1.5",
     "@electric-sql/prisma-generator": "workspace:*",
     "@op-engineering/op-sqlite": ">= 2.0.16",
     "@tauri-apps/plugin-sql": "2.0.0-alpha.5",
@@ -219,6 +233,7 @@
     "@types/lodash.throttle": "^4.1.7",
     "@types/lodash.uniqwith": "^4.5.9",
     "@types/node": "^18.8.4",
+    "@types/pg": "^8.11.0",
     "@types/prompts": "^2.4.9",
     "@types/react": "^18.0.18",
     "@types/tcp-port-used": "^1.0.2",
@@ -229,15 +244,17 @@
     "@vue/test-utils": "^2.4.4",
     "ava": "^4.3.1",
     "concurrently": "^8.2.2",
+    "embedded-postgres": "16.1.1-beta.9",
     "eslint": "^8.22.0",
-    "expo-sqlite": "^13.0.0",
+    "expo-sqlite": "^13.1.0",
     "glob": "^10.3.10",
     "global-jsdom": "24.0.0",
     "husky": "^8.0.3",
     "jsdom": "24.0.0",
     "lint-staged": "^13.1.0",
     "memorystorage": "^0.12.0",
     "nodemon": "^3.0.2",
+    "pg": "^8.11.3",
     "prettier": "2.8.2",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
@@ -255,9 +272,12 @@
   },
   "peerDependencies": {
     "@capacitor-community/sqlite": ">= 5.6.2",
+    "@electric-sql/pglite": ">= 0.1.5",
     "@op-engineering/op-sqlite": ">= 2.0.16",
     "@tauri-apps/plugin-sql": "2.0.0-alpha.5",
+    "embedded-postgres": "16.1.1-beta.9",
     "expo-sqlite": ">= 13.0.0",
+    "pg": "^8.11.3",
     "prisma": "4.8.1",
     "react": ">= 16.8.0",
     "react-dom": ">= 16.8.0",
@@ -268,18 +288,27 @@
     "zod": "3.21.1"
   },
   "peerDependenciesMeta": {
-    "@op-engineering/op-sqlite": {
+    "@capacitor-community/sqlite": {
       "optional": true
     },
-    "@capacitor-community/sqlite": {
+    "@electric-sql/pglite": {
+      "optional": true
+    },
+    "@op-engineering/op-sqlite": {
       "optional": true
     },
     "@tauri-apps/plugin-sql": {
       "optional": true
     },
+    "embedded-postgres": {
+      "optional": true
+    },
     "expo-sqlite": {
       "optional": true
     },
+    "pg": {
+      "optional": true
+    },
     "prisma": {
       "optional": true
     },