Skip to content

Commit

Permalink
chore(driver-adapters): remove external wasm deps hack for napi, lift…
Browse files Browse the repository at this point in the history
… `driver-adapters` from `query-engine/..` to `libs/..` (#5115)

* feat(schema-engine-wasm): created wasm-compatible schema-engine-wasm skeleton

* chore(schema-engine-wasm): added "make build-se-wasm" command

* fix(schema-engine-wasm): replace "SchemaEngineParams" class instance with JS object via tsify. Fix build.sh script

* feat(schema-engine-wasm): implement playground skeleton

* fix(schema-engine-wasm): clippy

* chore(schema-engine-wasm): lift "driver-adapters" from "query-engine" to "libs" folder

* Delete libs/driver-adapters/executor/bench/seed.sql.gz

* chore: update Cargo.lock

* chore: update driver-adapters path references

* chore: update query-engine-wasm path references

* chore: update query-engine-wasm path references

* chore: update query-engine-wasm and schema-engine-wasm path references

* feat(driver-adapters): get rid of external import hack, fix paths in Napi engine scenario

* Revert "Delete libs/driver-adapters/executor/bench/seed.sql.gz"

This reverts commit 0360567.

* chore: fix conflicts

* chore: fix pnpm version to avoid pnpm/pnpm#8953

* fix: use actual dist JS file

* chore: update tsup to latest

---------

Co-authored-by: jkomyno <[email protected]>
  • Loading branch information
jkomyno and jkomyno authored Jan 14, 2025
1 parent 4484558 commit 5700ed6
Show file tree
Hide file tree
Showing 80 changed files with 85 additions and 99 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/wasm-benchmarks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ jobs:
- name: "Setup pnpm"
uses: pnpm/[email protected]
with:
version: 9
version: 9.15.1

- name: "Login to Docker Hub"
uses: docker/login-action@v3
Expand Down
1 change: 0 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ members = [
"query-engine/core-tests",
"query-engine/black-box-tests",
"query-engine/dmmf",
"query-engine/driver-adapters",
"query-engine/query-structure",
"query-engine/query-engine",
"query-engine/query-engine-node-api",
Expand Down
6 changes: 3 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -218,13 +218,13 @@ test-driver-adapter-pg: test-pg-js
test-driver-adapter-pg-wasm: test-pg-wasm

start-pg-bench:
docker compose -f query-engine/driver-adapters/executor/bench/docker-compose.yml up --wait -d --remove-orphans postgres
docker compose -f libs/driver-adapters/executor/bench/docker-compose.yml up --wait -d --remove-orphans postgres

setup-pg-bench: start-pg-bench build-qe-napi build-qe-wasm build-driver-adapters-kit

run-bench:
DATABASE_URL="postgresql://postgres:postgres@localhost:5432/bench?schema=imdb_bench&sslmode=disable" \
node --experimental-wasm-modules query-engine/driver-adapters/executor/dist/bench.mjs
node --experimental-wasm-modules libs/driver-adapters/executor/dist/bench.mjs

bench-pg-js: setup-pg-bench run-bench

Expand Down Expand Up @@ -409,7 +409,7 @@ measure-qe-wasm: build-qe-wasm-gz
done;

build-driver-adapters-kit: build-driver-adapters
cd query-engine/driver-adapters && pnpm i && pnpm build
cd libs/driver-adapters && pnpm i && pnpm build

build-driver-adapters: ensure-prisma-present
@echo "Building driver adapters..."
Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -4,25 +4,16 @@
"pnpm": "^9.14.2"
},
"name": "executor",
"type": "module",
"version": "0.0.1",
"description": "",
"private": true,
"scripts": {
"build": "tsup ./src/testd-qe.ts ./src/bench.ts --format esm --dts",
"build": "tsup ./src/testd-qe.ts ./src/demo-se.ts ./src/bench.ts --format esm --dts",
"test:qe": "node --import tsx ./src/testd-qe.ts",
"demo:se": "node --import tsx ./src/demo-se.ts",
"clean:d1": "rm -rf ../../connector-test-kit-rs/query-engine-tests/.wrangler"
},
"tsup": {
"external": [
"../../../query-engine-wasm/pkg/postgresql/query_engine_bg.js",
"../../../query-engine-wasm/pkg/mysql/query_engine_bg.js",
"../../../query-engine-wasm/pkg/sqlite/query_engine_bg.js",
"../../../schema-engine-wasm/pkg/postgresql/schema_engine_bg.js",
"../../../schema-engine-wasm/pkg/mysql/schema_engine_bg.js",
"../../../schema-engine-wasm/pkg/sqlite/schema_engine_bg.js"
]
},
"keywords": [],
"author": "",
"sideEffects": false,
Expand All @@ -47,7 +38,7 @@
"devDependencies": {
"@cloudflare/workers-types": "4.20240405.0",
"@types/node": "20.12.7",
"tsup": "8.0.2",
"tsup": "8.3.5",
"tsx": "4.7.2",
"typescript": "5.4.5"
}
Expand Down
2 changes: 2 additions & 0 deletions libs/driver-adapters/executor/script/testd-qe.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
#!/usr/bin/env bash
node "$(dirname "${BASH_SOURCE[0]}")/../dist/testd-qe.js"
File renamed without changes.
33 changes: 33 additions & 0 deletions libs/driver-adapters/executor/src/query-engine-wasm.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import fs from 'node:fs/promises'
import path from 'node:path'
import { __dirname, normaliseProvider } from './utils'
import type { Queryable } from '@prisma/driver-adapter-utils'

const relativePath = '../../../../query-engine/query-engine-wasm/pkg'

const initializedModules = new Set<Queryable['provider']>()

export async function getQueryEngineForProvider(provider: Queryable['provider']) {
const normalisedProvider = normaliseProvider(provider)
const engine = await import(`${relativePath}/${normalisedProvider}/query_engine_bg.js`)

if (!initializedModules.has(provider)) {
const bytes = await fs.readFile(
path.resolve(
__dirname,
relativePath,
normalisedProvider,
'query_engine_bg.wasm',
),
)

const module = new WebAssembly.Module(bytes)
const instance = new WebAssembly.Instance(module, {
'./query_engine_bg.js': engine,
})
engine.__wbg_set_wasm(instance.exports)
initializedModules.add(provider)
}

return engine.QueryEngine
}
File renamed without changes.
31 changes: 31 additions & 0 deletions libs/driver-adapters/executor/src/schema-engine-wasm.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
import fs from 'node:fs/promises'
import path from 'node:path'
import { __dirname, normaliseProvider } from './utils.js'
import type { Queryable } from '@prisma/driver-adapter-utils'

const relativePath = '../../../../schema-engine/schema-engine-wasm/pkg'

const initializedModules = new Set<Queryable['provider']>()

export async function getSchemaEngineForProvider(provider: Queryable['provider']) {
const normalisedProvider = normaliseProvider(provider)
const engine = await import(`${relativePath}/${normalisedProvider}/schema_engine_bg.js`)

if (!initializedModules.has(provider)) {
const bytes = await fs.readFile(
path.resolve(
__dirname,
relativePath,
normalisedProvider,
'schema_engine_bg.wasm',
),
)

const module = new WebAssembly.Module(bytes)
const instance = new WebAssembly.Instance(module, { './schema_engine_bg.js': engine })
engine.__wbg_set_wasm(instance.exports)
initializedModules.add(provider)
}

return engine.SchemaEngine
}
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import type { D1Database, D1PreparedStatement, D1Result } from '@cloudflare/workers-types'
import type { Queryable } from '@prisma/driver-adapter-utils'
import path from 'node:path'
import { fileURLToPath } from 'node:url'

Expand All @@ -11,6 +12,10 @@ export function copyPathName({ fromURL, toURL }: { fromURL: string; toURL: strin
return toObj.toString()
}

export function normaliseProvider(provider: Queryable['provider']) {
return provider === 'postgres' ? 'postgresql' : provider
}

export function postgresSchemaName(url: string) {
return new URL(url).searchParams.get('schema') ?? undefined
}
Expand Down
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"build:prisma": "pnpm -r --parallel dev",
"build:executor": "pnpm -r --filter executor build",
"lint": "pnpm -r run lint",
"clean": "git clean -dXf -e !query-engine/driver-adapters"
"clean": "git clean -dXf -e !libs/driver-adapters"
},
"keywords": [],
"author": "",
Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
2 changes: 1 addition & 1 deletion query-engine/connector-test-kit-rs/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ To run tests through a driver adapters, you should also configure the following
Example:

```shell
export EXTERNAL_TEST_EXECUTOR="$WORKSPACE_ROOT/query-engine/driver-adapters/executor/script/testd-qe.sh"
export EXTERNAL_TEST_EXECUTOR="$WORKSPACE_ROOT/libs/driver-adapters/executor/script/testd-qe.sh"
export DRIVER_ADAPTER=neon
export ENGINE=wasm
export DRIVER_ADAPTER_CONFIG ='{ "proxyUrl": "127.0.0.1:5488/v1" }'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -365,7 +365,7 @@ impl TestConfig {
}

pub fn external_test_executor_path(&self) -> Option<String> {
const DEFAULT_TEST_EXECUTOR: &str = "query-engine/driver-adapters/executor/script/testd-qe.sh";
const DEFAULT_TEST_EXECUTOR: &str = "libs/driver-adapters/executor/script/testd-qe.sh";
self.with_driver_adapter()
.and_then(|_| {
Self::workspace_root().or_else(|| {
Expand Down
2 changes: 0 additions & 2 deletions query-engine/driver-adapters/executor/script/testd-qe.sh

This file was deleted.

43 changes: 0 additions & 43 deletions query-engine/driver-adapters/executor/src/query-engine-wasm.ts

This file was deleted.

30 changes: 0 additions & 30 deletions query-engine/driver-adapters/executor/src/schema-engine-wasm.ts

This file was deleted.

2 changes: 1 addition & 1 deletion query-engine/query-engine-node-api/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ sql-connector = { path = "../connectors/sql-query-connector", package = "sql-que
"all-native",
] }
query-structure = { path = "../query-structure" }
driver-adapters = { path = "../driver-adapters", features = [
driver-adapters = { path = "../../libs/driver-adapters", features = [
"postgresql",
"sqlite",
"mysql",
Expand Down
2 changes: 1 addition & 1 deletion query-engine/query-engine-wasm/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ request-handlers = { path = "../request-handlers", default-features = false, fea
"driver-adapters",
] }
query-core = { path = "../core" }
driver-adapters = { path = "../driver-adapters" }
driver-adapters = { path = "../../libs/driver-adapters" }
telemetry = { path = "../../libs/telemetry" }
quaint.workspace = true
connection-string.workspace = true
Expand Down
2 changes: 1 addition & 1 deletion renovate.json
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@
{
"groupName": "Driver Adapters directory",
"matchFileNames": [
"query-engine/driver-adapters/**"
"libs/driver-adapters/**"
]
},
{
Expand Down
2 changes: 1 addition & 1 deletion schema-engine/schema-engine-wasm/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ tsify.workspace = true
wasm-bindgen.workspace = true
wasm-bindgen-futures.workspace = true
wasm-rs-dbg.workspace = true
driver-adapters = { path = "../../query-engine/driver-adapters" }
driver-adapters = { path = "../../libs/driver-adapters" }

[build-dependencies]
build-utils.path = "../../libs/build-utils"
Expand Down

0 comments on commit 5700ed6

Please sign in to comment.