Merge pull request #126 from cipherstash/eql-v2

feat(protect): implement eql v2

Author: calvinbrewer

.changeset/shiny-jeans-clean.md

@@ -0,0 +1,9 @@
+---
+"@cipherstash/protect": major
+---
+
+Implement EQL v2 data structure.
+
+- Support for Protect.js searchable encryption when using Supabase.
+- Encrypted payloads are now composite types which support searchable encryption with EQL v2 functions.
+- The `data` property is an object that matches the EQL v2 data structure.

apps/drizzle/src/db/schema.ts

@@ -9,10 +9,10 @@ import {
 
 // Custom types will be implemented in the future - this is an example for now
 // ---
-// const cs_encrypted_v1 = <TData>(name: string) =>
+// const cs_encrypted_v2 = <TData>(name: string) =>
 //   customType<{ data: TData; driverData: string }>({
 //     dataType() {
-//       return 'cs_encrypted_v1'
+//       return 'cs_encrypted_v2'
 //     },
 //     toDriver(value: TData): string {
 //       return JSON.stringify(value)

apps/drizzle/src/select.ts

@@ -44,19 +44,19 @@ const fnForOp: (op: string) => BinaryOperator = (op) => {
 }
 
 const csEq: BinaryOperator = (left: SQLWrapper, right: unknown): SQL => {
-  return sql`cs_unique_v1(${left}) = cs_unique_v1(${bindIfParam(right, left)})`
+  return sql`cs_unique_v2(${left}) = cs_unique_v2(${bindIfParam(right, left)})`
 }
 
 const csGt: BinaryOperator = (left: SQLWrapper, right: unknown): SQL => {
-  return sql`cs_ore_64_8_v1(${left}) > cs_ore_64_8_v1(${bindIfParam(right, left)})`
+  return sql`cs_ore_64_8_v2(${left}) > cs_ore_64_8_v2(${bindIfParam(right, left)})`
 }
 
 const csLt: BinaryOperator = (left: SQLWrapper, right: unknown): SQL => {
-  return sql`cs_ore_64_8_v1(${left}) < cs_ore_64_8_v1(${bindIfParam(right, left)})`
+  return sql`cs_ore_64_8_v2(${left}) < cs_ore_64_8_v2(${bindIfParam(right, left)})`
 }
 
 const csMatch: BinaryOperator = (left: SQLWrapper, right: unknown): SQL => {
-  return sql`cs_match_v1(${left}) @> cs_match_v1(${bindIfParam(right, left)})`
+  return sql`cs_match_v2(${left}) @> cs_match_v2(${bindIfParam(right, left)})`
 }
 
 const filterInput = await protectClient.encrypt(filter, {
@@ -76,7 +76,7 @@ const query = db
   })
   .from(users)
   .where(filterFn(users.email_encrypted, filterInput.data))
-  .orderBy(sql`cs_ore_64_8_v1(users.email_encrypted)`)
+  .orderBy(sql`cs_ore_64_8_v2(users.email_encrypted)`)
 
 const sqlResult = query.toSQL()
 console.log('[INFO] SQL statement:', sqlResult)

docs/concepts/searchable-encryption.md

@@ -82,7 +82,7 @@ if (encryptedParam.failure) {
 const equalitySQL = `
   SELECT email
   FROM users
-  WHERE cs_unique_v1($1) = cs_unique_v1($2)
+  WHERE cs_unique_v2($1) = cs_unique_v2($2)
 `
 
 // 3) Execute the query, passing in the Postgres column name
@@ -96,7 +96,7 @@ Using the above approach, Protect.js is generating the EQL payloads and which me
 So does this solve the original problem of searching on encrypted data?
 
 ```sql
-# SELECT * FROM users WHERE WHERE cs_unique_v1(email) = cs_unique_v1(eql_payload_created_by_protect);
+# SELECT * FROM users WHERE WHERE cs_unique_v2(email) = cs_unique_v2(eql_payload_created_by_protect);
  id |      name      |           email
 ----+----------------+----------------------------
   1 | Alice Johnson  | mBbKmsMMkbKBSN...

docs/how-to/searchable-encryption.md

@@ -0,0 +1,5 @@
+# How to search encrypted data
+
+TODO: flesh this out (sorry it's not done yet!)
+
+In the meantime checkout the [EQL repo](https://github.com/cipherstash/encrypt-query-language) which is where these docs will get their inspiration from specifically for JavaScript/TypeScript implementations.

docs/reference/searchable-encryption-postgres.md

@@ -5,10 +5,7 @@ This reference guide outlines the different query patterns you can use to search
 ## Table of contents
 
 - [Before you start](#before-you-start)
-- [Equality query](#equality-query)
-- [Free-text search](#free-text-search)
-- [Sorting data](#sorting-data)
-- [Range queries](#range-queries)
+- [Query examples](#query-examples)
 
 ## Before you start
 
@@ -27,96 +24,7 @@ export const protectedUsers = csTable('users', {
 > [!TIP]
 > To see an example using the [Drizzle ORM](https://github.com/drizzle-team/drizzle-orm) see the example [here](../../apps/drizzle/src/select.ts).
 
-## Equality query
-
-For an equality query, use the `cs_unique_v1` function.
-
-```ts
-// 1) Encrypt the search term
-const searchTerm = 'alice@example.com'
-
-const encryptedParam = await protectClient.encrypt(searchTerm, {
-  column: protectedUsers.email, // Your Protect column definition
-  table: protectedUsers,        // Reference to the table schema
-})
-
-if (encryptedParam.failure) {
-  // Handle the failure
-}
-
-// 2) Build an equality query (cs_unique_v1)
-const equalitySQL = `
-  SELECT email
-  FROM users
-  WHERE cs_unique_v1($1) = cs_unique_v1($2)
-`
-
-// Use your flavor or ORM to execute the query. `client` is a PostgreSQL client.
-const result = await client.query(equalitySQL, [ protectedUser.email.getName(), encryptedParam.data ])
-```
-
-**Explanation:**
-
-`WHERE cs_unique_v1($1) = cs_unique_v1($2)`
-
-The first `$1` is the Postgres column name, and the second `$2` is the encrypted search term. 
-
-## Free-text search
-
-For partial matches or full-text searches, use the `cs_match_v1` function.
-
-```ts
-// Suppose you're searching for emails containing "alice"
-const searchTerm = 'alice'
-
-const encryptedParam = await protectClient.encrypt(searchTerm, {
-  column: protectedUsers.email,
-  table: protectedUsers,
-})
-
-if (encryptedParam.failure) {
-  // Handle the failure
-}
-
-// Build and execute a "match" query (cs_match_v1)
-const matchSQL = `
-  SELECT email
-  FROM users
-  WHERE cs_match_v1($1) @> cs_match_v1($2)
-`
-
-// Use your flavor or ORM to execute the query. `client` is a PostgreSQL client.
-const result = await client.query(matchSQL, [ protectedUser.email.getName(), encryptedParam.data ])
-```
-
-**Explanation:**
-
-`WHERE cs_match_v1($1) @> cs_match_v1($2)`
-
-The first `$1` is the Postgres column name, and the second `$2` is the encrypted search term.
-
-## Sorting data
-
-For `order by` queries, use the `cs_ore_64_8_v1` function.
-
-```ts
-// Suppose you're sorting by email address
-const orderSQL = `
-  SELECT email
-  FROM users
-  ORDER BY cs_ore_64_8_v1(email) ASC
-`
-const orderedResult = await client.query(orderSQL)
-
-// Use your flavor or ORM to execute the query. `client` is a PostgreSQL client.
-const result = await client.query(orderSQL)
-```
-
-### Explanation
-
-The `cs_ore_64_8_v1` function doesn't require any encrypted parameters, but rather the name of the Postgres column you want to sort by.
-
-## Range queries
+## Query examples
 
 TODO: flesh this out (sorry it's not done yet!)
 

packages/protect/README.md

@@ -94,7 +94,7 @@ yarn add @cipherstash/protect
 pnpm add @cipherstash/protect
 ```
 
-> [!TIP] 
+> [!TIP]
 > [Bun](https://bun.sh/) is not currently supported due to a lack of [Node-API compatibility](https://github.com/oven-sh/bun/issues/158). Under the hood, Protect.js uses [CipherStash Client](#cipherstash-client) which is written in Rust and embedded using [Neon](https://github.com/neon-bindings/neon).
 
 Lastly, install the CipherStash CLI:
@@ -111,7 +111,7 @@ Lastly, install the CipherStash CLI:
 
 ### Opt-out of bundling
 
-> [!IMPORTANT] 
+> [!IMPORTANT]
 > **You need to opt-out of bundling when using Protect.js.**
 
 Protect.js uses Node.js specific features and requires the use of the [native Node.js `require`](https://nodejs.org/api/modules.html#requireid).
@@ -122,9 +122,8 @@ Read more about [building and bundling with Protect.js](#builds-and-bundling).
 
 ## Getting started
 
-🆕 **Existing app?** Skip to [the next step](#configuration).
-
-🌱 **Clean slate?** Check out the [getting started tutorial](./docs/getting-started.md).
+- 🆕 **Existing app?** Skip to [the next step](#configuration).
+- 🌱 **Clean slate?** Check out the [getting started tutorial](./docs/getting-started.md).
 
 ### Configuration
 
@@ -244,8 +243,7 @@ if (encryptResult.failure) {
   );
 }
 
-const ciphertext = encryptResult.data;
-console.log("ciphertext:", ciphertext);
+console.log("EQL Payload containing ciphertexts:", encryptResult.data);
 ```
 
 The `encrypt` function will return a `Result` object with either a `data` key, or a `failure` key.
@@ -254,9 +252,7 @@ The `encryptResult` will return one of the following:
 ```typescript
 // Success
 {
-  data: {
-    c: 'mBbKmsMMkbKBSN}s1THy_NfQN892!dercyd0s...'
-  }
+  data: EncryptedPayload
 }
 
 // Failure
@@ -278,7 +274,8 @@ To start decrypting data, add the following to `src/index.ts`:
 ```typescript
 import { protectClient } from "./protect";
 
-const decryptResult = await protectClient.decrypt(ciphertext);
+// encryptResult is the EQL payload from the previous step
+const decryptResult = await protectClient.decrypt(encryptResult.data);
 
 if (decryptResult.failure) {
   // Handle the failure
@@ -313,7 +310,7 @@ The `decryptResult` will return one of the following:
 
 ### Working with models and objects
 
-Protect.js provides model-level encryption methods that make it easy to encrypt and decrypt entire objects. 
+Protect.js provides model-level encryption methods that make it easy to encrypt and decrypt entire objects.
 These methods automatically handle the encryption of fields defined in your schema.
 
 If you are working with a large data set, the model operations are significantly faster than encrypting and decrypting individual objects as they are able to perform bulk operations.
@@ -354,12 +351,12 @@ const encryptedUser = encryptedResult.data;
 console.log("encrypted user:", encryptedUser);
 ```
 
-The `encryptModel` function will only encrypt fields that are defined in your schema. 
+The `encryptModel` function will only encrypt fields that are defined in your schema.
 Other fields (like `id` and `createdAt` in the example above) will remain unchanged.
 
 #### Type safety with models
 
-Protect.js provides strong TypeScript support for model operations. 
+Protect.js provides strong TypeScript support for model operations.
 You can specify your model's type to ensure end-to-end type safety:
 
 ```typescript
@@ -523,7 +520,7 @@ if (decryptedResult.failure) {
 const decryptedUsers = decryptedResult.data;
 ```
 
-The model encryption methods provide a higher-level interface that's particularly useful when working with ORMs or when you need to encrypt multiple fields in an object. 
+The model encryption methods provide a higher-level interface that's particularly useful when working with ORMs or when you need to encrypt multiple fields in an object.
 They automatically handle the mapping between your model's structure and the encrypted fields defined in your schema.
 
 ### Store encrypted data in a database
@@ -549,21 +546,36 @@ To enable searchable encryption in PostgreSQL, [install the EQL custom types and
    curl -sLo cipherstash-encrypt.sql https://github.com/cipherstash/encrypt-query-language/releases/latest/download/cipherstash-encrypt.sql
    ```
 
+   Using [Supabase](https://supabase.com/)? We ship an EQL release specifically for Supabase.
+   Download the latest EQL install script:
+
+   ```sh
+   curl -sLo cipherstash-encrypt-supabase.sql https://github.com/cipherstash/encrypt-query-language/releases/latest/download/cipherstash-encrypt-supabase.sql
+   ```
+
 2. Run this command to install the custom types and functions:
 
    ```sh
    psql -f cipherstash-encrypt.sql
    ```
 
-EQL is now installed in your database and you can enable searchable encryption by adding the `cs_encrypted_v1` type to a column.
+   or with Supabase:
+
+   ```sh
+   psql -f cipherstash-encrypt-supabase.sql
+   ```
+
+EQL is now installed in your database and you can enable searchable encryption by adding the `eql_v2_encrypted` type to a column.
 
 ```sql
 CREATE TABLE users (
     id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
-    email cs_encrypted_v1
+    email eql_v2_encrypted
 );
 ```
 
+Read more about [how to search encrypted data](./docs/how-to/searchable-encryption.md) in the docs.
+
 ## Identity-aware encryption
 
 > [!IMPORTANT]
@@ -631,7 +643,7 @@ if (encryptResult.failure) {
   // Handle the failure
 }
 
-const ciphertext = encryptResult.data;
+console.log("EQL Payload containing ciphertexts:", encryptResult.data);
 ```
 
 ### Decrypting data with a lock context
@@ -642,7 +654,7 @@ To decrypt data with a lock context, call the optional `withLockContext` method
 import { protectClient } from "./protect";
 
 const decryptResult = await protectClient
-  .decrypt(ciphertext)
+  .decrypt(encryptResult.data)
   .withLockContext(lockContext);
 
 if (decryptResult.failure) {

packages/protect/__tests__/protect.test.ts

@@ -467,8 +467,8 @@ describe('performance', () => {
 
 // ------------------------
 // TODO get LockContext working in CI.
-// These tests pass locally, given you provide a valid JWT.
 // To manually test locally, uncomment the following lines and provide a valid JWT in the userJwt variable.
+// Last successful local test was 2025-05-20 by cj@cipherstash.com
 // ------------------------
 // const userJwt = ''
 // describe('encryption and decryption with lock context', () => {

packages/protect/generateEqlSchema.ts

@@ -3,7 +3,7 @@ import { execa } from 'execa'
 
 async function main() {
   const url =
-    'https://raw.githubusercontent.com/cipherstash/encrypt-query-language/main/sql/schemas/cs_encrypted_storage_v1.schema.json'
+    'https://raw.githubusercontent.com/cipherstash/encrypt-query-language/main/sql/schemas/cs_encrypted_storage_v2.schema.json'
 
   const response = await fetch(url)