Merge pull request #134 from cipherstash/spike/dynamo-db

Add DynamoDB examples

Author: calvinbrewer

.changeset/fifty-toes-carry.md

@@ -0,0 +1,7 @@
+---
+"@cipherstash/protect-dynamodb": minor
+"@cipherstash/protect": minor
+"@cipherstash/dynamo-example": minor
+---
+
+Released initial version of the DynamoDB helper interface.

examples/dynamo/.gitignore

@@ -0,0 +1,2 @@
+docker
+sql/cipherstash-encrypt.sql

examples/dynamo/README.md

@@ -0,0 +1,60 @@
+# DynamoDB Examples
+
+Examples of using Protect.js with DynamoDB.
+
+## Prereqs
+- [Node.js](https://nodejs.org/en) (tested with v22.11.0)
+- [pnpm](https://pnpm.io/) (tested with v9.15.3)
+- [Docker](https://www.docker.com/)
+- a CipherStash account and [credentials configured](../../README.md#configuration)
+
+## Setup
+
+Install the workspace dependencies and build Protect.js:
+```
+# change to the workspace root directory
+cd ../..
+
+pnpm install
+pnpm run build
+```
+
+Switch back to the DynamoDB examples
+```
+cd examples/dynamo
+```
+
+Start Docker services used by the DynamoDB examples:
+```
+docker compose up --detach
+```
+
+Download [EQL](https://github.com/cipherstash/encrypt-query-language) and install it into the PG DB (this is optional and only necessary for running the `export-to-pg` example):
+```
+pnpm run eql:download
+pnpm run eql:install
+```
+
+## Examples
+
+All examples run as scripts from [`package.json`](./package.json).
+You can run an example with the command `pnpm run [example_name]`.
+
+Each example runs against local DynamoDB in Docker.
+
+- `simple`
+  - `pnpm run simple`
+  - Round trip encryption/decryption through DynamoDB (no search on encrypted attributes).
+- `encrypted-partition-key`
+  - `pnpm run encrypted-partition-key`
+  - Uses an encrypted attribute as a partition key.
+- `encrypted-sort-key`
+  - `pnpm run encrypted-sort-key`
+  - Similar to the `encrypted-partition-key` example, but uses an encrypted attribute as a sort key instead.
+- `encrypted-key-in-gsi`
+  - `pnpm run encrypted-key-in-gsi`
+  - Uses an encrypted attribute as the partition key in a global secondary index.
+    The source ciphertext is projected into the index for decryption after querying the index.
+- `export-to-pg`
+  - `pnpm run export-to-pg`
+  - Encrypts an item, puts it in Dynamo, exports it to Postgres, and decrypts a result from Postgres.

examples/dynamo/docker-compose.yml

@@ -0,0 +1,41 @@
+
+services:
+  dynamodb-local:
+    command: "-jar DynamoDBLocal.jar -sharedDb -dbPath ./data"
+    image: "amazon/dynamodb-local:latest"
+    container_name: dynamodb-local
+    ports:
+      - "8000:8000"
+    volumes:
+      - "./docker/dynamodb:/home/dynamodblocal/data"
+    working_dir: /home/dynamodblocal
+
+  dynamodb-admin:
+    image: aaronshaf/dynamodb-admin
+    ports:
+      - 8001:8001
+    environment:
+      DYNAMO_ENDPOINT: http://dynamodb-local:8000
+
+  # used by export-to-pg example
+  postgres:
+    image: postgres:latest
+    environment:
+      PGPORT: 5432
+      POSTGRES_DB: "cipherstash"
+      POSTGRES_USER: "cipherstash"
+      PGUSER: "cipherstash"
+      POSTGRES_PASSWORD: password
+    ports:
+      - 5433:5432
+    deploy:
+      resources:
+        limits:
+          cpus: "${CPU_LIMIT:-2}"
+          memory: 2048mb
+    restart: always
+    healthcheck:
+      test: [ "CMD-SHELL", "pg_isready" ]
+      interval: 1s
+      timeout: 5s
+      retries: 10

examples/dynamo/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "@cipherstash/dynamo-example",
+  "private": true,
+  "version": "0.1.0",
+  "type": "module",
+  "scripts": {
+    "simple": "tsx src/simple.ts",
+    "bulk-operations": "tsx src/bulk-operations.ts",
+    "encrypted-partition-key": "tsx src/encrypted-partition-key.ts",
+    "encrypted-sort-key": "tsx src/encrypted-sort-key.ts",
+    "encrypted-key-in-gsi": "tsx src/encrypted-key-in-gsi.ts",
+    "export-to-pg": "tsx src/export-to-pg.ts",
+    "eql:download": "curl -sLo sql/cipherstash-encrypt.sql https://github.com/cipherstash/encrypt-query-language/releases/download/eql-2.0.2/cipherstash-encrypt.sql",
+    "eql:install": "cat sql/cipherstash-encrypt.sql | docker exec -i dynamo-postgres-1 psql postgresql://cipherstash:password@postgres:5432/cipherstash -f-"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "description": "",
+  "dependencies": {
+    "@aws-sdk/client-dynamodb": "^3.817.0",
+    "@aws-sdk/lib-dynamodb": "^3.817.0",
+    "@aws-sdk/util-dynamodb": "^3.817.0",
+    "@cipherstash/protect": "workspace:*",
+    "@cipherstash/protect-dynamodb": "workspace:*",
+    "pg": "^8.13.1"
+  },
+  "devDependencies": {
+    "@types/pg": "^8.11.10",
+    "tsx": "catalog:repo",
+    "typescript": "catalog:repo"
+  }
+}

examples/dynamo/sql/.gitkeep

@@ -0,0 +1,90 @@
+import { dynamoClient, docClient, createTable } from './common/dynamo'
+import { log } from './common/log'
+import { users, protectClient } from './common/protect'
+import { BatchGetCommand, BatchWriteCommand } from '@aws-sdk/lib-dynamodb'
+import { protectDynamoDB } from '@cipherstash/protect-dynamodb'
+
+const tableName = 'UsersBulkOperations'
+
+type User = {
+  pk: string
+  email: string
+}
+
+const main = async () => {
+  await createTable({
+    TableName: tableName,
+    AttributeDefinitions: [
+      {
+        AttributeName: 'pk',
+        AttributeType: 'S',
+      },
+    ],
+    KeySchema: [
+      {
+        AttributeName: 'pk',
+        KeyType: 'HASH',
+      },
+    ],
+  })
+
+  const protectDynamo = protectDynamoDB({
+    protectClient,
+  })
+
+  const items = [
+    {
+      // `pk` won't be encrypted because it's not included in the `users` protected table schema.
+      pk: 'user#1',
+      // `email` will be encrypted because it's included in the `users` protected table schema.
+      email: 'abc@example.com',
+    },
+    {
+      pk: 'user#2',
+      email: 'def@example.com',
+    },
+  ]
+
+  const encryptResult = await protectDynamo.bulkEncryptModels(items, users)
+
+  if (encryptResult.failure) {
+    throw new Error(`Failed to encrypt items: ${encryptResult.failure.message}`)
+  }
+
+  const putRequests = encryptResult.data.map(
+    (item: Record<string, unknown>) => ({
+      PutRequest: {
+        Item: item,
+      },
+    }),
+  )
+
+  log('encrypted items', encryptResult)
+
+  const batchWriteCommand = new BatchWriteCommand({
+    RequestItems: {
+      [tableName]: putRequests,
+    },
+  })
+
+  await dynamoClient.send(batchWriteCommand)
+
+  const batchGetCommand = new BatchGetCommand({
+    RequestItems: {
+      [tableName]: {
+        Keys: [{ pk: 'user#1' }, { pk: 'user#2' }],
+      },
+    },
+  })
+
+  const getResult = await docClient.send(batchGetCommand)
+
+  const decryptedItems = await protectDynamo.bulkDecryptModels<User>(
+    getResult.Responses?.[tableName],
+    users,
+  )
+
+  log('decrypted items', decryptedItems)
+}
+
+main()

examples/dynamo/src/bulk-operations.ts

@@ -0,0 +1,38 @@
+import {
+  CreateTableCommand,
+  DynamoDBClient,
+  type CreateTableCommandInput,
+} from '@aws-sdk/client-dynamodb'
+import { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'
+
+export const dynamoClient = new DynamoDBClient({
+  credentials: {
+    accessKeyId: 'fakeAccessKeyId',
+    secretAccessKey: 'fakeSecretAccessKey',
+  },
+  endpoint: 'http://localhost:8000',
+})
+
+export const docClient = DynamoDBDocumentClient.from(dynamoClient)
+
+// Creates a table with provisioned throughput set to 5 RCU and 5 WCU.
+// Ignores `ResourceInUseException`s if the table already exists.
+export async function createTable(
+  input: Omit<CreateTableCommandInput, 'ProvisionedThroughPut'>,
+) {
+  const command = new CreateTableCommand({
+    ProvisionedThroughput: {
+      ReadCapacityUnits: 5,
+      WriteCapacityUnits: 5,
+    },
+    ...input,
+  })
+
+  try {
+    await docClient.send(command)
+  } catch (err) {
+    if (err?.name! !== 'ResourceInUseException') {
+      throw err
+    }
+  }
+}

examples/dynamo/src/common/dynamo.ts

@@ -0,0 +1,3 @@
+export function log(description: string, data: unknown) {
+  console.log(`\n${description}:\n${JSON.stringify(data, null, 2)}`)
+}

examples/dynamo/src/common/log.ts

@@ -0,0 +1,9 @@
+import { protect, csColumn, csTable } from '@cipherstash/protect'
+
+export const users = csTable('users', {
+  email: csColumn('email').equality(),
+})
+
+export const protectClient = await protect({
+  schemas: [users],
+})