Add tbdex submodules to consume test vectors and JSON schemas (#129)

* Use git submodule for hosted schemas and test vectors

* Clean up extra test vector directory copy

* Copy test vectors into generated/

* Github Actions checkout submodule where appropriate

* Add chai-as-promised

* Use tbdex-interop-suite for submodule

* Revert "Use tbdex-interop-suite for submodule"

This reverts commit 3a5e39b.

* Sparse-checkout submodule tbdex/hosted

* Add submodule usage instructions to README

* PR comments

* Update README.md

Co-authored-by: Frank Hinek <[email protected]>

* Fix audit

* Move cloning to prereqs in README; remove chai-as-promised

* Update tbdex submod to latest main

---------

Co-authored-by: Frank Hinek <[email protected]>

.github/workflows/create-gh-release.yml

@@ -30,6 +30,8 @@ jobs:
     steps:
       - name: Checkout source
         uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+        with:
+          submodules: 'true'
 
       - name: Set up Node.js
         uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0

.github/workflows/docs-ci.yml

@@ -21,6 +21,8 @@ jobs:
     steps:
       - name: Checkout source
         uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+        with:
+          submodules: 'true'
 
       - name: install pnpm
         uses: pnpm/action-setup@v2

.github/workflows/integrity-check.yml

@@ -40,6 +40,8 @@ jobs:
     steps:
       - name: Checkout source
         uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+        with:
+          submodules: 'true'
 
       - name: install pnpm
         uses: pnpm/action-setup@v2

.github/workflows/npm-publish.yml

@@ -32,6 +32,8 @@ jobs:
     steps:
       - name: Checkout source
         uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+        with:
+          submodules: 'true'
 
       - name: install pnpm
         uses: pnpm/action-setup@v2

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "tbdex"]
+	path = tbdex
+	url = https://github.com/TBD54566975/tbdex.git

README.md

@@ -12,6 +12,20 @@ This multi-package repository uses [`pnpm` workspaces](https://pnpm.io/workspace
 
 ## Prerequisites
 
+### Cloning
+This repository uses git submodules. To clone this repo with submodules
+```sh
+git clone --recurse-submodules [email protected]:TBD54566975/tbdex-js.git
+```
+Or to add submodules after cloning
+```sh
+git submodule update --init
+```
+We recommend this config which will only checkout the files relevant to tbdex-js
+```sh
+git -C tbdex sparse-checkout set hosted
+```
+
 ### `node`
 
 This project is using `node v20.3.0`. You can verify your `node` and `npm` installation via the terminal:
@@ -93,3 +107,21 @@ Recap of the above changesets, plus the release process:
 5. Profit from the release automation:
    - [Create GH Release Workflow](./.github/workflows/create-gh-release.yml) will automatically create a new [GitHub Release](https://github.com/TBD54566975/tbdex-js/releases)
    - [NPM Publish Workflow](./.github/workflows/npm-publish.yml) will automatically publish a [new version to NPM](https://www.npmjs.com/package/@tbdex/protocol?activeTab=versions)
+
+## Working with the `tbdex` submodule
+
+### Pulling
+You may need to update the `tbdex` submodule after pulling.
+```sh
+git pull
+git submodule update
+```
+
+### Pushing
+If you have made changes to the `tbdex` submodule, you should push your changes to the `tbdex` remote as well as pushing changes to `tbdex-js`.
+```sh
+cd tbdex
+git push
+cd ..
+git push
+```

package.json

@@ -30,7 +30,8 @@
   },
   "pnpm": {
     "overrides": {
-      "browserify-sign@>=2.6.0 <=4.2.1": ">=4.2.2"
+      "browserify-sign@>=2.6.0 <=4.2.1": ">=4.2.2",
+      "follow-redirects": ">=1.15.4"
     }
   }
 }

packages/protocol/build/compile-validators.js

@@ -17,33 +17,26 @@ import standaloneCode from 'ajv/dist/standalone/index.js'
 
 import { mkdirp } from 'mkdirp'
 
-const schemaHostUrl = 'https://tbdex.dev/json-schemas'
-const schemaUrls = {
-  definitions : `${schemaHostUrl}/definitions.json`,
-  resource    : `${schemaHostUrl}/resource.schema.json`,
-  offering    : `${schemaHostUrl}/offering.schema.json`,
-  message     : `${schemaHostUrl}/message.schema.json`,
-  rfq         : `${schemaHostUrl}/rfq.schema.json`,
-  quote       : `${schemaHostUrl}/quote.schema.json`,
-  order       : `${schemaHostUrl}/order.schema.json`,
-  orderstatus : `${schemaHostUrl}/orderstatus.schema.json`,
-  close       : `${schemaHostUrl}/close.schema.json`,
-}
-
-// fetch schemas from https://tbdex.dev which pulls directly from
-// https://github.com/TBD54566975/tbdex/tree/main/json-schemas
-// TODO: cache schemas on disk
-const schemas = {}
-for (let schemaName in schemaUrls) {
-  const schemaUrl = schemaUrls[schemaName]
-  const response = await fetch(schemaUrl)
-
-  if (!response.ok) {
-    throw new Error(`failed to fetch ${schemaName} schema from ${schemaUrl}`)
-  }
-
-  const schema = await response.json()
-  schemas[schemaName] = schema
+import CloseSchema from '../../../tbdex/hosted/json-schemas/close.schema.json' assert { type: 'json' }
+import DefinitionsSchema from '../../../tbdex/hosted/json-schemas/definitions.json' assert { type: 'json' }
+import OfferingSchema from '../../../tbdex/hosted/json-schemas/offering.schema.json' assert { type: 'json' }
+import MessageSchema from '../../../tbdex/hosted/json-schemas/message.schema.json' assert { type: 'json' }
+import OrderSchema from '../../../tbdex/hosted/json-schemas/order.schema.json' assert { type: 'json' }
+import OrderstatusSchema from '../../../tbdex/hosted/json-schemas/orderstatus.schema.json' assert { type: 'json' }
+import QuoteSchema from '../../../tbdex/hosted/json-schemas/quote.schema.json' assert { type: 'json' }
+import ResourceSchema from '../../../tbdex/hosted/json-schemas/resource.schema.json' assert { type: 'json' }
+import RfqSchema from '../../../tbdex/hosted/json-schemas/rfq.schema.json' assert { type: 'json' }
+
+const schemas = {
+  close: CloseSchema,
+  definitions: DefinitionsSchema,
+  offering: OfferingSchema,
+  message: MessageSchema,
+  order: OrderSchema,
+  orderstatus: OrderstatusSchema,
+  quote: QuoteSchema,
+  resource: ResourceSchema,
+  rfq: RfqSchema,
 }
 
 const validator = new Ajv({ code: { source: true, esm: true } })

packages/protocol/package.json

@@ -85,12 +85,13 @@
     "build:esm": "rimraf dist/esm dist/types && tsc",
     "build:cjs": "rimraf dist/cjs && tsc -p tsconfig.cjs.json && echo '{\"type\": \"commonjs\"}' > ./dist/cjs/package.json",
     "build:browser": "rimraf dist/browser.mjs dist/browser.js && node build/bundles.js",
-    "test:node": "rimraf tests/compiled && pnpm compile-validators && tsc -p tests/tsconfig.json && mocha",
-    "test:browser": "pnpm compile-validators && karma start karma.conf.cjs",
-    "build": "pnpm clean && pnpm compile-validators && pnpm build:esm && pnpm build:cjs && pnpm build:browser",
+    "cp:test-vectors": "mkdir -p generated/test-vectors/ && cp -r ../../tbdex/hosted/test-vectors/protocol/vectors/ generated/test-vectors",
+    "test:node": "rimraf tests/compiled && pnpm compile-validators && pnpm cp:test-vectors && tsc -p tests/tsconfig.json && mocha",
+    "test:browser": "pnpm compile-validators && pnpm cp:test-vectors && karma start karma.conf.cjs",
+    "build": "pnpm clean && pnpm compile-validators && pnpm cp:test-vectors && pnpm build:esm && pnpm build:cjs && pnpm build:browser",
     "lint": "eslint . --ext .ts --max-warnings 0",
     "lint:fix": "eslint . --ext .ts --fix",
     "docs": "pnpm build:esm && typedoc --plugin typedoc-plugin-markdown --out docs src/main.ts",
-    "try": "pnpm compile-validators && pnpm build:esm && node dist/esm/src/try.js"
+    "try": "pnpm compile-validators && pnpm cp:test-vectors && pnpm build:esm && node dist/esm/src/try.js"
   }
 }

packages/protocol/src/message-kinds/rfq.ts

@@ -159,7 +159,9 @@ export class Rfq extends Message<'rfq'> {
    */
   toJSON() {
     const jsonMessage = super.toJSON()
-    jsonMessage['private'] = this._private
+    if (this._private) {
+      jsonMessage['private'] = this._private
+    }
 
     return jsonMessage
   }

packages/protocol/tests/test-vectors.spec.ts

@@ -0,0 +1,42 @@
+import { expect } from 'chai'
+import { Close, Order, OrderStatus, Quote, Rfq } from '../src/message-kinds/index.js'
+import { Offering } from '../src/resource-kinds/index.js'
+import ParseClose from '../generated/test-vectors/parse-close.json' assert { type: 'json' }
+import ParseOffering from '../generated/test-vectors/parse-offering.json' assert { type: 'json' }
+import ParseOrder from '../generated/test-vectors/parse-order.json' assert { type: 'json' }
+import ParseOrderStatus from '../generated/test-vectors/parse-orderstatus.json' assert { type: 'json' }
+import ParseQuote from '../generated/test-vectors/parse-quote.json' assert { type: 'json' }
+import ParseRfq from '../generated/test-vectors/parse-rfq.json' assert { type: 'json' }
+
+describe('Test vectors: parse and serialize', () => {
+  it('parse-close.json', async () => {
+    const close = await Close.parse(ParseClose.input)
+    expect(close.toJSON()).to.deep.eq(ParseClose.output)
+  })
+
+  it('parse-offering.json', async() => {
+    const offering = await Offering.parse(ParseOffering.input)
+    expect(offering.toJSON()).to.deep.eq(ParseOffering.output)
+  })
+
+  it('parse-order.json', async () => {
+    const order = await Order.parse(ParseOrder.input)
+    expect(order.toJSON()).to.deep.eq(ParseOrder.output)
+  })
+
+  it('parse-orderstatus.json', async () => {
+    const orderstatus = await OrderStatus.parse(ParseOrderStatus.input)
+    expect(orderstatus.toJSON()).to.deep.eq(ParseOrderStatus.output)
+  })
+
+  it('parse-quote.json', async () => {
+    const quote = await Quote.parse(ParseQuote.input)
+    expect(quote.toJSON()).to.deep.eq(ParseQuote.output)
+  })
+
+  it('parse-rfq.json', async () => {
+    const rfq = await Rfq.parse(ParseRfq.input)
+    expect(rfq.toJSON()).to.deep.eq(ParseRfq.output)
+  })
+})
+