-
Notifications
You must be signed in to change notification settings - Fork 7
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
To support users in languages that do not have existing UCAN invocation implementations, we are going to launch a bridge that allows them to make simple HTTP requests with JSON bodies that we transform into proper UCAN invocations. This follows the specification here: storacha/specs#112 Values for authorization headers can be generated using the `bridge generate-tokens` w3cli command proposed here: storacha/w3cli#175 - [x] factor core bridge logic out to a separate library (filed as #338) - [x] factor HTTP input wrangling out to a separate function - [x] rename `UPLOAD_API_DID` and `ACCESS_SERVICE_URL` environment variables to `W3UP_SERVICE_DID` and `W3UP_SERVICE_URL` (filed as #337) - [x] add tests - [x] expand and formalize bridge specification, move it to the specs repo (done - storacha/specs#112) - [x] document response format
- Loading branch information
Showing
10 changed files
with
886 additions
and
133 deletions.
There are no files selected for viewing
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,176 @@ | ||
| import { fetch } from '@web-std/fetch' | ||
| import { base64url } from 'multiformats/bases/base64' | ||
| import * as Signature from '@ipld/dag-ucan/signature' | ||
| import { ed25519 } from '@ucanto/principal' | ||
| import { CBOR } from '@ucanto/core' | ||
| import * as dagJSON from '@ipld/dag-json' | ||
| import pWaitFor from 'p-wait-for' | ||
| import { test } from './helpers/context.js' | ||
| import { | ||
| getApiEndpoint, | ||
| getDynamoDb | ||
| } from './helpers/deployment.js' | ||
| import { randomFile } from './helpers/random.js' | ||
| import { createMailSlurpInbox, setupNewClient } from './helpers/up-client.js' | ||
|
|
||
| test.before(t => { | ||
| t.context = { | ||
| apiEndpoint: getApiEndpoint(), | ||
| metricsDynamo: getDynamoDb('admin-metrics'), | ||
| spaceMetricsDynamo: getDynamoDb('space-metrics'), | ||
| rateLimitsDynamo: getDynamoDb('rate-limit') | ||
| } | ||
| }) | ||
|
|
||
| /** | ||
| * | ||
| * @param {string} apiEndpoint | ||
| * @returns | ||
| */ | ||
| async function getServicePublicKey(apiEndpoint) { | ||
| const serviceInfoResponse = await fetch(`${apiEndpoint}/version`) | ||
| const { publicKey } = await serviceInfoResponse.json() | ||
| return publicKey | ||
| } | ||
|
|
||
| /** | ||
| * | ||
| * @param {import('@web3-storage/w3up-client').Client} client | ||
| * @param {[import('@ucanto/interface').Capability, ...import('@ucanto/interface').Capability[]]} capabilities | ||
| * @param {number} expiration | ||
| * @param {string | undefined} password | ||
| * @returns | ||
| */ | ||
| async function generateAuthHeaders(client, capabilities, expiration, password = 'i am the very model of a modern major general') { | ||
| const coupon = await client.coupon.issue({ | ||
| capabilities, | ||
| expiration, | ||
| password | ||
| }) | ||
|
|
||
| const { ok: bytes, error } = await coupon.archive() | ||
| if (!bytes) { | ||
| console.error(error) | ||
| throw new Error(error.message) | ||
| } | ||
| return { | ||
| 'X-Auth-Secret': base64url.encode(new TextEncoder().encode(password)), | ||
| 'Authorization': base64url.encode(bytes) | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * | ||
| * @param {import('./helpers/context.js').Context} context | ||
| * @param {import('@web3-storage/w3up-client').Client} client | ||
| * @param {[import('@ucanto/interface').Capability, ...import('@ucanto/interface').Capability[]]} capabilities | ||
| * @param {number} expiration | ||
| * @param {any} requestBody | ||
| */ | ||
| async function makeBridgeRequest(context, client, capabilities, expiration, requestBody) { | ||
| return fetch(`${context.apiEndpoint}/bridge`, { | ||
| method: 'POST', | ||
| headers: { | ||
| 'content-type': 'application/json', | ||
| ...await generateAuthHeaders( | ||
| client, | ||
| capabilities, | ||
| expiration | ||
| ) | ||
| }, | ||
| body: dagJSON.stringify(requestBody), | ||
| }) | ||
| } | ||
|
|
||
| test('the bridge can make various types of requests', async t => { | ||
| const inbox = await createMailSlurpInbox() | ||
| const client = await setupNewClient(t.context.apiEndpoint, { inbox }) | ||
| const spaceDID = client.currentSpace()?.did() | ||
| if (!spaceDID) { | ||
| t.fail('client was set up but does not have a currentSpace - this is weird!') | ||
| return | ||
| } | ||
|
|
||
| const response = await makeBridgeRequest( | ||
| t.context, client, | ||
| [{ can: 'upload/list', with: spaceDID }], | ||
| Date.now() + (1000 * 60), | ||
| { | ||
| tasks: [ | ||
| ['upload/list', spaceDID, {}] | ||
| ] | ||
| } | ||
| ) | ||
|
|
||
| t.deepEqual(response.status, 200) | ||
| const receipts = dagJSON.parse(await response.text()) | ||
| t.deepEqual(receipts.length, 1) | ||
| t.assert(receipts[0].p.out.ok) | ||
| const result = receipts[0].p.out.ok | ||
| t.deepEqual(result.results, []) | ||
| t.deepEqual(result.size, 0) | ||
|
|
||
|
|
||
| // verify that uploading a file changes the upload/list response | ||
| // upload a file and wait for it to show up | ||
| const file = await randomFile(42) | ||
| const fileLink = await client.uploadFile(file) | ||
| let secondReceipts | ||
| await pWaitFor(async () => { | ||
| const secondResponse = await makeBridgeRequest( | ||
| t.context, client, | ||
| [{ can: 'upload/list', with: spaceDID }], | ||
| Date.now() + (1000 * 60), | ||
| { | ||
| tasks: [ | ||
| ['upload/list', spaceDID, {}] | ||
| ] | ||
| } | ||
| ) | ||
| const response = await secondResponse.text() | ||
| secondReceipts = dagJSON.parse(response) | ||
| const result = secondReceipts[0].p.out.ok.results[0] | ||
| return Boolean(result && result.root.equals(fileLink)) | ||
| }, { | ||
| interval: 100, | ||
| }) | ||
|
|
||
| t.assert(secondReceipts[0].p.out.ok) | ||
| t.deepEqual(secondReceipts[0].p.out.ok.results.length, 1) | ||
| // assert that the first item in the list is the item we just uploaded | ||
| t.deepEqual(secondReceipts[0].p.out.ok.results[0].root.toString(), fileLink.toString()) | ||
|
|
||
|
|
||
| // verify expired requests fail | ||
| const expiredResponse = await makeBridgeRequest( | ||
| t.context, client, | ||
| [{ can: 'upload/list', with: spaceDID }], | ||
| 0, | ||
| { | ||
| tasks: [ | ||
| ['upload/list', spaceDID, {}] | ||
| ] | ||
| } | ||
| ) | ||
| const expiredReceipts = dagJSON.parse(await expiredResponse.text()) | ||
| t.assert(expiredReceipts[0].p.out.error) | ||
|
|
||
|
|
||
| // ensure response is verifiable | ||
| const payload = receipts[0].p | ||
| const signature = Signature.view(receipts[0].s) | ||
|
|
||
| // we need to get the service key out of band because the receipt | ||
| // has a did:web as it's `iss` field but local development environments | ||
| // use the `did:web:staging` DID backed by different keys and therefore aren't | ||
| // resolvable using the normal `did:web` resolution algorithm | ||
| const publicKey = await getServicePublicKey(t.context.apiEndpoint) | ||
| const verifier = ed25519.Verifier.parse(publicKey) | ||
| const verification = await signature.verify(verifier, CBOR.encode(payload)) | ||
| if (verification.error) { | ||
| t.fail(verification.error.message) | ||
| console.error(verification.error) | ||
| } | ||
| t.assert(verification.ok) | ||
| }) | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -4,6 +4,7 @@ import pWaitFor from 'p-wait-for' | |
| import { HeadObjectCommand } from '@aws-sdk/client-s3' | ||
| import { PutItemCommand } from '@aws-sdk/client-dynamodb' | ||
| import { marshall } from '@aws-sdk/util-dynamodb' | ||
| import * as DidMailto from '@web3-storage/did-mailto' | ||
|
|
||
| import { METRICS_NAMES, SPACE_METRICS_NAMES } from '../upload-api/constants.js' | ||
|
|
||
|
|
@@ -104,7 +105,7 @@ test('authorizations can be blocked by email or domain', async t => { | |
|
|
||
| // it would be nice to use t.throwsAsync here, but that doesn't work with errors that aren't exceptions: https://github.com/avajs/ava/issues/2517 | ||
| try { | ||
| await client.authorize('[email protected]') | ||
| await client.login('[email protected]') | ||
| t.fail('authorize should fail with a blocked domain') | ||
| } catch (e) { | ||
| t.is(e.name, 'AccountBlocked') | ||
|
|
@@ -121,11 +122,12 @@ test('w3infra integration flow', async t => { | |
| if (!spaceDid) { | ||
| throw new Error('Testing space DID must be set') | ||
| } | ||
| const account = client.accounts()[DidMailto.fromEmail(inbox.email)] | ||
|
|
||
| // it should be possible to create more than one space | ||
| const space = await client.createSpace("2nd space") | ||
| await client.setCurrentSpace(space.did()) | ||
| await client.registerSpace(inbox.email) | ||
| await account.provision(space.did()) | ||
| await space.save() | ||
|
|
||
| // Get space metrics before upload | ||
| const spaceBeforeUploadAddMetrics = await getSpaceMetrics(t, spaceDid, SPACE_METRICS_NAMES.UPLOAD_ADD_TOTAL) | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,56 @@ | ||
| import { ReadableStream } from "@web-std/stream" | ||
|
|
||
| /** | ||
| * Stream utilities adapted from https://stackoverflow.com/questions/40385133/retrieve-data-from-a-readablestream-object | ||
| */ | ||
|
|
||
| /** | ||
| * | ||
| * @param {Uint8Array[]} chunks | ||
| * @returns {Uint8Array} | ||
| */ | ||
| function concatArrayBuffers(chunks) { | ||
| const result = new Uint8Array(chunks.reduce((a, c) => a + c.length, 0)) | ||
| let offset = 0 | ||
| for (const chunk of chunks) { | ||
| result.set(chunk, offset) | ||
| offset += chunk.length | ||
| } | ||
| return result | ||
| } | ||
|
|
||
| /** | ||
| * | ||
| * @param {ReadableStream<Uint8Array>} stream | ||
| * @returns {Promise<Uint8Array>} | ||
| */ | ||
| export async function streamToArrayBuffer(stream) { | ||
| const chunks = [] | ||
| const reader = stream.getReader() | ||
| while (true) { | ||
| const { done, value } = await reader.read() | ||
| if (done) { | ||
| break | ||
| } else { | ||
| chunks.push(value) | ||
| } | ||
| } | ||
| return concatArrayBuffers(chunks) | ||
| } | ||
|
|
||
| /** | ||
| * | ||
| * @param {string} str | ||
| * @returns {ReadableStream<Uint8Array>} | ||
| */ | ||
| export function stringToStream(str) { | ||
| const encoder = new TextEncoder() | ||
| const uint8Array = encoder.encode(str) | ||
|
|
||
| return new ReadableStream({ | ||
| start(controller) { | ||
| controller.enqueue(uint8Array) | ||
| controller.close() | ||
| } | ||
| }); | ||
| } |
Oops, something went wrong.