From 61782402bff06fcdadde91435cd64dc8cd3779c1 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Wed, 3 Apr 2024 11:03:40 +0200 Subject: [PATCH] feat: http put --- packages/capabilities/src/blob.js | 41 ++ packages/capabilities/src/index.js | 2 + packages/capabilities/src/types.ts | 12 + packages/capabilities/src/ucan.js | 53 ++ packages/capabilities/src/utils.js | 2 +- packages/filecoin-api/src/storefront/api.ts | 4 +- packages/filecoin-api/test/context/mocks.js | 4 +- packages/filecoin-api/test/context/service.js | 13 +- .../filecoin-api/test/events/storefront.js | 50 +- packages/filecoin-api/test/types.ts | 4 +- packages/upload-api/src/blob/add.js | 47 +- packages/upload-api/src/service.js | 2 +- packages/upload-api/src/types.ts | 13 +- packages/upload-api/src/ucan.js | 2 + packages/upload-api/src/ucan/conclude.js | 14 + packages/upload-api/test/handlers/blob.js | 657 +++++++++--------- packages/upload-api/test/helpers/context.js | 2 + 17 files changed, 567 insertions(+), 355 deletions(-) create mode 100644 packages/upload-api/src/ucan/conclude.js diff --git a/packages/capabilities/src/blob.js b/packages/capabilities/src/blob.js index 3a96a7e3e..331dd893b 100644 --- a/packages/capabilities/src/blob.js +++ b/packages/capabilities/src/blob.js @@ -188,6 +188,47 @@ export const allocate = capability({ }, }) +/** + * `http/put` capability invocation MAY be performed by any agent on behalf of the subject. + * The `blob/add` provider MUST add `/http/put` effect and capture private key of the + * `subject` in the `meta` field so that any agent could perform it. + */ +export const put = capability({ + can: 'http/put', + /** + * DID of the (memory) space where Blob is intended to + * be stored. + */ + with: SpaceDID, + nb: Schema.struct({ + /** + * A multihash digest of the blob payload bytes, uniquely identifying blob. + */ + content: Schema.bytes(), + /** + * Blob to accept. + */ + address: Schema.struct({ + /** + * HTTP(S) location that can receive blob content via HTTP PUT request. + */ + url: Schema.string(), + /** + * HTTP headers. + */ + headers: Schema.unknown(), + }).optional(), + }), + derives: (claim, from) => { + return ( + and(equalContent(claim, from)) || + and(equal(claim.nb.address?.url, from.nb.address, 'url')) || + and(equal(claim.nb.address?.headers, from.nb.address, 'headers')) || + ok({}) + ) + }, +}) + /** * `blob/accept` capability invocation should either succeed when content is * delivered on allocated address or fail if no content is allocation expires diff --git a/packages/capabilities/src/index.js b/packages/capabilities/src/index.js index 8c423bfe1..fc7e4bc7c 100644 --- a/packages/capabilities/src/index.js +++ b/packages/capabilities/src/index.js @@ -64,6 +64,7 @@ export const abilitiesAsStrings = [ Access.access.can, Access.authorize.can, UCAN.attest.can, + UCAN.conclude.can, Customer.get.can, Consumer.has.can, Consumer.get.can, @@ -92,6 +93,7 @@ export const abilitiesAsStrings = [ Blob.remove.can, Blob.list.can, Blob.serviceBlob.can, + Blob.put.can, Blob.allocate.can, Blob.accept.can, ] diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts index 9944af005..64c99825c 100644 --- a/packages/capabilities/src/types.ts +++ b/packages/capabilities/src/types.ts @@ -446,6 +446,7 @@ export type BlobAdd = InferInvokedCapability export type BlobRemove = InferInvokedCapability export type BlobList = InferInvokedCapability export type ServiceBlob = InferInvokedCapability +export type BlobPut = InferInvokedCapability export type BlobAllocate = InferInvokedCapability export type BlobAccept = InferInvokedCapability @@ -605,6 +606,7 @@ export interface UploadListSuccess extends ListResponse {} export type UCANRevoke = InferInvokedCapability export type UCANAttest = InferInvokedCapability +export type UCANConclude = InferInvokedCapability export interface Timestamp { /** @@ -615,6 +617,8 @@ export interface Timestamp { export type UCANRevokeSuccess = Timestamp +export type UCANConcludeSuccess = Timestamp + /** * Error is raised when `UCAN` being revoked is not supplied or it's proof chain * leading to supplied `scope` is not supplied. @@ -653,6 +657,12 @@ export type UCANRevokeFailure = | UnauthorizedRevocation | RevocationsStoreFailure +export interface InvocationNotFound extends Ucanto.Failure { + name: 'InvocationNotFound' +} + +export type UCANConcludeFailure = InvocationNotFound | Ucanto.Failure + // Admin export type Admin = InferInvokedCapability export type AdminUploadInspect = InferInvokedCapability< @@ -761,6 +771,7 @@ export type ServiceAbilityArray = [ Access['can'], AccessAuthorize['can'], UCANAttest['can'], + UCANConclude['can'], CustomerGet['can'], ConsumerHas['can'], ConsumerGet['can'], @@ -789,6 +800,7 @@ export type ServiceAbilityArray = [ BlobRemove['can'], BlobList['can'], ServiceBlob['can'], + BlobPut['can'], BlobAllocate['can'], BlobAccept['can'] ] diff --git a/packages/capabilities/src/ucan.js b/packages/capabilities/src/ucan.js index fe38b757e..83ce637fa 100644 --- a/packages/capabilities/src/ucan.js +++ b/packages/capabilities/src/ucan.js @@ -74,6 +74,59 @@ export const revoke = capability({ ), }) +/** + * `ucan/conclude` capability represents a receipt using a special UCAN capability. + * + * The UCAN invocation specification defines receipt record, that is cryptographically + * signed description of the invocation output and requested effects. Receipt + * structure is very similar to UCAN except it has no notion of expiry nor it is + * possible to delegate ability to issue receipt to another principal. + */ +export const conclude = capability({ + can: 'ucan/conclude', + /** + * DID of the principal representing the Conclusion Authority. + * MUST be the DID of the audience of the ran invocation. + */ + with: Schema.did(), + // TODO: Should this just have bytes? + nb: Schema.struct({ + /** + * A link to the UCAN invocation that this receipt is for. + */ + ran: UCANLink, + /** + * The value output of the invocation in Result format. + */ + out: Schema.unknown(), + /** + * Tasks that the invocation would like to enqueue. + */ + next: Schema.array(UCANLink), + /** + * Additional data about the receipt + */ + meta: Schema.unknown(), + /** + * The UTC Unix timestamp at which the Receipt was issued + */ + time: Schema.integer(), + }), + derives: (claim, from) => + // With field MUST be the same + and(equalWith(claim, from)) ?? + // invocation MUST be the same + and(checkLink(claim.nb.ran, from.nb.ran, 'nb.ran')) ?? + // value output MUST be the same + and(equal(claim.nb.out, from.nb.out, 'nb.out')) ?? + // tasks to enqueue MUST be the same + and(equal(claim.nb.next, from.nb.next, 'nb.next')) ?? + // additional data MUST be the same + and(equal(claim.nb.meta, from.nb.meta, 'nb.meta')) ?? + // the receipt issue time MUST be the same + equal(claim.nb.time, from.nb.time, 'nb.time'), +}) + /** * Issued by trusted authority (usually the one handling invocation) that attest * that specific UCAN delegation has been considered authentic. diff --git a/packages/capabilities/src/utils.js b/packages/capabilities/src/utils.js index 96c63c352..0db700c48 100644 --- a/packages/capabilities/src/utils.js +++ b/packages/capabilities/src/utils.js @@ -122,7 +122,7 @@ export const equalBlob = (claimed, delegated) => { } /** - * @template {Types.ParsedCapability<"blob/add"|"blob/remove"|"blob/allocate"|"blob/accept", Types.URI<'did:'>, {content: Uint8Array}>} T + * @template {Types.ParsedCapability<"blob/add"|"blob/remove"|"blob/allocate"|"blob/accept"|"http/put", Types.URI<'did:'>, {content: Uint8Array}>} T * @param {T} claimed * @param {T} delegated * @returns {Types.Result<{}, Types.Failure>} diff --git a/packages/filecoin-api/src/storefront/api.ts b/packages/filecoin-api/src/storefront/api.ts index aa4b4d712..6fb3584e2 100644 --- a/packages/filecoin-api/src/storefront/api.ts +++ b/packages/filecoin-api/src/storefront/api.ts @@ -124,7 +124,9 @@ export interface ClaimsClientContext { */ claimsService: { invocationConfig: ClaimsInvocationConfig - connection: ConnectionView + connection: ConnectionView< + import('@web3-storage/content-claims/server/service/api').Service + > } } diff --git a/packages/filecoin-api/test/context/mocks.js b/packages/filecoin-api/test/context/mocks.js index 8246b3e48..ff63b4df8 100644 --- a/packages/filecoin-api/test/context/mocks.js +++ b/packages/filecoin-api/test/context/mocks.js @@ -32,8 +32,8 @@ export function mockService(impl) { info: withCallParams(impl.deal?.info ?? notImplemented), }, assert: { - equals: withCallParams(impl.assert?.equals ?? notImplemented) - } + equals: withCallParams(impl.assert?.equals ?? notImplemented), + }, } } diff --git a/packages/filecoin-api/test/context/service.js b/packages/filecoin-api/test/context/service.js index 03ede702b..997903253 100644 --- a/packages/filecoin-api/test/context/service.js +++ b/packages/filecoin-api/test/context/service.js @@ -217,12 +217,15 @@ export function getMockService() { }), }, assert: { - equals: Server.provide(Assert.equals, async ({ capability, invocation }) => { - return { - ok: {} + equals: Server.provide( + Assert.equals, + async ({ capability, invocation }) => { + return { + ok: {}, + } } - }) - } + ), + }, }) } diff --git a/packages/filecoin-api/test/events/storefront.js b/packages/filecoin-api/test/events/storefront.js index 3f55699ce..58c9d464c 100644 --- a/packages/filecoin-api/test/events/storefront.js +++ b/packages/filecoin-api/test/events/storefront.js @@ -9,7 +9,7 @@ import * as StorefrontEvents from '../../src/storefront/events.js' import { StoreOperationErrorName, UnexpectedStateErrorName, - BlobNotFoundErrorName + BlobNotFoundErrorName, } from '../../src/errors.js' import { randomCargo, randomAggregate } from '../utils.js' @@ -53,23 +53,24 @@ export const test = { assert.ok(hasStoredPiece.ok) assert.equal(hasStoredPiece.ok?.status, 'submitted') }, - 'handles filecoin submit messages with error if blob of content is not stored': async (assert, context) => { - // Generate piece for test - const [cargo] = await randomCargo(1, 128) - - // Store piece into store - const message = { - piece: cargo.link.link(), - content: cargo.content.link(), - group: context.id.did(), - } + 'handles filecoin submit messages with error if blob of content is not stored': + async (assert, context) => { + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + + // Store piece into store + const message = { + piece: cargo.link.link(), + content: cargo.content.link(), + group: context.id.did(), + } - // Handle message - const handledMessageRes = - await StorefrontEvents.handleFilecoinSubmitMessage(context, message) - assert.ok(handledMessageRes.error) - assert.equal(handledMessageRes.error?.name, BlobNotFoundErrorName) - }, + // Handle message + const handledMessageRes = + await StorefrontEvents.handleFilecoinSubmitMessage(context, message) + assert.ok(handledMessageRes.error) + assert.equal(handledMessageRes.error?.name, BlobNotFoundErrorName) + }, 'handles filecoin submit messages deduping when stored': async ( assert, context @@ -255,7 +256,10 @@ export const test = { ) ) }, - 'handles piece insert event to issue equivalency claims successfully': async (assert, context) => { + 'handles piece insert event to issue equivalency claims successfully': async ( + assert, + context + ) => { // Generate piece for test const [cargo] = await randomCargo(1, 128) @@ -274,10 +278,11 @@ export const test = { } // Handle message - const handledMessageRes = await StorefrontEvents.handlePieceInsertToEquivalencyClaim( - context, - pieceRecord - ) + const handledMessageRes = + await StorefrontEvents.handlePieceInsertToEquivalencyClaim( + context, + pieceRecord + ) assert.ok(handledMessageRes.ok) // Verify invocation // @ts-expect-error not typed hooks @@ -294,7 +299,6 @@ export const test = { context.service.assert?.equals?._params[0].nb.equals ) ) - }, 'handles piece status update event successfully': async (assert, context) => { // Generate piece for test diff --git a/packages/filecoin-api/test/types.ts b/packages/filecoin-api/test/types.ts index 03079ac1e..d164eb73e 100644 --- a/packages/filecoin-api/test/types.ts +++ b/packages/filecoin-api/test/types.ts @@ -48,6 +48,8 @@ export interface StorefrontTestEventsContext piece: Partial aggregate: Partial deal: Partial - assert: Partial + assert: Partial< + import('@web3-storage/content-claims/server/service/api').AssertService + > }> } diff --git a/packages/upload-api/src/blob/add.js b/packages/upload-api/src/blob/add.js index 3c9825a36..9fc62c177 100644 --- a/packages/upload-api/src/blob/add.js +++ b/packages/upload-api/src/blob/add.js @@ -1,4 +1,5 @@ import * as Server from '@ucanto/server' +import { ed25519 } from '@ucanto/principal' import * as Blob from '@web3-storage/capabilities/blob' import * as API from '../types.js' @@ -20,12 +21,19 @@ export function blobAddProvider(context) { if (blob.size > maxUploadSize) { return { - error: new BlobItemSizeExceeded(maxUploadSize) + error: new BlobItemSizeExceeded(maxUploadSize), } } + const putSubject = await ed25519.derive(blob.content.slice(0, 32)) + const facts = Object.entries(putSubject.toArchive().keys).map( + ([key, value]) => ({ + did: key, + bytes: value, + }) + ) + // Create effects for receipt - // TODO: needs HTTP/PUT receipt const blobAllocate = Blob.allocate .invoke({ issuer: id, @@ -38,6 +46,17 @@ export function blobAddProvider(context) { }, expiration: Infinity }) + const blobPut = Blob.put + .invoke({ + issuer: putSubject, + audience: putSubject, + with: putSubject.toDIDKey(), + nb: { + content: blob.content, + }, + facts, + expiration: Infinity, + }) const blobAccept = Blob.accept .invoke({ issuer: id, @@ -49,11 +68,27 @@ export function blobAddProvider(context) { }, expiration: Infinity, }) - const [allocatefx, acceptfx] = await Promise.all([ + const [allocatefx, putfx, acceptfx] = await Promise.all([ + // 1. System attempts to allocate memory in user space for the blob. blobAllocate.delegate(), + // 2. System requests user agent (or anyone really) to upload the content + // corresponding to the blob + // via HTTP PUT to given location. + blobPut.delegate(), + // 3. System will attempt to accept uploaded content that matches blob + // multihash and size. blobAccept.delegate(), ]) + // TODO: store `http/put` invocation + const archiveRes = await putfx.archive() + if (archiveRes.error) { + return { + error: archiveRes.error + } + } + // write to store + // Schedule allocation if not allocated const allocated = await allocationStorage.exists(space, blob.content) if (!allocated.ok) { @@ -72,7 +107,11 @@ export function blobAddProvider(context) { 'await/ok': acceptfx.link(), }, }) - return result.fork(allocatefx.link()).join(acceptfx.link()) + // TODO: not pass links, but delegation + return result + .fork(allocatefx.link()) + .fork(putfx.link()) + .join(acceptfx.link()) }, }) } diff --git a/packages/upload-api/src/service.js b/packages/upload-api/src/service.js index bb9c503bc..e650e13d0 100644 --- a/packages/upload-api/src/service.js +++ b/packages/upload-api/src/service.js @@ -10,6 +10,6 @@ export function createService(context) { blob: { allocate: blobAllocateProvider(context), accept: blobAcceptProvider(context), - } + }, } } diff --git a/packages/upload-api/src/types.ts b/packages/upload-api/src/types.ts index 073d1a972..03291dd18 100644 --- a/packages/upload-api/src/types.ts +++ b/packages/upload-api/src/types.ts @@ -336,7 +336,8 @@ export type StoreServiceContext = SpaceServiceContext & { export type UploadServiceContext = ConsumerServiceContext & SpaceServiceContext & - RevocationServiceContext & { + RevocationServiceContext & + ConcludeServiceContext & { signer: EdSigner.Signer uploadTable: UploadTable dudewhereBucket: DudewhereBucket @@ -399,6 +400,13 @@ export interface RevocationServiceContext { revocationsStorage: RevocationsStorage } +export interface ConcludeServiceContext { + /** + * Stores receipts for tasks. + */ + receiptStorage: ReceiptStorage +} + export interface PlanServiceContext { plansStorage: PlansStorage } @@ -417,6 +425,7 @@ export interface ServiceContext SpaceServiceContext, StoreServiceContext, BlobServiceContext, + ConcludeServiceContext, SubscriptionServiceContext, RateLimitServiceContext, RevocationServiceContext, @@ -470,6 +479,8 @@ export interface BlobStorage { > } +export interface ReceiptStorage {} + export interface CarStoreBucket { has: (link: UnknownLink) => Promise createUploadUrl: ( diff --git a/packages/upload-api/src/ucan.js b/packages/upload-api/src/ucan.js index 9bf9b14a6..47bc64df8 100644 --- a/packages/upload-api/src/ucan.js +++ b/packages/upload-api/src/ucan.js @@ -1,4 +1,5 @@ import { ucanRevokeProvider } from './ucan/revoke.js' +import { ucanConcludeProvider } from './ucan/conclude.js' import * as API from './types.js' /** @@ -6,6 +7,7 @@ import * as API from './types.js' */ export const createService = (context) => { return { + conclude: ucanConcludeProvider(context), revoke: ucanRevokeProvider(context), } } diff --git a/packages/upload-api/src/ucan/conclude.js b/packages/upload-api/src/ucan/conclude.js new file mode 100644 index 000000000..e8b49102d --- /dev/null +++ b/packages/upload-api/src/ucan/conclude.js @@ -0,0 +1,14 @@ +import { provide } from '@ucanto/server' +import { conclude } from '@web3-storage/capabilities/ucan' +import * as API from '../types.js' + +/** + * @param {API.ConcludeServiceContext} context + * @returns {API.ServiceMethod} + */ +export const ucanConcludeProvider = ({ receiptStorage }) => + provide(conclude, async ({ capability, invocation }) => { + return { + ok: { time: Date.now() }, + } + }) diff --git a/packages/upload-api/test/handlers/blob.js b/packages/upload-api/test/handlers/blob.js index acee6a788..2fc20d020 100644 --- a/packages/upload-api/test/handlers/blob.js +++ b/packages/upload-api/test/handlers/blob.js @@ -119,7 +119,7 @@ export const test = { nb: { blob: { content, - size + size, }, }, proofs: [proof], @@ -156,7 +156,7 @@ export const test = { nb: { blob: { content, - size + size, }, }, proofs: [proof], @@ -170,7 +170,7 @@ export const test = { nb: { blob: { content, - size + size, }, cause: (await blobAddInvocation.delegate()).cid, space: spaceDid, @@ -187,13 +187,18 @@ export const test = { assert.ok(blobAllocate.out.ok.address) assert.ok(blobAllocate.out.ok.address?.headers) assert.ok(blobAllocate.out.ok.address?.url) - assert.equal(blobAllocate.out.ok.address?.headers?.['content-length'], String(size)) + assert.equal( + blobAllocate.out.ok.address?.headers?.['content-length'], + String(size) + ) assert.deepEqual( blobAllocate.out.ok.address?.headers?.['x-amz-checksum-sha256'], base64pad.baseEncode(digest) ) - const url = blobAllocate.out.ok.address?.url && new URL(blobAllocate.out.ok.address?.url) + const url = + blobAllocate.out.ok.address?.url && + new URL(blobAllocate.out.ok.address?.url) if (!url) { throw new Error('Expected presigned url in response') } @@ -226,325 +231,345 @@ export const test = { assert.equal(goodPut.status, 200, await goodPut.text()) }, - 'blob/allocate does not allocate more space to already allocated content': async (assert, context) => { - const { proof, spaceDid } = await registerSpace(alice, context) - // prepare data - const data = new Uint8Array([11, 22, 34, 44, 55]) - const multihash = await sha256.digest(data) - const content = multihash.bytes - const size = data.byteLength - - // create service connection - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // create `blob/add` invocation - const blobAddInvocation = BlobCapabilities.add.invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { - blob: { - content, - size + 'blob/allocate does not allocate more space to already allocated content': + async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const content = multihash.bytes + const size = data.byteLength + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // create `blob/add` invocation + const blobAddInvocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + content, + size, + }, }, - }, - proofs: [proof], - }) - - // invoke `service/blob/allocate` - const serviceBlobAllocate = BlobCapabilities.allocate.invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { - blob: { - content, - size + proofs: [proof], + }) + + // invoke `service/blob/allocate` + const serviceBlobAllocate = BlobCapabilities.allocate.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + content, + size, + }, + cause: (await blobAddInvocation.delegate()).cid, + space: spaceDid, }, - cause: (await blobAddInvocation.delegate()).cid, - space: spaceDid, - }, - proofs: [proof], - }) - const blobAllocate = await serviceBlobAllocate.execute(connection) - if (!blobAllocate.out.ok) { - throw new Error('invocation failed', { cause: blobAllocate }) - } - - // second blob allocate invocation - const secondBlobAllocate = await serviceBlobAllocate.execute(connection) - if (!secondBlobAllocate.out.ok) { - throw new Error('invocation failed', { cause: secondBlobAllocate }) - } - - // Validate response - assert.equal(secondBlobAllocate.out.ok.size, 0) - assert.ok(!!blobAllocate.out.ok.address) - }, - 'blob/allocate can allocate to different space after write to one space': async (assert, context) => { - const { proof: aliceProof, spaceDid: aliceSpaceDid } = await registerSpace(alice, context) - const { proof: bobProof, spaceDid: bobSpaceDid } = await registerSpace( - bob, - context, - 'bob' - ) - - // prepare data - const data = new Uint8Array([11, 22, 34, 44, 55]) - const multihash = await sha256.digest(data) - const content = multihash.bytes - const size = data.byteLength - - // create service connection - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // create `blob/add` invocations - const aliceBlobAddInvocation = BlobCapabilities.add.invoke({ - issuer: alice, - audience: context.id, - with: aliceSpaceDid, - nb: { - blob: { - content, - size + proofs: [proof], + }) + const blobAllocate = await serviceBlobAllocate.execute(connection) + if (!blobAllocate.out.ok) { + throw new Error('invocation failed', { cause: blobAllocate }) + } + + // second blob allocate invocation + const secondBlobAllocate = await serviceBlobAllocate.execute(connection) + if (!secondBlobAllocate.out.ok) { + throw new Error('invocation failed', { cause: secondBlobAllocate }) + } + + // Validate response + assert.equal(secondBlobAllocate.out.ok.size, 0) + assert.ok(!!blobAllocate.out.ok.address) + }, + 'blob/allocate can allocate to different space after write to one space': + async (assert, context) => { + const { proof: aliceProof, spaceDid: aliceSpaceDid } = + await registerSpace(alice, context) + const { proof: bobProof, spaceDid: bobSpaceDid } = await registerSpace( + bob, + context, + 'bob' + ) + + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const content = multihash.bytes + const size = data.byteLength + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // create `blob/add` invocations + const aliceBlobAddInvocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: aliceSpaceDid, + nb: { + blob: { + content, + size, + }, }, - }, - proofs: [aliceProof], - }) - const bobBlobAddInvocation = BlobCapabilities.add.invoke({ - issuer: bob, - audience: context.id, - with: bobSpaceDid, - nb: { - blob: { - content, - size + proofs: [aliceProof], + }) + const bobBlobAddInvocation = BlobCapabilities.add.invoke({ + issuer: bob, + audience: context.id, + with: bobSpaceDid, + nb: { + blob: { + content, + size, + }, }, - }, - proofs: [bobProof], - }) - - // invoke `service/blob/allocate` capabilities on alice space - const aliceServiceBlobAllocate = BlobCapabilities.allocate.invoke({ - issuer: alice, - audience: context.id, - with: aliceSpaceDid, - nb: { - blob: { - content, - size + proofs: [bobProof], + }) + + // invoke `service/blob/allocate` capabilities on alice space + const aliceServiceBlobAllocate = BlobCapabilities.allocate.invoke({ + issuer: alice, + audience: context.id, + with: aliceSpaceDid, + nb: { + blob: { + content, + size, + }, + cause: (await aliceBlobAddInvocation.delegate()).cid, + space: aliceSpaceDid, }, - cause: (await aliceBlobAddInvocation.delegate()).cid, - space: aliceSpaceDid, - }, - proofs: [aliceProof], - }) - const aliceBlobAllocate = await aliceServiceBlobAllocate.execute(connection) - if (!aliceBlobAllocate.out.ok) { - throw new Error('invocation failed', { cause: aliceBlobAllocate }) - } - // there is address to write - assert.ok(aliceBlobAllocate.out.ok.address) - assert.equal(aliceBlobAllocate.out.ok.size, size) - - // write to presigned url - const url = aliceBlobAllocate.out.ok.address?.url && new URL(aliceBlobAllocate.out.ok.address?.url) - if (!url) { - throw new Error('Expected presigned url in response') - } - const goodPut = await fetch(url, { - method: 'PUT', - mode: 'cors', - body: data, - headers: aliceBlobAllocate.out.ok.address?.headers, - }) - - assert.equal(goodPut.status, 200, await goodPut.text()) - - // invoke `service/blob/allocate` capabilities on bob space - const bobServiceBlobAllocate = BlobCapabilities.allocate.invoke({ - issuer: bob, - audience: context.id, - with: bobSpaceDid, - nb: { - blob: { - content, - size + proofs: [aliceProof], + }) + const aliceBlobAllocate = await aliceServiceBlobAllocate.execute( + connection + ) + if (!aliceBlobAllocate.out.ok) { + throw new Error('invocation failed', { cause: aliceBlobAllocate }) + } + // there is address to write + assert.ok(aliceBlobAllocate.out.ok.address) + assert.equal(aliceBlobAllocate.out.ok.size, size) + + // write to presigned url + const url = + aliceBlobAllocate.out.ok.address?.url && + new URL(aliceBlobAllocate.out.ok.address?.url) + if (!url) { + throw new Error('Expected presigned url in response') + } + const goodPut = await fetch(url, { + method: 'PUT', + mode: 'cors', + body: data, + headers: aliceBlobAllocate.out.ok.address?.headers, + }) + + assert.equal(goodPut.status, 200, await goodPut.text()) + + // invoke `service/blob/allocate` capabilities on bob space + const bobServiceBlobAllocate = BlobCapabilities.allocate.invoke({ + issuer: bob, + audience: context.id, + with: bobSpaceDid, + nb: { + blob: { + content, + size, + }, + cause: (await bobBlobAddInvocation.delegate()).cid, + space: bobSpaceDid, }, - cause: (await bobBlobAddInvocation.delegate()).cid, - space: bobSpaceDid, - }, - proofs: [bobProof], - }) - const bobBlobAllocate = await bobServiceBlobAllocate.execute(connection) - if (!bobBlobAllocate.out.ok) { - throw new Error('invocation failed', { cause: bobBlobAllocate }) - } - // there is no address to write - assert.ok(!bobBlobAllocate.out.ok.address) - assert.equal(bobBlobAllocate.out.ok.size, size) - - // Validate allocation state - const aliceSpaceAllocations = await context.allocationStorage.list(aliceSpaceDid) - assert.ok(aliceSpaceAllocations.ok) - assert.equal(aliceSpaceAllocations.ok?.size, 1) - - const bobSpaceAllocations = await context.allocationStorage.list(bobSpaceDid) - assert.ok(bobSpaceAllocations.ok) - assert.equal(bobSpaceAllocations.ok?.size, 1) - }, - 'blob/allocate creates presigned url that can only PUT a payload with right length': async (assert, context) => { - const { proof, spaceDid } = await registerSpace(alice, context) - - // prepare data - const data = new Uint8Array([11, 22, 34, 44, 55]) - const longer = new Uint8Array([11, 22, 34, 44, 55, 66]) - const multihash = await sha256.digest(data) - const content = multihash.bytes - const size = data.byteLength - - // create service connection - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // create `blob/add` invocation - const blobAddInvocation = BlobCapabilities.add.invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { - blob: { - content, - size + proofs: [bobProof], + }) + const bobBlobAllocate = await bobServiceBlobAllocate.execute(connection) + if (!bobBlobAllocate.out.ok) { + throw new Error('invocation failed', { cause: bobBlobAllocate }) + } + // there is no address to write + assert.ok(!bobBlobAllocate.out.ok.address) + assert.equal(bobBlobAllocate.out.ok.size, size) + + // Validate allocation state + const aliceSpaceAllocations = await context.allocationStorage.list( + aliceSpaceDid + ) + assert.ok(aliceSpaceAllocations.ok) + assert.equal(aliceSpaceAllocations.ok?.size, 1) + + const bobSpaceAllocations = await context.allocationStorage.list( + bobSpaceDid + ) + assert.ok(bobSpaceAllocations.ok) + assert.equal(bobSpaceAllocations.ok?.size, 1) + }, + 'blob/allocate creates presigned url that can only PUT a payload with right length': + async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const longer = new Uint8Array([11, 22, 34, 44, 55, 66]) + const multihash = await sha256.digest(data) + const content = multihash.bytes + const size = data.byteLength + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // create `blob/add` invocation + const blobAddInvocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + content, + size, + }, }, - }, - proofs: [proof], - }) - - // invoke `service/blob/allocate` - const serviceBlobAllocate = BlobCapabilities.allocate.invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { - blob: { - content, - size + proofs: [proof], + }) + + // invoke `service/blob/allocate` + const serviceBlobAllocate = BlobCapabilities.allocate.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + content, + size, + }, + cause: (await blobAddInvocation.delegate()).cid, + space: spaceDid, }, - cause: (await blobAddInvocation.delegate()).cid, - space: spaceDid, - }, - proofs: [proof], - }) - const blobAllocate = await serviceBlobAllocate.execute(connection) - if (!blobAllocate.out.ok) { - throw new Error('invocation failed', { cause: blobAllocate }) - } - // there is address to write - assert.ok(blobAllocate.out.ok.address) - assert.equal(blobAllocate.out.ok.size, size) - - // write to presigned url - const url = blobAllocate.out.ok.address?.url && new URL(blobAllocate.out.ok.address?.url) - if (!url) { - throw new Error('Expected presigned url in response') - } - const contentLengthFailSignature = await fetch(url, { - method: 'PUT', - mode: 'cors', - body: longer, - headers: { - ...blobAllocate.out.ok.address?.headers, - 'content-length': longer.byteLength.toString(10), - }, - }) - - assert.equal( - contentLengthFailSignature.status >= 400, - true, - 'should fail to upload as content-length differs from that used to sign the url' - ) - }, - 'blob/allocate creates presigned url that can only PUT a payload with exact bytes': async (assert, context) => { - const { proof, spaceDid } = await registerSpace(alice, context) - - // prepare data - const data = new Uint8Array([11, 22, 34, 44, 55]) - const other = new Uint8Array([10, 22, 34, 44, 55]) - const multihash = await sha256.digest(data) - const content = multihash.bytes - const size = data.byteLength - - // create service connection - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // create `blob/add` invocation - const blobAddInvocation = BlobCapabilities.add.invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { - blob: { - content, - size + proofs: [proof], + }) + const blobAllocate = await serviceBlobAllocate.execute(connection) + if (!blobAllocate.out.ok) { + throw new Error('invocation failed', { cause: blobAllocate }) + } + // there is address to write + assert.ok(blobAllocate.out.ok.address) + assert.equal(blobAllocate.out.ok.size, size) + + // write to presigned url + const url = + blobAllocate.out.ok.address?.url && + new URL(blobAllocate.out.ok.address?.url) + if (!url) { + throw new Error('Expected presigned url in response') + } + const contentLengthFailSignature = await fetch(url, { + method: 'PUT', + mode: 'cors', + body: longer, + headers: { + ...blobAllocate.out.ok.address?.headers, + 'content-length': longer.byteLength.toString(10), }, - }, - proofs: [proof], - }) - - // invoke `service/blob/allocate` - const serviceBlobAllocate = BlobCapabilities.allocate.invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { - blob: { - content, - size + }) + + assert.equal( + contentLengthFailSignature.status >= 400, + true, + 'should fail to upload as content-length differs from that used to sign the url' + ) + }, + 'blob/allocate creates presigned url that can only PUT a payload with exact bytes': + async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const other = new Uint8Array([10, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const content = multihash.bytes + const size = data.byteLength + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // create `blob/add` invocation + const blobAddInvocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + content, + size, + }, }, - cause: (await blobAddInvocation.delegate()).cid, - space: spaceDid, - }, - proofs: [proof], - }) - const blobAllocate = await serviceBlobAllocate.execute(connection) - if (!blobAllocate.out.ok) { - throw new Error('invocation failed', { cause: blobAllocate }) - } - // there is address to write - assert.ok(blobAllocate.out.ok.address) - assert.equal(blobAllocate.out.ok.size, size) - - // write to presigned url - const url = blobAllocate.out.ok.address?.url && new URL(blobAllocate.out.ok.address?.url) - if (!url) { - throw new Error('Expected presigned url in response') - } - const failChecksum = await fetch(url, { - method: 'PUT', - mode: 'cors', - body: other, - headers: blobAllocate.out.ok.address?.headers, - }) - - assert.equal( - failChecksum.status, - 400, - 'should fail to upload any other data.' - ) - }, - 'blob/allocate disallowed if invocation fails access verification': async (assert, context) => { + proofs: [proof], + }) + + // invoke `service/blob/allocate` + const serviceBlobAllocate = BlobCapabilities.allocate.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + content, + size, + }, + cause: (await blobAddInvocation.delegate()).cid, + space: spaceDid, + }, + proofs: [proof], + }) + const blobAllocate = await serviceBlobAllocate.execute(connection) + if (!blobAllocate.out.ok) { + throw new Error('invocation failed', { cause: blobAllocate }) + } + // there is address to write + assert.ok(blobAllocate.out.ok.address) + assert.equal(blobAllocate.out.ok.size, size) + + // write to presigned url + const url = + blobAllocate.out.ok.address?.url && + new URL(blobAllocate.out.ok.address?.url) + if (!url) { + throw new Error('Expected presigned url in response') + } + const failChecksum = await fetch(url, { + method: 'PUT', + mode: 'cors', + body: other, + headers: blobAllocate.out.ok.address?.headers, + }) + + assert.equal( + failChecksum.status, + 400, + 'should fail to upload any other data.' + ) + }, + 'blob/allocate disallowed if invocation fails access verification': async ( + assert, + context + ) => { const { proof, space, spaceDid } = await createSpace(alice) // prepare data @@ -567,7 +592,7 @@ export const test = { nb: { blob: { content, - size + size, }, }, proofs: [proof], @@ -581,7 +606,7 @@ export const test = { nb: { blob: { content, - size + size, }, cause: (await blobAddInvocation.delegate()).cid, space: spaceDid, diff --git a/packages/upload-api/test/helpers/context.js b/packages/upload-api/test/helpers/context.js index 6176b318b..8b610840d 100644 --- a/packages/upload-api/test/helpers/context.js +++ b/packages/upload-api/test/helpers/context.js @@ -50,6 +50,7 @@ export const createContext = async ( const subscriptionsStorage = new SubscriptionsStorage(provisionsStorage) const delegationsStorage = new DelegationsStorage() const rateLimitsStorage = new RateLimitsStorage() + const receiptStorage = {} const signer = await Signer.generate() const aggregatorSigner = await Signer.generate() const dealTrackerSigner = await Signer.generate() @@ -85,6 +86,7 @@ export const createContext = async ( plansStorage, usageStorage, revocationsStorage, + receiptStorage, errorReporter: { catch(error) { if (options.assert) {