diff --git a/.github/release-please-config.json b/.github/release-please-config.json index 748c101c9..0e20078dd 100644 --- a/.github/release-please-config.json +++ b/.github/release-please-config.json @@ -3,8 +3,8 @@ "bootstrap-sha": "c918ffc59eafa01fbc63d5df11ba621cc1888c64", "packages": { "packages/access-client": {}, - "packages/aggregate-api": {}, - "packages/aggregate-client": {}, + "packages/filecoin-api": {}, + "packages/filecoin-client": {}, "packages/capabilities": {}, "packages/did-mailto": {}, "packages/upload-api": {}, diff --git a/.github/release-please-manifest.json b/.github/release-please-manifest.json index 4bb411cfc..66bf6f183 100644 --- a/.github/release-please-manifest.json +++ b/.github/release-please-manifest.json @@ -1,7 +1,7 @@ { "packages/access-client": "15.0.0", - "packages/aggregate-api": "1.0.0", - "packages/aggregate-client": "1.0.0", + "packages/filecoin-api": "1.0.0", + "packages/filecoin-client": "1.0.0", "packages/capabilities": "7.0.0", "packages/upload-api": "4.1.0", "packages/upload-client": "9.1.1", diff --git a/.github/workflows/aggregate-api.yml b/.github/workflows/filecoin-api.yml similarity index 70% rename from .github/workflows/aggregate-api.yml rename to .github/workflows/filecoin-api.yml index ec83b63d6..e80d6ba38 100644 --- a/.github/workflows/aggregate-api.yml +++ b/.github/workflows/filecoin-api.yml @@ -1,4 +1,4 @@ -name: Aggregate API +name: Filecoin API env: CI: true FORCE_COLOR: 1 @@ -8,14 +8,14 @@ on: branches: - main paths: - - 'packages/aggregate-api/**' - - '.github/workflows/aggregate-api.yml' + - 'packages/filecoin-api/**' + - '.github/workflows/filecoin-api.yml' - 'pnpm-lock.yaml' - '.env.tpl' pull_request: paths: - - 'packages/aggregate-api/**' - - '.github/workflows/aggregate-api.yml' + - 'packages/filecoin-api/**' + - '.github/workflows/filecoin-api.yml' - 'pnpm-lock.yaml' - '.env.tpl' jobs: @@ -44,6 +44,6 @@ jobs: pnpm run --if-present build - name: Lint - run: pnpm -r --filter @web3-storage/aggregate-api run lint + run: pnpm -r --filter @web3-storage/filecoin-api run lint - name: Test - run: pnpm -r --filter @web3-storage/aggregate-api run test + run: pnpm -r --filter @web3-storage/filecoin-api run test diff --git a/.github/workflows/aggregate-client.yml b/.github/workflows/filecoin-client.yml similarity index 63% rename from .github/workflows/aggregate-client.yml rename to .github/workflows/filecoin-client.yml index 60df7c6c8..23bdf05d0 100644 --- a/.github/workflows/aggregate-client.yml +++ b/.github/workflows/filecoin-client.yml @@ -1,4 +1,4 @@ -name: Aggregate Client +name: Filecoin Client env: CI: true FORCE_COLOR: 1 @@ -7,13 +7,13 @@ on: branches: - main paths: - - 'packages/aggregate-client/**' - - '.github/workflows/aggregate-client.yml' + - 'packages/filecoin-client/**' + - '.github/workflows/filecoin-client.yml' - 'pnpm-lock.yaml' pull_request: paths: - - 'packages/aggregate-client/**' - - '.github/workflows/aggregate-client.yml' + - 'packages/filecoin-client/**' + - '.github/workflows/filecoin-client.yml' - 'pnpm-lock.yaml' jobs: test: @@ -34,5 +34,5 @@ jobs: cache: 'pnpm' - run: pnpm install - run: pnpm run build - - run: pnpm -r --filter @web3-storage/aggregate-client run lint - - run: pnpm -r --filter @web3-storage/aggregate-client run test + - run: pnpm -r --filter @web3-storage/filecoin-client run lint + - run: pnpm -r --filter @web3-storage/filecoin-client run test diff --git a/packages/aggregate-api/CHANGELOG.md b/packages/aggregate-api/CHANGELOG.md deleted file mode 100644 index 9d0d4046e..000000000 --- a/packages/aggregate-api/CHANGELOG.md +++ /dev/null @@ -1,18 +0,0 @@ -# Changelog - -## 1.0.0 (2023-07-11) - - -### ⚠ BREAKING CHANGES - -* aggregate capabilities now have different nb properties and aggregate client api was simplified - -### Features - -* w3 aggregate protocol client and api implementation ([#787](https://github.com/web3-storage/w3up/issues/787)) ([b58069d](https://github.com/web3-storage/w3up/commit/b58069d7960efe09283f3b23fed77515b62d4639)) - - -### Bug Fixes - -* aggregate api test link comparison type ([#816](https://github.com/web3-storage/w3up/issues/816)) ([81bdf1c](https://github.com/web3-storage/w3up/commit/81bdf1c08f7a99b55a8ff2d79af78bf161322737)) -* update aggregate spec in client and api ([#824](https://github.com/web3-storage/w3up/issues/824)) ([ebefd88](https://github.com/web3-storage/w3up/commit/ebefd889a028f325690370db8043c7b9e9fdf7bb)) diff --git a/packages/aggregate-api/src/aggregate.js b/packages/aggregate-api/src/aggregate.js deleted file mode 100644 index 43e7c733d..000000000 --- a/packages/aggregate-api/src/aggregate.js +++ /dev/null @@ -1,13 +0,0 @@ -import { provide as aggregateOfferProvider } from './aggregate/offer.js' -import { provide as aggregateGetProvider } from './aggregate/get.js' -import * as API from './types.js' - -/** - * @param {API.AggregateServiceContext} context - */ -export function createService(context) { - return { - offer: aggregateOfferProvider(context), - get: aggregateGetProvider(context), - } -} diff --git a/packages/aggregate-api/src/aggregate/get.js b/packages/aggregate-api/src/aggregate/get.js deleted file mode 100644 index 1804dd71b..000000000 --- a/packages/aggregate-api/src/aggregate/get.js +++ /dev/null @@ -1,38 +0,0 @@ -import * as Server from '@ucanto/server' -import * as Aggregate from '@web3-storage/capabilities/aggregate' -import * as API from '../types.js' - -/** - * @param {API.AggregateServiceContext} context - */ -export const provide = (context) => - Server.provide(Aggregate.get, (input) => claim(input, context)) - -/** - * @param {API.Input} input - * @param {API.AggregateServiceContext} context - * @returns {Promise>} - */ -export const claim = async ({ capability }, { aggregateStore }) => { - const subject = capability.nb.subject - - const aggregateArrangedResult = await aggregateStore.get(subject) - if (!aggregateArrangedResult) { - return { - error: new AggregateNotFound( - `aggregate not found for subject: ${subject}` - ), - } - } - return { - ok: { - deals: aggregateArrangedResult, - }, - } -} - -class AggregateNotFound extends Server.Failure { - get name() { - return /** @type {const} */ ('AggregateNotFound') - } -} diff --git a/packages/aggregate-api/src/aggregate/offer.js b/packages/aggregate-api/src/aggregate/offer.js deleted file mode 100644 index 340f7750c..000000000 --- a/packages/aggregate-api/src/aggregate/offer.js +++ /dev/null @@ -1,131 +0,0 @@ -import * as Server from '@ucanto/server' -import { CBOR } from '@ucanto/core' -import { Node, Piece, Aggregate as AggregateBuilder } from '@web3-storage/data-segment' -import * as Aggregate from '@web3-storage/capabilities/aggregate' -import * as Offer from '@web3-storage/capabilities/offer' -import * as API from '../types.js' - -// 16 GiB -export const MIN_SIZE = Piece.PaddedSize.from(2n ** 34n) -// 32 GiB -export const MAX_SIZE = Piece.PaddedSize.from(2n ** 35n) - -/** - * @param {API.AggregateServiceContext} context - */ -export const provide = (context) => - Server.provideAdvanced({ - capability: Aggregate.offer, - handler: (input) => claim(input, context), - }) - -/** - * @param {API.Input} input - * @param {API.AggregateServiceContext} context - * @returns {Promise | API.UcantoInterface.JoinBuilder>} - */ -export const claim = async ( - { capability, invocation, context }, - { offerStore } -) => { - // Get offer block - const offerCid = capability.nb.offer - const piece = capability.nb.piece - const offers = getOfferBlock(offerCid, invocation.iterateIPLDBlocks()) - - if (!offers) { - return { - error: new AggregateOfferBlockNotFoundError( - `missing offer block in invocation: ${offerCid.toString()}` - ), - } - } - - // Validate offer content - const aggregateLeafs = 2n ** BigInt(piece.height) - const aggregateSize = aggregateLeafs * BigInt(Node.Size) - - if (aggregateSize < MIN_SIZE) { - return { - error: new AggregateOfferInvalidSizeError( - `offer under size, offered: ${aggregateSize}, minimum: ${MIN_SIZE}` - ), - } - } else if (aggregateSize > MAX_SIZE) { - return { - error: new AggregateOfferInvalidSizeError( - `offer over size, offered: ${aggregateSize}, maximum: ${MAX_SIZE}` - ), - } - } - - // Validate commP of commPs - const aggregateBuild = AggregateBuilder.build({ - size: aggregateSize, - pieces: offers.map(offer => Piece.fromJSON({ - height: offer.height, - link: { '/': offer.link.toString() } - })) - }) - if (!aggregateBuild.link.equals(piece.link)) { - return { - error: new AggregateOfferInvalidSizeError( - `aggregate piece CID mismatch, specified: ${piece.link}, computed: ${aggregateBuild.link}` - ), - } - } else if (aggregateBuild.height !== piece.height) { - return { - error: new AggregateOfferInvalidSizeError( - `aggregate height mismatch, specified: ${piece.height}, computed: ${aggregateBuild.height}` - ), - } - } - - // Create effect for receipt - const fx = await Offer.arrange - .invoke({ - issuer: context.id, - audience: context.id, - with: context.id.did(), - nb: { - pieceLink: piece.link, - }, - }) - .delegate() - - // Write offer to store - await offerStore.queue({ piece, offers }) - - return Server.ok({ - status: 'queued', - }).join(fx.link()) -} - -/** - * @param {Server.API.Link} offerCid - * @param {IterableIterator>} blockIterator - */ -function getOfferBlock(offerCid, blockIterator) { - for (const block of blockIterator) { - if (block.cid.equals(offerCid)) { - const decoded = - /** @type {import('@web3-storage/data-segment').PieceView[]} */ ( - CBOR.decode(block.bytes) - ) - return decoded - // TODO: Validate with schema - } - } -} - -class AggregateOfferInvalidSizeError extends Server.Failure { - get name() { - return /** @type {const} */ ('AggregateOfferInvalidSize') - } -} - -class AggregateOfferBlockNotFoundError extends Server.Failure { - get name() { - return /** @type {const} */ ('AggregateOfferBlockNotFound') - } -} diff --git a/packages/aggregate-api/src/lib.js b/packages/aggregate-api/src/lib.js deleted file mode 100644 index 16cfef454..000000000 --- a/packages/aggregate-api/src/lib.js +++ /dev/null @@ -1,46 +0,0 @@ -import * as Server from '@ucanto/server' -import * as Client from '@ucanto/client' -import * as Types from './types.js' -import * as CAR from '@ucanto/transport/car' -import { createService as createAggregateService } from './aggregate.js' -import { createService as createOfferService } from './offer.js' -export * from './types.js' - -/** - * @param {Types.UcantoServerContext} options - */ -export const createServer = ({ id, codec = CAR.inbound, ...context }) => - Server.create({ - id, - codec: CAR.inbound, - service: createService(context), - catch: (error) => context.errorReporter.catch(error), - }) - -/** - * @param {Types.ServiceContext} context - * @returns {Types.Service} - */ -export const createService = (context) => ({ - aggregate: createAggregateService(context), - offer: createOfferService(context), -}) - -/** - * @param {object} options - * @param {Types.UcantoInterface.Principal} options.id - * @param {Types.UcantoInterface.Transport.Channel} options.channel - * @param {Types.UcantoInterface.OutboundCodec} [options.codec] - */ -export const connect = ({ id, channel, codec = CAR.outbound }) => - Client.connect({ - id, - channel, - codec, - }) - -export { - createService as createUploadService, - createServer as createUploadServer, - connect as createUploadClient, -} diff --git a/packages/aggregate-api/src/offer.js b/packages/aggregate-api/src/offer.js deleted file mode 100644 index 07da0c321..000000000 --- a/packages/aggregate-api/src/offer.js +++ /dev/null @@ -1,11 +0,0 @@ -import { provide as offerArrangeProvider } from './offer/arrange.js' -import * as API from './types.js' - -/** - * @param {API.OfferServiceContext} context - */ -export function createService(context) { - return { - arrange: offerArrangeProvider(context), - } -} diff --git a/packages/aggregate-api/src/offer/arrange.js b/packages/aggregate-api/src/offer/arrange.js deleted file mode 100644 index dd552cf21..000000000 --- a/packages/aggregate-api/src/offer/arrange.js +++ /dev/null @@ -1,40 +0,0 @@ -import * as Server from '@ucanto/server' -import * as Offer from '@web3-storage/capabilities/offer' -import * as API from '../types.js' - -/** - * @param {API.OfferServiceContext} context - */ -export const provide = (context) => - Server.provide(Offer.arrange, (input) => claim(input, context)) - -/** - * @param {API.Input} input - * @param {API.OfferServiceContext} context - * @returns {Promise>} - */ -export const claim = async ({ capability }, { arrangedOfferStore }) => { - const pieceLink = capability.nb.pieceLink - - const status = await arrangedOfferStore.get(pieceLink) - - if (!status) { - return { - error: new OfferArrangeNotFound( - `arranged offer not found for piece: ${pieceLink}` - ), - } - } - - return { - ok: { - status, - }, - } -} - -class OfferArrangeNotFound extends Server.Failure { - get name() { - return /** @type {const} */ ('OfferArrangeNotFound') - } -} diff --git a/packages/aggregate-api/src/types.ts b/packages/aggregate-api/src/types.ts deleted file mode 100644 index 5dca45e4c..000000000 --- a/packages/aggregate-api/src/types.ts +++ /dev/null @@ -1,89 +0,0 @@ -import type { - HandlerExecutionError, - Signer, - InboundCodec, - CapabilityParser, - ParsedCapability, - InferInvokedCapability, - Match, - Link, -} from '@ucanto/interface' -import type { ProviderInput } from '@ucanto/server' - -import type { PieceLink, PieceView } from '@web3-storage/data-segment' -export * from '@web3-storage/aggregate-client/types' - -export * from '@web3-storage/capabilities/types' -export * as UcantoInterface from '@ucanto/interface' - -export interface AggregateServiceContext { - aggregateStore: AggregateStore - offerStore: OfferStore -} - -export interface OfferServiceContext { - arrangedOfferStore: ArrangedOfferStore -} - -export interface ServiceContext - extends AggregateServiceContext, - OfferServiceContext {} - -export interface ArrangedOfferStore { - get: (pieceLink: PieceLink) => Promise -} - -export interface OfferStore { - queue: (aggregateOffer: OfferToQueue) => Promise -} - -export interface OfferToQueue { - piece: PieceView - offers: PieceView[] -} - -export interface AggregateStore { - get: (pieceLink: PieceLink) => Promise -} - -export interface UcantoServerContext extends ServiceContext { - id: Signer - codec?: InboundCodec - errorReporter: ErrorReporter -} - -export interface ErrorReporter { - catch: (error: HandlerExecutionError) => void -} - -export interface Assert { - equal: ( - actual: Actual, - expected: Expected, - message?: string - ) => unknown - deepEqual: ( - actual: Actual, - expected: Expected, - message?: string - ) => unknown - ok: (actual: Actual, message?: string) => unknown -} - -export interface AggregateStoreBackend { - put: ( - pieceLink: Link, - aggregateInfo: unknown - ) => Promise -} - -export interface UcantoServerContextTest extends UcantoServerContext { - // to enable tests to insert data in aggregateStore memory db - aggregateStoreBackend: AggregateStoreBackend -} - -export type Test = (assert: Assert, context: UcantoServerContextTest) => unknown -export type Tests = Record - -export type Input>> = - ProviderInput & ParsedCapability> diff --git a/packages/aggregate-api/test/aggregate.js b/packages/aggregate-api/test/aggregate.js deleted file mode 100644 index d7c1936b1..000000000 --- a/packages/aggregate-api/test/aggregate.js +++ /dev/null @@ -1,386 +0,0 @@ -import { Aggregate, Offer } from '@web3-storage/capabilities' -import { Piece } from '@web3-storage/data-segment' - -import { CBOR, parseLink } from '@ucanto/core' -import * as Signer from '@ucanto/principal/ed25519' - -import { MIN_SIZE, MAX_SIZE } from '../src/aggregate/offer.js' -import * as API from '../src/types.js' -import { randomAggregate } from './utils.js' -import { createServer, connect } from '../src/lib.js' - -/** - * @type {API.Tests} - */ -export const test = { - // aggregate/offer tests - 'aggregate/offer inserts valid offer into bucket': async ( - assert, - context - ) => { - const { storeFront } = await getServiceContext() - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // Generate Pieces for offer - const { pieces, aggregate } = await randomAggregate(100, 128) - const block = await CBOR.write(pieces) - const aggregateOfferInvocation = Aggregate.offer.invoke({ - issuer: storeFront, - audience: connection.id, - with: storeFront.did(), - nb: { - offer: block.cid, - piece: aggregate, - }, - }) - aggregateOfferInvocation.attach(block) - - const aggregateOffer = await aggregateOfferInvocation.execute(connection) - if (aggregateOffer.out.error) { - throw new Error('invocation failed', { cause: aggregateOffer.out.error }) - } - assert.ok(aggregateOffer.out.ok) - assert.deepEqual(aggregateOffer.out.ok.status, 'queued') - - // Validate effect in receipt - const fx = await Offer.arrange - .invoke({ - issuer: context.id, - audience: context.id, - with: context.id.did(), - nb: { - pieceLink: aggregate.link, - }, - }) - .delegate() - - assert.ok(aggregateOffer.fx.join) - assert.ok(fx.link().equals(aggregateOffer.fx.join?.link())) - }, - 'aggregate/offer fails when offer block is not attached': async ( - assert, - context - ) => { - const { storeFront } = await getServiceContext() - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // Generate Pieces for offer - const { pieces, aggregate } = await randomAggregate(100, 128) - const block = await CBOR.write(pieces) - const aggregateOfferInvocation = Aggregate.offer.invoke({ - issuer: storeFront, - audience: connection.id, - with: storeFront.did(), - nb: { - offer: block.cid, - piece: aggregate, - }, - }) - - const aggregateOffer = await aggregateOfferInvocation.execute(connection) - assert.ok(aggregateOffer.out.error) - assert.deepEqual( - aggregateOffer.out.error?.message, - `missing offer block in invocation: ${block.cid.toString()}` - ) - - // Validate effect in receipt does not exist - assert.ok(!aggregateOffer.fx.join) - }, - 'aggregate/offer fails when size is not enough for offer': async ( - assert, - context - ) => { - const { storeFront } = await getServiceContext() - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // Generate Pieces for offer - const { pieces, aggregate } = await randomAggregate(100, 128) - const badHeight = 3 - const size = Piece.PaddedSize.fromHeight(badHeight) - - const block = await CBOR.write(pieces) - const aggregateOfferInvocation = Aggregate.offer.invoke({ - issuer: storeFront, - audience: connection.id, - with: storeFront.did(), - nb: { - offer: block.cid, - piece: { - ...aggregate, - height: badHeight, - }, - }, - }) - aggregateOfferInvocation.attach(block) - - const aggregateOffer = await aggregateOfferInvocation.execute(connection) - assert.ok(aggregateOffer.out.error) - // TODO: compute size - assert.deepEqual( - aggregateOffer.out.error?.message, - `offer under size, offered: ${Number(size)}, minimum: ${MIN_SIZE}` - ) - - // Validate effect in receipt does not exist - assert.ok(!aggregateOffer.fx.join) - }, - 'aggregate/offer fails when size is above limit for offer': async ( - assert, - context - ) => { - const { storeFront } = await getServiceContext() - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // Generate Pieces for offer - const { pieces, aggregate } = await randomAggregate(100, 128) - const badHeight = 31 - const size = Piece.PaddedSize.fromHeight(badHeight) - - const block = await CBOR.write(pieces) - const aggregateOfferInvocation = Aggregate.offer.invoke({ - issuer: storeFront, - audience: connection.id, - with: storeFront.did(), - nb: { - offer: block.cid, - piece: { - ...aggregate, - height: badHeight, - }, - }, - }) - aggregateOfferInvocation.attach(block) - - const aggregateOffer = await aggregateOfferInvocation.execute(connection) - assert.ok(aggregateOffer.out.error) - assert.deepEqual( - aggregateOffer.out.error?.message, - `offer over size, offered: ${size}, maximum: ${MAX_SIZE}` - ) - - // Validate effect in receipt does not exist - assert.ok(!aggregateOffer.fx.join) - }, - 'aggregate/offer fails when provided height is different than computed': - async (assert, context) => { - const { storeFront } = await getServiceContext() - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // Generate Pieces for offer - const { pieces, aggregate } = await randomAggregate(100, 128) - const badHeight = 29 - - const block = await CBOR.write(pieces) - const aggregateOfferInvocation = Aggregate.offer.invoke({ - issuer: storeFront, - audience: connection.id, - with: storeFront.did(), - nb: { - offer: block.cid, - piece: { - link: aggregate.link, - height: badHeight, - }, - }, - }) - aggregateOfferInvocation.attach(block) - - const aggregateOffer = await aggregateOfferInvocation.execute(connection) - assert.ok(aggregateOffer.out.error) - - // Validate effect in receipt does not exist - assert.ok(!aggregateOffer.fx.join) - }, - 'aggregate/offer fails when provided piece CID is different than computed': - async (assert, context) => { - const { storeFront } = await getServiceContext() - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // Generate Pieces for offer - const { pieces, aggregate } = await randomAggregate(100, 128) - const badLink = - /** @type {import('@web3-storage/data-segment').PieceLink} */ ( - parseLink( - 'baga6ea4seaqm2u43527zehkqqcpyyopgsw2c4mapyy2vbqzqouqtzhxtacueeki' - ) - ) - - const block = await CBOR.write(pieces) - const aggregateOfferInvocation = Aggregate.offer.invoke({ - issuer: storeFront, - audience: connection.id, - with: storeFront.did(), - nb: { - offer: block.cid, - piece: { - link: badLink, - height: aggregate.height, - }, - }, - }) - aggregateOfferInvocation.attach(block) - - const aggregateOffer = await aggregateOfferInvocation.execute(connection) - assert.ok(aggregateOffer.out.error) - assert.deepEqual( - aggregateOffer.out.error?.message, - `aggregate piece CID mismatch, specified: ${badLink}, computed: ${aggregate.link}` - ) - - // Validate effect in receipt does not exist - assert.ok(!aggregateOffer.fx.join) - }, - // offer/arrange tests - 'aggregate/arrange can be invoked after aggregate/offer': async ( - assert, - context - ) => { - const { storeFront } = await getServiceContext() - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // Generate Pieces for offer - const { pieces, aggregate } = await randomAggregate(100, 128) - // TODO: Inflate size for testing - - const block = await CBOR.write(pieces) - const aggregateOfferInvocation = Aggregate.offer.invoke({ - issuer: storeFront, - audience: connection.id, - with: storeFront.did(), - nb: { - offer: block.cid, - piece: aggregate, - }, - }) - aggregateOfferInvocation.attach(block) - - const aggregateOffer = await aggregateOfferInvocation.execute(connection) - if (aggregateOffer.out.error) { - throw new Error('invocation failed', { cause: aggregateOffer.out.error }) - } - assert.ok(aggregateOffer.out.ok) - - // Validate effect in receipt - const fx = await Offer.arrange - .invoke({ - issuer: context.id, - audience: context.id, - with: context.id.did(), - nb: { - pieceLink: aggregate.link, - }, - }) - .delegate() - - const offerArrangeInvocation = Offer.arrange.invoke({ - issuer: context.id, - audience: context.id, - with: context.id.did(), - nb: { - pieceLink: aggregate.link, - }, - }) - - const offerArrange = await offerArrangeInvocation.execute(connection) - if (offerArrange.out.error) { - throw new Error('invocation failed', { cause: offerArrange.out.error }) - } - assert.ok(offerArrange.out.ok) - assert.ok(offerArrange.ran.link().equals(fx.link())) - }, - // aggregate/get tests - 'aggregate/get fails when requested aggregate does not exist': async ( - assert, - context - ) => { - const { storeFront } = await getServiceContext() - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - const subject = - /** @type {import('@web3-storage/data-segment').PieceLink} */ ( - parseLink( - 'baga6ea4seaqm2u43527zehkqqcpyyopgsw2c4mapyy2vbqzqouqtzhxtacueeki' - ) - ) - const aggregateGetInvocation = Aggregate.get.invoke({ - issuer: storeFront, - audience: connection.id, - with: storeFront.did(), - nb: { - subject, - }, - }) - - const aggregateGet = await aggregateGetInvocation.execute(connection) - assert.ok(aggregateGet.out.error) - }, - // aggregate/get tests - 'aggregate/get returns known deals for given commitment proof': async ( - assert, - context - ) => { - const { storeFront } = await getServiceContext() - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - const subject = - /** @type {import('@web3-storage/data-segment').PieceLink} */ ( - parseLink( - 'baga6ea4seaqm2u43527zehkqqcpyyopgsw2c4mapyy2vbqzqouqtzhxtacueeki' - ) - ) - const deal = { - status: 'done', - } - await context.aggregateStoreBackend.put(subject, deal) - - const aggregateGetInvocation = Aggregate.get.invoke({ - issuer: storeFront, - audience: connection.id, - with: storeFront.did(), - nb: { - subject, - }, - }) - - const aggregateGet = await aggregateGetInvocation.execute(connection) - if (aggregateGet.out.error) { - throw new Error('invocation failed', { cause: aggregateGet.out.error }) - } - assert.equal(aggregateGet.out.ok.deals.length, 1) - assert.deepEqual(aggregateGet.out.ok.deals[0], deal) - }, -} - -async function getServiceContext() { - const storeFront = await Signer.generate() - - return { storeFront } -} diff --git a/packages/aggregate-api/test/aggregate.spec.js b/packages/aggregate-api/test/aggregate.spec.js deleted file mode 100644 index 1e7ae0551..000000000 --- a/packages/aggregate-api/test/aggregate.spec.js +++ /dev/null @@ -1,43 +0,0 @@ -/* eslint-disable no-only-tests/no-only-tests */ -import * as assert from 'assert' -import * as Aggregate from './aggregate.js' -import * as Signer from '@ucanto/principal/ed25519' - -import { OfferStore } from './context/offer-store.js' -import { AggregateStore } from './context/aggregate-store.js' - -describe('aggregate/*', () => { - for (const [name, test] of Object.entries(Aggregate.test)) { - const define = name.startsWith('only ') - ? it.only - : name.startsWith('skip ') - ? it.skip - : it - - define(name, async () => { - const signer = await Signer.generate() - const id = signer.withDID('did:web:test.web3.storage') - const aggregateStore = new AggregateStore() - - await test( - { - equal: assert.strictEqual, - deepEqual: assert.deepStrictEqual, - ok: assert.ok, - }, - { - id, - errorReporter: { - catch(error) { - assert.fail(error) - }, - }, - offerStore: new OfferStore(), - arrangedOfferStore: new OfferStore(), - aggregateStore, - aggregateStoreBackend: aggregateStore, - } - ) - }) - } -}) diff --git a/packages/aggregate-api/test/context/aggregate-store.js b/packages/aggregate-api/test/context/aggregate-store.js deleted file mode 100644 index 939270612..000000000 --- a/packages/aggregate-api/test/context/aggregate-store.js +++ /dev/null @@ -1,36 +0,0 @@ -import * as API from '../../src/types.js' - -/** - * @implements {API.AggregateStore} - */ -export class AggregateStore { - constructor() { - /** @type {Map} */ - this.items = new Map() - } - - /** - * @param {import('@ucanto/interface').Link} pieceLink - * @param {unknown} deal - */ - put(pieceLink, deal) { - const dealEntries = this.items.get(pieceLink.toString()) - let newEntries - if (dealEntries) { - newEntries = [...dealEntries, deal] - this.items.set(pieceLink.toString(), newEntries) - } else { - newEntries = [deal] - this.items.set(pieceLink.toString(), newEntries) - } - - return Promise.resolve() - } - - /** - * @param {import('@ucanto/interface').Link} pieceLink - */ - get(pieceLink) { - return Promise.resolve(this.items.get(pieceLink.toString())) - } -} diff --git a/packages/aggregate-api/test/context/offer-store.js b/packages/aggregate-api/test/context/offer-store.js deleted file mode 100644 index 16724785b..000000000 --- a/packages/aggregate-api/test/context/offer-store.js +++ /dev/null @@ -1,24 +0,0 @@ -/** - * @typedef {import('@web3-storage/data-segment').PieceView[]} Offers - */ - -export class OfferStore { - constructor() { - /** @type {Map} */ - this.offers = new Map() - } - /** - * @param {import('../../src/types').OfferToQueue} offerToQueue - */ - async queue(offerToQueue) { - this.offers.set(offerToQueue.piece.link.toString(), offerToQueue.offers) - } - - /** - * @param {import('@ucanto/interface').Link} pieceLink - * @returns {Promise} - */ - async get(pieceLink) { - return Promise.resolve(`todo:${pieceLink.toString()}`) - } -} diff --git a/packages/aggregate-api/test/lib.js b/packages/aggregate-api/test/lib.js deleted file mode 100644 index 98b21c0f1..000000000 --- a/packages/aggregate-api/test/lib.js +++ /dev/null @@ -1,8 +0,0 @@ -import * as Aggregate from './aggregate.js' -export * from './utils.js' - -export const test = { - ...Aggregate.test, -} - -export { Aggregate } diff --git a/packages/aggregate-client/CHANGELOG.md b/packages/aggregate-client/CHANGELOG.md deleted file mode 100644 index 49f857792..000000000 --- a/packages/aggregate-client/CHANGELOG.md +++ /dev/null @@ -1,17 +0,0 @@ -# Changelog - -## 1.0.0 (2023-07-11) - - -### ⚠ BREAKING CHANGES - -* aggregate capabilities now have different nb properties and aggregate client api was simplified - -### Features - -* w3 aggregate protocol client and api implementation ([#787](https://github.com/web3-storage/w3up/issues/787)) ([b58069d](https://github.com/web3-storage/w3up/commit/b58069d7960efe09283f3b23fed77515b62d4639)) - - -### Bug Fixes - -* update aggregate spec in client and api ([#824](https://github.com/web3-storage/w3up/issues/824)) ([ebefd88](https://github.com/web3-storage/w3up/commit/ebefd889a028f325690370db8043c7b9e9fdf7bb)) diff --git a/packages/aggregate-client/README.md b/packages/aggregate-client/README.md deleted file mode 100644 index 3f637f391..000000000 --- a/packages/aggregate-client/README.md +++ /dev/null @@ -1,75 +0,0 @@ -

⁂
web3.storage

-

The aggregate client for https://web3.storage

- -## About - -The `@web3-storage/aggregate-client` package provides the "low level" client API for aggregating data uploaded with the w3up platform. It is based on [web3-storage/specs/w3-aggregation.md])https://github.com/web3-storage/specs/blob/feat/filecoin-spec/w3-aggregation.md) and is not intended for web3.storage end users. - -## Install - -Install the package using npm: - -```bash -npm install @web3-storage/aggregate-client -``` - -## Usage - -### `aggregateOffer` - -```ts -function aggregateOffer( - conf: InvocationConfig, - piece: Piece, - offer: Piece[], -): Promise<{ status: string }> -``` - -Ask the service to create an aggregate offer and put it available for Storage Providers. - -More information: [`InvocationConfig`](#invocationconfig) - -### `aggregateGet` - -```ts -function aggregateGet( - conf: InvocationConfig, - subject: PieceCID, -): Promise -``` - -Ask the service to get deal details of an aggregate. - -More information: [`InvocationConfig`](#invocationconfig) - -## Types - -### `Piece` - -An offered CAR to be part of an Aggregate. - -```ts -export interface Piece { - link: PieceCID - size: number -} - -export type PieceCID = ReturnType -``` - -### `InvocationConfig` - -This is the configuration for the UCAN invocation. It is an object with `issuer`, `audience`, `resource` and `proofs`: - -- The `issuer` is the signing authority that is issuing the UCAN invocation(s). -- The `audience` is the principal authority that the UCAN is delegated to. -- The `resource` (`with` field) points to a storage space. -- The `proofs` are a set of capability delegations that prove the issuer has the capability to perform the action. These might not be required. - -## Contributing - -Feel free to join in. All welcome. Please [open an issue](https://github.com/web3-storage/w3protocol/issues)! - -## License - -Dual-licensed under [MIT + Apache 2.0](https://github.com/web3-storage/w3protocol/blob/main/license.md) diff --git a/packages/aggregate-client/src/aggregate.js b/packages/aggregate-client/src/aggregate.js deleted file mode 100644 index 3164a9e26..000000000 --- a/packages/aggregate-client/src/aggregate.js +++ /dev/null @@ -1,70 +0,0 @@ -import * as AggregateCapabilities from '@web3-storage/capabilities/aggregate' -import { CBOR } from '@ucanto/core' - -import { servicePrincipal, connection } from './service.js' - -export const MIN_SIZE = 1 + 127 * (1 << 27) -export const MAX_SIZE = 127 * (1 << 28) - -/** - * Offer an aggregate to be assembled and stored. - * - * @param {import('./types').InvocationConfig} conf - Configuration - * @param {import('@web3-storage/data-segment').PieceView} piece - * @param {import('@web3-storage/data-segment').PieceView[]} offer - * @param {import('./types').RequestOptions} [options] - */ -export async function aggregateOffer( - { issuer, with: resource, proofs, audience }, - piece, - offer, - options = {} -) { - /* c8 ignore next */ - const conn = options.connection ?? connection - - const block = await CBOR.write(offer) - const invocation = AggregateCapabilities.offer.invoke({ - issuer, - /* c8 ignore next */ - audience: audience ?? servicePrincipal, - with: resource, - nb: { - offer: block.cid, - piece, - }, - proofs, - }) - invocation.attach(block) - - return await invocation.execute(conn) -} - -/** - * Get details of an aggregate. - * - * @param {import('./types').InvocationConfig} conf - Configuration - * @param {import('@web3-storage/data-segment').PieceLink} subject - * @param {import('./types').RequestOptions} [options] - */ -export async function aggregateGet( - { issuer, with: resource, proofs, audience }, - subject, - options = {} -) { - /* c8 ignore next */ - const conn = options.connection ?? connection - - return await AggregateCapabilities.get - .invoke({ - issuer, - /* c8 ignore next */ - audience: audience ?? servicePrincipal, - with: resource, - nb: { - subject, - }, - proofs, - }) - .execute(conn) -} diff --git a/packages/aggregate-client/src/index.js b/packages/aggregate-client/src/index.js deleted file mode 100644 index a8aca01bf..000000000 --- a/packages/aggregate-client/src/index.js +++ /dev/null @@ -1 +0,0 @@ -export * as Aggregate from './aggregate.js' diff --git a/packages/aggregate-client/src/service.js b/packages/aggregate-client/src/service.js deleted file mode 100644 index 587ec5160..000000000 --- a/packages/aggregate-client/src/service.js +++ /dev/null @@ -1,16 +0,0 @@ -import { connect } from '@ucanto/client' -import { CAR, HTTP } from '@ucanto/transport' -import * as DID from '@ipld/dag-ucan/did' - -export const serviceURL = new URL('https://spade-proxy.web3.storage') -export const servicePrincipal = DID.parse('did:web:web3.storage') - -/** @type {import('@ucanto/interface').ConnectionView} */ -export const connection = connect({ - id: servicePrincipal, - codec: CAR.outbound, - channel: HTTP.open({ - url: serviceURL, - method: 'POST', - }), -}) diff --git a/packages/aggregate-client/src/types.ts b/packages/aggregate-client/src/types.ts deleted file mode 100644 index 976768445..000000000 --- a/packages/aggregate-client/src/types.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { Link } from 'multiformats/link' -import { CAR } from '@ucanto/transport' -import { - ConnectionView, - ServiceMethod, - Signer, - Proof, - DID, - Principal, -} from '@ucanto/interface' -import { - AggregateOffer, - AggregateOfferSuccess, - AggregateOfferFailure, - AggregateGet, - AggregateGetSuccess, - AggregateGetFailure, - OfferArrange, - OfferArrangeSuccess, - OfferArrangeFailure, -} from '@web3-storage/capabilities/types' - -export interface InvocationConfig { - /** - * Signing authority that is issuing the UCAN invocation(s). - */ - issuer: Signer - /** - * The principal delegated to in the current UCAN. - */ - audience?: Principal - /** - * The resource the invocation applies to. - */ - with: DID - /** - * Proof(s) the issuer has the capability to perform the action. - */ - proofs?: Proof[] -} - -export interface Service { - aggregate: { - offer: ServiceMethod< - AggregateOffer, - AggregateOfferSuccess, - AggregateOfferFailure - > - get: ServiceMethod - } - offer: { - arrange: ServiceMethod< - OfferArrange, - OfferArrangeSuccess, - OfferArrangeFailure - > - } -} - -export interface RequestOptions extends Connectable {} - -export interface Connectable { - connection?: ConnectionView -} - -/** - * An IPLD Link that has the CAR codec code. - */ -export type CARLink = Link diff --git a/packages/aggregate-client/test/aggregate.test.js b/packages/aggregate-client/test/aggregate.test.js deleted file mode 100644 index c90684c69..000000000 --- a/packages/aggregate-client/test/aggregate.test.js +++ /dev/null @@ -1,155 +0,0 @@ -import assert from 'assert' -import * as Client from '@ucanto/client' -import * as Server from '@ucanto/server' -import * as Signer from '@ucanto/principal/ed25519' -import * as CAR from '@ucanto/transport/car' -import { CBOR, parseLink } from '@ucanto/core' -import * as AggregateCapabilities from '@web3-storage/capabilities/aggregate' -import * as OfferCapabilities from '@web3-storage/capabilities/offer' - -import * as Aggregate from '../src/aggregate.js' - -import { serviceProvider } from './fixtures.js' -import { mockService } from './helpers/mocks.js' -import { randomAggregate } from './helpers/random.js' - -describe('aggregate.offer', () => { - it('places a valid offer with the service', async () => { - const { storeFront } = await getContext() - - // Generate Pieces for offer - const { pieces, aggregate } = await randomAggregate(100, 100) - - const offerBlock = await CBOR.write(pieces) - /** @type {import('@web3-storage/capabilities/types').AggregateOfferSuccess} */ - const aggregateOfferResponse = { - status: 'queued', - } - - // Create Ucanto service - const service = mockService({ - aggregate: { - offer: Server.provideAdvanced({ - capability: AggregateCapabilities.offer, - // @ts-expect-error not failure type expected because of assert throw - handler: async ({ invocation, context }) => { - assert.strictEqual(invocation.issuer.did(), storeFront.did()) - assert.strictEqual(invocation.capabilities.length, 1) - const invCap = invocation.capabilities[0] - assert.strictEqual(invCap.can, AggregateCapabilities.offer.can) - assert.equal(invCap.with, invocation.issuer.did()) - // size - assert.strictEqual(invCap.nb?.piece.height, aggregate.height) - assert.ok(invCap.nb?.piece.link) - // TODO: Validate commitmemnt proof - assert.ok(invCap.nb?.offer) - // Validate block inline exists - const invocationBlocks = Array.from(invocation.iterateIPLDBlocks()) - assert.ok( - invocationBlocks.find((b) => b.cid.equals(offerBlock.cid)) - ) - - // Create effect for receipt - const fx = await OfferCapabilities.arrange - .invoke({ - issuer: context.id, - audience: context.id, - with: context.id.did(), - nb: { - pieceLink: invCap.nb?.piece.link, - }, - }) - .delegate() - - return Server.ok(aggregateOfferResponse).join(fx.link()) - }, - }), - }, - }) - const res = await Aggregate.aggregateOffer( - { - issuer: storeFront, - with: storeFront.did(), - audience: serviceProvider, - }, - aggregate, - pieces, - // @ts-expect-error no full service implemented - { connection: getConnection(service).connection } - ) - assert.ok(res.out.ok) - assert.deepEqual(res.out.ok, aggregateOfferResponse) - // includes effect fx in receipt - assert.ok(res.fx.join) - }) -}) - -describe('aggregate.get', () => { - it('places a valid offer with the service', async () => { - const { storeFront } = await getContext() - const subject = - /** @type {import('@web3-storage/data-segment').PieceLink} */ ( - parseLink( - 'baga6ea4seaqm2u43527zehkqqcpyyopgsw2c4mapyy2vbqzqouqtzhxtacueeki' - ) - ) - /** @type {unknown[]} */ - const deals = [] - - // Create Ucanto service - const service = mockService({ - aggregate: { - get: Server.provide(AggregateCapabilities.get, ({ invocation }) => { - assert.strictEqual(invocation.issuer.did(), storeFront.did()) - assert.strictEqual(invocation.capabilities.length, 1) - const invCap = invocation.capabilities[0] - assert.strictEqual(invCap.can, AggregateCapabilities.get.can) - assert.equal(invCap.with, invocation.issuer.did()) - assert.ok(invCap.nb?.subject) - return { ok: { deals } } - }), - }, - }) - - const res = await Aggregate.aggregateGet( - { - issuer: storeFront, - with: storeFront.did(), - audience: serviceProvider, - }, - subject, - // @ts-expect-error no full service implemented - { connection: getConnection(service).connection } - ) - - assert.ok(res.out.ok) - assert.deepEqual(res.out.ok.deals, deals) - }) -}) - -async function getContext() { - const storeFront = await Signer.generate() - - return { storeFront } -} - -/** - * @param {Partial<{ - * aggregate: Partial - * offer: Partial - * }>} service - */ -function getConnection(service) { - const server = Server.create({ - id: serviceProvider, - service, - codec: CAR.inbound, - }) - const connection = Client.connect({ - id: serviceProvider, - codec: CAR.outbound, - channel: server, - }) - - return { connection } -} diff --git a/packages/capabilities/src/aggregate.js b/packages/capabilities/src/aggregate.js deleted file mode 100644 index fabb9bf31..000000000 --- a/packages/capabilities/src/aggregate.js +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Aggregate Capabilities - * - * These can be imported directly with: - * ```js - * import * as Aggregate from '@web3-storage/capabilities/aggregate' - * ``` - * - * @module - */ -import { capability, Schema, ok } from '@ucanto/validator' -import { checkLink, equalWith, equal, and } from './utils.js' - -/** - * @see https://github.com/multiformats/go-multihash/blob/dc3bd6897fcd17f6acd8d4d6ffd2cea3d4d3ebeb/multihash.go#L73 - */ -const SHA2_256_TRUNC254_PADDED = 0x1012 -/** - * @see https://github.com/ipfs/go-cid/blob/829c826f6be23320846f4b7318aee4d17bf8e094/cid.go#L104 - */ -const FilCommitmentUnsealed = 0xf101 - -/** - * `aggregate/offer` capability allows agent to create an offer to get an aggregate - * of CARs files in the market to be fetched and stored by a Storage provider. - */ -export const offer = capability({ - can: 'aggregate/offer', - /** - * did:key identifier of the broker authority where offer is made available. - */ - with: Schema.did(), - nb: Schema.struct({ - /** - * CID of the DAG-CBOR encoded block with offer details. - * Service will queue given offer to be validated and handled. - */ - offer: Schema.link(), - /** - * Commitment proof for the aggregate being offered. - * https://github.com/filecoin-project/go-state-types/blob/1e6cf0d47cdda75383ef036fc2725d1cf51dbde8/abi/piece.go#L47-L50 - */ - piece: Schema.struct({ - /** - * CID of the aggregate piece. - */ - link: /** @type {import('./types').PieceLinkSchema} */ ( - Schema.link({ - code: FilCommitmentUnsealed, - version: 1, - multihash: { - code: SHA2_256_TRUNC254_PADDED, - }, - }) - ), - /** - * Height of the perfect binary tree for the piece. - * It can be used to derive leafCount and consequently `size` of the piece. - */ - height: Schema.integer(), - }), - }), - derives: (claim, from) => { - return ( - and(equalWith(claim, from)) || - and(checkLink(claim.nb.offer, from.nb.offer, 'nb.offer')) || - and( - checkLink(claim.nb.piece.link, from.nb.piece.link, 'nb.piece.link') - ) || - and( - equal(claim.nb.piece.height, from.nb.piece.height, 'nb.piece.height') - ) || - ok({}) - ) - }, -}) - -/** - * Capability can be used to get information about previously stored aggregates. - * space identified by `with` field. - */ -export const get = capability({ - can: 'aggregate/get', - with: Schema.did(), - nb: Schema.struct({ - /** - * Commitment proof for the aggregate being requested. - */ - subject: /** @type {import('./types').PieceLinkSchema} */ (Schema.link()), - }), - derives: (claim, from) => { - return ( - and(equalWith(claim, from)) || - and(checkLink(claim.nb.subject, from.nb.subject, 'nb.subject')) || - ok({}) - ) - }, -}) - -// ⚠️ We export imports here so they are not omitted in generated typedes -// @see https://github.com/microsoft/TypeScript/issues/51548 -export { Schema } diff --git a/packages/capabilities/src/filecoin.js b/packages/capabilities/src/filecoin.js new file mode 100644 index 000000000..fc970ea71 --- /dev/null +++ b/packages/capabilities/src/filecoin.js @@ -0,0 +1,185 @@ +/** + * Filecoin Capabilities + * + * These can be imported directly with: + * ```js + * import * as Filecoin from '@web3-storage/capabilities/filecoin' + * ``` + * + * @module + */ + +import { capability, Schema, ok } from '@ucanto/validator' +import { equal, equalWith, checkLink, and } from './utils.js' + +/** + * @see https://github.com/multiformats/go-multihash/blob/dc3bd6897fcd17f6acd8d4d6ffd2cea3d4d3ebeb/multihash.go#L73 + */ +const SHA2_256_TRUNC254_PADDED = 0x1012 +/** + * @see https://github.com/ipfs/go-cid/blob/829c826f6be23320846f4b7318aee4d17bf8e094/cid.go#L104 + */ +const FilCommitmentUnsealed = 0xf101 + +/** + * `filecoin/add` capability allows agent to add a filecoin piece to be aggregated + * so that it can be stored by a Storage provider on a future time. + */ +export const filecoinAdd = capability({ + can: 'filecoin/add', + /** + * did:key identifier of the broker authority where offer is made available. + */ + with: Schema.did(), + nb: Schema.struct({ + /** + * CID of the content that resulted in Filecoin piece. + */ + content: Schema.link(), + /** + * CID of the piece. + */ + piece: /** @type {import('./types').PieceLinkSchema} */ ( + Schema.link({ + code: FilCommitmentUnsealed, + version: 1, + multihash: { + code: SHA2_256_TRUNC254_PADDED, + }, + }) + ), + }), + derives: (claim, from) => { + return ( + and(equalWith(claim, from)) || + and(checkLink(claim.nb.content, from.nb.content, 'nb.content')) || + and(checkLink(claim.nb.piece, from.nb.piece, 'nb.piece')) || + ok({}) + ) + }, +}) + +/** + * `aggregate/add` capability allows agent to add a piece to be aggregated + * so that it can be stored by a Storage provider on a future time. + */ +export const aggregateAdd = capability({ + can: 'aggregate/add', + /** + * did:key identifier of the broker authority where offer is made available. + */ + with: Schema.did(), + nb: Schema.struct({ + /** + * CID of the piece. + */ + piece: /** @type {import('./types').PieceLinkSchema} */ ( + Schema.link({ + code: FilCommitmentUnsealed, + version: 1, + multihash: { + code: SHA2_256_TRUNC254_PADDED, + }, + }) + ), + /** + * Storefront requestin piece to be aggregated + */ + storefront: Schema.text(), + /** + * Grouping for the piece to be aggregated + */ + group: Schema.text(), + }), + derives: (claim, from) => { + return ( + and(equalWith(claim, from)) || + and(checkLink(claim.nb.piece, from.nb.piece, 'nb.piece')) || + and(equal(claim.nb.storefront, from.nb.storefront, 'nb.storefront')) || + and(equal(claim.nb.group, from.nb.group, 'nb.group')) || + ok({}) + ) + }, +}) + +/** + * `deal/add` capability allows agent to create a deal offer to get an aggregate + * of CARs files in the market to be fetched and stored by a Storage provider. + */ +export const dealAdd = capability({ + can: 'deal/add', + /** + * did:key identifier of the broker authority where offer is made available. + */ + with: Schema.did(), + nb: Schema.struct({ + /** + * CID of the DAG-CBOR encoded block with offer details. + * Service will queue given offer to be validated and handled. + */ + pieces: Schema.link(), + /** + * Commitment proof for the aggregate being offered. + * https://github.com/filecoin-project/go-state-types/blob/1e6cf0d47cdda75383ef036fc2725d1cf51dbde8/abi/piece.go#L47-L50 + */ + aggregate: /** @type {import('./types').PieceLinkSchema} */ ( + Schema.link({ + code: FilCommitmentUnsealed, + version: 1, + multihash: { + code: SHA2_256_TRUNC254_PADDED, + }, + }) + ), + /** + * Storefront requesting deal + */ + storefront: Schema.text(), + /** + * arbitrary label to be added to the deal on chain + */ + label: Schema.text().optional(), + }), + derives: (claim, from) => { + return ( + and(equalWith(claim, from)) || + and(checkLink(claim.nb.aggregate, from.nb.aggregate, 'nb.aggregate')) || + and(checkLink(claim.nb.pieces, from.nb.pieces, 'nb.pieces')) || + and(equal(claim.nb.storefront, from.nb.storefront, 'nb.storefront')) || + and(equal(claim.nb.label, from.nb.label, 'nb.label')) || + ok({}) + ) + }, +}) + +/** + * `chain-tracker/info` capability allows agent to get chain info of a given piece. + */ +export const chainTrackerInfo = capability({ + can: 'chain-tracker/info', + /** + * did:key identifier of the broker authority where offer is made available. + */ + with: Schema.did(), + nb: Schema.struct({ + /** + * CID of the piece. + */ + piece: /** @type {import('./types').PieceLinkSchema} */ ( + Schema.link({ + code: FilCommitmentUnsealed, + version: 1, + multihash: { + code: SHA2_256_TRUNC254_PADDED, + }, + }) + ), + }), + derives: (claim, from) => { + return ( + and(equalWith(claim, from)) || + and(checkLink(claim.nb.piece, from.nb.piece, 'nb.piece')) || + ok({}) + ) + }, +}) diff --git a/packages/capabilities/src/index.js b/packages/capabilities/src/index.js index 7f9119439..b7af3b1d8 100644 --- a/packages/capabilities/src/index.js +++ b/packages/capabilities/src/index.js @@ -9,8 +9,7 @@ import * as Utils from './utils.js' import * as Consumer from './consumer.js' import * as Customer from './customer.js' import * as Console from './console.js' -import * as Offer from './offer.js' -import * as Aggregate from './aggregate.js' +import * as Filecoin from './filecoin.js' export { Access, @@ -24,8 +23,7 @@ export { Customer, Console, Utils, - Aggregate, - Offer, + Filecoin, } /** @type {import('./types.js').AbilitiesArray} */ @@ -49,7 +47,8 @@ export const abilitiesAsStrings = [ Access.access.can, Access.authorize.can, Access.session.can, - Aggregate.offer.can, - Aggregate.get.can, - Offer.arrange.can, + Filecoin.filecoinAdd.can, + Filecoin.aggregateAdd.can, + Filecoin.dealAdd.can, + Filecoin.chainTrackerInfo.can, ] diff --git a/packages/capabilities/src/offer.js b/packages/capabilities/src/offer.js deleted file mode 100644 index 5546fabf7..000000000 --- a/packages/capabilities/src/offer.js +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Offer Capabilities - * - * These can be imported directly with: - * ```js - * import * as Offer from '@web3-storage/capabilities/offer' - * ``` - * - * @module - */ -import { capability, Schema, ok } from '@ucanto/validator' -import { equalWith, checkLink, and } from './utils.js' - -/** - * Capability can be used to arrange an offer with an aggregate of CARs. - */ -export const arrange = capability({ - can: 'offer/arrange', - with: Schema.did(), - nb: Schema.struct({ - /** - * Commitment proof for the aggregate being requested. - */ - pieceLink: /** @type {import('./types').PieceLinkSchema} */ (Schema.link()), - }), - derives: (claim, from) => { - return ( - and(equalWith(claim, from)) || - and(checkLink(claim.nb.pieceLink, from.nb.pieceLink, 'nb.pieceLink')) || - ok({}) - ) - }, -}) - -// ⚠️ We export imports here so they are not omitted in generated typedes -// @see https://github.com/microsoft/TypeScript/issues/51548 -export { Schema } diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts index 42dbbf1d9..d5864287d 100644 --- a/packages/capabilities/src/types.ts +++ b/packages/capabilities/src/types.ts @@ -10,8 +10,7 @@ import { add, list, remove, store } from './store.js' import * as UploadCaps from './upload.js' import { claim, redeem } from './voucher.js' import * as AccessCaps from './access.js' -import * as AggregateCaps from './aggregate.js' -import * as OfferCaps from './offer.js' +import * as FilecoinCaps from './filecoin.js' export type { Unit } /** @@ -76,29 +75,50 @@ export type SpaceRecoverValidation = InferInvokedCapability< > export type SpaceRecover = InferInvokedCapability -// Aggregate -export interface AggregateGetSuccess { - deals: unknown[] +// filecoin +export type FILECOIN_PROCESSING_STATUS = 'pending' | 'done' +export interface FilecoinAddSuccess { + piece: PieceLink } -export interface AggregateGetFailure extends Ucanto.Failure { - name: 'AggregateNotFound' +export interface FilecoinAddFailure extends Ucanto.Failure { + name: string } -export interface AggregateOfferSuccess { - status: string +export interface AggregateAddSuccess { + piece: PieceLink + aggregate?: PieceLink } -export interface AggregateOfferFailure extends Ucanto.Failure { - name: - | 'AggregateOfferInvalidSize' - | 'AggregateOfferBlockNotFound' - | 'AggregateOfferInvalidUrl' +export interface AggregateAddFailure extends Ucanto.Failure { + name: string } -export interface OfferArrangeSuccess { - status: string +export interface DealAddSuccess { + aggregate?: PieceLink } -export interface OfferArrangeFailure extends Ucanto.Failure { - name: 'OfferArrangeNotFound' + +export type DealAddFailure = DealAddParseFailure | DealAddFailureWithBadPiece + +export interface DealAddParseFailure extends Ucanto.Failure { + name: string +} + +export interface DealAddFailureWithBadPiece extends Ucanto.Failure { + piece?: PieceLink + cause?: DealAddFailureCause[] | unknown +} + +export interface DealAddFailureCause { + piece: PieceLink + reason: string +} + +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface ChainTrackerInfoSuccess { + // TODO +} + +export interface ChainTrackerInfoFailure extends Ucanto.Failure { + // TODO } // Voucher Protocol @@ -114,12 +134,17 @@ export type Store = InferInvokedCapability export type StoreAdd = InferInvokedCapability export type StoreRemove = InferInvokedCapability export type StoreList = InferInvokedCapability -// Aggregate -export type AggregateOffer = InferInvokedCapability -export type AggregateGet = InferInvokedCapability -// Offer -export type OfferArrange = InferInvokedCapability - +// Filecoin +export type FilecoinAdd = InferInvokedCapability< + typeof FilecoinCaps.filecoinAdd +> +export type AggregateAdd = InferInvokedCapability< + typeof FilecoinCaps.aggregateAdd +> +export type DealAdd = InferInvokedCapability +export type ChainTrackerInfo = InferInvokedCapability< + typeof FilecoinCaps.chainTrackerInfo +> // Top export type Top = InferInvokedCapability @@ -145,7 +170,8 @@ export type AbilitiesArray = [ Access['can'], AccessAuthorize['can'], AccessSession['can'], - AggregateOffer['can'], - AggregateGet['can'], - OfferArrange['can'] + FilecoinAdd['can'], + AggregateAdd['can'], + DealAdd['can'], + ChainTrackerInfo['can'] ] diff --git a/packages/aggregate-api/package.json b/packages/filecoin-api/package.json similarity index 66% rename from packages/aggregate-api/package.json rename to packages/filecoin-api/package.json index 64be82874..128c08668 100644 --- a/packages/aggregate-api/package.json +++ b/packages/filecoin-api/package.json @@ -1,6 +1,6 @@ { - "name": "@web3-storage/aggregate-api", - "version": "1.0.0", + "name": "@web3-storage/filecoin-api", + "version": "0.0.0", "type": "module", "main": "./src/lib.js", "files": [ @@ -14,11 +14,20 @@ "src/lib.js": [ "dist/src/lib.d.ts" ], - "aggregate": [ - "dist/src/aggregate.d.ts" + "aggregator": [ + "dist/src/aggregator.d.ts" ], - "offer": [ - "dist/src/offer.d.ts" + "dealer": [ + "dist/src/dealer.d.ts" + ], + "chain-tracker": [ + "dist/src/chain-tracker.d.ts" + ], + "errors": [ + "dist/src/errors.d.ts" + ], + "storefront": [ + "dist/src/storefront.d.ts" ], "types": [ "dist/src/types.d.ts" @@ -37,13 +46,25 @@ "types": "./dist/src/types.d.ts", "import": "./src/types.js" }, - "./aggregate": { - "types": "./dist/src/aggregate.d.ts", - "import": "./src/aggregate.js" + "./aggregator": { + "types": "./dist/src/aggregator.d.ts", + "import": "./src/aggregator.js" + }, + "./dealer": { + "types": "./dist/src/dealer.d.ts", + "import": "./src/dealer.js" }, - "./offer": { - "types": "./dist/src/offer.d.ts", - "import": "./src/offer.js" + "./chain-tracker": { + "types": "./dist/src/chain-tracker.d.ts", + "import": "./src/chain-tracker.js" + }, + "./storefront": { + "types": "./dist/src/storefront.d.ts", + "import": "./src/storefront.js" + }, + "./errors": { + "types": "./dist/src/errors.d.ts", + "import": "./src/errors.js" }, "./test": { "types": "./dist/test/lib.d.ts", @@ -58,6 +79,7 @@ "test-watch": "pnpm build && mocha --bail --timeout 10s --watch --parallel -n no-warnings -n experimental-vm-modules -n experimental-fetch --watch-files src,test" }, "dependencies": { + "@ipld/dag-ucan": "^3.3.2", "@ucanto/client": "^8.0.0", "@ucanto/core": "^8.0.0", "@ucanto/interface": "^8.0.0", @@ -69,12 +91,14 @@ "devDependencies": { "@ipld/car": "^5.1.1", "@types/mocha": "^10.0.1", + "@ucanto/client": "^8.0.0", "@ucanto/principal": "^8.0.0", "@web-std/blob": "^3.0.4", - "@web3-storage/aggregate-client": "workspace:^", + "@web3-storage/filecoin-client": "workspace:^", "hd-scripts": "^4.1.0", "mocha": "^10.2.0", - "multiformats": "^11.0.2" + "multiformats": "^11.0.2", + "p-wait-for": "^5.0.2" }, "eslintConfig": { "extends": [ diff --git a/packages/filecoin-api/src/aggregator.js b/packages/filecoin-api/src/aggregator.js new file mode 100644 index 000000000..1cf0fb65b --- /dev/null +++ b/packages/filecoin-api/src/aggregator.js @@ -0,0 +1,127 @@ +import * as Server from '@ucanto/server' +import * as Client from '@ucanto/client' +import * as CAR from '@ucanto/transport/car' +import { Filecoin as FilecoinCapabilities } from '@web3-storage/capabilities' + +import * as API from './types.js' +import { QueueOperationFailed, StoreOperationFailed } from './errors.js' + +/** + * @param {API.Input} input + * @param {API.AggregatorServiceContext} context + * @returns {Promise | API.UcantoInterface.JoinBuilder>} + */ +export const add = async ({ capability }, context) => { + const { piece, storefront, group } = capability.nb + + // If self issued we accept without verification + return context.id.did() === capability.with + ? accept(piece, storefront, group, context) + : enqueue(piece, storefront, group, context) +} + +/** + * @param {import('@web3-storage/data-segment').PieceLink} piece + * @param {string} storefront + * @param {string} group + * @param {API.AggregatorServiceContext} context + * @returns {Promise | API.UcantoInterface.JoinBuilder>} + */ +async function enqueue(piece, storefront, group, context) { + const queued = await context.addQueue.add({ + piece, + storefront, + group, + insertedAt: Date.now(), + }) + if (queued.error) { + return { + error: new QueueOperationFailed(queued.error.message), + } + } + + // Create effect for receipt + const fx = await FilecoinCapabilities.aggregateAdd + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + piece, + storefront, + group, + }, + }) + .delegate() + + return Server.ok({ + piece, + }).join(fx.link()) +} + +/** + * @param {import('@web3-storage/data-segment').PieceLink} piece + * @param {string} storefront + * @param {string} group + * @param {API.AggregatorServiceContext} context + * @returns {Promise | API.UcantoInterface.JoinBuilder>} + */ +async function accept(piece, storefront, group, context) { + // Store piece into the store. Store events MAY be used to propagate piece over + const put = await context.pieceStore.put({ + piece, + storefront, + group, + insertedAt: Date.now(), + }) + + if (put.error) { + return { + error: new StoreOperationFailed(put.error.message), + } + } + + return { + ok: { + piece, + }, + } +} + +/** + * @param {API.AggregatorServiceContext} context + */ +export function createService(context) { + return { + aggregate: { + add: Server.provideAdvanced({ + capability: FilecoinCapabilities.aggregateAdd, + handler: (input) => add(input, context), + }), + }, + } +} + +/** + * @param {API.UcantoServerContext & API.AggregatorServiceContext} context + */ +export const createServer = (context) => + Server.create({ + id: context.id, + codec: context.codec || CAR.inbound, + service: createService(context), + catch: (error) => context.errorReporter.catch(error), + }) + +/** + * @param {object} options + * @param {API.UcantoInterface.Principal} options.id + * @param {API.UcantoInterface.Transport.Channel} options.channel + * @param {API.UcantoInterface.OutboundCodec} [options.codec] + */ +export const connect = ({ id, channel, codec = CAR.outbound }) => + Client.connect({ + id, + channel, + codec, + }) diff --git a/packages/filecoin-api/src/dealer.js b/packages/filecoin-api/src/dealer.js new file mode 100644 index 000000000..a13284dfa --- /dev/null +++ b/packages/filecoin-api/src/dealer.js @@ -0,0 +1,172 @@ +import * as Server from '@ucanto/server' +import * as Client from '@ucanto/client' +import * as CAR from '@ucanto/transport/car' +import { CBOR } from '@ucanto/core' +import { Filecoin as FilecoinCapabilities } from '@web3-storage/capabilities' + +import * as API from './types.js' +import { + QueueOperationFailed, + StoreOperationFailed, + DecodeBlockOperationFailed, +} from './errors.js' + +/** + * @param {API.Input} input + * @param {API.DealerServiceContext} context + * @returns {Promise | API.UcantoInterface.JoinBuilder>} + */ +export const add = async ({ capability, invocation }, context) => { + const { aggregate, pieces: offerCid, storefront, label } = capability.nb + const pieces = getOfferBlock(offerCid, invocation.iterateIPLDBlocks()) + + if (!pieces) { + return { + error: new DecodeBlockOperationFailed( + `missing offer block in invocation: ${offerCid.toString()}` + ), + } + } + + // If self issued we accept without verification + return context.id.did() === capability.with + ? accept(aggregate, pieces, storefront, label, context) + : enqueue(aggregate, offerCid, storefront, label, pieces, context) +} + +/** + * @param {import('@web3-storage/data-segment').PieceLink} aggregate + * @param {Server.API.Link} offerCid + * @param {string} storefront + * @param {string | undefined} label + * @param {import('@web3-storage/data-segment').PieceLink[]} pieces + * @param {API.DealerServiceContext} context + * @returns {Promise | API.UcantoInterface.JoinBuilder>} + */ +async function enqueue( + aggregate, + offerCid, + storefront, + label, + pieces, + context +) { + const queued = await context.addQueue.add({ + aggregate, + pieces, // add queue can opt to store offers in separate datastore + storefront, + label, + insertedAt: Date.now(), + }) + if (queued.error) { + return { + error: new QueueOperationFailed(queued.error.message), + } + } + + // Create effect for receipt + const fx = await FilecoinCapabilities.dealAdd + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + aggregate, + pieces: offerCid, + storefront, + label, + }, + }) + .delegate() + + return Server.ok({ + aggregate, + }).join(fx.link()) +} + +/** + * @param {import('@web3-storage/data-segment').PieceLink} aggregate + * @param {import('@web3-storage/data-segment').PieceLink[]} pieces + * @param {string} storefront + * @param {string | undefined} label + * @param {API.DealerServiceContext} context + * @returns {Promise | API.UcantoInterface.JoinBuilder>} + */ +async function accept(aggregate, pieces, storefront, label, context) { + // TODO: failure - needs to read from store + + // Store aggregate into the store. Store events MAY be used to propagate aggregate over + const put = await context.offerStore.put({ + aggregate, + pieces, + storefront, + label, + insertedAt: Date.now(), + }) + if (put.error) { + return { + error: new StoreOperationFailed(put.error.message), + } + } + + return { + ok: { + aggregate, + }, + } +} + +/** + * @param {Server.API.Link} offerCid + * @param {IterableIterator>} blockIterator + */ +function getOfferBlock(offerCid, blockIterator) { + for (const block of blockIterator) { + if (block.cid.equals(offerCid)) { + const decoded = + /** @type {import('@web3-storage/data-segment').PieceLink[]} */ ( + CBOR.decode(block.bytes) + ) + return decoded + // TODO: Validate with schema + } + } +} + +/** + * @param {API.DealerServiceContext} context + */ +export function createService(context) { + return { + deal: { + add: Server.provideAdvanced({ + capability: FilecoinCapabilities.dealAdd, + handler: (input) => add(input, context), + }), + }, + } +} + +/** + * @param {API.UcantoServerContext & API.DealerServiceContext} context + */ +export const createServer = (context) => + Server.create({ + id: context.id, + codec: context.codec || CAR.inbound, + service: createService(context), + catch: (error) => context.errorReporter.catch(error), + }) + +/** + * @param {object} options + * @param {API.UcantoInterface.Principal} options.id + * @param {API.UcantoInterface.Transport.Channel} options.channel + * @param {API.UcantoInterface.OutboundCodec} [options.codec] + */ +export const connect = ({ id, channel, codec = CAR.outbound }) => + Client.connect({ + id, + channel, + codec, + }) diff --git a/packages/filecoin-api/src/errors.js b/packages/filecoin-api/src/errors.js new file mode 100644 index 000000000..a1f627ee9 --- /dev/null +++ b/packages/filecoin-api/src/errors.js @@ -0,0 +1,87 @@ +import * as Server from '@ucanto/server' + +export const QueueOperationErrorName = /** @type {const} */ ( + 'QueueOperationFailed' +) +export class QueueOperationFailed extends Server.Failure { + /** + * @param {string} message + */ + constructor(message) { + super(message) + } + get reason() { + return this.message + } + get name() { + return QueueOperationErrorName + } +} + +export const StoreOperationErrorName = /** @type {const} */ ( + 'StoreOperationFailed' +) +export class StoreOperationFailed extends Server.Failure { + /** + * @param {string} message + */ + constructor(message) { + super(message) + } + get reason() { + return this.message + } + get name() { + return StoreOperationErrorName + } +} + +export const StoreNotFoundErrorName = /** @type {const} */ ('StoreNotFound') +export class StoreNotFound extends Server.Failure { + /** + * @param {string} message + */ + constructor(message) { + super(message) + } + get reason() { + return this.message + } + get name() { + return StoreNotFoundErrorName + } +} + +export const EncodeRecordErrorName = /** @type {const} */ ('EncodeRecordFailed') +export class EncodeRecordFailed extends Server.Failure { + /** + * @param {string} message + */ + constructor(message) { + super(message) + } + get reason() { + return this.message + } + get name() { + return EncodeRecordErrorName + } +} + +export const DecodeBlockOperationErrorName = /** @type {const} */ ( + 'DecodeBlockOperationFailed' +) +export class DecodeBlockOperationFailed extends Server.Failure { + /** + * @param {string} message + */ + constructor(message) { + super(message) + } + get reason() { + return this.message + } + get name() { + return DecodeBlockOperationErrorName + } +} diff --git a/packages/filecoin-api/src/lib.js b/packages/filecoin-api/src/lib.js new file mode 100644 index 000000000..6401e13e8 --- /dev/null +++ b/packages/filecoin-api/src/lib.js @@ -0,0 +1,3 @@ +export * as Storefront from './storefront.js' +export * as Aggregator from './aggregator.js' +export * as Dealer from './dealer.js' diff --git a/packages/filecoin-api/src/storefront.js b/packages/filecoin-api/src/storefront.js new file mode 100644 index 000000000..3e731b821 --- /dev/null +++ b/packages/filecoin-api/src/storefront.js @@ -0,0 +1,122 @@ +import * as Server from '@ucanto/server' +import * as Client from '@ucanto/client' +import * as CAR from '@ucanto/transport/car' +import { Filecoin as FilecoinCapabilities } from '@web3-storage/capabilities' + +import * as API from './types.js' +import { QueueOperationFailed, StoreOperationFailed } from './errors.js' + +/** + * @param {API.Input} input + * @param {API.StorefrontServiceContext} context + * @returns {Promise | API.UcantoInterface.JoinBuilder>} + */ +export const add = async ({ capability }, context) => { + // TODO: source + const { piece, content } = capability.nb + + // If self issued we accept without verification + return context.id.did() === capability.with + ? accept(piece, content, context) + : enqueue(piece, content, context) +} + +/** + * @param {import('@web3-storage/data-segment').PieceLink} piece + * @param {import('multiformats').UnknownLink} content + * @param {API.StorefrontServiceContext} context + * @returns {Promise | API.UcantoInterface.JoinBuilder>} + */ +async function enqueue(piece, content, context) { + const queued = await context.addQueue.add({ + piece, + content, + insertedAt: Date.now(), + }) + if (queued.error) { + return { + error: new QueueOperationFailed(queued.error.message), + } + } + + // Create effect for receipt + const fx = await FilecoinCapabilities.filecoinAdd + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + piece, + content, + }, + }) + .delegate() + + return Server.ok({ + piece, + }).join(fx.link()) +} + +/** + * @param {import('@web3-storage/data-segment').PieceLink} piece + * @param {import('multiformats').UnknownLink} content + * @param {API.StorefrontServiceContext} context + * @returns {Promise | API.UcantoInterface.JoinBuilder>} + */ +async function accept(piece, content, context) { + // Store piece into the store. Store events MAY be used to propagate piece over + const put = await context.pieceStore.put({ + content, + piece, + insertedAt: Date.now(), + }) + if (put.error) { + return { + error: new StoreOperationFailed(put.error.message), + } + } + + return { + ok: { + piece, + }, + } +} + +/** + * @param {API.StorefrontServiceContext} context + */ +export function createService(context) { + return { + filecoin: { + add: Server.provideAdvanced({ + capability: FilecoinCapabilities.filecoinAdd, + handler: (input) => add(input, context), + }), + }, + } +} + +/** + * @param {API.UcantoServerContext & API.StorefrontServiceContext} context + */ +export const createServer = (context) => + Server.create({ + id: context.id, + codec: context.codec || CAR.inbound, + service: createService(context), + catch: (error) => context.errorReporter.catch(error), + }) + +/** + * @param {object} options + * @param {API.UcantoInterface.Principal} options.id + * @param {API.UcantoInterface.Transport.Channel} options.channel + * @param {API.UcantoInterface.OutboundCodec} [options.codec] + */ +export const connect = ({ id, channel, codec = CAR.outbound }) => + Client.connect({ + id, + channel, + codec, + }) diff --git a/packages/aggregate-api/src/types.js b/packages/filecoin-api/src/types.js similarity index 100% rename from packages/aggregate-api/src/types.js rename to packages/filecoin-api/src/types.js diff --git a/packages/filecoin-api/src/types.ts b/packages/filecoin-api/src/types.ts new file mode 100644 index 000000000..ad5a2f132 --- /dev/null +++ b/packages/filecoin-api/src/types.ts @@ -0,0 +1,189 @@ +import type { + HandlerExecutionError, + Signer, + InboundCodec, + CapabilityParser, + ParsedCapability, + InferInvokedCapability, + Match, +} from '@ucanto/interface' +import type { ProviderInput } from '@ucanto/server' +import { PieceLink } from '@web3-storage/data-segment' +import { UnknownLink } from '@ucanto/interface' + +export * as UcantoInterface from '@ucanto/interface' +export * from '@web3-storage/filecoin-client/types' +export * from '@web3-storage/capabilities/types' + +// Resources +export interface Queue { + add: ( + record: Record, + options?: QueueMessageOptions + ) => Promise> +} + +export interface Store { + put: (record: Record) => Promise> + /** + * Gets content data from the store. + */ + get(key: any): Promise> +} + +export interface QueueMessageOptions { + messageGroupId?: string +} + +// Services +export interface StorefrontServiceContext { + id: Signer + addQueue: Queue + pieceStore: Store +} + +export interface AggregatorServiceContext { + id: Signer + addQueue: Queue + pieceStore: Store +} + +export interface DealerServiceContext { + id: Signer + addQueue: Queue + offerStore: Store +} + +// Service Types + +export interface StorefrontRecord { + piece: PieceLink + content: UnknownLink + insertedAt: number +} + +export interface AggregatorRecord { + piece: PieceLink + storefront: string + group: string + insertedAt: number +} + +export interface DealerRecord { + aggregate: PieceLink + pieces: PieceLink[] + storefront: string + label?: string + insertedAt: number +} + +// Errors + +export type QueueAddError = QueueOperationError | EncodeRecordFailed +export type StorePutError = StoreOperationError | EncodeRecordFailed +export type StoreGetError = + | StoreOperationError + | EncodeRecordFailed + | StoreNotFound + +export interface QueueOperationError extends Error { + name: 'QueueOperationFailed' +} + +export interface StoreOperationError extends Error { + name: 'StoreOperationFailed' +} + +export interface StoreNotFound extends Error { + name: 'StoreNotFound' +} + +export interface EncodeRecordFailed extends Error { + name: 'EncodeRecordFailed' +} + +// Service utils + +export interface UcantoServerContext { + id: Signer + codec?: InboundCodec + errorReporter: ErrorReporter +} + +export interface ErrorReporter { + catch: (error: HandlerExecutionError) => void +} + +export type Result = Variant<{ + ok: T + error: X +}> + +/** + * Utility type for defining a [keyed union] type as in IPLD Schema. In practice + * this just works around typescript limitation that requires discriminant field + * on all variants. + * + * ```ts + * type Result = + * | { ok: T } + * | { error: X } + * + * const demo = (result: Result) => { + * if (result.ok) { + * // ^^^^^^^^^ Property 'ok' does not exist on type '{ error: Error; }` + * } + * } + * ``` + * + * Using `Variant` type we can define same union type that works as expected: + * + * ```ts + * type Result = Variant<{ + * ok: T + * error: X + * }> + * + * const demo = (result: Result) => { + * if (result.ok) { + * result.ok.toUpperCase() + * } + * } + * ``` + * + * [keyed union]:https://ipld.io/docs/schemas/features/representation-strategies/#union-keyed-representation + */ +export type Variant> = { + [Key in keyof U]: { [K in Exclude]?: never } & { + [K in Key]: U[Key] + } +}[keyof U] + +// test + +export interface UcantoServerContextTest extends UcantoServerContext { + queuedMessages: unknown[] +} + +export type Test = ( + assert: Assert, + context: UcantoServerContextTest & S +) => unknown +export type Tests = Record> + +export type Input>> = + ProviderInput & ParsedCapability> + +export interface Assert { + equal: ( + actual: Actual, + expected: Expected, + message?: string + ) => unknown + deepEqual: ( + actual: Actual, + expected: Expected, + message?: string + ) => unknown + ok: (actual: Actual, message?: string) => unknown +} diff --git a/packages/filecoin-api/src/utils.js b/packages/filecoin-api/src/utils.js new file mode 100644 index 000000000..0019d7bb1 --- /dev/null +++ b/packages/filecoin-api/src/utils.js @@ -0,0 +1,22 @@ +import * as DID from '@ipld/dag-ucan/did' +import { connect } from '@ucanto/client' +import { CAR, HTTP } from '@ucanto/transport' + +/** + * @param {{ did: string, url: string }} config + */ +export function getServiceConnection(config) { + const servicePrincipal = DID.parse(config.did) + const serviceURL = new URL(config.url) + + const serviceConnection = connect({ + id: servicePrincipal, + codec: CAR.outbound, + channel: HTTP.open({ + url: serviceURL, + method: 'POST', + }), + }) + + return serviceConnection +} diff --git a/packages/filecoin-api/test/aggregator.spec.js b/packages/filecoin-api/test/aggregator.spec.js new file mode 100644 index 000000000..ac89d6b9a --- /dev/null +++ b/packages/filecoin-api/test/aggregator.spec.js @@ -0,0 +1,55 @@ +/* eslint-disable no-only-tests/no-only-tests */ +import * as assert from 'assert' +import * as Aggregator from './services/aggregator.js' +import * as Signer from '@ucanto/principal/ed25519' + +import { Store } from './context/store.js' +import { Queue } from './context/queue.js' + +describe('aggregate/*', () => { + for (const [name, test] of Object.entries(Aggregator.test)) { + const define = name.startsWith('only ') + ? it.only + : name.startsWith('skip ') + ? it.skip + : it + + define(name, async () => { + const signer = await Signer.generate() + const id = signer.withDID('did:web:test.aggregator.web3.storage') + + // resources + /** @type {unknown[]} */ + const queuedMessages = [] + const addQueue = new Queue({ + onMessage: (message) => queuedMessages.push(message), + }) + const pieceLookupFn = ( + /** @type {Iterable | ArrayLike} */ items, + /** @type {any} */ record + ) => { + return Array.from(items).find((i) => i.piece.equals(record.piece)) + } + const pieceStore = new Store(pieceLookupFn) + + await test( + { + equal: assert.strictEqual, + deepEqual: assert.deepStrictEqual, + ok: assert.ok, + }, + { + id, + errorReporter: { + catch(error) { + assert.fail(error) + }, + }, + addQueue, + pieceStore, + queuedMessages, + } + ) + }) + } +}) diff --git a/packages/filecoin-api/test/context/queue.js b/packages/filecoin-api/test/context/queue.js new file mode 100644 index 000000000..54139a87b --- /dev/null +++ b/packages/filecoin-api/test/context/queue.js @@ -0,0 +1,30 @@ +import * as API from '../../src/types.js' + +/** + * @template T + * @implements {API.Queue} + */ +export class Queue { + /** + * @param {object} [options] + * @param {(message: T) => void} [options.onMessage] + */ + constructor(options = {}) { + /** @type {Set} */ + this.items = new Set() + + this.onMessage = options.onMessage || (() => {}) + } + + /** + * @param {T} record + */ + async add(record) { + this.items.add(record) + + this.onMessage(record) + return Promise.resolve({ + ok: {}, + }) + } +} diff --git a/packages/filecoin-api/test/context/store.js b/packages/filecoin-api/test/context/store.js new file mode 100644 index 000000000..795ca050b --- /dev/null +++ b/packages/filecoin-api/test/context/store.js @@ -0,0 +1,39 @@ +import * as API from '../../src/types.js' + +/** + * @template T + * @implements {API.Store} + */ +export class Store { + /** + * @param {(items: Set, item: any) => T} lookupFn + */ + constructor(lookupFn) { + /** @type {Set} */ + this.items = new Set() + + this.lookupFn = lookupFn + } + + /** + * @param {T} record + */ + async put(record) { + this.items.add(record) + + return Promise.resolve({ + ok: {}, + }) + } + + /** + * + * @param {any} item + * @returns boolean + */ + async get(item) { + return { + ok: this.lookupFn(this.items, item), + } + } +} diff --git a/packages/filecoin-api/test/dealer.spec.js b/packages/filecoin-api/test/dealer.spec.js new file mode 100644 index 000000000..b9fbf93ac --- /dev/null +++ b/packages/filecoin-api/test/dealer.spec.js @@ -0,0 +1,57 @@ +/* eslint-disable no-only-tests/no-only-tests */ +import * as assert from 'assert' +import * as Broker from './services/dealer.js' +import * as Signer from '@ucanto/principal/ed25519' + +import { Store } from './context/store.js' +import { Queue } from './context/queue.js' + +describe('deal/*', () => { + for (const [name, test] of Object.entries(Broker.test)) { + const define = name.startsWith('only ') + ? it.only + : name.startsWith('skip ') + ? it.skip + : it + + define(name, async () => { + const signer = await Signer.generate() + const id = signer.withDID('did:web:test.spade-proxy.web3.storage') + + // resources + /** @type {unknown[]} */ + const queuedMessages = [] + const addQueue = new Queue({ + onMessage: (message) => queuedMessages.push(message), + }) + const offerLookupFn = ( + /** @type {Iterable | ArrayLike} */ items, + /** @type {any} */ record + ) => { + return Array.from(items).find((i) => + i.aggregate.equals(record.aggregate) + ) + } + const offerStore = new Store(offerLookupFn) + + await test( + { + equal: assert.strictEqual, + deepEqual: assert.deepStrictEqual, + ok: assert.ok, + }, + { + id, + errorReporter: { + catch(error) { + assert.fail(error) + }, + }, + addQueue, + offerStore, + queuedMessages, + } + ) + }) + } +}) diff --git a/packages/filecoin-api/test/lib.js b/packages/filecoin-api/test/lib.js new file mode 100644 index 000000000..0ff2d6c31 --- /dev/null +++ b/packages/filecoin-api/test/lib.js @@ -0,0 +1,12 @@ +import * as Aggregator from './services/aggregator.js' +import * as Dealer from './services/dealer.js' +import * as Storefront from './services/storefront.js' +export * from './utils.js' + +export const test = { + ...Aggregator.test, + ...Dealer.test, + ...Storefront.test, +} + +export { Aggregator, Dealer, Storefront } diff --git a/packages/filecoin-api/test/services/aggregator.js b/packages/filecoin-api/test/services/aggregator.js new file mode 100644 index 000000000..a999ce8f6 --- /dev/null +++ b/packages/filecoin-api/test/services/aggregator.js @@ -0,0 +1,161 @@ +import { Filecoin } from '@web3-storage/capabilities' +import * as Signer from '@ucanto/principal/ed25519' +import pWaitFor from 'p-wait-for' + +import * as API from '../../src/types.js' + +import { randomCargo } from '../utils.js' +import { createServer, connect } from '../../src/aggregator.js' + +/** + * @type {API.Tests} + */ +export const test = { + 'piece/add inserts piece into processing queue': async (assert, context) => { + const { storefront } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + const group = 'did:web:free.web3.storage' + + // storefront invocation + const pieceAddInv = Filecoin.aggregateAdd.invoke({ + issuer: storefront, + audience: connection.id, + with: storefront.did(), + nb: { + piece: cargo.link.link(), + storefront: storefront.did(), + group, + }, + }) + + const response = await pieceAddInv.execute(connection) + if (response.out.error) { + throw new Error('invocation failed', { cause: response.out.error }) + } + assert.ok(response.out.ok) + assert.deepEqual(response.out.ok.piece, cargo.link.link()) + + // Validate effect in receipt + const fx = await Filecoin.aggregateAdd + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + piece: cargo.link.link(), + storefront: storefront.did(), + group, + }, + }) + .delegate() + + assert.ok(response.fx.join) + assert.ok(fx.link().equals(response.fx.join?.link())) + + // Validate queue and store + await pWaitFor(() => context.queuedMessages.length === 1) + + const hasStoredPiece = await context.pieceStore.get({ + piece: cargo.link.link(), + storefront: storefront.did(), + }) + assert.ok(!hasStoredPiece.ok) + }, + 'piece/add from signer inserts piece into store and returns accepted': async ( + assert, + context + ) => { + const { storefront } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + const group = 'did:web:free.web3.storage' + + // aggregator invocation + const pieceAddInv = Filecoin.aggregateAdd.invoke({ + issuer: context.id, + audience: connection.id, + with: context.id.did(), + nb: { + piece: cargo.link.link(), + storefront: storefront.did(), + group, + }, + }) + + const response = await pieceAddInv.execute(connection) + if (response.out.error) { + throw new Error('invocation failed', { cause: response.out.error }) + } + assert.ok(response.out.ok) + assert.deepEqual(response.out.ok.piece, cargo.link.link()) + + // Validate queue and store + await pWaitFor(() => context.queuedMessages.length === 0) + + const hasStoredPiece = await context.pieceStore.get({ + piece: cargo.link.link(), + storefront: storefront.did(), + }) + assert.ok(hasStoredPiece.ok) + assert.ok(hasStoredPiece.ok?.piece.equals(cargo.link.link())) + assert.deepEqual(hasStoredPiece.ok?.group, group) + assert.deepEqual(hasStoredPiece.ok?.storefront, storefront.did()) + }, + 'skip piece/add from signer inserts piece into store and returns rejected': + async (assert, context) => { + const { storefront } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + const group = 'did:web:free.web3.storage' + + // aggregator invocation + const pieceAddInv = Filecoin.aggregateAdd.invoke({ + issuer: context.id, + audience: connection.id, + with: context.id.did(), + nb: { + piece: cargo.link.link(), + storefront: storefront.did(), + group, + }, + }) + + const response = await pieceAddInv.execute(connection) + if (response.out.error) { + throw new Error('invocation failed', { cause: response.out.error }) + } + assert.ok(response.out.ok) + assert.deepEqual(response.out.ok.piece, cargo.link.link()) + + // Validate queue and store + await pWaitFor(() => context.queuedMessages.length === 0) + + const hasStoredPiece = await context.pieceStore.get({ + piece: cargo.link.link(), + storefront: storefront.did(), + }) + assert.ok(!hasStoredPiece.ok) + }, +} + +async function getServiceContext() { + const storefront = await Signer.generate() + + return { storefront } +} diff --git a/packages/filecoin-api/test/services/dealer.js b/packages/filecoin-api/test/services/dealer.js new file mode 100644 index 000000000..18996e697 --- /dev/null +++ b/packages/filecoin-api/test/services/dealer.js @@ -0,0 +1,175 @@ +import { Filecoin } from '@web3-storage/capabilities' +import * as Signer from '@ucanto/principal/ed25519' +import pWaitFor from 'p-wait-for' +import { CBOR } from '@ucanto/core' + +import * as API from '../../src/types.js' + +import { randomAggregate } from '../utils.js' +import { createServer, connect } from '../../src/dealer.js' + +/** + * @type {API.Tests} + */ +export const test = { + 'aggregate/add inserts piece into processing queue': async ( + assert, + context + ) => { + const { aggregator, storefront: storefrontSigner } = + await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Generate piece for test + const { pieces, aggregate } = await randomAggregate(100, 128) + const offer = pieces.map((p) => p.link) + const piecesBlock = await CBOR.write(offer) + const storefront = storefrontSigner.did() + const label = 'label' + + // aggregator invocation + const pieceAddInv = Filecoin.dealAdd.invoke({ + issuer: aggregator, + audience: connection.id, + with: aggregator.did(), + nb: { + aggregate: aggregate.link, + pieces: piecesBlock.cid, + storefront, + label, + }, + }) + pieceAddInv.attach(piecesBlock) + + const response = await pieceAddInv.execute(connection) + if (response.out.error) { + throw new Error('invocation failed', { cause: response.out.error }) + } + assert.ok(response.out.ok) + assert.deepEqual(response.out.ok.aggregate, aggregate.link) + + // Validate effect in receipt + const fx = await Filecoin.dealAdd + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + aggregate: aggregate.link, + pieces: piecesBlock.cid, + storefront, + label, + }, + }) + .delegate() + + assert.ok(response.fx.join) + assert.ok(fx.link().equals(response.fx.join?.link())) + + // Validate queue and store + await pWaitFor(() => context.queuedMessages.length === 1) + + const hasStoredOffer = await context.offerStore.get({ + piece: aggregate.link.link(), + }) + assert.ok(!hasStoredOffer.ok) + }, + 'aggregate/add from signer inserts piece into store and returns accepted': + async (assert, context) => { + const { storefront: storefrontSigner } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Generate piece for test + const { pieces, aggregate } = await randomAggregate(100, 128) + const offer = pieces.map((p) => p.link) + const piecesBlock = await CBOR.write(offer) + const storefront = storefrontSigner.did() + const label = 'label' + + // aggregator invocation + const pieceAddInv = Filecoin.dealAdd.invoke({ + issuer: context.id, + audience: connection.id, + with: context.id.did(), + nb: { + aggregate: aggregate.link, + pieces: piecesBlock.cid, + storefront, + label, + }, + }) + pieceAddInv.attach(piecesBlock) + + const response = await pieceAddInv.execute(connection) + if (response.out.error) { + throw new Error('invocation failed', { cause: response.out.error }) + } + assert.ok(response.out.ok) + assert.deepEqual(response.out.ok.aggregate, aggregate.link) + + // Validate queue and store + await pWaitFor(() => context.queuedMessages.length === 0) + + const hasStoredOffer = await context.offerStore.get({ + aggregate: aggregate.link.link(), + }) + assert.ok(hasStoredOffer.ok) + }, + 'skip aggregate/add from signer inserts piece into store and returns rejected': + async (assert, context) => { + const { storefront: storefrontSigner } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Generate piece for test + const { pieces, aggregate } = await randomAggregate(100, 128) + const offer = pieces.map((p) => p.link) + const piecesBlock = await CBOR.write(offer) + const storefront = storefrontSigner.did() + const label = 'label' + + // aggregator invocation + const pieceAddInv = Filecoin.dealAdd.invoke({ + issuer: context.id, + audience: connection.id, + with: context.id.did(), + nb: { + aggregate: aggregate.link, + pieces: piecesBlock.cid, + storefront, + label, + }, + }) + pieceAddInv.attach(piecesBlock) + + const response = await pieceAddInv.execute(connection) + if (response.out.error) { + throw new Error('invocation failed', { cause: response.out.error }) + } + assert.ok(response.out.ok) + assert.deepEqual(response.out.ok.aggregate, aggregate.link) + + // Validate queue and store + await pWaitFor(() => context.queuedMessages.length === 0) + + const hasStoredOffer = await context.offerStore.get({ + aggregate: aggregate.link.link(), + }) + assert.ok(!hasStoredOffer.ok) + }, +} + +async function getServiceContext() { + const aggregator = await Signer.generate() + const storefront = await Signer.generate() + + return { aggregator, storefront } +} diff --git a/packages/filecoin-api/test/services/storefront.js b/packages/filecoin-api/test/services/storefront.js new file mode 100644 index 000000000..71cb505f7 --- /dev/null +++ b/packages/filecoin-api/test/services/storefront.js @@ -0,0 +1,147 @@ +import { Filecoin } from '@web3-storage/capabilities' +import * as Signer from '@ucanto/principal/ed25519' +import pWaitFor from 'p-wait-for' + +import * as API from '../../src/types.js' + +import { randomCargo } from '../utils.js' +import { createServer, connect } from '../../src/storefront.js' + +/** + * @type {API.Tests} + */ +export const test = { + 'filecoin/add inserts piece into verification queue': async ( + assert, + context + ) => { + const { agent } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + + // agent invocation + const filecoinAddInv = Filecoin.filecoinAdd.invoke({ + issuer: agent, + audience: connection.id, + with: agent.did(), + nb: { + piece: cargo.link.link(), + content: cargo.content.link(), + }, + }) + + const response = await filecoinAddInv.execute(connection) + if (response.out.error) { + throw new Error('invocation failed', { cause: response.out.error }) + } + assert.ok(response.out.ok) + assert.deepEqual(response.out.ok.piece, cargo.link.link()) + + // Validate effect in receipt + const fx = await Filecoin.filecoinAdd + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + piece: cargo.link.link(), + content: cargo.content.link(), + }, + }) + .delegate() + + assert.ok(response.fx.join) + assert.ok(fx.link().equals(response.fx.join?.link())) + + // Validate queue and store + await pWaitFor(() => context.queuedMessages.length === 1) + + const hasStoredPiece = await context.pieceStore.get({ + piece: cargo.link.link(), + }) + assert.ok(!hasStoredPiece.ok) + }, + 'filecoin/add from signer inserts piece into store and returns accepted': + async (assert, context) => { + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + + // storefront invocation + const filecoinAddInv = Filecoin.filecoinAdd.invoke({ + issuer: context.id, + audience: connection.id, + with: context.id.did(), + nb: { + piece: cargo.link.link(), + content: cargo.content.link(), + }, + }) + + const response = await filecoinAddInv.execute(connection) + if (response.out.error) { + throw new Error('invocation failed', { cause: response.out.error }) + } + assert.ok(response.out.ok) + assert.deepEqual(response.out.ok.piece, cargo.link.link()) + + // Validate queue and store + await pWaitFor(() => context.queuedMessages.length === 0) + + const hasStoredPiece = await context.pieceStore.get({ + piece: cargo.link.link(), + }) + assert.ok(hasStoredPiece.ok) + }, + 'skip filecoin/add from signer inserts piece into store and returns rejected': + async (assert, context) => { + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + + // storefront invocation + const filecoinAddInv = Filecoin.filecoinAdd.invoke({ + issuer: context.id, + audience: connection.id, + with: context.id.did(), + nb: { + piece: cargo.link.link(), + content: cargo.content.link(), + }, + }) + + const response = await filecoinAddInv.execute(connection) + if (response.out.error) { + throw new Error('invocation failed', { cause: response.out.error }) + } + assert.ok(response.out.ok) + assert.deepEqual(response.out.ok.piece, cargo.link.link()) + + // Validate queue and store + await pWaitFor(() => context.queuedMessages.length === 0) + + const hasStoredPiece = await context.pieceStore.get({ + piece: cargo.link.link(), + }) + assert.ok(!hasStoredPiece.ok) + }, +} + +async function getServiceContext() { + const agent = await Signer.generate() + + return { agent } +} diff --git a/packages/filecoin-api/test/storefront.spec.js b/packages/filecoin-api/test/storefront.spec.js new file mode 100644 index 000000000..9f563e0cf --- /dev/null +++ b/packages/filecoin-api/test/storefront.spec.js @@ -0,0 +1,56 @@ +/* eslint-disable no-only-tests/no-only-tests */ +import * as assert from 'assert' +import * as Signer from '@ucanto/principal/ed25519' + +import * as Storefront from './services/storefront.js' + +import { Store } from './context/store.js' +import { Queue } from './context/queue.js' + +describe('filecoin/*', () => { + for (const [name, test] of Object.entries(Storefront.test)) { + const define = name.startsWith('only ') + ? it.only + : name.startsWith('skip ') + ? it.skip + : it + + define(name, async () => { + const signer = await Signer.generate() + const id = signer.withDID('did:web:test.web3.storage') + + // resources + /** @type {unknown[]} */ + const queuedMessages = [] + const addQueue = new Queue({ + onMessage: (message) => queuedMessages.push(message), + }) + const pieceLookupFn = ( + /** @type {Iterable | ArrayLike} */ items, + /** @type {any} */ record + ) => { + return Array.from(items).find((i) => i.piece.equals(record.piece)) + } + const pieceStore = new Store(pieceLookupFn) + + await test( + { + equal: assert.strictEqual, + deepEqual: assert.deepStrictEqual, + ok: assert.ok, + }, + { + id, + errorReporter: { + catch(error) { + assert.fail(error) + }, + }, + addQueue, + pieceStore, + queuedMessages, + } + ) + }) + } +}) diff --git a/packages/aggregate-api/test/utils.js b/packages/filecoin-api/test/utils.js similarity index 98% rename from packages/aggregate-api/test/utils.js rename to packages/filecoin-api/test/utils.js index 663d7685b..47452a6cf 100644 --- a/packages/aggregate-api/test/utils.js +++ b/packages/filecoin-api/test/utils.js @@ -67,6 +67,7 @@ export async function randomCargo(length, size) { return { link: piece.link, + content: car.cid, height: piece.height, size: piece.size, } diff --git a/packages/aggregate-api/tsconfig.json b/packages/filecoin-api/tsconfig.json similarity index 100% rename from packages/aggregate-api/tsconfig.json rename to packages/filecoin-api/tsconfig.json diff --git a/packages/filecoin-client/README.md b/packages/filecoin-client/README.md new file mode 100644 index 000000000..327a13393 --- /dev/null +++ b/packages/filecoin-client/README.md @@ -0,0 +1,138 @@ +

⁂
web3.storage

+

The w3filecoin client for https://web3.storage

+ +## About + +The `@web3-storage/filecoin-client` package provides the "low level" client API to make data uploaded with the w3up platform available in Filecoin Storage providers. It is based on [web3-storage/specs/w3-filecoin.md])https://github.com/web3-storage/specs/blob/feat/filecoin-spec/w3-filecoin.md) and is not intended for web3.storage end users. + +## Install + +Install the package using npm: + +```bash +npm install @web3-storage/filecoin-client +``` + +## Usage + +### `Storefront.filecoinAdd` + +Request a Storefront service to add computed filecoin piece into Filecoin Storage Providers. + +```js +import { Storefront } from '@web3-storage/filecoin-client' + +const add = await Storefront.filecoinAdd( + invocationConfig, + piece, + content +) +``` + +```typescript +function filecoinAdd( + conf: InvocationConfig, + piece: Piece, // Filecoin piece + content: Link, // Content CID +): Promise +``` + +More information: [`InvocationConfig`](#invocationconfig) + +### `Aggregator.pieceAdd` + +Request an Aggregator service to add a filecoin piece into an aggregate to be offered to Filecoin Storage Providers. + +```js +import { Aggregator } from '@web3-storage/filecoin-client' + +const add = await Aggregator.pieceAdd( + invocationConfig, + piece, + storefront, + group +) +``` + +```typescript +function pieceAdd( + conf: InvocationConfig, + piece: Piece, // Filecoin piece + storefront: string, // Storefront identifier + group: string, // Aggregate grouping with different criterium +): Promise +``` + +More information: [`InvocationConfig`](#invocationconfig) + +### `Broker.aggregateAdd` + +Request a Broker service to offer a filecoin piece (larger aggregate of pieces) to Filecoin Storage Providers. + +```js +import { Broker } from '@web3-storage/filecoin-client' + +const add = await Broker.aggregateAdd( + invocationConfig, + piece, + offer, + deal +) +``` + +```typescript +function pieceAdd( + conf: InvocationConfig, + piece: Piece, // Filecoin piece representing aggregate + offer: Piece[], // Filecoin pieces part of the aggregate (sorted) + deal: DealConfig, // Aggregate grouping with different criterium +): Promise + +interface DealConfig { + tenantId: string // Identifier of the tenant (storefront) for Broker + label?: string // optionaal label for deal +} +``` + +More information: [`InvocationConfig`](#invocationconfig) + +### `Chain.chainInfo` + +Request a Chain service to find chain information of a given piece. It will return deals where given piece is present in Receipt. + +```js +import { Chain } from '@web3-storage/filecoin-client' + +const add = await Broker.chainInfo( + invocationConfig, + piece +) +``` + +```typescript +function chainInfo( + conf: InvocationConfig, + piece: Piece, // Filecoin piece to check +): Promise +``` + +More information: [`InvocationConfig`](#invocationconfig) + +## Types + +### `InvocationConfig` + +This is the configuration for the UCAN invocation. It is an object with `issuer`, `audience`, `resource` and `proofs`: + +- The `issuer` is the signing authority that is issuing the UCAN invocation(s). +- The `audience` is the principal authority that the UCAN is delegated to. +- The `resource` (`with` field) points to a storage space. +- The `proofs` are a set of capability delegations that prove the issuer has the capability to perform the action. These might not be required. + +## Contributing + +Feel free to join in. All welcome. Please [open an issue](https://github.com/web3-storage/w3protocol/issues)! + +## License + +Dual-licensed under [MIT + Apache 2.0](https://github.com/web3-storage/w3protocol/blob/main/license.md) \ No newline at end of file diff --git a/packages/aggregate-client/package.json b/packages/filecoin-client/package.json similarity index 79% rename from packages/aggregate-client/package.json rename to packages/filecoin-client/package.json index 451974cf8..c72eed351 100644 --- a/packages/aggregate-client/package.json +++ b/packages/filecoin-client/package.json @@ -1,12 +1,12 @@ { - "name": "@web3-storage/aggregate-client", - "version": "1.0.0", - "description": "The web3.storage aggregate client", - "homepage": "https://github.com/web3-storage/w3up/tree/main/packages/aggregate-client", + "name": "@web3-storage/filecoin-client", + "version": "0.0.0", + "description": "The w3filecoin client for web3.storage", + "homepage": "https://github.com/web3-storage/w3up/tree/main/packages/filecoin-client", "repository": { "type": "git", "url": "https://github.com/web3-storage/w3up.git", - "directory": "packages/aggregate-client" + "directory": "packages/w3filecoin-client" }, "author": "Vasco Santos", "license": "Apache-2.0 OR MIT", @@ -24,7 +24,10 @@ }, "exports": { ".": "./src/index.js", - "./aggregate": "./src/aggregate.js", + "./aggregator": "./src/aggregator.js", + "./dealer": "./src/dealer.js", + "./chain-tracker": "./src/chain-tracker.js", + "./storefront": "./src/storefront.js", "./types": "./src/types.js" }, "typesVersions": { @@ -32,8 +35,17 @@ "types": [ "dist/src/types.d.ts" ], - "aggregate": [ - "dist/src/aggregate.d.ts" + "aggregator": [ + "dist/src/aggregator.d.ts" + ], + "dealer": [ + "dist/src/dealer.d.ts" + ], + "chain-tracker": [ + "dist/src/chain-tracker.d.ts" + ], + "storefront": [ + "dist/src/storefront.d.ts" ] } }, diff --git a/packages/filecoin-client/src/aggregator.js b/packages/filecoin-client/src/aggregator.js new file mode 100644 index 000000000..2fc1e1e55 --- /dev/null +++ b/packages/filecoin-client/src/aggregator.js @@ -0,0 +1,56 @@ +import { connect } from '@ucanto/client' +import { CAR, HTTP } from '@ucanto/transport' + +import { Filecoin as FilecoinCapabilities } from '@web3-storage/capabilities' + +import { services } from './service.js' + +/** + * @typedef {import('./types.js').AggregatorService} AggregatorService + * @typedef {import('@ucanto/interface').ConnectionView} ConnectionView + */ + +/** @type {ConnectionView} */ +export const connection = connect({ + id: services.AGGREGATOR.principal, + codec: CAR.outbound, + channel: HTTP.open({ + url: services.AGGREGATOR.url, + method: 'POST', + }), +}) + +/** + * Add a piece to the aggregator system of the filecoin pipeline. + * + * @param {import('./types.js').InvocationConfig} conf - Configuration + * @param {import('@web3-storage/data-segment').PieceLink} piece + * @param {string} storefront + * @param {string} group + * @param {import('./types.js').RequestOptions} [options] + */ +export async function aggregateAdd( + { issuer, with: resource, proofs, audience }, + piece, + storefront, + group, + options = {} +) { + /* c8 ignore next */ + const conn = options.connection ?? connection + + const invocation = FilecoinCapabilities.aggregateAdd.invoke({ + issuer, + /* c8 ignore next */ + audience: audience ?? services.AGGREGATOR.principal, + with: resource, + nb: { + piece, + storefront, + group, + }, + proofs, + }) + + return await invocation.execute(conn) +} diff --git a/packages/filecoin-client/src/chain-tracker.js b/packages/filecoin-client/src/chain-tracker.js new file mode 100644 index 000000000..d06dfb7b5 --- /dev/null +++ b/packages/filecoin-client/src/chain-tracker.js @@ -0,0 +1,50 @@ +import { connect } from '@ucanto/client' +import { CAR, HTTP } from '@ucanto/transport' + +import { Filecoin as FilecoinCapabilities } from '@web3-storage/capabilities' + +import { services } from './service.js' + +/** + * @typedef {import('./types.js').ChainTrackerService} ChainTrackerService + * @typedef {import('@ucanto/interface').ConnectionView} ConnectionView + */ + +/** @type {ConnectionView} */ +export const connection = connect({ + id: services.CHAIN_TRACKER.principal, + codec: CAR.outbound, + channel: HTTP.open({ + url: services.CHAIN_TRACKER.url, + method: 'POST', + }), +}) + +/** + * Get chain information for a given a piece.. + * + * @param {import('./types.js').InvocationConfig} conf - Configuration + * @param {import('@web3-storage/data-segment').PieceLink} piece + * @param {import('./types.js').RequestOptions} [options] + */ +export async function chainInfo( + { issuer, with: resource, proofs, audience }, + piece, + options = {} +) { + /* c8 ignore next */ + const conn = options.connection ?? connection + + const invocation = FilecoinCapabilities.chainTrackerInfo.invoke({ + issuer, + /* c8 ignore next */ + audience: audience ?? services.CHAIN_TRACKER.principal, + with: resource, + nb: { + piece, + }, + proofs, + }) + + return await invocation.execute(conn) +} diff --git a/packages/filecoin-client/src/dealer.js b/packages/filecoin-client/src/dealer.js new file mode 100644 index 000000000..4d3cc412a --- /dev/null +++ b/packages/filecoin-client/src/dealer.js @@ -0,0 +1,62 @@ +import { connect } from '@ucanto/client' +import { CAR, HTTP } from '@ucanto/transport' +import { CBOR } from '@ucanto/core' + +import { Filecoin as FilecoinCapabilities } from '@web3-storage/capabilities' + +import { services } from './service.js' + +/** + * @typedef {import('./types.js').DealerService} DealerService + * @typedef {import('@ucanto/interface').ConnectionView} ConnectionView + */ + +/** @type {ConnectionView} */ +export const connection = connect({ + id: services.DEALER.principal, + codec: CAR.outbound, + channel: HTTP.open({ + url: services.DEALER.url, + method: 'POST', + }), +}) + +/** + * Add a piece (aggregate) to the dealer system of the filecoin pipeline to offer to SPs. + * + * @param {import('./types.js').InvocationConfig} conf - Configuration + * @param {import('@web3-storage/data-segment').PieceLink} aggregate + * @param {import('@web3-storage/data-segment').PieceLink[]} pieces + * @param {string} storefront + * @param {string} label + * @param {import('./types.js').RequestOptions} [options] + */ +export async function dealAdd( + { issuer, with: resource, proofs, audience }, + aggregate, + pieces, + storefront, + label, + options = {} +) { + /* c8 ignore next */ + const conn = options.connection ?? connection + + const block = await CBOR.write(pieces) + const invocation = FilecoinCapabilities.dealAdd.invoke({ + issuer, + /* c8 ignore next */ + audience: audience ?? services.AGGREGATOR.principal, + with: resource, + nb: { + aggregate, + pieces: block.cid, + storefront, + label, + }, + proofs, + }) + invocation.attach(block) + + return await invocation.execute(conn) +} diff --git a/packages/filecoin-client/src/index.js b/packages/filecoin-client/src/index.js new file mode 100644 index 000000000..4ac9cae0b --- /dev/null +++ b/packages/filecoin-client/src/index.js @@ -0,0 +1,4 @@ +export * as Storefront from './storefront.js' +export * as Aggregator from './aggregator.js' +export * as Dealer from './dealer.js' +export * as Chain from './chain-tracker.js' diff --git a/packages/filecoin-client/src/service.js b/packages/filecoin-client/src/service.js new file mode 100644 index 000000000..c5e62c2ee --- /dev/null +++ b/packages/filecoin-client/src/service.js @@ -0,0 +1,28 @@ +import * as DID from '@ipld/dag-ucan/did' + +/** + * @typedef {import('./types').SERVICE} Service + * @typedef {import('./types').ServiceConfig} ServiceConfig + */ + +/** + * @type {Record} + */ +export const services = { + STOREFRONT: { + url: new URL('https://up.web3.storage'), + principal: DID.parse('did:web:web3.storage'), + }, + AGGREGATOR: { + url: new URL('https://aggregator.web3.storage'), + principal: DID.parse('did:web:web3.storage'), + }, + DEALER: { + url: new URL('https://spade-proxy.web3.storage'), + principal: DID.parse('did:web:spade.web3.storage'), + }, + CHAIN_TRACKER: { + url: new URL('https://spade-proxy.web3.storage'), + principal: DID.parse('did:web:spade.web3.storage'), + }, +} diff --git a/packages/filecoin-client/src/storefront.js b/packages/filecoin-client/src/storefront.js new file mode 100644 index 000000000..d847b1e1a --- /dev/null +++ b/packages/filecoin-client/src/storefront.js @@ -0,0 +1,53 @@ +import { connect } from '@ucanto/client' +import { CAR, HTTP } from '@ucanto/transport' + +import { Filecoin as FilecoinCapabilities } from '@web3-storage/capabilities' + +import { services } from './service.js' + +/** + * @typedef {import('./types.js').StorefrontService} StorefrontService + * @typedef {import('@ucanto/interface').ConnectionView} ConnectionView + */ + +/** @type {ConnectionView} */ +export const connection = connect({ + id: services.STOREFRONT.principal, + codec: CAR.outbound, + channel: HTTP.open({ + url: services.STOREFRONT.url, + method: 'POST', + }), +}) + +/** + * Add a piece to the filecoin pipeline. + * + * @param {import('./types.js').InvocationConfig} conf - Configuration + * @param {import('@web3-storage/data-segment').PieceLink} piece + * @param {import('multiformats').UnknownLink} content + * @param {import('./types.js').RequestOptions} [options] + */ +export async function filecoinAdd( + { issuer, with: resource, proofs, audience }, + piece, + content, + options = {} +) { + /* c8 ignore next */ + const conn = options.connection ?? connection + + const invocation = FilecoinCapabilities.filecoinAdd.invoke({ + issuer, + /* c8 ignore next */ + audience: audience ?? services.STOREFRONT.principal, + with: resource, + nb: { + content: content, + piece, + }, + proofs, + }) + + return await invocation.execute(conn) +} diff --git a/packages/filecoin-client/src/types.js b/packages/filecoin-client/src/types.js new file mode 100644 index 000000000..336ce12bb --- /dev/null +++ b/packages/filecoin-client/src/types.js @@ -0,0 +1 @@ +export {} diff --git a/packages/filecoin-client/src/types.ts b/packages/filecoin-client/src/types.ts new file mode 100644 index 000000000..4ed973665 --- /dev/null +++ b/packages/filecoin-client/src/types.ts @@ -0,0 +1,83 @@ +import { + ConnectionView, + ServiceMethod, + Signer, + Proof, + DID, + Principal, +} from '@ucanto/interface' +import { + FilecoinAdd, + FilecoinAddSuccess, + FilecoinAddFailure, + AggregateAdd, + AggregateAddSuccess, + AggregateAddFailure, + DealAddSuccess, + DealAdd, + DealAddFailure, + ChainTrackerInfo, + ChainTrackerInfoSuccess, + ChainTrackerInfoFailure, +} from '@web3-storage/capabilities/types' + +export type SERVICE = 'STOREFRONT' | 'AGGREGATOR' | 'DEALER' | 'CHAIN_TRACKER' +export interface ServiceConfig { + url: URL + principal: Principal +} + +export interface InvocationConfig { + /** + * Signing authority that is issuing the UCAN invocation(s). + */ + issuer: Signer + /** + * The principal delegated to in the current UCAN. + */ + audience?: Principal + /** + * The resource the invocation applies to. + */ + with: DID + /** + * Proof(s) the issuer has the capability to perform the action. + */ + proofs?: Proof[] +} + +export interface StorefrontService { + filecoin: { + add: ServiceMethod + } +} + +export interface AggregatorService { + aggregate: { + add: ServiceMethod + } +} + +export interface DealerService { + deal: { + add: ServiceMethod + } +} + +export interface ChainTrackerService { + 'chain-tracker': { + info: ServiceMethod< + ChainTrackerInfo, + ChainTrackerInfoSuccess, + ChainTrackerInfoFailure + > + } +} + +export interface RequestOptions> { + connection?: ConnectionView +} + +export interface Connectable> { + connection?: ConnectionView +} diff --git a/packages/filecoin-client/test/aggregator.test.js b/packages/filecoin-client/test/aggregator.test.js new file mode 100644 index 000000000..4ba3ad41d --- /dev/null +++ b/packages/filecoin-client/test/aggregator.test.js @@ -0,0 +1,223 @@ +import assert from 'assert' +import * as Signer from '@ucanto/principal/ed25519' +import * as Client from '@ucanto/client' +import * as Server from '@ucanto/server' +import * as CAR from '@ucanto/transport/car' +import { Filecoin as FilecoinCapabilities } from '@web3-storage/capabilities' + +import { aggregateAdd } from '../src/aggregator.js' + +import { randomCargo } from './helpers/random.js' +import { mockService } from './helpers/mocks.js' +import { OperationFailed, OperationErrorName } from './helpers/errors.js' +import { serviceProvider as aggregatorService } from './fixtures.js' + +describe('aggregate/add', () => { + it('storefront adds a filecoin piece to aggregator, getting the piece queued', async () => { + const { storefront } = await getContext() + + // Generate cargo to add + const [cargo] = await randomCargo(1, 100) + const storefrontId = storefront.did() + const group = 'did:web:free.web3.storage' + + /** @type {import('@web3-storage/capabilities/types').AggregateAddSuccess} */ + const pieceAddResponse = { + piece: cargo.link, + } + + // Create Ucanto service + const service = mockService({ + aggregate: { + add: Server.provideAdvanced({ + capability: FilecoinCapabilities.aggregateAdd, + handler: async ({ invocation, context }) => { + assert.strictEqual(invocation.issuer.did(), storefront.did()) + assert.strictEqual(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.strictEqual( + invCap.can, + FilecoinCapabilities.aggregateAdd.can + ) + assert.equal(invCap.with, invocation.issuer.did()) + // piece link + assert.ok(invCap.nb?.piece.equals(cargo.link.link())) + // group + assert.strictEqual(invCap.nb?.group, group) + + // Create effect for receipt with self signed queued operation + const fx = await FilecoinCapabilities.aggregateAdd + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: invCap.nb, + }) + .delegate() + + return Server.ok(pieceAddResponse).join(fx.link()) + }, + }), + }, + }) + + // invoke piece add from storefront + const res = await aggregateAdd( + { + issuer: storefront, + with: storefront.did(), + audience: aggregatorService, + }, + cargo.link.link(), + storefrontId, + group, + { connection: getConnection(service).connection } + ) + + assert.ok(res.out.ok) + assert.deepEqual(res.out.ok, pieceAddResponse) + // includes effect fx in receipt + assert.ok(res.fx.join) + }) + + it('aggregator self invokes add a filecoin piece to accept the piece queued', async () => { + const { storefront } = await getContext() + + // Generate cargo to add + const [cargo] = await randomCargo(1, 100) + const storefrontId = storefront.did() + const group = 'did:web:free.web3.storage' + + /** @type {import('@web3-storage/capabilities/types').AggregateAddSuccess} */ + const pieceAddResponse = { + piece: cargo.link, + } + + // Create Ucanto service + const service = mockService({ + aggregate: { + add: Server.provideAdvanced({ + capability: FilecoinCapabilities.aggregateAdd, + handler: async ({ invocation }) => { + assert.strictEqual(invocation.issuer.did(), aggregatorService.did()) + assert.strictEqual(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.strictEqual( + invCap.can, + FilecoinCapabilities.aggregateAdd.can + ) + assert.equal(invCap.with, invocation.issuer.did()) + // piece link + assert.ok(invCap.nb?.piece.equals(cargo.link.link())) + // group + assert.strictEqual(invCap.nb?.group, group) + + return Server.ok(pieceAddResponse) + }, + }), + }, + }) + + // self invoke piece/add from aggregator + const res = await aggregateAdd( + { + issuer: aggregatorService, + with: aggregatorService.did(), + audience: aggregatorService, + }, + cargo.link.link(), + storefrontId, + group, + { connection: getConnection(service).connection } + ) + + assert.ok(res.out.ok) + assert.deepEqual(res.out.ok, pieceAddResponse) + // does not include effect fx in receipt + assert.ok(!res.fx.join) + }) + + it('aggregator self invokes add a filecoin piece to reject the piece queued', async () => { + const { storefront } = await getContext() + + // Generate cargo to add + const [cargo] = await randomCargo(1, 100) + const storefrontId = storefront.did() + const group = 'did:web:free.web3.storage' + + /** @type {import('@web3-storage/capabilities/types').AggregateAddFailure} */ + const pieceAddResponse = new OperationFailed( + 'failed to add to aggregate', + cargo.link + ) + + // Create Ucanto service + const service = mockService({ + aggregate: { + add: Server.provideAdvanced({ + capability: FilecoinCapabilities.aggregateAdd, + handler: async ({ invocation }) => { + assert.strictEqual(invocation.issuer.did(), aggregatorService.did()) + assert.strictEqual(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.strictEqual( + invCap.can, + FilecoinCapabilities.aggregateAdd.can + ) + assert.equal(invCap.with, invocation.issuer.did()) + // piece link + assert.ok(invCap.nb?.piece.equals(cargo.link.link())) + // group + assert.strictEqual(invCap.nb?.group, group) + + return { + error: pieceAddResponse, + } + }, + }), + }, + }) + + // self invoke piece add from aggregator + const res = await aggregateAdd( + { + issuer: aggregatorService, + with: aggregatorService.did(), + audience: aggregatorService, + }, + cargo.link.link(), + storefrontId, + group, + { connection: getConnection(service).connection } + ) + + assert.ok(res.out.error) + assert.deepEqual(res.out.error.name, OperationErrorName) + // does not include effect fx in receipt + assert.ok(!res.fx.join) + }) +}) + +async function getContext() { + const storefront = await Signer.generate() + + return { storefront } +} + +/** + * @param {import('../src/types').AggregatorService} service + */ +function getConnection(service) { + const server = Server.create({ + id: aggregatorService, + service, + codec: CAR.inbound, + }) + const connection = Client.connect({ + id: aggregatorService, + codec: CAR.outbound, + channel: server, + }) + + return { connection } +} diff --git a/packages/filecoin-client/test/chain-tracker.test.js b/packages/filecoin-client/test/chain-tracker.test.js new file mode 100644 index 000000000..885abb3c6 --- /dev/null +++ b/packages/filecoin-client/test/chain-tracker.test.js @@ -0,0 +1,88 @@ +import assert from 'assert' +import * as Signer from '@ucanto/principal/ed25519' +import * as Client from '@ucanto/client' +import * as Server from '@ucanto/server' +import * as CAR from '@ucanto/transport/car' +import { Filecoin as FilecoinCapabilities } from '@web3-storage/capabilities' + +import { chainInfo } from '../src/chain-tracker.js' + +import { randomCargo } from './helpers/random.js' +import { mockService } from './helpers/mocks.js' +import { serviceProvider as chainService } from './fixtures.js' + +describe('chain.info', () => { + it('storefront gets info of a filecoin piece from chain', async () => { + const { storefront } = await getContext() + + // Generate cargo to get info + const [cargo] = await randomCargo(1, 100) + + /** @type {import('@web3-storage/capabilities/types').ChainTrackerInfoSuccess} */ + const chainInfoResponse = { + piece: cargo.link, + } + + // Create Ucanto service + const service = mockService({ + 'chain-tracker': { + info: Server.provideAdvanced({ + capability: FilecoinCapabilities.chainTrackerInfo, + handler: async ({ invocation, context }) => { + assert.strictEqual(invocation.issuer.did(), storefront.did()) + assert.strictEqual(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.strictEqual( + invCap.can, + FilecoinCapabilities.chainTrackerInfo.can + ) + assert.equal(invCap.with, invocation.issuer.did()) + + // piece link + assert.ok(invCap.nb?.piece.equals(cargo.link.link())) + + return Server.ok(chainInfoResponse) + }, + }), + }, + }) + + // invoke piece add from storefront + const res = await chainInfo( + { + issuer: storefront, + with: storefront.did(), + audience: chainService, + }, + cargo.link.link(), + { connection: getConnection(service).connection } + ) + + assert.ok(res.out.ok) + assert.deepEqual(res.out.ok, chainInfoResponse) + }) +}) + +async function getContext() { + const storefront = await Signer.generate() + + return { storefront } +} + +/** + * @param {import('../src/types.js').ChainTrackerService} service + */ +function getConnection(service) { + const server = Server.create({ + id: chainService, + service, + codec: CAR.inbound, + }) + const connection = Client.connect({ + id: chainService, + codec: CAR.outbound, + channel: server, + }) + + return { connection } +} diff --git a/packages/filecoin-client/test/dealer.test.js b/packages/filecoin-client/test/dealer.test.js new file mode 100644 index 000000000..228ac4686 --- /dev/null +++ b/packages/filecoin-client/test/dealer.test.js @@ -0,0 +1,242 @@ +import assert from 'assert' +import * as Signer from '@ucanto/principal/ed25519' +import * as Client from '@ucanto/client' +import * as Server from '@ucanto/server' +import * as CAR from '@ucanto/transport/car' +import { CBOR } from '@ucanto/core' +import { Filecoin as FilecoinCapabilities } from '@web3-storage/capabilities' + +import { dealAdd } from '../src/dealer.js' + +import { randomAggregate } from './helpers/random.js' +import { mockService } from './helpers/mocks.js' +import { OperationFailed, OperationErrorName } from './helpers/errors.js' +import { serviceProvider as brokerService } from './fixtures.js' + +describe('aggregate.add', () => { + it('aggregator adds an aggregate piece to the broker, getting the piece queued', async () => { + const { aggregator, storefront: storefrontSigner } = await getContext() + + // generate aggregate to add + const { pieces, aggregate } = await randomAggregate(100, 100) + const offer = pieces.map((p) => p.link) + const piecesBlock = await CBOR.write(offer) + const storefront = storefrontSigner.did() + const label = 'label' + /** @type {import('@web3-storage/capabilities/types').DealAddSuccess} */ + const dealAddResponse = { + aggregate: aggregate.link, + } + + // Create Ucanto service + const service = mockService({ + deal: { + add: Server.provideAdvanced({ + capability: FilecoinCapabilities.dealAdd, + handler: async ({ invocation, context }) => { + assert.strictEqual(invocation.issuer.did(), aggregator.did()) + assert.strictEqual(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.strictEqual(invCap.can, FilecoinCapabilities.dealAdd.can) + assert.equal(invCap.with, invocation.issuer.did()) + assert.ok(invCap.nb) + + // piece link + assert.ok(invCap.nb.aggregate.equals(aggregate.link.link())) + + // Validate block inline exists + const invocationBlocks = Array.from(invocation.iterateIPLDBlocks()) + assert.ok( + invocationBlocks.find((b) => b.cid.equals(piecesBlock.cid)) + ) + + // Create effect for receipt with self signed queued operation + const fx = await FilecoinCapabilities.dealAdd + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: invCap.nb, + }) + .delegate() + + return Server.ok(dealAddResponse).join(fx.link()) + }, + }), + }, + }) + + // invoke piece add from storefront + const res = await dealAdd( + { + issuer: aggregator, + with: aggregator.did(), + audience: brokerService, + }, + aggregate.link.link(), + offer, + storefront, + label, + { connection: getConnection(service).connection } + ) + + assert.ok(res.out.ok) + assert.deepEqual(res.out.ok, dealAddResponse) + // includes effect fx in receipt + assert.ok(res.fx.join) + }) + + it('broker self invokes add an aggregate piece to accept the piece queued', async () => { + const { storefront: storefrontSigner } = await getContext() + + // generate aggregate to add + const { pieces, aggregate } = await randomAggregate(100, 100) + const offer = pieces.map((p) => p.link) + const piecesBlock = await CBOR.write(offer) + const storefront = storefrontSigner.did() + const label = 'label' + + /** @type {import('@web3-storage/capabilities/types').DealAddSuccess} */ + const dealAddResponse = { + aggregate: aggregate.link, + } + + // Create Ucanto service + const service = mockService({ + deal: { + add: Server.provideAdvanced({ + capability: FilecoinCapabilities.dealAdd, + handler: async ({ invocation }) => { + assert.strictEqual(invocation.issuer.did(), brokerService.did()) + assert.strictEqual(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.strictEqual(invCap.can, FilecoinCapabilities.dealAdd.can) + assert.equal(invCap.with, invocation.issuer.did()) + assert.ok(invCap.nb) + + // piece link + assert.ok(invCap.nb.aggregate.equals(aggregate.link.link())) + + // Validate block inline exists + const invocationBlocks = Array.from(invocation.iterateIPLDBlocks()) + assert.ok( + invocationBlocks.find((b) => b.cid.equals(piecesBlock.cid)) + ) + + return Server.ok(dealAddResponse) + }, + }), + }, + }) + + // invoke piece add from storefront + const res = await dealAdd( + { + issuer: brokerService, + with: brokerService.did(), + audience: brokerService, + }, + aggregate.link.link(), + offer, + storefront, + label, + { connection: getConnection(service).connection } + ) + + assert.ok(res.out.ok) + assert.deepEqual(res.out.ok, dealAddResponse) + // does not include effect fx in receipt + assert.ok(!res.fx.join) + }) + + it('broker self invokes add an aggregate piece to reject the piece queued', async () => { + const { storefront: storefrontSigner } = await getContext() + + // generate aggregate to add + const { pieces, aggregate } = await randomAggregate(100, 100) + const offer = pieces.map((p) => p.link) + const piecesBlock = await CBOR.write(offer) + const storefront = storefrontSigner.did() + const label = 'label' + + /** @type {import('@web3-storage/capabilities/types').DealAddFailure} */ + const dealAddResponse = new OperationFailed( + 'failed to add to aggregate', + aggregate.link + ) + + // Create Ucanto service + const service = mockService({ + deal: { + add: Server.provideAdvanced({ + capability: FilecoinCapabilities.dealAdd, + handler: async ({ invocation, context }) => { + assert.strictEqual(invocation.issuer.did(), brokerService.did()) + assert.strictEqual(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.strictEqual(invCap.can, FilecoinCapabilities.dealAdd.can) + assert.equal(invCap.with, invocation.issuer.did()) + assert.ok(invCap.nb) + + // piece link + assert.ok(invCap.nb.aggregate.equals(aggregate.link.link())) + + // Validate block inline exists + const invocationBlocks = Array.from(invocation.iterateIPLDBlocks()) + assert.ok( + invocationBlocks.find((b) => b.cid.equals(piecesBlock.cid)) + ) + + return { + error: dealAddResponse, + } + }, + }), + }, + }) + + // invoke piece add from storefront + const res = await dealAdd( + { + issuer: brokerService, + with: brokerService.did(), + audience: brokerService, + }, + aggregate.link.link(), + offer, + storefront, + label, + { connection: getConnection(service).connection } + ) + + assert.ok(res.out.error) + assert.deepEqual(res.out.error.name, OperationErrorName) + // does not include effect fx in receipt + assert.ok(!res.fx.join) + }) +}) + +async function getContext() { + const aggregator = await Signer.generate() + const storefront = await Signer.generate() + + return { aggregator, storefront } +} + +/** + * @param {import('../src/types.js').DealerService} service + */ +function getConnection(service) { + const server = Server.create({ + id: brokerService, + service, + codec: CAR.inbound, + }) + const connection = Client.connect({ + id: brokerService, + codec: CAR.outbound, + channel: server, + }) + + return { connection } +} diff --git a/packages/aggregate-client/test/fixtures.js b/packages/filecoin-client/test/fixtures.js similarity index 100% rename from packages/aggregate-client/test/fixtures.js rename to packages/filecoin-client/test/fixtures.js diff --git a/packages/aggregate-client/test/helpers/block.js b/packages/filecoin-client/test/helpers/block.js similarity index 100% rename from packages/aggregate-client/test/helpers/block.js rename to packages/filecoin-client/test/helpers/block.js diff --git a/packages/aggregate-client/test/helpers/car.js b/packages/filecoin-client/test/helpers/car.js similarity index 100% rename from packages/aggregate-client/test/helpers/car.js rename to packages/filecoin-client/test/helpers/car.js diff --git a/packages/filecoin-client/test/helpers/errors.js b/packages/filecoin-client/test/helpers/errors.js new file mode 100644 index 000000000..eac436079 --- /dev/null +++ b/packages/filecoin-client/test/helpers/errors.js @@ -0,0 +1,21 @@ +import * as Server from '@ucanto/server' + +export const OperationErrorName = /** @type {const} */ ('OperationFailed') +export class OperationFailed extends Server.Failure { + /** + * @param {string} message + * @param {import('@web3-storage/data-segment').PieceLink} piece + */ + constructor(message, piece) { + super(message) + this.piece = piece + } + + get reason() { + return this.message + } + + get name() { + return OperationErrorName + } +} diff --git a/packages/aggregate-client/test/helpers/mocks.js b/packages/filecoin-client/test/helpers/mocks.js similarity index 50% rename from packages/aggregate-client/test/helpers/mocks.js rename to packages/filecoin-client/test/helpers/mocks.js index cfe92fe76..d06948dcc 100644 --- a/packages/aggregate-client/test/helpers/mocks.js +++ b/packages/filecoin-client/test/helpers/mocks.js @@ -5,19 +5,26 @@ const notImplemented = () => { } /** - * @param {Partial<{ - * aggregate: Partial - * offer: Partial - * }>} impl + * @param {Partial< + * import('../../src/types').StorefrontService & + * import('../../src/types').AggregatorService & + * import('../../src/types').DealerService & + * import('../../src/types').ChainTrackerService + * >} impl */ export function mockService(impl) { return { + filecoin: { + add: withCallCount(impl.filecoin?.add ?? notImplemented), + }, aggregate: { - offer: withCallCount(impl.aggregate?.offer ?? notImplemented), - get: withCallCount(impl.aggregate?.get ?? notImplemented), + add: withCallCount(impl.aggregate?.add ?? notImplemented), + }, + deal: { + add: withCallCount(impl.deal?.add ?? notImplemented), }, - offer: { - arrange: withCallCount(impl.offer?.arrange ?? notImplemented), + 'chain-tracker': { + info: withCallCount(impl['chain-tracker']?.info ?? notImplemented), }, } } diff --git a/packages/aggregate-client/test/helpers/random.js b/packages/filecoin-client/test/helpers/random.js similarity index 98% rename from packages/aggregate-client/test/helpers/random.js rename to packages/filecoin-client/test/helpers/random.js index f9e9c6368..5694cbec5 100644 --- a/packages/aggregate-client/test/helpers/random.js +++ b/packages/filecoin-client/test/helpers/random.js @@ -61,6 +61,7 @@ export async function randomCargo(length, size) { return { link: piece.link, height: piece.height, + content: car.cid, size: piece.size, } }) diff --git a/packages/filecoin-client/test/storefront.test.js b/packages/filecoin-client/test/storefront.test.js new file mode 100644 index 000000000..b09088b0d --- /dev/null +++ b/packages/filecoin-client/test/storefront.test.js @@ -0,0 +1,203 @@ +import assert from 'assert' +import * as Signer from '@ucanto/principal/ed25519' +import * as Client from '@ucanto/client' +import * as Server from '@ucanto/server' +import * as CAR from '@ucanto/transport/car' +import { Filecoin as FilecoinCapabilities } from '@web3-storage/capabilities' + +import { filecoinAdd } from '../src/storefront.js' + +import { randomCargo } from './helpers/random.js' +import { mockService } from './helpers/mocks.js' +import { OperationFailed, OperationErrorName } from './helpers/errors.js' +import { serviceProvider as storefrontService } from './fixtures.js' + +describe('filecoin/add', () => { + it('agent adds a filecoin piece to a storefront, getting the piece queued', async () => { + const { agent } = await getContext() + + // Generate cargo to add + const [cargo] = await randomCargo(1, 100) + + /** @type {import('@web3-storage/capabilities/types').FilecoinAddSuccess} */ + const filecoinAddResponse = { + piece: cargo.link, + } + + // Create Ucanto service + const service = mockService({ + filecoin: { + add: Server.provideAdvanced({ + capability: FilecoinCapabilities.filecoinAdd, + handler: async ({ invocation, context }) => { + assert.strictEqual(invocation.issuer.did(), agent.did()) + assert.strictEqual(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.strictEqual(invCap.can, FilecoinCapabilities.filecoinAdd.can) + assert.equal(invCap.with, invocation.issuer.did()) + assert.ok(invCap.nb) + // piece link + assert.ok(invCap.nb.piece.equals(cargo.link.link())) + // content + assert.ok(invCap.nb.content.equals(cargo.content.link())) + + // Create effect for receipt with self signed queued operation + const fx = await FilecoinCapabilities.filecoinAdd + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: invCap.nb, + }) + .delegate() + + return Server.ok(filecoinAddResponse).join(fx.link()) + }, + }), + }, + }) + + const res = await filecoinAdd( + { + issuer: agent, + with: agent.did(), + audience: storefrontService, + }, + cargo.link.link(), + cargo.content.link(), + { connection: getConnection(service).connection } + ) + + assert.ok(res.out.ok) + assert.deepEqual(res.out.ok, filecoinAddResponse) + // includes effect fx in receipt + assert.ok(res.fx.join) + }) + + it('storefront self invokes add a filecoin piece to accept the piece queued', async () => { + // Generate cargo to add + const [cargo] = await randomCargo(1, 100) + + /** @type {import('@web3-storage/capabilities/types').FilecoinAddSuccess} */ + const filecoinAddResponse = { + piece: cargo.link, + } + + // Create Ucanto service + const service = mockService({ + filecoin: { + add: Server.provideAdvanced({ + capability: FilecoinCapabilities.filecoinAdd, + handler: async ({ invocation, context }) => { + assert.strictEqual(invocation.issuer.did(), storefrontService.did()) + assert.strictEqual(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.strictEqual(invCap.can, FilecoinCapabilities.filecoinAdd.can) + assert.equal(invCap.with, invocation.issuer.did()) + assert.ok(invCap.nb) + // piece link + assert.ok(invCap.nb.piece.equals(cargo.link.link())) + // content + assert.ok(invCap.nb.content.equals(cargo.content.link())) + + return Server.ok(filecoinAddResponse) + }, + }), + }, + }) + + // self invoke filecoin/add from storefront + const res = await filecoinAdd( + { + issuer: storefrontService, + with: storefrontService.did(), + audience: storefrontService, + }, + cargo.link.link(), + cargo.content.link(), + { connection: getConnection(service).connection } + ) + + assert.ok(res.out.ok) + assert.deepEqual(res.out.ok, filecoinAddResponse) + // does not include effect fx in receipt + assert.ok(!res.fx.join) + }) + + it('storefront self invokes add a filecoin piece to reject the piece queued', async () => { + // Generate cargo to add + const [cargo] = await randomCargo(1, 100) + + /** @type {import('@web3-storage/capabilities/types').FilecoinAddFailure} */ + const filecoinAddResponse = new OperationFailed( + 'failed to verify piece', + cargo.link + ) + + // Create Ucanto service + const service = mockService({ + filecoin: { + add: Server.provideAdvanced({ + capability: FilecoinCapabilities.filecoinAdd, + handler: async ({ invocation, context }) => { + assert.strictEqual(invocation.issuer.did(), storefrontService.did()) + assert.strictEqual(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.strictEqual(invCap.can, FilecoinCapabilities.filecoinAdd.can) + assert.equal(invCap.with, invocation.issuer.did()) + assert.ok(invCap.nb) + // piece link + assert.ok(invCap.nb.piece.equals(cargo.link.link())) + // content + assert.ok(invCap.nb.content.equals(cargo.content.link())) + + return { + error: filecoinAddResponse, + } + }, + }), + }, + }) + + // self invoke filecoin/add from storefront + const res = await filecoinAdd( + { + issuer: storefrontService, + with: storefrontService.did(), + audience: storefrontService, + }, + cargo.link.link(), + cargo.content.link(), + { connection: getConnection(service).connection } + ) + + assert.ok(res.out.error) + assert.deepEqual(res.out.error.name, OperationErrorName) + // does not include effect fx in receipt + assert.ok(!res.fx.join) + }) +}) + +async function getContext() { + const agent = await Signer.generate() + + return { agent } +} + +/** + * @param {import('../src/types').StorefrontService} service + */ +function getConnection(service) { + const server = Server.create({ + id: storefrontService, + service, + codec: CAR.inbound, + }) + const connection = Client.connect({ + id: storefrontService, + codec: CAR.outbound, + channel: server, + }) + + return { connection } +} diff --git a/packages/aggregate-client/tsconfig.json b/packages/filecoin-client/tsconfig.json similarity index 100% rename from packages/aggregate-client/tsconfig.json rename to packages/filecoin-client/tsconfig.json diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2a30efc80..18f11f96b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -161,8 +161,78 @@ importers: specifier: ^1.0.2 version: 1.0.2 - packages/aggregate-api: + packages/capabilities: + dependencies: + '@ucanto/core': + specifier: ^8.0.0 + version: 8.0.0 + '@ucanto/interface': + specifier: ^8.0.0 + version: 8.0.0 + '@ucanto/principal': + specifier: ^8.0.0 + version: 8.0.0 + '@ucanto/transport': + specifier: ^8.0.0 + version: 8.0.0 + '@ucanto/validator': + specifier: ^8.0.0 + version: 8.0.0 + '@web3-storage/data-segment': + specifier: ^2.2.0 + version: 2.2.0 + devDependencies: + '@types/assert': + specifier: ^1.5.6 + version: 1.5.6 + '@types/mocha': + specifier: ^10.0.0 + version: 10.0.1 + '@types/node': + specifier: ^18.11.18 + version: 18.11.18 + assert: + specifier: ^2.0.0 + version: 2.0.0 + hd-scripts: + specifier: ^4.0.0 + version: 4.1.0 + mocha: + specifier: ^10.2.0 + version: 10.2.0 + playwright-test: + specifier: ^8.1.2 + version: 8.1.2 + type-fest: + specifier: ^3.3.0 + version: 3.3.0 + typescript: + specifier: 4.9.5 + version: 4.9.5 + watch: + specifier: ^1.0.2 + version: 1.0.2 + + packages/did-mailto: + devDependencies: + '@types/mocha': + specifier: ^10.0.1 + version: 10.0.1 + '@types/node': + specifier: ^18.11.18 + version: 18.11.18 + hd-scripts: + specifier: ^4.1.0 + version: 4.1.0 + mocha: + specifier: ^10.2.0 + version: 10.2.0 + + packages/filecoin-api: dependencies: + '@ipld/dag-ucan': + specifier: ^3.3.2 + version: 3.3.2 '@ucanto/client': specifier: ^8.0.0 version: 8.0.0 @@ -197,9 +267,9 @@ importers: '@web-std/blob': specifier: ^3.0.4 version: 3.0.4 - '@web3-storage/aggregate-client': + '@web3-storage/filecoin-client': specifier: workspace:^ - version: link:../aggregate-client + version: link:../filecoin-client hd-scripts: specifier: ^4.1.0 version: 4.1.0 @@ -209,8 +279,11 @@ importers: multiformats: specifier: ^11.0.2 version: 11.0.2 + p-wait-for: + specifier: ^5.0.2 + version: 5.0.2 - packages/aggregate-client: + packages/filecoin-client: dependencies: '@ipld/dag-cbor': specifier: ^9.0.0 @@ -280,73 +353,6 @@ importers: specifier: 4.9.5 version: 4.9.5 - packages/capabilities: - dependencies: - '@ucanto/core': - specifier: ^8.0.0 - version: 8.0.0 - '@ucanto/interface': - specifier: ^8.0.0 - version: 8.0.0 - '@ucanto/principal': - specifier: ^8.0.0 - version: 8.0.0 - '@ucanto/transport': - specifier: ^8.0.0 - version: 8.0.0 - '@ucanto/validator': - specifier: ^8.0.0 - version: 8.0.0 - '@web3-storage/data-segment': - specifier: ^2.2.0 - version: 2.2.0 - devDependencies: - '@types/assert': - specifier: ^1.5.6 - version: 1.5.6 - '@types/mocha': - specifier: ^10.0.0 - version: 10.0.1 - '@types/node': - specifier: ^18.11.18 - version: 18.11.18 - assert: - specifier: ^2.0.0 - version: 2.0.0 - hd-scripts: - specifier: ^4.0.0 - version: 4.1.0 - mocha: - specifier: ^10.2.0 - version: 10.2.0 - playwright-test: - specifier: ^8.1.2 - version: 8.1.2 - type-fest: - specifier: ^3.3.0 - version: 3.3.0 - typescript: - specifier: 4.9.5 - version: 4.9.5 - watch: - specifier: ^1.0.2 - version: 1.0.2 - - packages/did-mailto: - devDependencies: - '@types/mocha': - specifier: ^10.0.1 - version: 10.0.1 - '@types/node': - specifier: ^18.11.18 - version: 18.11.18 - hd-scripts: - specifier: ^4.1.0 - version: 4.1.0 - mocha: - specifier: ^10.2.0 - version: 10.2.0 - packages/upload-api: dependencies: '@ucanto/client': @@ -8808,7 +8814,6 @@ packages: /p-timeout@6.1.2: resolution: {integrity: sha512-UbD77BuZ9Bc9aABo74gfXhNvzC9Tx7SxtHSh1fxvx3jTLLYvmVhiQZZrJzqqU0jKbN32kb5VOKiLEQI/3bIjgQ==} engines: {node: '>=14.16'} - dev: false /p-try@2.2.0: resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==} @@ -8828,6 +8833,13 @@ packages: p-timeout: 6.1.2 dev: false + /p-wait-for@5.0.2: + resolution: {integrity: sha512-lwx6u1CotQYPVju77R+D0vFomni/AqRfqLmqQ8hekklqZ6gAY9rONh7lBQ0uxWMkC2AuX9b2DVAl8To0NyP1JA==} + engines: {node: '>=12'} + dependencies: + p-timeout: 6.1.2 + dev: true + /package-json@6.5.0: resolution: {integrity: sha512-k3bdm2n25tkyxcjSKzB5x8kfVxlMdgsbPr0GkZcwHsLpba6cBjqCt1KlcChKEvxHIcTB1FVMuwoijZ26xex5MQ==} engines: {node: '>=8'} diff --git a/tsconfig.json b/tsconfig.json index afabcba71..b848c7969 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -39,6 +39,7 @@ "packages/access-client", "packages/capabilities", "packages/upload-client", + "packages/filecoin-client", "packages/w3up-client" ], "excludeExternals": true,