From edb76a34b4aaf73dc85655e1082f9705064b84a0 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 23 Oct 2023 22:38:29 +0200 Subject: [PATCH] refactor!: filecoin api services events and tests (#974) Implements `filecoin-api` services, events and tests all over the place. BREAKING CHANGE: see latest specs https://github.com/web3-storage/specs/blob/cbdb706f18567900c5c24d7fb16ccbaf93d0d023/w3-filecoin.md --- packages/capabilities/src/filecoin/dealer.js | 2 +- packages/capabilities/src/types.ts | 30 +- packages/filecoin-api/package.json | 14 +- packages/filecoin-api/src/aggregator.js | 119 -- packages/filecoin-api/src/aggregator/api.js | 1 + packages/filecoin-api/src/aggregator/api.ts | 302 ++++ .../src/aggregator/buffer-reducing.js | 248 ++++ .../filecoin-api/src/aggregator/events.js | 358 +++++ .../filecoin-api/src/aggregator/service.js | 151 ++ packages/filecoin-api/src/deal-tracker/api.js | 1 + packages/filecoin-api/src/deal-tracker/api.ts | 34 + .../filecoin-api/src/deal-tracker/service.js | 77 + packages/filecoin-api/src/dealer.js | 157 --- packages/filecoin-api/src/dealer/api.js | 1 + packages/filecoin-api/src/dealer/api.ts | 116 ++ .../filecoin-api/src/dealer/deal-track.js | 0 packages/filecoin-api/src/dealer/events.js | 165 +++ packages/filecoin-api/src/dealer/service.js | 187 +++ packages/filecoin-api/src/errors.js | 52 +- packages/filecoin-api/src/lib.js | 6 +- packages/filecoin-api/src/storefront.js | 114 -- packages/filecoin-api/src/storefront/api.js | 1 + packages/filecoin-api/src/storefront/api.ts | 158 +++ .../filecoin-api/src/storefront/events.js | 264 ++++ .../filecoin-api/src/storefront/service.js | 290 ++++ packages/filecoin-api/src/types.ts | 159 +-- packages/filecoin-api/test/aggregator.spec.js | 240 +++- packages/filecoin-api/test/context/mocks.js | 54 + packages/filecoin-api/test/context/queue.js | 17 + .../filecoin-api/test/context/receipts.js | 145 ++ packages/filecoin-api/test/context/service.js | 231 +++ .../test/context/store-implementations.js | 242 ++++ packages/filecoin-api/test/context/store.js | 171 ++- packages/filecoin-api/test/context/types.ts | 8 + .../filecoin-api/test/deal-tracker.spec.js | 51 + packages/filecoin-api/test/dealer.spec.js | 160 ++- .../filecoin-api/test/events/aggregator.js | 1238 +++++++++++++++++ packages/filecoin-api/test/events/dealer.js | 448 ++++++ .../filecoin-api/test/events/storefront.js | 496 +++++++ packages/filecoin-api/test/lib.js | 18 +- .../filecoin-api/test/services/aggregator.js | 462 ++++-- .../test/services/deal-tracker.js | 143 ++ packages/filecoin-api/test/services/dealer.js | 315 +++-- .../filecoin-api/test/services/storefront.js | 514 ++++++- packages/filecoin-api/test/storefront.spec.js | 181 ++- packages/filecoin-api/test/types.js | 1 + packages/filecoin-api/test/types.ts | 50 + packages/filecoin-client/src/dealer.js | 7 +- .../filecoin-client/test/aggregator.test.js | 23 +- packages/filecoin-client/test/dealer.test.js | 37 +- .../filecoin-client/test/storefront.test.js | 39 +- 51 files changed, 7234 insertions(+), 1064 deletions(-) delete mode 100644 packages/filecoin-api/src/aggregator.js create mode 100644 packages/filecoin-api/src/aggregator/api.js create mode 100644 packages/filecoin-api/src/aggregator/api.ts create mode 100644 packages/filecoin-api/src/aggregator/buffer-reducing.js create mode 100644 packages/filecoin-api/src/aggregator/events.js create mode 100644 packages/filecoin-api/src/aggregator/service.js create mode 100644 packages/filecoin-api/src/deal-tracker/api.js create mode 100644 packages/filecoin-api/src/deal-tracker/api.ts create mode 100644 packages/filecoin-api/src/deal-tracker/service.js delete mode 100644 packages/filecoin-api/src/dealer.js create mode 100644 packages/filecoin-api/src/dealer/api.js create mode 100644 packages/filecoin-api/src/dealer/api.ts create mode 100644 packages/filecoin-api/src/dealer/deal-track.js create mode 100644 packages/filecoin-api/src/dealer/events.js create mode 100644 packages/filecoin-api/src/dealer/service.js delete mode 100644 packages/filecoin-api/src/storefront.js create mode 100644 packages/filecoin-api/src/storefront/api.js create mode 100644 packages/filecoin-api/src/storefront/api.ts create mode 100644 packages/filecoin-api/src/storefront/events.js create mode 100644 packages/filecoin-api/src/storefront/service.js create mode 100644 packages/filecoin-api/test/context/mocks.js create mode 100644 packages/filecoin-api/test/context/receipts.js create mode 100644 packages/filecoin-api/test/context/service.js create mode 100644 packages/filecoin-api/test/context/store-implementations.js create mode 100644 packages/filecoin-api/test/context/types.ts create mode 100644 packages/filecoin-api/test/deal-tracker.spec.js create mode 100644 packages/filecoin-api/test/events/aggregator.js create mode 100644 packages/filecoin-api/test/events/dealer.js create mode 100644 packages/filecoin-api/test/events/storefront.js create mode 100644 packages/filecoin-api/test/services/deal-tracker.js create mode 100644 packages/filecoin-api/test/types.js create mode 100644 packages/filecoin-api/test/types.ts diff --git a/packages/capabilities/src/filecoin/dealer.js b/packages/capabilities/src/filecoin/dealer.js index 8b7b8c822..ca456fe65 100644 --- a/packages/capabilities/src/filecoin/dealer.js +++ b/packages/capabilities/src/filecoin/dealer.js @@ -32,7 +32,7 @@ export const aggregateOffer = capability({ * CID of the DAG-CBOR encoded block with offer details. * Service will queue given offer to be validated and handled. */ - pieces: Schema.link(), + pieces: Schema.link({ version: 1 }), }), derives: (claim, from) => { return ( diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts index 114f3ed50..2384561bb 100644 --- a/packages/capabilities/src/types.ts +++ b/packages/capabilities/src/types.ts @@ -181,14 +181,20 @@ export type Space = InferInvokedCapability export type SpaceInfo = InferInvokedCapability // filecoin +export interface DealMetadata { + dataType: uint64 + dataSource: SingletonMarketSource +} /** @see https://github.com/filecoin-project/go-data-segment/blob/e3257b64fa2c84e0df95df35de409cfed7a38438/datasegment/verifier.go#L8-L14 */ export interface DataAggregationProof { /** * Proof the piece is included in the aggregate. */ inclusion: InclusionProof - auxDataType: uint64 - auxDataSource: SingletonMarketSource + /** + * Filecoin deal metadata. + */ + aux: DealMetadata } /** @see https://github.com/filecoin-project/go-data-segment/blob/e3257b64fa2c84e0df95df35de409cfed7a38438/datasegment/inclusion.go#L30-L39 */ export interface InclusionProof { @@ -233,15 +239,25 @@ export interface FilecoinSubmitSuccess { export type FilecoinSubmitFailure = InvalidPieceCID | Ucanto.Failure -export type FilecoinAcceptSuccess = DataAggregationProof +export interface FilecoinAcceptSuccess extends DataAggregationProof { + aggregate: PieceLink + piece: PieceLink +} -export type FilecoinAcceptFailure = InvalidContentPiece | Ucanto.Failure +export type FilecoinAcceptFailure = + | InvalidContentPiece + | ProofNotFound + | Ucanto.Failure export interface InvalidContentPiece extends Ucanto.Failure { name: 'InvalidContentPiece' content: PieceLink } +export interface ProofNotFound extends Ucanto.Failure { + name: 'ProofNotFound' +} + // filecoin aggregator export interface PieceOfferSuccess { /** @@ -276,7 +292,9 @@ export interface AggregateOfferSuccess { } export type AggregateOfferFailure = Ucanto.Failure -export type AggregateAcceptSuccess = DataAggregationProof +export interface AggregateAcceptSuccess extends DealMetadata { + aggregate: PieceLink +} export type AggregateAcceptFailure = InvalidPiece | Ucanto.Failure export interface InvalidPiece extends Ucanto.Failure { @@ -303,7 +321,7 @@ export interface DealDetails { // TODO: start/end epoch? etc. } -export type FilecoinAddress = `f${string}` +export type FilecoinAddress = string export type DealInfoFailure = DealNotFound | Ucanto.Failure diff --git a/packages/filecoin-api/package.json b/packages/filecoin-api/package.json index 0a667e8ce..865ec2a5c 100644 --- a/packages/filecoin-api/package.json +++ b/packages/filecoin-api/package.json @@ -20,8 +20,8 @@ "dealer": [ "dist/src/dealer.d.ts" ], - "chain-tracker": [ - "dist/src/chain-tracker.d.ts" + "deal-tracker": [ + "dist/src/deal-tracker.d.ts" ], "errors": [ "dist/src/errors.d.ts" @@ -54,9 +54,9 @@ "types": "./dist/src/dealer.d.ts", "import": "./src/dealer.js" }, - "./chain-tracker": { - "types": "./dist/src/chain-tracker.d.ts", - "import": "./src/chain-tracker.js" + "./deal-tracker": { + "types": "./dist/src/deal-tracker.d.ts", + "import": "./src/deal-tracker.js" }, "./storefront": { "types": "./dist/src/storefront.d.ts", @@ -108,7 +108,9 @@ "project": "./tsconfig.json" }, "rules": { - "unicorn/expiring-todo-comments": "off" + "unicorn/expiring-todo-comments": "off", + "unicorn/prefer-number-properties": "off", + "jsdoc/check-indentation": "off" }, "env": { "mocha": true diff --git a/packages/filecoin-api/src/aggregator.js b/packages/filecoin-api/src/aggregator.js deleted file mode 100644 index ad1f8b7e6..000000000 --- a/packages/filecoin-api/src/aggregator.js +++ /dev/null @@ -1,119 +0,0 @@ -import * as Server from '@ucanto/server' -import * as Client from '@ucanto/client' -import * as CAR from '@ucanto/transport/car' -import { Filecoin as FilecoinCapabilities } from '@web3-storage/capabilities' - -import * as API from './types.js' -import { QueueOperationFailed, StoreOperationFailed } from './errors.js' - -/** - * @param {API.Input} input - * @param {API.AggregatorServiceContext} context - * @returns {Promise | API.UcantoInterface.JoinBuilder>} - */ -export const add = async ({ capability }, context) => { - const { piece, storefront, group } = capability.nb - - // Store piece into the store. Store events MAY be used to propagate piece over - const put = await context.pieceStore.put({ - piece, - storefront, - group, - insertedAt: Date.now(), - }) - - if (put.error) { - return { - error: new StoreOperationFailed(put.error.message), - } - } - - return { - ok: { - piece, - }, - } -} - -/** - * @param {API.Input} input - * @param {API.AggregatorServiceContext} context - * @returns {Promise | API.UcantoInterface.JoinBuilder>} - */ -export const queue = async ({ capability }, context) => { - const { piece, group } = capability.nb - const storefront = capability.with - - const queued = await context.addQueue.add({ - piece, - storefront, - group, - insertedAt: Date.now(), - }) - if (queued.error) { - return { - error: new QueueOperationFailed(queued.error.message), - } - } - - // Create effect for receipt - const fx = await FilecoinCapabilities.aggregateAdd - .invoke({ - issuer: context.id, - audience: context.id, - with: context.id.did(), - nb: { - piece, - storefront, - group, - }, - }) - .delegate() - - return Server.ok({ - piece, - }).join(fx.link()) -} - -/** - * @param {API.AggregatorServiceContext} context - */ -export function createService(context) { - return { - aggregate: { - queue: Server.provideAdvanced({ - capability: FilecoinCapabilities.aggregateQueue, - handler: (input) => queue(input, context), - }), - add: Server.provideAdvanced({ - capability: FilecoinCapabilities.aggregateAdd, - handler: (input) => add(input, context), - }), - }, - } -} - -/** - * @param {API.UcantoServerContext & API.AggregatorServiceContext} context - */ -export const createServer = (context) => - Server.create({ - id: context.id, - codec: context.codec || CAR.inbound, - service: createService(context), - catch: (error) => context.errorReporter.catch(error), - validateAuthorization: () => ({ ok: {} }), - }) - -/** - * @param {object} options - * @param {API.UcantoInterface.Principal} options.id - * @param {API.UcantoInterface.Transport.Channel} options.channel - * @param {API.UcantoInterface.OutboundCodec} [options.codec] - */ -export const connect = ({ id, channel, codec = CAR.outbound }) => - Client.connect({ - id, - channel, - codec, - }) diff --git a/packages/filecoin-api/src/aggregator/api.js b/packages/filecoin-api/src/aggregator/api.js new file mode 100644 index 000000000..336ce12bb --- /dev/null +++ b/packages/filecoin-api/src/aggregator/api.js @@ -0,0 +1 @@ +export {} diff --git a/packages/filecoin-api/src/aggregator/api.ts b/packages/filecoin-api/src/aggregator/api.ts new file mode 100644 index 000000000..50335472b --- /dev/null +++ b/packages/filecoin-api/src/aggregator/api.ts @@ -0,0 +1,302 @@ +import type { Signer, Principal, Link } from '@ucanto/interface' +import { InclusionProof } from '@web3-storage/capabilities/types' +import { PieceLink } from '@web3-storage/data-segment' +import { + AggregatorService, + DealerService, +} from '@web3-storage/filecoin-client/types' +import { + Store, + UpdatableStore, + QueryableStore, + Queue, + ServiceConfig, +} from '../types.js' + +export interface ServiceContext { + /** + * Service signer + */ + id: Signer + /** + * Principal for dealer service + */ + dealerId: Principal + /** + * Stores pieces that have been offered to the aggregator. + */ + pieceStore: UpdatableStore + /** + * Queues pieces being buffered into an aggregate. + */ + pieceQueue: Queue + /** + * Queues pieces being buffered into an aggregate. + */ + bufferQueue: Queue + /** + * Store of CID => Buffer Record + */ + bufferStore: Store + /** + * Stores fully buffered aggregates. + */ + aggregateStore: Store + /** + * Queues pieces, their aggregate and their inclusion proofs. + */ + pieceAcceptQueue: Queue + /** + * Stores inclusion proofs for pieces included in an aggregate. + */ + inclusionStore: QueryableStore< + InclusionRecordKey, + InclusionRecord, + InclusionRecordQueryByGroup + > + /** + * Queues buffered aggregates to be offered to the Dealer. + */ + aggregateOfferQueue: Queue +} + +export interface PieceMessageContext + extends Pick {} + +export interface PieceAcceptMessageContext + extends Pick {} + +export interface AggregateOfferMessageContext + extends Pick {} + +export interface PieceInsertEventContext + extends Pick {} + +export interface InclusionInsertEventToUpdateState + extends Pick {} + +export interface InclusionInsertEventToIssuePieceAccept { + /** + * Aggregator connection to moves pieces into the pipeline. + */ + aggregatorService: ServiceConfig +} + +export interface AggregateInsertEventToPieceAcceptQueueContext + extends Pick { + /** + * Buffer configuration for aggregation. + */ + config: AggregateConfig +} + +export interface AggregateInsertEventToAggregateOfferContext + extends Pick { + /** + * Dealer connection to moves pieces into the pipeline. + */ + dealerService: ServiceConfig +} + +export interface BufferMessageContext + extends Pick< + ServiceContext, + 'bufferStore' | 'bufferQueue' | 'aggregateOfferQueue' + > { + /** + * Buffer configuration for aggregation. + */ + config: AggregateConfig +} + +export interface PieceRecord { + /** + * Piece CID for the content. + */ + piece: PieceLink + /** + * Grouping information for submitted piece. + */ + group: string + /** + * Status of the offered piece. + * - offered = acknowledged received for aggregation. + * - accepted = accepted into an aggregate and offered for inclusion in filecoin deal(s). + */ + status: 'offered' | 'accepted' + /** + * Insertion date ISO string. + */ + insertedAt: string + /** + * Update date ISO string. + */ + updatedAt: string +} + +export interface PieceRecordKey extends Pick {} + +export interface PieceMessage { + /** + * Piece CID for the content. + */ + piece: PieceLink + /** + * Grouping information for submitted piece. + */ + group: string +} + +export interface AggregateRecord { + /** + * `bagy...aggregate` Piece CID of an aggregate + */ + aggregate: PieceLink + /** + * `bafy...cbor` as CID of dag-cbor block with list of pieces in an aggregate. + */ + pieces: Link + /** + * Grouping information for submitted piece. + */ + group: string + /** + * Insertion date ISO string. + */ + insertedAt: string +} + +// TODO: probably group should also be key! +export interface AggregateRecordKey + extends Pick {} + +export interface InclusionRecord { + /** + * Piece CID for the content. + */ + piece: PieceLink + /** + * Piece CID of an aggregate. + */ + aggregate: PieceLink + /** + * Grouping information for submitted piece. + */ + group: string + /** + * Proof that the piece is included in the aggregate. + */ + inclusion: InclusionProof + /** + * Insertion date ISO string. + */ + insertedAt: string +} + +export interface InclusionRecordKey + extends Pick {} + +export interface InclusionRecordQueryByGroup + extends Pick {} + +export type BufferedPiece = { + /** + * Piece CID for the content. + */ + piece: PieceLink + /** + * Policies that this piece is under + */ + policy: PiecePolicy + /** + * Insertion date ISO string. + */ + insertedAt: string +} + +export interface Buffer { + /** + * `bagy...aggregate` Piece CID of an aggregate + */ + aggregate?: PieceLink + /** + * Pieces inside the buffer record. + */ + pieces: BufferedPiece[] + /** + * Grouping information for submitted buffer. + */ + group: string +} + +export interface BufferRecord { + /** + * Buffer with a set of Filecoin pieces pending aggregation. + */ + buffer: Buffer + /** + * `bafy...cbor` as CID of dag-cbor block with list of pieces in an aggregate. + */ + block: Link +} + +export interface BufferMessage { + /** + * `bagy...aggregate` Piece CID of an aggregate + */ + aggregate?: PieceLink + /** + * `bafy...cbor` as CID of dag-cbor block with Buffer + */ + pieces: Link + /** + * Grouping information for submitted buffer. + */ + group: string +} + +export interface PieceAcceptMessage { + /** + * Piece CID. + */ + piece: PieceLink + /** + * Piece CID of an aggregate + */ + aggregate: PieceLink + /** + * Grouping information for submitted piece. + */ + group: string + /** + * Proof that the piece is included in the aggregate. + */ + inclusion: InclusionProof +} + +export interface AggregateOfferMessage { + /** + * Piece CID of an aggregate. + */ + aggregate: PieceLink + /** + * List of pieces in an aggregate. + */ + pieces: Link + /** + * Grouping information for submitted piece. + */ + group: string +} + +export interface AggregateConfig { + maxAggregateSize: number + minAggregateSize: number + minUtilizationFactor: number +} + +// Enums +export type PiecePolicy = NORMAL | RETRY + +type NORMAL = 0 +type RETRY = 1 diff --git a/packages/filecoin-api/src/aggregator/buffer-reducing.js b/packages/filecoin-api/src/aggregator/buffer-reducing.js new file mode 100644 index 000000000..091928f29 --- /dev/null +++ b/packages/filecoin-api/src/aggregator/buffer-reducing.js @@ -0,0 +1,248 @@ +import { Aggregate, Piece, NODE_SIZE, Index } from '@web3-storage/data-segment' +import { CBOR } from '@ucanto/core' + +import { UnexpectedState } from '../errors.js' + +/** + * @typedef {import('@ucanto/interface').Link} Link + * @typedef {import('@web3-storage/data-segment').AggregateView} AggregateView + * + * @typedef {import('./api').BufferedPiece} BufferedPiece + * @typedef {import('./api').BufferRecord} BufferRecord + * @typedef {import('./api').BufferMessage} BufferMessage + * @typedef {import('./api').AggregateOfferMessage} AggregateOfferMessage + * @typedef {import('../types').StoreGetError} StoreGetError + * @typedef {{ bufferedPieces: BufferedPiece[], group: string }} GetBufferedPieces + * @typedef {import('../types.js').Result} GetBufferedPiecesResult + * + * @typedef {object} AggregateInfo + * @property {BufferedPiece[]} addedBufferedPieces + * @property {BufferedPiece[]} remainingBufferedPieces + * @property {AggregateView} aggregate + */ + +/** + * @param {object} props + * @param {AggregateInfo} props.aggregateInfo + * @param {import('../types').Store} props.bufferStore + * @param {import('../types').Queue} props.bufferQueue + * @param {import('../types').Queue} props.aggregateOfferQueue + * @param {string} props.group + */ +export async function handleBufferReducingWithAggregate({ + aggregateInfo, + bufferStore, + bufferQueue, + aggregateOfferQueue, + group, +}) { + // If aggregate has enough space + // store buffered pieces that are part of aggregate and queue aggregate + // store remaining pieces and queue them to be reduced + /** @type {import('./api.js').Buffer} */ + const aggregateReducedBuffer = { + aggregate: aggregateInfo.aggregate.link, + pieces: aggregateInfo.addedBufferedPieces, + group, + } + const aggregateBlock = await CBOR.write(aggregateReducedBuffer) + + // Store buffered pieces for aggregate + const bufferStoreAggregatePut = await bufferStore.put({ + buffer: aggregateReducedBuffer, + block: aggregateBlock.cid, + }) + if (bufferStoreAggregatePut.error) { + return bufferStoreAggregatePut + } + + // Propagate message for aggregate offer queue + const aggregateOfferQueueAdd = await aggregateOfferQueue.add({ + aggregate: aggregateInfo.aggregate.link, + pieces: aggregateBlock.cid, + group, + }) + if (aggregateOfferQueueAdd.error) { + return aggregateOfferQueueAdd + } + + // Store remaining buffered pieces to reduce if they exist + if (!aggregateInfo.remainingBufferedPieces.length) { + return { ok: {} } + } + + const remainingReducedBuffer = { + pieces: aggregateInfo.remainingBufferedPieces, + group: group, + } + const remainingBlock = await CBOR.write(remainingReducedBuffer) + + // Store remaining buffered pieces + const bufferStoreRemainingPut = await bufferStore.put({ + buffer: remainingReducedBuffer, + block: remainingBlock.cid, + }) + if (bufferStoreRemainingPut.error) { + return bufferStoreRemainingPut + } + + // Propagate message for buffer queue + const bufferQueueAdd = await bufferQueue.add({ + pieces: remainingBlock.cid, + group: group, + }) + if (bufferQueueAdd.error) { + return bufferQueueAdd + } + + return { ok: {}, error: undefined } +} + +/** + * Store given buffer into store and queue it to further reducing. + * + * @param {object} props + * @param {import('./api.js').Buffer} props.buffer + * @param {import('../types').Store} props.bufferStore + * @param {import('../types').Queue} props.bufferQueue + */ +export async function handleBufferReducingWithoutAggregate({ + buffer, + bufferStore, + bufferQueue, +}) { + const block = await CBOR.write(buffer) + + // Store block in buffer store + const bufferStorePut = await bufferStore.put({ + buffer, + block: block.cid, + }) + if (bufferStorePut.error) { + return bufferStorePut + } + + // Propagate message + const bufferQueueAdd = await bufferQueue.add({ + pieces: block.cid, + group: buffer.group, + }) + if (bufferQueueAdd.error) { + return bufferQueueAdd + } + + return { ok: {}, error: undefined } +} + +/** + * Attempt to build an aggregate with buffered pieces within ranges. + * + * @param {BufferedPiece[]} bufferedPieces + * @param {object} sizes + * @param {number} sizes.maxAggregateSize + * @param {number} sizes.minAggregateSize + * @param {number} sizes.minUtilizationFactor + */ +export function aggregatePieces(bufferedPieces, sizes) { + // Guarantee buffered pieces total size is bigger than the minimum utilization + const bufferUtilizationSize = bufferedPieces.reduce((total, p) => { + const piece = Piece.fromLink(p.piece) + total += piece.size + return total + }, 0n) + if ( + bufferUtilizationSize < + sizes.maxAggregateSize / sizes.minUtilizationFactor + ) { + return + } + + // Create builder with maximum size and try to fill it up + const builder = Aggregate.createBuilder({ + size: Piece.PaddedSize.from(sizes.maxAggregateSize), + }) + + // add pieces to an aggregate until there is no more space, or no more pieces + /** @type {BufferedPiece[]} */ + const addedBufferedPieces = [] + /** @type {BufferedPiece[]} */ + const remainingBufferedPieces = [] + + for (const bufferedPiece of bufferedPieces) { + const p = Piece.fromLink(bufferedPiece.piece) + if (builder.estimate(p).error) { + remainingBufferedPieces.push(bufferedPiece) + continue + } + builder.write(p) + addedBufferedPieces.push(bufferedPiece) + } + const totalUsedSpace = + builder.offset * BigInt(NODE_SIZE) + + BigInt(builder.limit) * BigInt(Index.EntrySize) + + // If not enough space return undefined + if (totalUsedSpace < BigInt(sizes.minAggregateSize)) { + return + } + + const aggregate = builder.build() + + return { + addedBufferedPieces, + remainingBufferedPieces, + aggregate, + } +} + +/** + * Get buffered pieces from queue buffer records. + * + * @param {Link[]} bufferPieces + * @param {import('../types').Store} bufferStore + * @returns {Promise} + */ +export async function getBufferedPieces(bufferPieces, bufferStore) { + if (!bufferPieces.length) { + return { + error: new UnexpectedState('received buffer pieces are empty'), + } + } + + const getBufferRes = await Promise.all( + bufferPieces.map((bufferPiece) => bufferStore.get(bufferPiece)) + ) + + // Concatenate pieces and sort them by policy and size + /** @type {BufferedPiece[]} */ + let bufferedPieces = [] + for (const b of getBufferRes) { + if (b.error) return b + // eslint-disable-next-line unicorn/prefer-spread + bufferedPieces = bufferedPieces.concat(b.ok.buffer.pieces || []) + } + + bufferedPieces.sort(sortPieces) + + return { + ok: { + bufferedPieces, + // extract group from one entry + // TODO: needs to change to support multi group buffering + // @ts-expect-error typescript does not understand with find that no error and group MUST exist + group: getBufferRes[0].ok.buffer.group, + }, + } +} + +/** + * Sort given buffered pieces by policy and then by size. + * + * @param {BufferedPiece} a + * @param {BufferedPiece} b + */ +export function sortPieces(a, b) { + return a.policy !== b.policy + ? a.policy - b.policy + : Piece.fromLink(a.piece).height - Piece.fromLink(b.piece).height +} diff --git a/packages/filecoin-api/src/aggregator/events.js b/packages/filecoin-api/src/aggregator/events.js new file mode 100644 index 000000000..1d62bbdea --- /dev/null +++ b/packages/filecoin-api/src/aggregator/events.js @@ -0,0 +1,358 @@ +import { Aggregator, Dealer } from '@web3-storage/filecoin-client' +import { Aggregate, Piece } from '@web3-storage/data-segment' +import { CBOR } from '@ucanto/core' + +import { + getBufferedPieces, + aggregatePieces, + handleBufferReducingWithoutAggregate, + handleBufferReducingWithAggregate, +} from './buffer-reducing.js' +import { + StoreOperationFailed, + QueueOperationFailed, + UnexpectedState, +} from '../errors.js' + +/** + * On piece queue messages, store piece. + * + * @param {import('./api').PieceMessageContext} context + * @param {import('./api').PieceMessage} message + */ +export const handlePieceMessage = async (context, message) => { + const { piece, group } = message + + // Store piece into the store. Store events MAY be used to propagate piece over + const putRes = await context.pieceStore.put({ + piece, + group, + status: 'offered', + insertedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }) + if (putRes.error) { + return { + error: new StoreOperationFailed(putRes.error.message), + } + } + + return { ok: {} } +} + +/** + * On Piece store insert batch, buffer pieces together to resume buffer processing. + * + * @param {import('./api').PieceInsertEventContext} context + * @param {import('./api').PieceRecord[]} records + */ +export const handlePiecesInsert = async (context, records) => { + // TODO: separate buffers per group after MVP + const { group } = records[0] + + /** @type {import('./api.js').Buffer} */ + const buffer = { + pieces: records.map((p) => ({ + piece: p.piece, + insertedAt: p.insertedAt, + // Set policy as insertion + policy: /** @type {import('./api.js').PiecePolicy} */ (0), + })), + group, + } + const block = await CBOR.write(buffer) + + // Store block in buffer store + const bufferStorePut = await context.bufferStore.put({ + buffer, + block: block.cid, + }) + if (bufferStorePut.error) { + return bufferStorePut + } + + // Propagate message + const bufferQueueAdd = await context.bufferQueue.add({ + pieces: block.cid, + group, + }) + if (bufferQueueAdd.error) { + return { + error: new QueueOperationFailed(bufferQueueAdd.error.message), + } + } + + return { ok: {} } +} + +/** + * On buffer queue messages, reduce received buffer records into a bigger buffer. + * - If new buffer does not have enough load to build an aggregate, it is stored + * and requeued for buffer reducing + * - If new buffer has enough load to build an aggregate, it is stored and queued + * into aggregateOfferQueue. Remaining of the new buffer (in case buffer bigger + * than maximum aggregate size) is re-queued into the buffer queue. + * + * @param {import('./api').BufferMessageContext} context + * @param {import('./api').BufferMessage[]} records + */ +export const handleBufferQueueMessage = async (context, records) => { + // Get reduced buffered pieces + const buffers = records.map((r) => r.pieces) + const { error: errorGetBufferedPieces, ok: okGetBufferedPieces } = + await getBufferedPieces(buffers, context.bufferStore) + if (errorGetBufferedPieces) { + return { error: errorGetBufferedPieces } + } + + const { bufferedPieces, group } = okGetBufferedPieces + + // Attempt to aggregate buffered pieces within the ranges. + // In case it is possible, included pieces and remaining pieces are returned + // so that they can be propagated to respective stores/queues. + const aggregateInfo = aggregatePieces(bufferedPieces, { + maxAggregateSize: context.config.maxAggregateSize, + minAggregateSize: context.config.minAggregateSize, + minUtilizationFactor: context.config.minUtilizationFactor, + }) + + // Store buffered pieces if not enough to do aggregate and re-queue them + if (!aggregateInfo) { + const { error: errorHandleBufferReducingWithoutAggregate } = + await handleBufferReducingWithoutAggregate({ + buffer: { + pieces: bufferedPieces, + group, + }, + bufferStore: context.bufferStore, + bufferQueue: context.bufferQueue, + }) + + if (errorHandleBufferReducingWithoutAggregate) { + return { error: errorHandleBufferReducingWithoutAggregate } + } + + // No pieces were aggregate + return { + ok: { + aggregatedPieces: 0, + }, + } + } + + // Store buffered pieces to do aggregate and re-queue remaining ones + const { error: errorHandleBufferReducingWithAggregate } = + await handleBufferReducingWithAggregate({ + aggregateInfo, + bufferStore: context.bufferStore, + bufferQueue: context.bufferQueue, + aggregateOfferQueue: context.aggregateOfferQueue, + group, + }) + + if (errorHandleBufferReducingWithAggregate) { + return { error: errorHandleBufferReducingWithAggregate } + } + + return { + ok: { + aggregatedPieces: aggregateInfo.addedBufferedPieces.length, + }, + } +} + +/** + * On aggregate offer queue message, store aggregate record in store. + * + * @param {import('./api').AggregateOfferMessageContext} context + * @param {import('./api').AggregateOfferMessage} message + */ +export const handleAggregateOfferMessage = async (context, message) => { + const { pieces, aggregate, group } = message + + // Store aggregate information into the store. Store events MAY be used to propagate aggregate over + const putRes = await context.aggregateStore.put({ + pieces, + aggregate, + group, + insertedAt: new Date().toISOString(), + }) + + // TODO: should we ignore error already there? + if (putRes.error) { + return putRes + } + + return { ok: {}, error: undefined } +} + +/** + * On Aggregate store insert, offer inserted aggregate for deal. + * + * @param {import('./api').AggregateInsertEventToPieceAcceptQueueContext} context + * @param {import('./api').AggregateRecord} record + */ +export const handleAggregateInsertToPieceAcceptQueue = async ( + context, + record +) => { + const bufferStoreRes = await context.bufferStore.get(record.pieces) + if (bufferStoreRes.error) { + return bufferStoreRes + } + + // Get pieces from buffer + const pieces = bufferStoreRes.ok.buffer.pieces.map((p) => + Piece.fromLink(p.piece) + ) + const aggregate = bufferStoreRes.ok.buffer.aggregate + + const aggregateBuilder = Aggregate.build({ + pieces, + size: Piece.PaddedSize.from(context.config.maxAggregateSize), + }) + + if (aggregate && !aggregateBuilder.link.equals(aggregate)) { + return { + error: new UnexpectedState( + `invalid aggregate computed for ${bufferStoreRes.ok.block.link}` + ), + } + } + + // TODO: Batch per a maximum to queue + for (const piece of pieces) { + const inclusionProof = aggregateBuilder.resolveProof(piece.link) + if (inclusionProof.error) { + return inclusionProof + } + const addMessage = await context.pieceAcceptQueue.add({ + piece: piece.link, + aggregate: aggregateBuilder.link, + group: bufferStoreRes.ok.buffer.group, + inclusion: { + subtree: inclusionProof.ok[0], + index: inclusionProof.ok[1], + }, + }) + + if (addMessage.error) { + return addMessage + } + } + + return { + ok: {}, + } +} + +/** + * On piece accept queue message, store inclusion record in store. + * + * @param {import('./api').PieceAcceptMessageContext} context + * @param {import('./api').PieceAcceptMessage} message + */ +export const handlePieceAcceptMessage = async (context, message) => { + const { piece, aggregate, group, inclusion } = message + + // Store inclusion information into the store. Store events MAY be used to propagate inclusion over + const putRes = await context.inclusionStore.put({ + piece, + aggregate, + group, + inclusion, + insertedAt: new Date().toISOString(), + }) + + // TODO: should we ignore error already there? + if (putRes.error) { + return putRes + } + + return { ok: {}, error: undefined } +} + +/** + * On Inclusion store insert, piece table can be updated to reflect piece state. + * + * @param {import('./api').InclusionInsertEventToUpdateState} context + * @param {import('./api').InclusionRecord} record + */ +export const handleInclusionInsertToUpdateState = async (context, record) => { + const updateRes = await context.pieceStore.update( + { + piece: record.piece, + group: record.group, + }, + { + status: 'accepted', + updatedAt: new Date().toISOString(), + } + ) + if (updateRes.error) { + return updateRes + } + + return { ok: {}, error: undefined } +} + +/** + * @param {import('./api').InclusionInsertEventToIssuePieceAccept} context + * @param {import('./api').InclusionRecord} record + */ +export const handleInclusionInsertToIssuePieceAccept = async ( + context, + record +) => { + // invoke piece/accept to issue receipt + const pieceAcceptInv = await Aggregator.pieceAccept( + context.aggregatorService.invocationConfig, + record.piece, + record.group, + { connection: context.aggregatorService.connection } + ) + + if (pieceAcceptInv.out.error) { + return { + error: pieceAcceptInv.out.error, + } + } + + return { ok: {} } +} + +/** + * On Aggregate store insert, offer inserted aggregate for deal. + * + * @param {import('./api').AggregateInsertEventToAggregateOfferContext} context + * @param {import('./api').AggregateRecord} record + */ +export const handleAggregateInsertToAggregateOffer = async ( + context, + record +) => { + const bufferStoreRes = await context.bufferStore.get(record.pieces) + if (bufferStoreRes.error) { + return { + error: bufferStoreRes.error, + } + } + // Get pieces from buffer + const pieces = bufferStoreRes.ok.buffer.pieces.map((p) => p.piece) + + // invoke aggregate/offer + const aggregateOfferInv = await Dealer.aggregateOffer( + context.dealerService.invocationConfig, + record.aggregate, + pieces, + { connection: context.dealerService.connection } + ) + + if (aggregateOfferInv.out.error) { + return { + error: aggregateOfferInv.out.error, + } + } + + return { ok: {} } +} diff --git a/packages/filecoin-api/src/aggregator/service.js b/packages/filecoin-api/src/aggregator/service.js new file mode 100644 index 000000000..f8fa04986 --- /dev/null +++ b/packages/filecoin-api/src/aggregator/service.js @@ -0,0 +1,151 @@ +import * as Server from '@ucanto/server' +import * as Client from '@ucanto/client' +import * as CAR from '@ucanto/transport/car' +import * as AggregatorCaps from '@web3-storage/capabilities/filecoin/aggregator' +import * as DealerCaps from '@web3-storage/capabilities/filecoin/dealer' +// eslint-disable-next-line no-unused-vars +import * as API from '../types.js' +import { + QueueOperationFailed, + StoreOperationFailed, + UnexpectedState, + RecordNotFoundErrorName, +} from '../errors.js' + +/** + * @param {API.Input} input + * @param {import('./api').ServiceContext} context + * @returns {Promise | API.UcantoInterface.JoinBuilder>} + */ +export const pieceOffer = async ({ capability }, context) => { + const { piece, group } = capability.nb + + // dedupe + const hasRes = await context.pieceStore.has({ piece, group }) + let exists = true + if (hasRes.error?.name === RecordNotFoundErrorName) { + exists = false + } else if (hasRes.error) { + return { + error: new StoreOperationFailed(hasRes.error.message), + } + } + + if (!exists) { + const addRes = await context.pieceQueue.add({ piece, group }) + if (addRes.error) { + return { + error: new QueueOperationFailed(addRes.error.message), + } + } + } + + const fx = await AggregatorCaps.pieceAccept + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + piece, + group, + }, + expiration: Infinity, + }) + .delegate() + + /** @type {API.UcantoInterface.OkBuilder} */ + const result = Server.ok({ piece }) + return result.join(fx.link()) +} + +/** + * @param {API.Input} input + * @param {import('./api').ServiceContext} context + * @returns {Promise | API.UcantoInterface.JoinBuilder>} + */ +export const pieceAccept = async ({ capability }, context) => { + const { piece, group } = capability.nb + + // Get inclusion proof for piece associated with this group + const getInclusionRes = await context.inclusionStore.query({ piece, group }) + if (getInclusionRes.error) { + return { + error: new StoreOperationFailed(getInclusionRes.error?.message), + } + } + if (!getInclusionRes.ok.length) { + return { + error: new UnexpectedState( + `no inclusion proof found for pair {${piece}, ${group}}` + ), + } + } + + const [{ aggregate, inclusion }] = getInclusionRes.ok + const getAggregateRes = await context.aggregateStore.get({ aggregate }) + if (getAggregateRes.error) { + return { + error: new StoreOperationFailed(getAggregateRes.error.message), + } + } + const { pieces } = getAggregateRes.ok + + // Create effect for receipt + const fx = await DealerCaps.aggregateOffer + .invoke({ + issuer: context.id, + audience: context.dealerId, + with: context.id.did(), + nb: { + aggregate, + pieces, + }, + }) + .delegate() + + /** @type {API.UcantoInterface.OkBuilder} */ + const result = Server.ok({ piece, aggregate, inclusion }) + return result.join(fx.link()) +} + +/** + * @param {import('./api').ServiceContext} context + */ +export function createService(context) { + return { + piece: { + offer: Server.provideAdvanced({ + capability: AggregatorCaps.pieceOffer, + handler: (input) => pieceOffer(input, context), + }), + accept: Server.provideAdvanced({ + capability: AggregatorCaps.pieceAccept, + handler: (input) => pieceAccept(input, context), + }), + }, + } +} + +/** + * @param {API.UcantoServerContext & import('./api').ServiceContext} context + */ +export const createServer = (context) => + Server.create({ + id: context.id, + codec: context.codec || CAR.inbound, + service: createService(context), + catch: (error) => context.errorReporter.catch(error), + }) + +/** + * @param {object} options + * @param {API.UcantoInterface.Principal} options.id + * @param {API.UcantoInterface.Transport.Channel} options.channel + * @param {API.UcantoInterface.OutboundCodec} [options.codec] + */ +export const connect = ({ id, channel, codec = CAR.outbound }) => + Client.connect({ + id, + channel, + codec, + }) diff --git a/packages/filecoin-api/src/deal-tracker/api.js b/packages/filecoin-api/src/deal-tracker/api.js new file mode 100644 index 000000000..336ce12bb --- /dev/null +++ b/packages/filecoin-api/src/deal-tracker/api.js @@ -0,0 +1 @@ +export {} diff --git a/packages/filecoin-api/src/deal-tracker/api.ts b/packages/filecoin-api/src/deal-tracker/api.ts new file mode 100644 index 000000000..7ee1e2e4c --- /dev/null +++ b/packages/filecoin-api/src/deal-tracker/api.ts @@ -0,0 +1,34 @@ +import type { Signer } from '@ucanto/interface' +import { PieceLink } from '@web3-storage/data-segment' +import { QueryableStore } from '../types.js' + +export interface ServiceContext { + /** + * Service signer + */ + id: Signer + + /** + * Stores information about deals for a given aggregate piece CID. + */ + dealStore: QueryableStore +} + +export interface DealRecord { + // PieceCid of an Aggregate `bagy...aggregate` + piece: PieceLink + // address of the Filecoin storage provider storing deal + provider: string + // deal identifier + dealId: number + // epoch of deal expiration + expirationEpoch: number + // source of the deal information + source: string + // Date when deal was added as ISO string + insertedAt: string +} + +export interface DealRecordKey extends Pick {} + +export interface DealRecordQueryByPiece extends Pick {} diff --git a/packages/filecoin-api/src/deal-tracker/service.js b/packages/filecoin-api/src/deal-tracker/service.js new file mode 100644 index 000000000..4507d2ad5 --- /dev/null +++ b/packages/filecoin-api/src/deal-tracker/service.js @@ -0,0 +1,77 @@ +import * as Server from '@ucanto/server' +import * as Client from '@ucanto/client' +import * as CAR from '@ucanto/transport/car' +import * as DealTrackerCaps from '@web3-storage/capabilities/filecoin/deal-tracker' + +// eslint-disable-next-line no-unused-vars +import * as API from '../types.js' +import { StoreOperationFailed } from '../errors.js' + +/** + * @typedef {import('@web3-storage/capabilities/types.js').DealDetails} DealDetails + */ + +/** + * @param {API.Input} input + * @param {import('./api').ServiceContext} context + * @returns {Promise>} + */ +export const dealInfo = async ({ capability }, context) => { + const { piece } = capability.nb + + const storeGet = await context.dealStore.query({ piece }) + if (storeGet.error) { + return { + error: new StoreOperationFailed(storeGet.error.message), + } + } + + return { + ok: { + deals: storeGet.ok.reduce((acc, curr) => { + acc[`${curr.dealId}`] = { + provider: curr.provider, + } + + return acc + }, /** @type {Record} */ ({})), + }, + } +} + +/** + * @param {import('./api').ServiceContext} context + */ +export function createService(context) { + return { + deal: { + info: Server.provide(DealTrackerCaps.dealInfo, (input) => + dealInfo(input, context) + ), + }, + } +} + +/** + * @param {API.UcantoServerContext & import('./api').ServiceContext} context + */ +export const createServer = (context) => + Server.create({ + id: context.id, + codec: context.codec || CAR.inbound, + service: createService(context), + catch: (error) => context.errorReporter.catch(error), + }) + +/** + * @param {object} options + * @param {API.UcantoInterface.Principal} options.id + * @param {API.UcantoInterface.Transport.Channel} options.channel + * @param {API.UcantoInterface.OutboundCodec} [options.codec] + */ +export const connect = ({ id, channel, codec = CAR.outbound }) => + Client.connect({ + id, + channel, + codec, + }) diff --git a/packages/filecoin-api/src/dealer.js b/packages/filecoin-api/src/dealer.js deleted file mode 100644 index 1a33ed519..000000000 --- a/packages/filecoin-api/src/dealer.js +++ /dev/null @@ -1,157 +0,0 @@ -import * as Server from '@ucanto/server' -import * as Client from '@ucanto/client' -import * as CAR from '@ucanto/transport/car' -import { CBOR } from '@ucanto/core' -import { Filecoin as FilecoinCapabilities } from '@web3-storage/capabilities' - -import * as API from './types.js' -import { - QueueOperationFailed, - StoreOperationFailed, - DecodeBlockOperationFailed, -} from './errors.js' - -/** - * @param {API.Input} input - * @param {API.DealerServiceContext} context - * @returns {Promise | API.UcantoInterface.JoinBuilder>} - */ -export const queue = async ({ capability, invocation }, context) => { - const { aggregate, pieces: offerCid, storefront, label } = capability.nb - const pieces = getOfferBlock(offerCid, invocation.iterateIPLDBlocks()) - - if (!pieces) { - return { - error: new DecodeBlockOperationFailed( - `missing offer block in invocation: ${offerCid.toString()}` - ), - } - } - - const queued = await context.addQueue.add({ - aggregate, - pieces, // add queue can opt to store offers in separate datastore - storefront, - label, - insertedAt: Date.now(), - }) - if (queued.error) { - return { - error: new QueueOperationFailed(queued.error.message), - } - } - - // Create effect for receipt - const fx = await FilecoinCapabilities.dealAdd - .invoke({ - issuer: context.id, - audience: context.id, - with: context.id.did(), - nb: { - aggregate, - pieces: offerCid, - storefront, - label, - }, - }) - .delegate() - - return Server.ok({ - aggregate, - }).join(fx.link()) -} - -/** - * @param {API.Input} input - * @param {API.DealerServiceContext} context - * @returns {Promise | API.UcantoInterface.JoinBuilder>} - */ -export const add = async ({ capability, invocation }, context) => { - const { aggregate, pieces: offerCid, storefront } = capability.nb - const pieces = getOfferBlock(offerCid, invocation.iterateIPLDBlocks()) - - if (!pieces) { - return { - error: new DecodeBlockOperationFailed( - `missing offer block in invocation: ${offerCid.toString()}` - ), - } - } - - // Get deal status from the store. - const get = await context.dealStore.get({ - aggregate, - storefront, - }) - if (get.error) { - return { - error: new StoreOperationFailed(get.error.message), - } - } - - return { - ok: { - aggregate, - }, - } -} - -/** - * @param {Server.API.Link} offerCid - * @param {IterableIterator>} blockIterator - */ -function getOfferBlock(offerCid, blockIterator) { - for (const block of blockIterator) { - if (block.cid.equals(offerCid)) { - const decoded = - /** @type {import('@web3-storage/data-segment').PieceLink[]} */ ( - CBOR.decode(block.bytes) - ) - return decoded - // TODO: Validate with schema - } - } -} - -/** - * @param {API.DealerServiceContext} context - */ -export function createService(context) { - return { - deal: { - queue: Server.provideAdvanced({ - capability: FilecoinCapabilities.dealQueue, - handler: (input) => queue(input, context), - }), - add: Server.provideAdvanced({ - capability: FilecoinCapabilities.dealAdd, - handler: (input) => add(input, context), - }), - }, - } -} - -/** - * @param {API.UcantoServerContext & API.DealerServiceContext} context - */ -export const createServer = (context) => - Server.create({ - id: context.id, - codec: context.codec || CAR.inbound, - service: createService(context), - catch: (error) => context.errorReporter.catch(error), - validateAuthorization: (auth) => context.validateAuthorization(auth), - }) - -/** - * @param {object} options - * @param {API.UcantoInterface.Principal} options.id - * @param {API.UcantoInterface.Transport.Channel} options.channel - * @param {API.UcantoInterface.OutboundCodec} [options.codec] - */ -export const connect = ({ id, channel, codec = CAR.outbound }) => - Client.connect({ - id, - channel, - codec, - }) diff --git a/packages/filecoin-api/src/dealer/api.js b/packages/filecoin-api/src/dealer/api.js new file mode 100644 index 000000000..336ce12bb --- /dev/null +++ b/packages/filecoin-api/src/dealer/api.js @@ -0,0 +1 @@ +export {} diff --git a/packages/filecoin-api/src/dealer/api.ts b/packages/filecoin-api/src/dealer/api.ts new file mode 100644 index 000000000..e0c6f1e74 --- /dev/null +++ b/packages/filecoin-api/src/dealer/api.ts @@ -0,0 +1,116 @@ +import type { Signer, Link } from '@ucanto/interface' +import { DealMetadata } from '@web3-storage/capabilities/types' +import { PieceLink } from '@web3-storage/data-segment' +import { + DealerService, + DealTrackerService, +} from '@web3-storage/filecoin-client/types' +import { + UpdatableStore, + UpdatableAndQueryableStore, + ServiceConfig, +} from '../types.js' + +export type OfferStore = UpdatableStore +export type AggregateStore = UpdatableAndQueryableStore< + AggregateRecordKey, + AggregateRecord, + Pick +> + +export interface ServiceContext { + id: Signer + /** + * Stores serialized broker specific offer document containing details of the + * aggregate and it's pieces. + */ + offerStore: OfferStore + /** + * Stores aggregates and their deal proofs. + */ + aggregateStore: AggregateStore +} + +export interface AggregateInsertEventContext + extends Pick {} + +export interface AggregateUpdatedStatusEventContext { + /** + * Dealer connection to offer aggregates for deals. + */ + dealerService: ServiceConfig +} + +export interface CronContext extends Pick { + /** + * Deal tracker connection to find out available deals for an aggregate. + */ + dealTrackerService: ServiceConfig +} + +export interface AggregateRecord { + /** + * Piece CID of an aggregate. + */ + aggregate: PieceLink + /** + * List of pieces in an aggregate. + */ + pieces: Link + /** + * Filecoin deal where aggregate is present. + */ + deal?: DealMetadata + /** + * Status of the offered aggregate piece. + * - offered = acknowledged received for inclusion in filecoin deals. + * - accepted = accepted and included a filecoin deal(s). + * - invalid = not valid for storage. + */ + status: 'offered' | 'accepted' | 'invalid' + /** + * Insertion date ISO string. + */ + insertedAt: string + /** + * Update date ISO string. + */ + updatedAt: string +} + +export interface AggregateRecordKey { + /** + * Piece CID of an aggregate. + */ + aggregate: PieceLink + /** + * Filecoin deal where aggregate is present. + */ + deal?: DealMetadata +} + +export interface OfferDocument { + /** + * Key of the offer document + */ + key: string + /** + * Value of the offer document + */ + value: OfferValue +} + +export interface OfferValue { + /** + * Issuer of the aggregate offer. + */ + issuer: `did:${string}:${string}` + /** + * Piece CID of an aggregate. + */ + aggregate: PieceLink + /** + * Pieces part of the aggregate + */ + pieces: PieceLink[] +} diff --git a/packages/filecoin-api/src/dealer/deal-track.js b/packages/filecoin-api/src/dealer/deal-track.js new file mode 100644 index 000000000..e69de29bb diff --git a/packages/filecoin-api/src/dealer/events.js b/packages/filecoin-api/src/dealer/events.js new file mode 100644 index 000000000..cff4f0de1 --- /dev/null +++ b/packages/filecoin-api/src/dealer/events.js @@ -0,0 +1,165 @@ +import { Dealer, DealTracker } from '@web3-storage/filecoin-client' + +import { StoreOperationFailed } from '../errors.js' + +/** + * @typedef {import('./api').AggregateRecord} AggregateRecord + * @typedef {import('./api').AggregateRecordKey} AggregateRecordKey + */ + +/** + * On aggregate insert event, update offer key with date to be retrievable by broker. + * + * @param {import('./api').AggregateInsertEventContext} context + * @param {AggregateRecord} record + */ +export const handleAggregateInsert = async (context, record) => { + const updateRes = await context.offerStore.update(record.pieces.toString(), { + key: `${new Date( + record.insertedAt + ).toISOString()} ${record.aggregate.toString()}.json`, + }) + if (updateRes.error) { + return { error: new StoreOperationFailed(updateRes.error.message) } + } + + return { ok: {} } +} + +/** + * On Aggregate update status event, issue aggregate accept receipt. + * + * @param {import('./api').AggregateUpdatedStatusEventContext} context + * @param {AggregateRecord} record + */ +export const handleAggregatUpdatedStatus = async (context, record) => { + const aggregateAcceptInv = await Dealer.aggregateAccept( + context.dealerService.invocationConfig, + record.aggregate, + record.pieces, + { connection: context.dealerService.connection } + ) + if (aggregateAcceptInv.out.error) { + return { + error: aggregateAcceptInv.out.error, + } + } + + return { ok: {} } +} + +/** + * On cron tick event, get aggregates without deals, and verify if there are updates on them. + * If there are deals for pending aggregates, their state can be updated. + * + * @param {import('./api').CronContext} context + */ +export const handleCronTick = async (context) => { + // Get offered deals pending approval/rejection + const offeredDeals = await context.aggregateStore.query({ + status: 'offered', + }) + if (offeredDeals.error) { + return { + error: offeredDeals.error, + } + } + + // Update approved deals from the ones resolved + const updatedResponses = await Promise.all( + offeredDeals.ok.map((deal) => + updateApprovedDeals({ + deal, + aggregateStore: context.aggregateStore, + dealTrackerServiceConnection: context.dealTrackerService.connection, + dealTrackerInvocationConfig: + context.dealTrackerService.invocationConfig, + }) + ) + ) + + // Fail if one or more update operations did not succeed. + // The successful ones are still valid, but we should keep track of errors for monitoring/alerting. + const updateErrorResponse = updatedResponses.find((r) => r.error) + if (updateErrorResponse) { + return { + error: updateErrorResponse.error, + } + } + + // Return successful update operation + // Include in response the ones that were Updated, and the ones still pending response. + const updatedDealsCount = updatedResponses.filter((r) => r.ok?.updated).length + return { + ok: { + updatedCount: updatedDealsCount, + pendingCount: updatedResponses.length - updatedDealsCount, + }, + } +} + +/** + * Find out if deal is on chain. When on chain, updates its status in store. + * + * @param {object} context + * @param {AggregateRecord} context.deal + * @param {import('../types.js').UpdatableAndQueryableStore>} context.aggregateStore + * @param {import('@ucanto/interface').ConnectionView} context.dealTrackerServiceConnection + * @param {import('@web3-storage/filecoin-client/types').InvocationConfig} context.dealTrackerInvocationConfig + */ +async function updateApprovedDeals({ + deal, + aggregateStore, + dealTrackerServiceConnection, + dealTrackerInvocationConfig, +}) { + // Query current state + const info = await DealTracker.dealInfo( + dealTrackerInvocationConfig, + deal.aggregate, + { connection: dealTrackerServiceConnection } + ) + + if (info.out.error) { + return { + error: info.out.error, + } + } + + // If there are no deals for it, we can skip + const deals = Object.keys(info.out.ok.deals || {}) + if (!deals.length) { + return { + ok: { + updated: false, + }, + } + } + + // Update status and deal information + const updateAggregate = await aggregateStore.update( + { aggregate: deal.aggregate }, + { + status: 'accepted', + updatedAt: new Date().toISOString(), + deal: { + dataType: 0n, + dataSource: { + dealID: BigInt(deals[0]), + }, + }, + } + ) + + if (updateAggregate.error) { + return { + error: updateAggregate.error, + } + } + + return { + ok: { + updated: true, + }, + } +} diff --git a/packages/filecoin-api/src/dealer/service.js b/packages/filecoin-api/src/dealer/service.js new file mode 100644 index 000000000..56e4debc1 --- /dev/null +++ b/packages/filecoin-api/src/dealer/service.js @@ -0,0 +1,187 @@ +import * as Server from '@ucanto/server' +import * as Client from '@ucanto/client' +import * as CAR from '@ucanto/transport/car' +import { CBOR } from '@ucanto/core' +import { sha256 } from 'multiformats/hashes/sha2' +import * as Block from 'multiformats/block' +import * as DealerCaps from '@web3-storage/capabilities/filecoin/dealer' +// eslint-disable-next-line no-unused-vars +import * as API from '../types.js' +import { + StoreOperationFailed, + DecodeBlockOperationFailed, + RecordNotFoundErrorName, +} from '../errors.js' + +/** + * @param {API.Input} input + * @param {import('./api').ServiceContext} context + * @returns {Promise | API.UcantoInterface.JoinBuilder>} + */ +export const aggregateOffer = async ({ capability, invocation }, context) => { + const issuer = invocation.issuer.did() + const { aggregate, pieces } = capability.nb + + const hasRes = await context.aggregateStore.has({ aggregate }) + let exists = true + if (hasRes.error?.name === RecordNotFoundErrorName) { + exists = false + } else if (hasRes.error) { + return { + error: new StoreOperationFailed(hasRes.error.message), + } + } + + if (!exists) { + const piecesBlockRes = await findCBORBlock( + pieces, + invocation.iterateIPLDBlocks() + ) + if (piecesBlockRes.error) { + return piecesBlockRes + } + + // Write Spade formatted doc to offerStore before putting aggregate for tracking + const putOfferRes = await context.offerStore.put({ + key: piecesBlockRes.ok.cid.toString(), + value: { + issuer, + aggregate, + pieces: piecesBlockRes.ok.value, + }, + }) + if (putOfferRes.error) { + return { + error: new StoreOperationFailed(putOfferRes.error.message), + } + } + + // Put aggregate offered into store + const putRes = await context.aggregateStore.put({ + aggregate, + pieces, + status: 'offered', + insertedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }) + if (putRes.error) { + return { + error: new StoreOperationFailed(putRes.error.message), + } + } + } + + // Effect + const fx = await DealerCaps.aggregateAccept + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + aggregate, + pieces, + }, + expiration: Infinity, + }) + .delegate() + + /** @type {API.UcantoInterface.OkBuilder} */ + const result = Server.ok({ aggregate }) + return result.join(fx.link()) +} + +/** + * @param {API.Input} input + * @param {import('./api').ServiceContext} context + * @returns {Promise>} + */ +export const aggregateAccept = async ({ capability }, context) => { + const { aggregate } = capability.nb + + // Get deal status from the store. + const get = await context.aggregateStore.get({ + aggregate, + }) + if (get.error) { + return { + error: new StoreOperationFailed(get.error.message), + } + } + if (!get.ok.deal) { + return { + error: new StoreOperationFailed('no deal available'), + } + } + + return { + ok: { + aggregate, + dataSource: get.ok.deal.dataSource, + dataType: get.ok.deal.dataType, + }, + } +} + +/** + * @param {import('multiformats').Link} cid + * @param {IterableIterator>} blocks + * @returns {Promise>} + */ +const findCBORBlock = async (cid, blocks) => { + let bytes + for (const b of blocks) { + if (b.cid.equals(cid)) { + bytes = b.bytes + } + } + if (!bytes) { + return { + error: new DecodeBlockOperationFailed(`missing block: ${cid}`), + } + } + return { + ok: await Block.create({ cid, bytes, codec: CBOR, hasher: sha256 }), + } +} + +/** + * @param {import('./api').ServiceContext} context + */ +export function createService(context) { + return { + aggregate: { + offer: Server.provideAdvanced({ + capability: DealerCaps.aggregateOffer, + handler: (input) => aggregateOffer(input, context), + }), + accept: Server.provideAdvanced({ + capability: DealerCaps.aggregateAccept, + handler: (input) => aggregateAccept(input, context), + }), + }, + } +} + +/** + * @param {API.UcantoServerContext & import('./api').ServiceContext} context + */ +export const createServer = (context) => + Server.create({ + id: context.id, + codec: context.codec || CAR.inbound, + service: createService(context), + catch: (error) => context.errorReporter.catch(error), + }) + +/** + * @param {object} options + * @param {API.UcantoInterface.Principal} options.id + * @param {API.UcantoInterface.Transport.Channel} options.channel + * @param {API.UcantoInterface.OutboundCodec} [options.codec] + */ +export const connect = ({ id, channel, codec = CAR.outbound }) => + Client.connect({ + id, + channel, + codec, + }) diff --git a/packages/filecoin-api/src/errors.js b/packages/filecoin-api/src/errors.js index a1f627ee9..94f27277e 100644 --- a/packages/filecoin-api/src/errors.js +++ b/packages/filecoin-api/src/errors.js @@ -1,18 +1,24 @@ import * as Server from '@ucanto/server' +export const UnexpectedStateErrorName = /** @type {const} */ ('UnexpectedState') +export class UnexpectedState extends Server.Failure { + get reason() { + return this.message + } + + get name() { + return UnexpectedStateErrorName + } +} + export const QueueOperationErrorName = /** @type {const} */ ( 'QueueOperationFailed' ) export class QueueOperationFailed extends Server.Failure { - /** - * @param {string} message - */ - constructor(message) { - super(message) - } get reason() { return this.message } + get name() { return QueueOperationErrorName } @@ -22,47 +28,32 @@ export const StoreOperationErrorName = /** @type {const} */ ( 'StoreOperationFailed' ) export class StoreOperationFailed extends Server.Failure { - /** - * @param {string} message - */ - constructor(message) { - super(message) - } get reason() { return this.message } + get name() { return StoreOperationErrorName } } -export const StoreNotFoundErrorName = /** @type {const} */ ('StoreNotFound') -export class StoreNotFound extends Server.Failure { - /** - * @param {string} message - */ - constructor(message) { - super(message) - } +export const RecordNotFoundErrorName = /** @type {const} */ ('RecordNotFound') +export class RecordNotFound extends Server.Failure { get reason() { return this.message } + get name() { - return StoreNotFoundErrorName + return RecordNotFoundErrorName } } export const EncodeRecordErrorName = /** @type {const} */ ('EncodeRecordFailed') export class EncodeRecordFailed extends Server.Failure { - /** - * @param {string} message - */ - constructor(message) { - super(message) - } get reason() { return this.message } + get name() { return EncodeRecordErrorName } @@ -72,15 +63,10 @@ export const DecodeBlockOperationErrorName = /** @type {const} */ ( 'DecodeBlockOperationFailed' ) export class DecodeBlockOperationFailed extends Server.Failure { - /** - * @param {string} message - */ - constructor(message) { - super(message) - } get reason() { return this.message } + get name() { return DecodeBlockOperationErrorName } diff --git a/packages/filecoin-api/src/lib.js b/packages/filecoin-api/src/lib.js index 6401e13e8..ad09c8eae 100644 --- a/packages/filecoin-api/src/lib.js +++ b/packages/filecoin-api/src/lib.js @@ -1,3 +1,3 @@ -export * as Storefront from './storefront.js' -export * as Aggregator from './aggregator.js' -export * as Dealer from './dealer.js' +export * as Storefront from './storefront/service.js' +export * as Aggregator from './aggregator/service.js' +export * as Dealer from './dealer/service.js' diff --git a/packages/filecoin-api/src/storefront.js b/packages/filecoin-api/src/storefront.js deleted file mode 100644 index 55c55e1a8..000000000 --- a/packages/filecoin-api/src/storefront.js +++ /dev/null @@ -1,114 +0,0 @@ -import * as Server from '@ucanto/server' -import * as Client from '@ucanto/client' -import * as CAR from '@ucanto/transport/car' -import { Filecoin as FilecoinCapabilities } from '@web3-storage/capabilities' - -import * as API from './types.js' -import { QueueOperationFailed, StoreOperationFailed } from './errors.js' - -/** - * @param {API.Input} input - * @param {API.StorefrontServiceContext} context - * @returns {Promise | API.UcantoInterface.JoinBuilder>} - */ -export const add = async ({ capability }, context) => { - const { piece, content } = capability.nb - - /// Store piece into the store. Store events MAY be used to propagate piece over - const put = await context.pieceStore.put({ - content, - piece, - insertedAt: Date.now(), - }) - if (put.error) { - return { - error: new StoreOperationFailed(put.error.message), - } - } - - return { - ok: { - piece, - }, - } -} - -/** - * @param {API.Input} input - * @param {API.StorefrontServiceContext} context - * @returns {Promise | API.UcantoInterface.JoinBuilder>} - */ -export const queue = async ({ capability }, context) => { - const { piece, content } = capability.nb - - const queued = await context.addQueue.add({ - piece, - content, - insertedAt: Date.now(), - }) - if (queued.error) { - return { - error: new QueueOperationFailed(queued.error.message), - } - } - - // Create effect for receipt - const fx = await FilecoinCapabilities.filecoinAdd - .invoke({ - issuer: context.id, - audience: context.id, - with: context.id.did(), - nb: { - piece, - content, - }, - }) - .delegate() - - return Server.ok({ - piece, - }).join(fx.link()) -} - -/** - * @param {API.StorefrontServiceContext} context - */ -export function createService(context) { - return { - filecoin: { - queue: Server.provideAdvanced({ - capability: FilecoinCapabilities.filecoinQueue, - handler: (input) => queue(input, context), - }), - add: Server.provideAdvanced({ - capability: FilecoinCapabilities.filecoinAdd, - handler: (input) => add(input, context), - }), - }, - } -} - -/** - * @param {API.UcantoServerContext & API.StorefrontServiceContext} context - */ -export const createServer = (context) => - Server.create({ - id: context.id, - codec: context.codec || CAR.inbound, - service: createService(context), - catch: (error) => context.errorReporter.catch(error), - validateAuthorization: (auth) => context.validateAuthorization(auth), - }) - -/** - * @param {object} options - * @param {API.UcantoInterface.Principal} options.id - * @param {API.UcantoInterface.Transport.Channel} options.channel - * @param {API.UcantoInterface.OutboundCodec} [options.codec] - */ -export const connect = ({ id, channel, codec = CAR.outbound }) => - Client.connect({ - id, - channel, - codec, - }) diff --git a/packages/filecoin-api/src/storefront/api.js b/packages/filecoin-api/src/storefront/api.js new file mode 100644 index 000000000..336ce12bb --- /dev/null +++ b/packages/filecoin-api/src/storefront/api.js @@ -0,0 +1 @@ +export {} diff --git a/packages/filecoin-api/src/storefront/api.ts b/packages/filecoin-api/src/storefront/api.ts new file mode 100644 index 000000000..c9c2ecc6a --- /dev/null +++ b/packages/filecoin-api/src/storefront/api.ts @@ -0,0 +1,158 @@ +import type { + Signer, + Principal, + UnknownLink, + Receipt, + Invocation, + Failure, +} from '@ucanto/interface' +import { PieceLink } from '@web3-storage/data-segment' +import { + AggregatorService, + StorefrontService, +} from '@web3-storage/filecoin-client/types' +import { + Store, + UpdatableAndQueryableStore, + Queue, + ServiceConfig, +} from '../types.js' + +export interface ServiceOptions { + /** + * Implementer MAY handle submission without user request. + */ + skipFilecoinSubmitQueue?: boolean +} + +export interface ServiceContext { + /** + * Service signer + */ + id: Signer + /** + * Principal for aggregator service + */ + aggregatorId: Principal + /** + * Stores pieces that have been offered to the Storefront. + */ + pieceStore: UpdatableAndQueryableStore< + PieceRecordKey, + PieceRecord, + Pick + > + /** + * Queues pieces for verification. + */ + filecoinSubmitQueue: Queue + /** + * Queues pieces for offering to an Aggregator. + */ + pieceOfferQueue: Queue + /** + * Stores task invocations. + */ + taskStore: Store + /** + * Stores receipts for tasks. + */ + receiptStore: Store + /** + * Service options. + */ + options?: ServiceOptions +} + +export interface FilecoinSubmitMessageContext + extends Pick {} + +export interface PieceOfferMessageContext { + /** + * Aggregator connection to moves pieces into the pipeline. + */ + aggregatorService: ServiceConfig +} + +export interface StorefrontClientContext { + /** + * Storefront own connection to issue receipts. + */ + storefrontService: ServiceConfig +} + +export interface CronContext + extends Pick< + ServiceContext, + 'id' | 'pieceStore' | 'receiptStore' | 'taskStore' + > { + /** + * Principal for aggregator service + */ + aggregatorId: Signer +} + +export interface PieceRecord { + /** + * Piece CID for the content. + */ + piece: PieceLink + /** + * CAR shard CID. + */ + content: UnknownLink + /** + * Grouping information for submitted piece. + */ + group: string + /** + * Status of the offered filecoin piece. + * - submitted = verified valid piece and submitted to the aggregation pipeline + * - accepted = accepted and included in filecoin deal(s) + * - invalid = content/piece CID mismatch + */ + status: 'submitted' | 'accepted' | 'invalid' + /** + * Insertion date ISO string. + */ + insertedAt: string + /** + * Update date ISO string. + */ + updatedAt: string +} +export interface PieceRecordKey extends Pick {} + +export interface FilecoinSubmitMessage { + /** + * Piece CID for the content. + */ + piece: PieceLink + /** + * CAR shard CID. + */ + content: UnknownLink + /** + * Grouping information for submitted piece. + */ + group: string +} + +export interface PieceOfferMessage { + /** + * Piece CID. + */ + piece: PieceLink + /** + * CAR shard CID. + */ + content: UnknownLink + /** + * Grouping information for submitted piece. + */ + group: string +} + +export interface DataAggregationProofNotFound extends Failure { + name: 'DataAggregationProofNotFound' +} diff --git a/packages/filecoin-api/src/storefront/events.js b/packages/filecoin-api/src/storefront/events.js new file mode 100644 index 000000000..5d05777fe --- /dev/null +++ b/packages/filecoin-api/src/storefront/events.js @@ -0,0 +1,264 @@ +import { Storefront, Aggregator } from '@web3-storage/filecoin-client' +import * as AggregatorCaps from '@web3-storage/capabilities/filecoin/aggregator' + +// eslint-disable-next-line no-unused-vars +import * as API from '../types.js' +import { + RecordNotFoundErrorName, + StoreOperationFailed, + UnexpectedState, +} from '../errors.js' + +/** + * @typedef {import('./api').PieceRecord} PieceRecord + * @typedef {import('./api').PieceRecordKey} PieceRecordKey + * @typedef {import('../types.js').UpdatableAndQueryableStore>} PieceStore + */ + +/** + * On filecoin submit queue messages, validate piece for given content and store it in store. + * + * @param {import('./api').FilecoinSubmitMessageContext} context + * @param {import('./api').FilecoinSubmitMessage} message + */ +export const handleFilecoinSubmitMessage = async (context, message) => { + // dedupe concurrent writes + const hasRes = await context.pieceStore.has({ piece: message.piece }) + if (hasRes.error && hasRes.error.name !== RecordNotFoundErrorName) { + return { error: new StoreOperationFailed(hasRes.error.message) } + } + + // TODO: verify piece + + const putRes = await context.pieceStore.put({ + piece: message.piece, + content: message.content, + group: message.group, + status: 'submitted', + insertedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }) + if (putRes.error) { + return { error: new StoreOperationFailed(putRes.error.message) } + } + return { ok: {} } +} + +/** + * On piece offer queue message, offer piece for aggregation. + * + * @param {import('./api').PieceOfferMessageContext} context + * @param {import('./api').PieceOfferMessage} message + */ +export const handlePieceOfferMessage = async (context, message) => { + const pieceOfferInv = await Aggregator.pieceOffer( + context.aggregatorService.invocationConfig, + message.piece, + message.group, + { connection: context.aggregatorService.connection } + ) + if (pieceOfferInv.out.error) { + return { + error: pieceOfferInv.out.error, + } + } + + return { ok: {} } +} + +/** + * On piece inserted into store, invoke submit to queue piece to be offered for aggregate. + * + * @param {import('./api').StorefrontClientContext} context + * @param {PieceRecord} record + */ +export const handlePieceInsert = async (context, record) => { + const filecoinSubmitInv = await Storefront.filecoinSubmit( + context.storefrontService.invocationConfig, + record.content, + record.piece, + { connection: context.storefrontService.connection } + ) + + if (filecoinSubmitInv.out.error) { + return { + error: filecoinSubmitInv.out.error, + } + } + + return { ok: {} } +} + +/** + * @param {import('./api').StorefrontClientContext} context + * @param {PieceRecord} record + */ +export const handlePieceStatusUpdate = async (context, record) => { + // Validate expected status + if (record.status === 'submitted') { + return { + error: new UnexpectedState( + `record status for ${record.piece} is "${record.status}"` + ), + } + } + + const filecoinAcceptInv = await Storefront.filecoinAccept( + context.storefrontService.invocationConfig, + record.content, + record.piece, + { connection: context.storefrontService.connection } + ) + + if (filecoinAcceptInv.out.error) { + return { + error: filecoinAcceptInv.out.error, + } + } + + return { ok: {} } +} + +/** + * @param {import('./api').CronContext} context + */ +export const handleCronTick = async (context) => { + const submittedPieces = await context.pieceStore.query({ + status: 'submitted', + }) + if (submittedPieces.error) { + return { + error: submittedPieces.error, + } + } + // Update approved pieces from the ones resolved + const updatedResponses = await Promise.all( + submittedPieces.ok.map((pieceRecord) => + updatePiecesWithDeal({ + id: context.id, + aggregatorId: context.aggregatorId, + pieceRecord, + pieceStore: context.pieceStore, + taskStore: context.taskStore, + receiptStore: context.receiptStore, + }) + ) + ) + + // Fail if one or more update operations did not succeed. + // The successful ones are still valid, but we should keep track of errors for monitoring/alerting. + const updateErrorResponse = updatedResponses.find((r) => r.error) + if (updateErrorResponse) { + return { + error: updateErrorResponse.error, + } + } + + // Return successful update operation + // Include in response the ones that were Updated, and the ones still pending response. + const updatedPiecesCount = updatedResponses.filter( + (r) => r.ok?.updated + ).length + return { + ok: { + updatedCount: updatedPiecesCount, + pendingCount: updatedResponses.length - updatedPiecesCount, + }, + } +} + +/** + * Read receipt chain to determine if an aggregate was accepted for the piece. + * Update its status if there is an accepted aggregate. + * + * @param {object} context + * @param {import('@ucanto/interface').Signer} context.id + * @param {import('@ucanto/interface').Principal} context.aggregatorId + * @param {PieceRecord} context.pieceRecord + * @param {PieceStore} context.pieceStore + * @param {API.Store} context.taskStore + * @param {API.Store} context.receiptStore + */ +async function updatePiecesWithDeal({ + id, + aggregatorId, + pieceRecord, + pieceStore, + taskStore, + receiptStore, +}) { + let aggregateAcceptReceipt + + let task = /** @type {API.UcantoInterface.Link} */ ( + ( + await AggregatorCaps.pieceOffer + .invoke({ + issuer: id, + audience: aggregatorId, + with: id.did(), + nb: { + piece: pieceRecord.piece, + group: pieceRecord.group, + }, + expiration: Infinity, + }) + .delegate() + ).cid + ) + + while (true) { + const [taskRes, receiptRes] = await Promise.all([ + taskStore.get(task), + receiptStore.get(task), + ]) + // Should fail if errored and not with StoreNotFound Error + if ( + (taskRes.error && taskRes.error.name !== RecordNotFoundErrorName) || + (receiptRes.error && receiptRes.error.name !== RecordNotFoundErrorName) + ) { + return { + error: taskRes.error || receiptRes.error, + } + } + // Might not be available still, as piece is in progress to get into a deal + if (taskRes.error || receiptRes.error) { + // Store not found + break + } + + // Save very last receipt - aggregate/accept + const ability = taskRes.ok.capabilities[0]?.can + if (ability === 'aggregate/accept') { + aggregateAcceptReceipt = receiptRes.ok + } + if (!receiptRes.ok.fx.join) break + task = receiptRes.ok.fx.join.link() + } + + // If there is a receipt, status can be updated + if (aggregateAcceptReceipt) { + const updateRes = await pieceStore.update( + { + piece: pieceRecord.piece, + }, + { + status: !!aggregateAcceptReceipt.out.ok ? 'accepted' : 'invalid', + updatedAt: new Date().toISOString(), + } + ) + + if (updateRes.ok) { + return { + ok: { + updated: true, + }, + } + } + } + + return { + ok: { + updated: false, + }, + } +} diff --git a/packages/filecoin-api/src/storefront/service.js b/packages/filecoin-api/src/storefront/service.js new file mode 100644 index 000000000..39f4dfaad --- /dev/null +++ b/packages/filecoin-api/src/storefront/service.js @@ -0,0 +1,290 @@ +import * as Server from '@ucanto/server' +import * as Client from '@ucanto/client' +import * as CAR from '@ucanto/transport/car' +import * as StorefrontCaps from '@web3-storage/capabilities/filecoin/storefront' +import * as AggregatorCaps from '@web3-storage/capabilities/filecoin/aggregator' +// eslint-disable-next-line no-unused-vars +import * as API from '../types.js' +import { + QueueOperationFailed, + RecordNotFoundErrorName, + StoreOperationFailed, +} from '../errors.js' + +/** + * @param {API.Input} input + * @param {import('./api').ServiceContext} context + * @returns {Promise | API.UcantoInterface.JoinBuilder>} + */ +export const filecoinOffer = async ({ capability }, context) => { + const { piece, content } = capability.nb + + // Queue offer for filecoin submission + if (!context.options?.skipFilecoinSubmitQueue) { + // dedupe + const hasRes = await context.pieceStore.has({ piece }) + let exists = true + if (hasRes.error?.name === RecordNotFoundErrorName) { + exists = false + } else if (hasRes.error) { + return { error: new StoreOperationFailed(hasRes.error.message) } + } + + const group = context.id.did() + if (!exists) { + // Queue the piece for validation etc. + const queueRes = await context.filecoinSubmitQueue.add({ + piece, + content, + group, + }) + if (queueRes.error) { + return { + error: new QueueOperationFailed(queueRes.error.message), + } + } + } + } + + // Create effect for receipt + const [submitfx, acceptfx] = await Promise.all([ + StorefrontCaps.filecoinSubmit + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + piece, + content, + }, + expiration: Infinity, + }) + .delegate(), + StorefrontCaps.filecoinAccept + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + piece, + content, + }, + expiration: Infinity, + }) + .delegate(), + ]) + + // TODO: receipt timestamp? + /** @type {API.UcantoInterface.OkBuilder} */ + const result = Server.ok({ piece }) + return result.fork(submitfx.link()).join(acceptfx.link()) +} + +/** + * @param {API.Input} input + * @param {import('./api').ServiceContext} context + * @returns {Promise | API.UcantoInterface.JoinBuilder>} + */ +export const filecoinSubmit = async ({ capability }, context) => { + const { piece, content } = capability.nb + const group = context.id.did() + + // Queue `piece/offer` invocation + const res = await context.pieceOfferQueue.add({ + piece, + content, + group, + }) + if (res.error) { + return { + error: new QueueOperationFailed(res.error.message), + } + } + + // Create effect for receipt + const fx = await AggregatorCaps.pieceOffer + .invoke({ + issuer: context.id, + audience: context.aggregatorId, + with: context.id.did(), + nb: { + piece, + group, + }, + expiration: Infinity, + }) + .delegate() + + // TODO: receipt timestamp? + /** @type {API.UcantoInterface.OkBuilder} */ + const result = Server.ok({ piece }) + return result.join(fx.link()) +} + +/** + * @param {API.Input} input + * @param {import('./api').ServiceContext} context + * @returns {Promise>} + */ +export const filecoinAccept = async ({ capability }, context) => { + const { piece } = capability.nb + const getPieceRes = await context.pieceStore.get({ piece }) + if (getPieceRes.error) { + return { error: new StoreOperationFailed(getPieceRes.error.message) } + } + + const { group } = getPieceRes.ok + const fx = await AggregatorCaps.pieceOffer + .invoke({ + issuer: context.id, + audience: context.aggregatorId, + with: context.id.did(), + nb: { + piece, + group, + }, + expiration: Infinity, + }) + .delegate() + + const dataAggregationProof = await findDataAggregationProof( + context, + fx.link() + ) + if (dataAggregationProof.error) { + return { error: new ProofNotFound(dataAggregationProof.error.message) } + } + + return { + ok: { + aux: dataAggregationProof.ok.aux, + inclusion: dataAggregationProof.ok.inclusion, + piece, + aggregate: dataAggregationProof.ok.aggregate, + }, + } +} + +/** + * Find a DataAggregationProof by following the receipt chain for a piece + * offered to the Filecoin pipeline. Starts on `piece/offer` and issued `piece/accept` receipt, + * making its way into `aggregate/offer` and `aggregate/accept` receipt for getting DealAggregationProof. + * + * @param {{ + * taskStore: API.Store + * receiptStore: API.Store + * }} stores + * @param {API.UcantoInterface.Link} task + * @returns {Promise>} + */ +async function findDataAggregationProof({ taskStore, receiptStore }, task) { + /** @type {API.InclusionProof|undefined} */ + let inclusion + /** @type {API.AggregateAcceptSuccess|undefined} */ + let aggregateAcceptReceipt + while (true) { + const [taskRes, receiptRes] = await Promise.all([ + taskStore.get(task), + receiptStore.get(task), + ]) + if (taskRes.error) { + return { + error: new StoreOperationFailed( + `failed to fetch task: ${task}: ${taskRes.error.message}` + ), + } + } + if (receiptRes.error) { + return { + error: new StoreOperationFailed( + `failed to fetch receipt for task: ${task}: ${receiptRes.error.message}` + ), + } + } + const ability = taskRes.ok.capabilities[0]?.can + if (ability === 'piece/accept' && receiptRes.ok.out.ok) { + inclusion = receiptRes.ok.out.ok.inclusion + } else if (ability === 'aggregate/accept' && receiptRes.ok.out.ok) { + aggregateAcceptReceipt = receiptRes.ok.out.ok + } + if (!receiptRes.ok.fx.join) break + task = receiptRes.ok.fx.join + } + if (!inclusion) { + return { + error: new ProofNotFound( + 'missing inclusion proof for piece in aggregate' + ), + } + } + if (!aggregateAcceptReceipt) { + return { error: new ProofNotFound('missing data aggregation proof') } + } + return { + ok: { + aux: { + dataSource: aggregateAcceptReceipt.dataSource, + dataType: aggregateAcceptReceipt.dataType, + }, + aggregate: aggregateAcceptReceipt.aggregate, + inclusion, + }, + } +} + +export const ProofNotFoundName = /** @type {const} */ ('ProofNotFound') +export class ProofNotFound extends Server.Failure { + get reason() { + return this.message + } + + get name() { + return ProofNotFoundName + } +} + +/** + * @param {import('./api').ServiceContext} context + */ +export function createService(context) { + return { + filecoin: { + offer: Server.provideAdvanced({ + capability: StorefrontCaps.filecoinOffer, + handler: (input) => filecoinOffer(input, context), + }), + submit: Server.provideAdvanced({ + capability: StorefrontCaps.filecoinSubmit, + handler: (input) => filecoinSubmit(input, context), + }), + accept: Server.provideAdvanced({ + capability: StorefrontCaps.filecoinAccept, + handler: (input) => filecoinAccept(input, context), + }), + }, + } +} + +/** + * @param {API.UcantoServerContext & import('./api').ServiceContext} context + */ +export const createServer = (context) => + Server.create({ + id: context.id, + codec: context.codec || CAR.inbound, + service: createService(context), + catch: (error) => context.errorReporter.catch(error), + }) + +/** + * @param {object} options + * @param {API.UcantoInterface.Principal} options.id + * @param {API.UcantoInterface.Transport.Channel} options.channel + * @param {API.UcantoInterface.OutboundCodec} [options.codec] + */ +export const connect = ({ id, channel, codec = CAR.outbound }) => + Client.connect({ + id, + channel, + codec, + }) diff --git a/packages/filecoin-api/src/types.ts b/packages/filecoin-api/src/types.ts index d504f8ac3..3fc13536d 100644 --- a/packages/filecoin-api/src/types.ts +++ b/packages/filecoin-api/src/types.ts @@ -7,90 +7,69 @@ import type { InferInvokedCapability, RevocationChecker, Match, + Unit, + Result, + ConnectionView, } from '@ucanto/interface' import type { ProviderInput } from '@ucanto/server' -import { PieceLink } from '@web3-storage/data-segment' -import { UnknownLink } from '@ucanto/interface' +import { InvocationConfig } from '@web3-storage/filecoin-client/types' export * as UcantoInterface from '@ucanto/interface' +export type { Result, Variant } from '@ucanto/interface' export * from '@web3-storage/filecoin-client/types' export * from '@web3-storage/capabilities/types' // Resources -export interface Queue { +export interface Queue { add: ( - record: Record, + message: Message, options?: QueueMessageOptions - ) => Promise> + ) => Promise> } -export interface Store { - put: (record: Record) => Promise> +export interface Store { /** - * Gets content data from the store. + * Puts a record in the store. */ - get: (key: any) => Promise> -} - -export interface QueueMessageOptions { - messageGroupId?: string -} - -// Services -export interface StorefrontServiceContext { - id: Signer - addQueue: Queue - pieceStore: Store -} - -export interface AggregatorServiceContext { - id: Signer - addQueue: Queue - pieceStore: Store -} - -export interface DealerServiceContext { - id: Signer - addQueue: Queue - dealStore: Store + put: (record: Rec) => Promise> + /** + * Gets a record from the store. + */ + get: (key: RecKey) => Promise> + /** + * Determine if a record already exists in the store for the given key. + */ + has: (key: RecKey) => Promise> } -// Service Types - -export interface StorefrontRecord { - piece: PieceLink - content: UnknownLink - insertedAt: number +export interface UpdatableStore extends Store { + /** + * Updates a record from the store. + */ + update: ( + key: RecKey, + record: Partial + ) => Promise> } -export interface AggregatorMessageRecord { - piece: PieceLink - storefront: string - group: string - insertedAt: number +export interface QueryableStore extends Store { + /** + * Queries for record matching a given criterium. + */ + query: (search: Query) => Promise> } -export interface AggregatorRecord { - piece: PieceLink - storefront: string - group: string - insertedAt: number -} +export interface UpdatableAndQueryableStore + extends UpdatableStore, + QueryableStore {} -export interface DealerMessageRecord { - aggregate: PieceLink - pieces: PieceLink[] - storefront: string - label?: string - insertedAt: number +export interface QueueMessageOptions { + messageGroupId?: string } -export interface DealerRecord { - aggregate: PieceLink - storefront: string - offer: string - stat: number - insertedAt: number +export interface ServiceConfig> { + connection: ConnectionView + invocationConfig: InvocationConfig } // Errors @@ -99,11 +78,8 @@ export type StorePutError = StoreOperationError | EncodeRecordFailed export type StoreGetError = | StoreOperationError | EncodeRecordFailed - | StoreNotFound -export type QueueAddError = - | QueueOperationError - | EncodeRecordFailed - | StorePutError + | RecordNotFound +export type QueueAddError = QueueOperationError | EncodeRecordFailed export interface QueueOperationError extends Error { name: 'QueueOperationFailed' @@ -113,8 +89,8 @@ export interface StoreOperationError extends Error { name: 'StoreOperationFailed' } -export interface StoreNotFound extends Error { - name: 'StoreNotFound' +export interface RecordNotFound extends Error { + name: 'RecordNotFound' } export interface EncodeRecordFailed extends Error { @@ -133,55 +109,10 @@ export interface ErrorReporter { catch: (error: HandlerExecutionError) => void } -export type Result = Variant<{ - ok: T - error: X -}> - -/** - * Utility type for defining a [keyed union] type as in IPLD Schema. In practice - * this just works around typescript limitation that requires discriminant field - * on all variants. - * - * ```ts - * type Result = - * | { ok: T } - * | { error: X } - * - * const demo = (result: Result) => { - * if (result.ok) { - * // ^^^^^^^^^ Property 'ok' does not exist on type '{ error: Error; }` - * } - * } - * ``` - * - * Using `Variant` type we can define same union type that works as expected: - * - * ```ts - * type Result = Variant<{ - * ok: T - * error: X - * }> - * - * const demo = (result: Result) => { - * if (result.ok) { - * result.ok.toUpperCase() - * } - * } - * ``` - * - * [keyed union]:https://ipld.io/docs/schemas/features/representation-strategies/#union-keyed-representation - */ -export type Variant> = { - [Key in keyof U]: { [K in Exclude]?: never } & { - [K in Key]: U[Key] - } -}[keyof U] - // test export interface UcantoServerContextTest extends UcantoServerContext { - queuedMessages: unknown[] + queuedMessages: Map } export type Test = ( diff --git a/packages/filecoin-api/test/aggregator.spec.js b/packages/filecoin-api/test/aggregator.spec.js index f01c14c23..00db93a44 100644 --- a/packages/filecoin-api/test/aggregator.spec.js +++ b/packages/filecoin-api/test/aggregator.spec.js @@ -1,57 +1,199 @@ /* eslint-disable no-only-tests/no-only-tests */ import * as assert from 'assert' -import * as Aggregator from './services/aggregator.js' import * as Signer from '@ucanto/principal/ed25519' -import { Store } from './context/store.js' +import * as AggregatorService from './services/aggregator.js' +import * as AggregatorEvents from './events/aggregator.js' + +import { getStoreImplementations } from './context/store-implementations.js' import { Queue } from './context/queue.js' -import { validateAuthorization } from './helpers/utils.js' - -describe('aggregate/*', () => { - for (const [name, test] of Object.entries(Aggregator.test)) { - const define = name.startsWith('only ') - ? it.only - : name.startsWith('skip ') - ? it.skip - : it - - define(name, async () => { - const signer = await Signer.generate() - const id = signer.withDID('did:web:test.aggregator.web3.storage') - - // resources - /** @type {unknown[]} */ - const queuedMessages = [] - const addQueue = new Queue({ - onMessage: (message) => queuedMessages.push(message), - }) - const pieceLookupFn = ( - /** @type {Iterable | ArrayLike} */ items, - /** @type {any} */ record - ) => { - return Array.from(items).find((i) => i.piece.equals(record.piece)) - } - const pieceStore = new Store(pieceLookupFn) - - await test( - { - equal: assert.strictEqual, - deepEqual: assert.deepStrictEqual, - ok: assert.ok, - }, - { - id, - errorReporter: { - catch(error) { - assert.fail(error) +import { getMockService, getConnection } from './context/service.js' + +describe('Aggregator', () => { + describe('piece/*', () => { + for (const [name, test] of Object.entries(AggregatorService.test)) { + const define = name.startsWith('only ') + ? it.only + : name.startsWith('skip ') + ? it.skip + : it + + define(name, async () => { + const aggregatorSigner = await Signer.generate() + const dealerSigner = await Signer.generate() + + // resources + /** @type {Map} */ + const queuedMessages = new Map() + const { + pieceQueue, + bufferQueue, + pieceAcceptQueue, + aggregateOfferQueue, + } = getQueues(queuedMessages) + const { + aggregator: { + pieceStore, + bufferStore, + aggregateStore, + inclusionStore, + }, + } = getStoreImplementations() + + await test( + { + equal: assert.strictEqual, + deepEqual: assert.deepStrictEqual, + ok: assert.ok, + }, + { + id: aggregatorSigner, + dealerId: dealerSigner, + errorReporter: { + catch(error) { + assert.fail(error) + }, }, + pieceStore, + bufferStore, + aggregateStore, + inclusionStore, + pieceQueue, + bufferQueue, + pieceAcceptQueue, + aggregateOfferQueue, + queuedMessages, + } + ) + }) + } + }) + + describe('events', () => { + for (const [name, test] of Object.entries(AggregatorEvents.test)) { + const define = name.startsWith('only ') + ? it.only + : name.startsWith('skip ') + ? it.skip + : it + + define(name, async () => { + const aggregatorSigner = await Signer.generate() + const dealerSigner = await Signer.generate() + + const service = getMockService() + const aggregatorConnection = getConnection( + aggregatorSigner, + service + ).connection + const dealerConnection = getConnection(dealerSigner, service).connection + + // resources + /** @type {Map} */ + const queuedMessages = new Map() + const { bufferQueue, pieceAcceptQueue, aggregateOfferQueue } = + getQueues(queuedMessages) + const { + aggregator: { + pieceStore, + bufferStore, + aggregateStore, + inclusionStore, }, - addQueue, - pieceStore, - queuedMessages, - validateAuthorization, - } - ) - }) - } + } = getStoreImplementations() + + await test( + { + equal: assert.strictEqual, + deepEqual: assert.deepStrictEqual, + ok: assert.ok, + }, + { + id: aggregatorSigner, + pieceStore, + bufferStore, + aggregateStore, + inclusionStore, + bufferQueue, + pieceAcceptQueue, + aggregateOfferQueue, + dealerService: { + connection: dealerConnection, + invocationConfig: { + issuer: aggregatorSigner, + with: aggregatorSigner.did(), + audience: dealerSigner, + }, + }, + aggregatorService: { + connection: aggregatorConnection, + invocationConfig: { + issuer: aggregatorSigner, + with: aggregatorSigner.did(), + audience: aggregatorSigner, + }, + }, + queuedMessages, + service, + errorReporter: { + catch(error) { + assert.fail(error) + }, + }, + config: { + maxAggregateSize: 2 ** 35, + minAggregateSize: 2 ** 34, + minUtilizationFactor: 4, + }, + } + ) + }) + } + }) }) + +/** + * @param {Map} queuedMessages + */ +function getQueues(queuedMessages) { + queuedMessages.set('filecoinSubmitQueue', []) + queuedMessages.set('pieceQueue', []) + queuedMessages.set('bufferQueue', []) + queuedMessages.set('pieceAcceptQueue', []) + queuedMessages.set('aggregateOfferQueue', []) + const pieceQueue = new Queue({ + onMessage: (message) => { + const messages = queuedMessages.get('pieceQueue') || [] + messages.push(message) + queuedMessages.set('pieceQueue', messages) + }, + }) + const bufferQueue = new Queue({ + onMessage: (message) => { + const messages = queuedMessages.get('bufferQueue') || [] + messages.push(message) + queuedMessages.set('bufferQueue', messages) + }, + }) + const pieceAcceptQueue = new Queue({ + onMessage: (message) => { + const messages = queuedMessages.get('pieceAcceptQueue') || [] + messages.push(message) + queuedMessages.set('pieceAcceptQueue', messages) + }, + }) + const aggregateOfferQueue = new Queue({ + onMessage: (message) => { + const messages = queuedMessages.get('aggregateOfferQueue') || [] + messages.push(message) + queuedMessages.set('aggregateOfferQueue', messages) + }, + }) + + return { + pieceQueue, + bufferQueue, + pieceAcceptQueue, + aggregateOfferQueue, + } +} diff --git a/packages/filecoin-api/test/context/mocks.js b/packages/filecoin-api/test/context/mocks.js new file mode 100644 index 000000000..fc17443eb --- /dev/null +++ b/packages/filecoin-api/test/context/mocks.js @@ -0,0 +1,54 @@ +import * as Server from '@ucanto/server' + +const notImplemented = () => { + throw new Server.Failure('not implemented') +} + +/** + * @param {Partial<{ + * filecoin: Partial + * piece: Partial + * aggregate: Partial + * deal: Partial + * }>} impl + */ +export function mockService(impl) { + return { + filecoin: { + offer: withCallParams(impl.filecoin?.offer ?? notImplemented), + submit: withCallParams(impl.filecoin?.submit ?? notImplemented), + accept: withCallParams(impl.filecoin?.accept ?? notImplemented), + }, + piece: { + offer: withCallParams(impl.piece?.offer ?? notImplemented), + accept: withCallParams(impl.piece?.accept ?? notImplemented), + }, + aggregate: { + offer: withCallParams(impl.aggregate?.offer ?? notImplemented), + accept: withCallParams(impl.aggregate?.accept ?? notImplemented), + }, + deal: { + info: withCallParams(impl.deal?.info ?? notImplemented), + }, + } +} + +/** + * @template {Function} T + * @param {T} fn + */ +function withCallParams(fn) { + /** @param {T extends (...args: infer A) => any ? A : never} args */ + const annotatedParamsFn = (...args) => { + // @ts-expect-error not typed param + annotatedParamsFn._params.push(args[0].capabilities[0]) + annotatedParamsFn.called = true + annotatedParamsFn.callCount++ + return fn(...args) + } + /** @type {any[]} */ + annotatedParamsFn._params = [] + annotatedParamsFn.called = false + annotatedParamsFn.callCount = 0 + return annotatedParamsFn +} diff --git a/packages/filecoin-api/test/context/queue.js b/packages/filecoin-api/test/context/queue.js index 54139a87b..3d0605247 100644 --- a/packages/filecoin-api/test/context/queue.js +++ b/packages/filecoin-api/test/context/queue.js @@ -1,5 +1,7 @@ import * as API from '../../src/types.js' +import { QueueOperationFailed } from '../../src/errors.js' + /** * @template T * @implements {API.Queue} @@ -28,3 +30,18 @@ export class Queue { }) } } + +/** + * @template T + * @implements {API.Queue} + */ +export class FailingQueue { + /** + * @param {T} record + */ + async add(record) { + return Promise.resolve({ + error: new QueueOperationFailed('failed to add to queue'), + }) + } +} diff --git a/packages/filecoin-api/test/context/receipts.js b/packages/filecoin-api/test/context/receipts.js new file mode 100644 index 000000000..6f5f859ed --- /dev/null +++ b/packages/filecoin-api/test/context/receipts.js @@ -0,0 +1,145 @@ +import { Receipt } from '@ucanto/core' +import * as AggregatorCaps from '@web3-storage/capabilities/filecoin/aggregator' +import * as DealerCaps from '@web3-storage/capabilities/filecoin/dealer' + +import * as API from '../../src/types.js' + +/** + * @param {object} context + * @param {import('@ucanto/interface').Signer} context.storefront + * @param {import('@ucanto/interface').Signer} context.aggregator + * @param {import('@ucanto/interface').Signer} context.dealer + * @param {API.PieceLink} context.aggregate + * @param {string} context.group + * @param {API.PieceLink} context.piece + * @param {import('@ucanto/interface').Block} context.piecesBlock + * @param {API.InclusionProof} context.inclusionProof + * @param {API.AggregateAcceptSuccess} context.aggregateAcceptStatus + */ +export async function createInvocationsAndReceiptsForDealDataProofChain({ + storefront, + aggregator, + dealer, + aggregate, + group, + piece, + piecesBlock, + inclusionProof, + aggregateAcceptStatus, +}) { + const pieceOfferInvocation = await AggregatorCaps.pieceOffer + .invoke({ + issuer: storefront, + audience: aggregator, + with: storefront.did(), + nb: { + piece, + group, + }, + expiration: Infinity, + }) + .delegate() + const pieceAcceptInvocation = await AggregatorCaps.pieceAccept + .invoke({ + issuer: aggregator, + audience: aggregator, + with: aggregator.did(), + nb: { + piece, + group, + }, + expiration: Infinity, + }) + .delegate() + const aggregateOfferInvocation = await DealerCaps.aggregateOffer + .invoke({ + issuer: aggregator, + audience: dealer, + with: aggregator.did(), + nb: { + pieces: piecesBlock.cid, + aggregate, + }, + expiration: Infinity, + }) + .delegate() + aggregateOfferInvocation.attach(piecesBlock) + const aggregateAcceptInvocation = await DealerCaps.aggregateAccept + .invoke({ + issuer: aggregator, + audience: dealer, + with: aggregator.did(), + nb: { + pieces: piecesBlock.cid, + aggregate, + }, + expiration: Infinity, + }) + .delegate() + const pieceOfferReceipt = await Receipt.issue({ + issuer: aggregator, + ran: pieceOfferInvocation.cid, + result: { + ok: /** @type {API.PieceOfferSuccess} */ ({ + piece, + }), + }, + fx: { + join: pieceAcceptInvocation.cid, + fork: [], + }, + }) + + const pieceAcceptReceipt = await Receipt.issue({ + issuer: aggregator, + ran: pieceAcceptInvocation.cid, + result: { + ok: /** @type {API.PieceAcceptSuccess} */ ({ + piece, + aggregate, + inclusion: inclusionProof, + }), + }, + fx: { + join: aggregateOfferInvocation.cid, + fork: [], + }, + }) + + const aggregateOfferReceipt = await Receipt.issue({ + issuer: aggregator, + ran: aggregateOfferInvocation.cid, + result: { + ok: /** @type {API.AggregateOfferSuccess} */ ({ + aggregate, + }), + }, + fx: { + join: aggregateAcceptInvocation.cid, + fork: [], + }, + }) + + const aggregateAcceptReceipt = await Receipt.issue({ + issuer: dealer, + ran: aggregateAcceptInvocation.cid, + result: { + ok: /** @type {API.AggregateAcceptSuccess} */ (aggregateAcceptStatus), + }, + }) + + return { + invocations: { + pieceOfferInvocation, + pieceAcceptInvocation, + aggregateOfferInvocation, + aggregateAcceptInvocation, + }, + receipts: { + pieceOfferReceipt, + pieceAcceptReceipt, + aggregateOfferReceipt, + aggregateAcceptReceipt, + }, + } +} diff --git a/packages/filecoin-api/test/context/service.js b/packages/filecoin-api/test/context/service.js new file mode 100644 index 000000000..2a61e0860 --- /dev/null +++ b/packages/filecoin-api/test/context/service.js @@ -0,0 +1,231 @@ +import * as Client from '@ucanto/client' +import * as Server from '@ucanto/server' +import * as CAR from '@ucanto/transport/car' + +import * as StorefrontCaps from '@web3-storage/capabilities/filecoin/storefront' +import * as AggregatorCaps from '@web3-storage/capabilities/filecoin/aggregator' +import * as DealerCaps from '@web3-storage/capabilities/filecoin/dealer' +import * as DealTrackerCaps from '@web3-storage/capabilities/filecoin/deal-tracker' + +// eslint-disable-next-line no-unused-vars +import * as API from '../../src/types.js' + +import { mockService } from './mocks.js' + +export function getMockService() { + return mockService({ + aggregate: { + offer: Server.provideAdvanced({ + capability: DealerCaps.aggregateOffer, + handler: async ({ invocation, context }) => { + const invCap = invocation.capabilities[0] + if (!invCap.nb?.pieces || !invCap.nb.aggregate) { + throw new Error() + } + const fx = await DealerCaps.aggregateAccept + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + aggregate: invCap.nb.aggregate, + pieces: invCap.nb?.pieces, + }, + expiration: Infinity, + }) + .delegate() + + return Server.ok({ aggregate: invCap.nb.aggregate }).join(fx.link()) + }, + }), + accept: Server.provideAdvanced({ + capability: DealerCaps.aggregateAccept, + handler: async ({ invocation, context }) => { + const invCap = invocation.capabilities[0] + if (!invCap.nb?.aggregate) { + throw new Error() + } + + return Server.ok({ + aggregate: invCap.nb.aggregate, + dataSource: { + dealID: 15151n, + }, + dataType: 1n, + }) + }, + }), + }, + piece: { + offer: Server.provideAdvanced({ + capability: AggregatorCaps.pieceOffer, + handler: async ({ invocation, context }) => { + const invCap = invocation.capabilities[0] + if (!invCap.nb?.piece) { + throw new Error() + } + + // Create effect for receipt with self signed queued operation + const fx = await AggregatorCaps.pieceAccept + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + ...invCap.nb, + }, + }) + .delegate() + + return Server.ok({ + piece: invCap.nb?.piece, + }).join(fx.link()) + }, + }), + accept: Server.provideAdvanced({ + capability: AggregatorCaps.pieceAccept, + // @ts-expect-error inclusion types + handler: async ({ invocation, context }) => { + const invCap = invocation.capabilities[0] + if (!invCap.nb?.piece) { + throw new Error() + } + + // Create effect for receipt + const fx = await DealerCaps.aggregateOffer + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + aggregate: invCap.nb.piece, + pieces: invCap.nb.piece, + }, + }) + .delegate() + + return Server.ok({ + piece: invCap.nb.piece, + aggregate: invCap.nb.piece, + inclusion: { + subtree: + /** @type {import('@web3-storage/data-segment').ProofData} */ [ + 0n, + /** @type {import('@web3-storage/data-segment').MerkleTreePath} */ ([]), + ], + index: + /** @type {import('@web3-storage/data-segment').ProofData} */ [ + 0n, + /** @type {import('@web3-storage/data-segment').MerkleTreePath} */ ([]), + ], + }, + }).join(fx.link()) + }, + }), + }, + filecoin: { + submit: Server.provideAdvanced({ + capability: StorefrontCaps.filecoinSubmit, + handler: async ({ invocation, context }) => { + const invCap = invocation.capabilities[0] + if (!invCap.nb?.piece) { + throw new Error() + } + + // Create effect for receipt with self signed queued operation + const fx = await AggregatorCaps.pieceOffer + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + ...invCap.nb, + group: context.id.did(), + }, + }) + .delegate() + + return Server.ok({ + piece: invCap.nb?.piece, + }).join(fx.link()) + }, + }), + accept: Server.provideAdvanced({ + capability: StorefrontCaps.filecoinAccept, + handler: async ({ invocation, context }) => { + const invCap = invocation.capabilities[0] + if (!invCap.nb?.piece) { + throw new Error() + } + + /** @type {API.UcantoInterface.OkBuilder} */ + const result = Server.ok({ + piece: invCap.nb.piece, + aggregate: invCap.nb.piece, + inclusion: { + subtree: + /** @type {import('@web3-storage/data-segment').ProofData} */ [ + 0n, + /** @type {import('@web3-storage/data-segment').MerkleTreePath} */ ([]), + ], + index: + /** @type {import('@web3-storage/data-segment').ProofData} */ [ + 0n, + /** @type {import('@web3-storage/data-segment').MerkleTreePath} */ ([]), + ], + }, + aux: { + dataType: 0n, + dataSource: { + dealID: 1138n, + }, + }, + }) + + return result + }, + }), + }, + deal: { + info: Server.provideAdvanced({ + capability: DealTrackerCaps.dealInfo, + handler: async ({ invocation, context }) => { + const invCap = invocation.capabilities[0] + if (!invCap.nb?.piece) { + throw new Error() + } + + /** @type {API.UcantoInterface.OkBuilder} */ + const result = Server.ok({ + deals: { + 111: { + provider: 'f11111', + }, + }, + }) + + return result + }, + }), + }, + }) +} + +/** + * @param {any} service + * @param {any} id + */ +export function getConnection(id, service) { + const server = Server.create({ + id: id, + service, + codec: CAR.inbound, + }) + const connection = Client.connect({ + id: id, + codec: CAR.outbound, + channel: server, + }) + + return { connection } +} diff --git a/packages/filecoin-api/test/context/store-implementations.js b/packages/filecoin-api/test/context/store-implementations.js new file mode 100644 index 000000000..23c4ac000 --- /dev/null +++ b/packages/filecoin-api/test/context/store-implementations.js @@ -0,0 +1,242 @@ +import { UpdatableStore } from './store.js' + +/** + * @typedef {import('@ucanto/interface').Link} Link + * @typedef {import('../../src/storefront/api.js').PieceRecord} PieceRecord + * @typedef {import('../../src/storefront/api.js').PieceRecordKey} PieceRecordKey + * @typedef {import('../../src/aggregator/api.js').PieceRecord} AggregatorPieceRecord + * @typedef {import('../../src/aggregator/api.js').PieceRecordKey} AggregatorPieceRecordKey + * @typedef {import('../../src/aggregator/api.js').BufferRecord} BufferRecord + * @typedef {import('../../src/aggregator/api.js').AggregateRecord} AggregateRecord + * @typedef {import('../../src/aggregator/api.js').AggregateRecordKey} AggregateRecordKey + * @typedef {import('../../src/aggregator/api.js').InclusionRecord} InclusionRecord + * @typedef {import('../../src/aggregator/api.js').InclusionRecordKey} InclusionRecordKey + * @typedef {import('../../src/dealer/api.js').AggregateRecord} DealerAggregateRecord + * @typedef {import('../../src/dealer/api.js').AggregateRecordKey} DealerAggregateRecordKey + * @typedef {import('../../src/dealer/api.js').OfferDocument} OfferDocument + * @typedef {import('../../src/deal-tracker/api.js').DealRecord} DealRecord + * @typedef {import('../../src/deal-tracker/api.js').DealRecordKey} DealRecordKey + */ +export const getStoreImplementations = ( + StoreImplementation = UpdatableStore +) => ({ + storefront: { + pieceStore: new StoreImplementation({ + getFn: ( + /** @type {Set} */ items, + /** @type {PieceRecordKey} */ record + ) => { + return Array.from(items).find((i) => i?.piece.equals(record.piece)) + }, + queryFn: ( + /** @type {Set} */ items, + /** @type {Partial} */ search + ) => { + const filteredItems = Array.from(items).filter((i) => { + if (i.status === search.status) { + return true + } + return true + }) + return filteredItems + }, + updateFn: ( + /** @type {Set} */ items, + /** @type {PieceRecordKey} */ key, + /** @type {Partial} */ item + ) => { + const itemToUpdate = Array.from(items).find((i) => + i?.piece.equals(key.piece) + ) + if (!itemToUpdate) { + throw new Error('not found') + } + const updatedItem = { + ...itemToUpdate, + ...item, + } + items.delete(itemToUpdate) + items.add(updatedItem) + return updatedItem + }, + }), + taskStore: new StoreImplementation({ + getFn: ( + /** @type {Set} */ items, + /** @type {import('@ucanto/interface').UnknownLink} */ record + ) => { + return Array.from(items).find((i) => i.cid.equals(record)) + }, + }), + receiptStore: new StoreImplementation({ + getFn: ( + /** @type {Set} */ items, + /** @type {import('@ucanto/interface').UnknownLink} */ record + ) => { + return Array.from(items).find((i) => i.ran.link().equals(record)) + }, + }), + }, + aggregator: { + pieceStore: new StoreImplementation({ + getFn: ( + /** @type {Set} */ items, + /** @type {AggregatorPieceRecordKey} */ record + ) => { + return Array.from(items).find((i) => i?.piece.equals(record.piece)) + }, + updateFn: ( + /** @type {Set} */ items, + /** @type {AggregatorPieceRecordKey} */ key, + /** @type {Partial} */ item + ) => { + const itemToUpdate = Array.from(items).find( + (i) => i?.piece.equals(key.piece) && i.group === key.group + ) + if (!itemToUpdate) { + throw new Error('not found') + } + const updatedItem = { + ...itemToUpdate, + ...item, + } + items.delete(itemToUpdate) + items.add(updatedItem) + return updatedItem + }, + }), + bufferStore: new StoreImplementation({ + getFn: ( + /** @type {Set} */ items, + /** @type {Link} */ record + ) => { + // Return first item + return Array.from(items).find((i) => i.block.equals(record)) + }, + }), + aggregateStore: new StoreImplementation({ + getFn: ( + /** @type {Set} */ items, + /** @type {AggregateRecordKey} */ record + ) => { + return Array.from(items).find((i) => + i?.aggregate.equals(record.aggregate) + ) + }, + }), + inclusionStore: new StoreImplementation({ + getFn: ( + /** @type {Set} */ items, + /** @type {InclusionRecordKey} */ record + ) => { + return Array.from(items).find( + (i) => + i?.aggregate.equals(record.aggregate) && + i?.piece.equals(record.piece) + ) + }, + queryFn: ( + /** @type {Set} */ items, + /** @type {Partial} */ search + ) => { + const filteredItems = Array.from(items).filter((i) => { + if (search.piece && !i.piece.equals(search.piece)) { + return false + } else if ( + search.aggregate && + !i.aggregate.equals(search.aggregate) + ) { + return false + } else if (search.group && i.group !== search.group) { + return false + } + return true + }) + return filteredItems + }, + }), + }, + dealer: { + aggregateStore: new StoreImplementation({ + getFn: ( + /** @type {Set} */ items, + /** @type {DealerAggregateRecordKey} */ record + ) => { + return Array.from(items).find((i) => + i?.aggregate.equals(record.aggregate) + ) + }, + queryFn: ( + /** @type {Set} */ items, + /** @type {Partial} */ search + ) => { + return Array.from(items).filter((i) => i.status === search.status) + }, + updateFn: ( + /** @type {Set} */ items, + /** @type {DealerAggregateRecordKey} */ key, + /** @type {Partial} */ item + ) => { + const itemToUpdate = Array.from(items).find( + (i) => i.aggregate.equals(key.aggregate) && i.deal === key.deal + ) + if (!itemToUpdate) { + throw new Error('not found') + } + const updatedItem = { + ...itemToUpdate, + ...item, + } + items.delete(itemToUpdate) + items.add(updatedItem) + return updatedItem + }, + }), + offerStore: new StoreImplementation({ + getFn: ( + /** @type {Set} */ items, + /** @type {string} */ record + ) => { + return Array.from(items).find((i) => i.key === record) + }, + updateFn: ( + /** @type {Set} */ items, + /** @type {string} */ key, + /** @type {Partial} */ item + ) => { + const lastItem = Array.from(items).pop() + if (!lastItem) { + throw new Error('not found') + } + + const nItem = { + ...lastItem, + ...item, + } + + items.delete(lastItem) + items.add(nItem) + + return nItem + }, + }), + }, + dealTracker: { + dealStore: new StoreImplementation({ + getFn: ( + /** @type {Set} */ items, + /** @type {DealRecordKey} */ record + ) => { + return Array.from(items).find( + (i) => i?.piece.equals(record.piece) && i.dealId === record.dealId + ) + }, + queryFn: ( + /** @type {Set} */ items, + /** @type {Partial} */ search + ) => { + return Array.from(items).filter((i) => i.piece.equals(search.piece)) + }, + }), + }, +}) diff --git a/packages/filecoin-api/test/context/store.js b/packages/filecoin-api/test/context/store.js index 795ca050b..9bfc3999d 100644 --- a/packages/filecoin-api/test/context/store.js +++ b/packages/filecoin-api/test/context/store.js @@ -1,22 +1,31 @@ import * as API from '../../src/types.js' +import { RecordNotFound, StoreOperationFailed } from '../../src/errors.js' /** - * @template T - * @implements {API.Store} + * @typedef {import('../../src/types.js').StorePutError} StorePutError + * @typedef {import('../../src/types.js').StoreGetError} StoreGetError + */ + +/** + * @template K + * @template V + * @implements {API.Store} */ export class Store { /** - * @param {(items: Set, item: any) => T} lookupFn + * @param {import('./types.js').StoreOptions} options */ - constructor(lookupFn) { - /** @type {Set} */ + constructor(options) { + /** @type {Set} */ this.items = new Set() - this.lookupFn = lookupFn + this.getFn = options.getFn + this.queryFn = options.queryFn } /** - * @param {T} record + * @param {V} record + * @returns {Promise>} */ async put(record) { this.items.add(record) @@ -28,12 +37,154 @@ export class Store { /** * - * @param {any} item - * @returns boolean + * @param {K} item + * @returns {Promise>} */ async get(item) { + if (!this.getFn) { + throw new Error('get not supported') + } + const t = this.getFn(this.items, item) + if (!t) { + return { + error: new RecordNotFound(), + } + } + return { + ok: t, + } + } + + /** + * @param {K} item + * @returns {Promise>} + */ + async has(item) { + if (!this.getFn) { + throw new Error('has not supported') + } + const t = this.getFn(this.items, item) + if (!t) { + return { + error: new RecordNotFound(), + } + } + return { + ok: Boolean(t), + } + } + + /** + * @param {Partial} search + * @returns {Promise>} + */ + async query(search) { + if (!this.queryFn) { + throw new Error('query not supported') + } + const t = this.queryFn(this.items, search) + if (!t) { + return { + error: new RecordNotFound(), + } + } + return { + ok: t, + } + } +} + +/** + * @template K + * @template V + * @implements {API.UpdatableStore} + * @extends {Store} + */ +export class UpdatableStore extends Store { + /** + * @param {import('./types.js').UpdatableStoreOptions} options + */ + constructor(options) { + super(options) + + this.updateFn = options.updateFn + } + + /** + * @param {K} key + * @param {Partial} item + * @returns {Promise>} + */ + async update(key, item) { + if (!this.updateFn) { + throw new Error('query not supported') + } + + const t = this.updateFn(this.items, key, item) + if (!t) { + return { + error: new RecordNotFound(), + } + } + return { + ok: t, + } + } +} + +/** + * @template K + * @template V + * @extends {UpdatableStore} + */ +export class FailingStore extends UpdatableStore { + /** + * @param {V} record + */ + async put(record) { + return { + error: new StoreOperationFailed('failed to put to store'), + } + } + + /** + * @param {K} item + * @returns {Promise>} + */ + async get(item) { + return { + error: new StoreOperationFailed('failed to get from store'), + } + } + + /** + * @param {K} item + * @returns {Promise>} + */ + async has(item) { + return { + error: new StoreOperationFailed('failed to check store'), + } + } + + /** + * @param {Partial} search + * @returns {Promise>} + */ + async query(search) { + return { + error: new StoreOperationFailed('failed to query store'), + } + } + + /** + * @param {K} key + * @param {Partial} item + * @returns {Promise>} + */ + async update(key, item) { return { - ok: this.lookupFn(this.items, item), + error: new StoreOperationFailed('failed to update store'), } } } diff --git a/packages/filecoin-api/test/context/types.ts b/packages/filecoin-api/test/context/types.ts new file mode 100644 index 000000000..1c183c3a6 --- /dev/null +++ b/packages/filecoin-api/test/context/types.ts @@ -0,0 +1,8 @@ +export interface StoreOptions { + getFn?: (items: Set, item: K) => V | undefined + queryFn?: (items: Set, item: Partial) => V[] +} + +export interface UpdatableStoreOptions extends StoreOptions { + updateFn?: (items: Set, key: K, item: Partial) => V +} diff --git a/packages/filecoin-api/test/deal-tracker.spec.js b/packages/filecoin-api/test/deal-tracker.spec.js new file mode 100644 index 000000000..57748c70d --- /dev/null +++ b/packages/filecoin-api/test/deal-tracker.spec.js @@ -0,0 +1,51 @@ +/* eslint-disable no-only-tests/no-only-tests */ +import * as assert from 'assert' +import * as Signer from '@ucanto/principal/ed25519' + +import * as DealTrackerService from './services/deal-tracker.js' + +import { getStoreImplementations } from './context/store-implementations.js' + +/** + * @typedef {import('../src/deal-tracker/api.js').DealRecord} DealRecord + * @typedef {import('../src/deal-tracker/api.js').DealRecordKey} DealRecordKey + */ + +describe('deal-tracker', () => { + describe('deal/*', () => { + for (const [name, test] of Object.entries(DealTrackerService.test)) { + const define = name.startsWith('only ') + ? it.only + : name.startsWith('skip ') + ? it.skip + : it + + define(name, async () => { + const dealTrackerSigner = await Signer.generate() + + // resources + const { + dealTracker: { dealStore }, + } = getStoreImplementations() + + await test( + { + equal: assert.strictEqual, + deepEqual: assert.deepStrictEqual, + ok: assert.ok, + }, + { + id: dealTrackerSigner, + dealStore, + queuedMessages: new Map(), + errorReporter: { + catch(error) { + assert.fail(error) + }, + }, + } + ) + }) + } + }) +}) diff --git a/packages/filecoin-api/test/dealer.spec.js b/packages/filecoin-api/test/dealer.spec.js index 507f9e243..66f2dc913 100644 --- a/packages/filecoin-api/test/dealer.spec.js +++ b/packages/filecoin-api/test/dealer.spec.js @@ -1,59 +1,115 @@ /* eslint-disable no-only-tests/no-only-tests */ import * as assert from 'assert' -import * as Broker from './services/dealer.js' import * as Signer from '@ucanto/principal/ed25519' -import { Store } from './context/store.js' -import { Queue } from './context/queue.js' -import { validateAuthorization } from './helpers/utils.js' - -describe('deal/*', () => { - for (const [name, test] of Object.entries(Broker.test)) { - const define = name.startsWith('only ') - ? it.only - : name.startsWith('skip ') - ? it.skip - : it - - define(name, async () => { - const signer = await Signer.generate() - const id = signer.withDID('did:web:test.spade-proxy.web3.storage') - - // resources - /** @type {unknown[]} */ - const queuedMessages = [] - const addQueue = new Queue({ - onMessage: (message) => queuedMessages.push(message), - }) - const dealLookupFn = ( - /** @type {Iterable | ArrayLike} */ items, - /** @type {any} */ record - ) => { - return Array.from(items).find((i) => - i.aggregate.equals(record.aggregate) - ) - } - const dealStore = new Store(dealLookupFn) - - await test( - { - equal: assert.strictEqual, - deepEqual: assert.deepStrictEqual, - ok: assert.ok, - }, - { - id, - errorReporter: { - catch(error) { - assert.fail(error) +import * as DealerService from './services/dealer.js' +import * as DealerEvents from './events/dealer.js' + +import { getStoreImplementations } from './context/store-implementations.js' +import { getMockService, getConnection } from './context/service.js' + +describe('Dealer', () => { + describe('aggregate/*', () => { + for (const [name, test] of Object.entries(DealerService.test)) { + const define = name.startsWith('only ') + ? it.only + : name.startsWith('skip ') + ? it.skip + : it + + define(name, async () => { + const dealerSigner = await Signer.generate() + + // resources + /** @type {Map} */ + const queuedMessages = new Map() + const { + dealer: { aggregateStore, offerStore }, + } = getStoreImplementations() + + await test( + { + equal: assert.strictEqual, + deepEqual: assert.deepStrictEqual, + ok: assert.ok, + }, + { + id: dealerSigner, + errorReporter: { + catch(error) { + assert.fail(error) + }, }, + aggregateStore, + offerStore, + queuedMessages, + } + ) + }) + } + }) + + describe('events', () => { + for (const [name, test] of Object.entries(DealerEvents.test)) { + const define = name.startsWith('only ') + ? it.only + : name.startsWith('skip ') + ? it.skip + : it + + define(name, async () => { + const dealerSigner = await Signer.generate() + const dealTrackerSigner = await Signer.generate() + const service = getMockService() + const dealerConnection = getConnection(dealerSigner, service).connection + const dealTrackerConnection = getConnection( + dealTrackerSigner, + service + ).connection + + // resources + /** @type {Map} */ + const queuedMessages = new Map() + const { + dealer: { aggregateStore, offerStore }, + } = getStoreImplementations() + + await test( + { + equal: assert.strictEqual, + deepEqual: assert.deepStrictEqual, + ok: assert.ok, }, - addQueue, - dealStore, - queuedMessages, - validateAuthorization, - } - ) - }) - } + { + id: dealerSigner, + errorReporter: { + catch(error) { + assert.fail(error) + }, + }, + aggregateStore, + offerStore, + queuedMessages, + dealerService: { + connection: dealerConnection, + invocationConfig: { + issuer: dealerSigner, + with: dealerSigner.did(), + audience: dealerSigner, + }, + }, + dealTrackerService: { + connection: dealTrackerConnection, + invocationConfig: { + issuer: dealerSigner, + with: dealerSigner.did(), + audience: dealTrackerSigner, + }, + }, + service, + } + ) + }) + } + }) }) diff --git a/packages/filecoin-api/test/events/aggregator.js b/packages/filecoin-api/test/events/aggregator.js new file mode 100644 index 000000000..3810b2bcb --- /dev/null +++ b/packages/filecoin-api/test/events/aggregator.js @@ -0,0 +1,1238 @@ +import * as Server from '@ucanto/server' +import * as AggregatorCaps from '@web3-storage/capabilities/filecoin/aggregator' +import pWaitFor from 'p-wait-for' +import { CBOR } from '@ucanto/core' + +import * as API from '../../src/types.js' +import * as TestAPI from '../types.js' +import * as AggregatorEvents from '../../src/aggregator/events.js' + +import { FailingStore } from '../context/store.js' +import { FailingQueue } from '../context/queue.js' +import { mockService } from '../context/mocks.js' +import { getConnection } from '../context/service.js' +import { getStoreImplementations } from '../context/store-implementations.js' +import { randomAggregate, randomCargo } from '../utils.js' +import { + QueueOperationErrorName, + RecordNotFoundErrorName, + StoreOperationErrorName, +} from '../../src/errors.js' + +/** + * @typedef {import('../../src/aggregator/api.js').Buffer} Buffer + * @typedef {import('../../src/aggregator/api.js').PiecePolicy} PiecePolicy + * + * @typedef {import('../../src/aggregator/api.js').PieceMessage} PieceMessage + * @typedef {import('../../src/aggregator/api.js').BufferMessage} BufferMessage + * @typedef {import('../../src/aggregator/api.js').AggregateOfferMessage} AggregateOfferMessage + * @typedef {import('../../src/aggregator/api.js').PieceAcceptMessage} PieceAcceptMessage + */ + +/** + * @type {API.Tests} + */ +export const test = { + 'handles piece queue messages successfully': async (assert, context) => { + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + /** @type {PieceMessage} */ + const message = { + piece: cargo.link.link(), + group: context.id.did(), + } + + // Handle message + const handledMessageRes = await AggregatorEvents.handlePieceMessage( + context, + message + ) + assert.ok(handledMessageRes.ok) + + // Verify store + const hasStoredPiece = await context.pieceStore.get({ + piece: message.piece, + group: message.group, + }) + assert.ok(hasStoredPiece.ok) + assert.equal(hasStoredPiece.ok?.status, 'offered') + assert.ok(hasStoredPiece.ok?.insertedAt) + assert.ok(hasStoredPiece.ok?.updatedAt) + }, + 'handles piece queue message errors when fails to access piece store': + wichMockableContext( + async (assert, context) => { + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + /** @type {PieceMessage} */ + const message = { + piece: cargo.link.link(), + group: context.id.did(), + } + + // Handle message + const handledMessageRes = await AggregatorEvents.handlePieceMessage( + context, + message + ) + assert.ok(handledMessageRes.error) + assert.equal(handledMessageRes.error?.name, StoreOperationErrorName) + }, + async (context) => ({ + ...context, + pieceStore: getStoreImplementations(FailingStore).aggregator.pieceStore, + }) + ), + 'handles pieces insert batch successfully': async (assert, context) => { + const group = context.id.did() + const { pieces } = await randomAggregate(100, 128) + + // Handle event + const handledPieceInsertsRes = await AggregatorEvents.handlePiecesInsert( + context, + pieces.map((p) => ({ + piece: p.link, + group, + status: 'offered', + insertedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + })) + ) + assert.ok(handledPieceInsertsRes.ok) + + // Validate queue and store + await pWaitFor( + () => context.queuedMessages.get('bufferQueue')?.length === 1 + ) + /** @type {BufferMessage} */ + // @ts-expect-error cannot infer buffer message + const message = context.queuedMessages.get('bufferQueue')?.[0] + assert.equal(message.group, group) + + const bufferGet = await context.bufferStore.get(message.pieces) + assert.ok(bufferGet.ok) + assert.ok(bufferGet.ok?.block.equals(message.pieces)) + assert.deepEqual( + bufferGet.ok?.buffer.pieces.map((p) => p.piece), + pieces.map((p) => p.link) + ) + }, + 'handles piece insert event errors when fails to access buffer store': + wichMockableContext( + async (assert, context) => { + const group = context.id.did() + const { pieces } = await randomAggregate(100, 128) + + // Handle event + const handledPieceInsertsRes = + await AggregatorEvents.handlePiecesInsert( + context, + pieces.map((p) => ({ + piece: p.link, + group, + status: 'offered', + insertedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + })) + ) + assert.ok(handledPieceInsertsRes.error) + assert.equal( + handledPieceInsertsRes.error?.name, + StoreOperationErrorName + ) + }, + async (context) => ({ + ...context, + bufferStore: + getStoreImplementations(FailingStore).aggregator.bufferStore, + }) + ), + 'handles piece insert event errors when fails to access buffer queue': + wichMockableContext( + async (assert, context) => { + const group = context.id.did() + const { pieces } = await randomAggregate(100, 128) + + // Handle event + const handledPieceInsertsRes = + await AggregatorEvents.handlePiecesInsert( + context, + pieces.map((p) => ({ + piece: p.link, + group, + status: 'offered', + insertedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + })) + ) + assert.ok(handledPieceInsertsRes.error) + assert.equal( + handledPieceInsertsRes.error?.name, + QueueOperationErrorName + ) + }, + async (context) => ({ + ...context, + bufferQueue: new FailingQueue(), + }) + ), + 'handles buffer queue messages successfully to requeue bigger buffer': async ( + assert, + context + ) => { + const group = context.id.did() + const { buffers, blocks } = await getBuffers(2, group) + + // Store buffers + for (let i = 0; i < blocks.length; i++) { + const putBufferRes = await context.bufferStore.put({ + buffer: buffers[i], + block: blocks[i].cid, + }) + assert.ok(putBufferRes.ok) + } + + // Handle messages + const handledMessageRes = await AggregatorEvents.handleBufferQueueMessage( + { + ...context, + config: { + minAggregateSize: 2 ** 34, + minUtilizationFactor: 4, + maxAggregateSize: 2 ** 35, + }, + }, + blocks.map((b) => ({ + pieces: b.cid, + group, + })) + ) + assert.ok(handledMessageRes.ok) + assert.equal(handledMessageRes.ok?.aggregatedPieces, 0) + + // Validate queue and store + await pWaitFor( + () => context.queuedMessages.get('bufferQueue')?.length === 1 + ) + /** @type {BufferMessage} */ + // @ts-expect-error cannot infer buffer message + const message = context.queuedMessages.get('bufferQueue')?.[0] + assert.equal(message.group, group) + + const bufferGet = await context.bufferStore.get(message.pieces) + assert.ok(bufferGet.ok) + assert.ok(bufferGet.ok?.block.equals(message.pieces)) + assert.equal(bufferGet.ok?.buffer.group, group) + assert.ok(!bufferGet.ok?.buffer.aggregate) + assert.equal( + bufferGet.ok?.buffer.pieces.length, + buffers.reduce((acc, v) => { + acc += v.pieces.length + return acc + }, 0) + ) + }, + 'handles buffer queue messages with failure when fails to read them from store': + async (assert, context) => { + const group = context.id.did() + const { blocks } = await getBuffers(2, group) + + // Handle messages + const handledMessageRes = await AggregatorEvents.handleBufferQueueMessage( + context, + blocks.map((b) => ({ + pieces: b.cid, + group, + })) + ) + assert.ok(handledMessageRes.error) + assert.equal(handledMessageRes.error?.name, RecordNotFoundErrorName) + }, + 'handles buffer queue messages successfully to requeue bigger buffer if does not have minimum utilization': + async (assert, context) => { + const group = context.id.did() + const { buffers, blocks } = await getBuffers(2, group, { + length: 10, + size: 1024, + }) + + // Store buffers + for (let i = 0; i < blocks.length; i++) { + const putBufferRes = await context.bufferStore.put({ + buffer: buffers[i], + block: blocks[i].cid, + }) + assert.ok(putBufferRes.ok) + } + + // Handle messages + const handledMessageRes = await AggregatorEvents.handleBufferQueueMessage( + { + ...context, + config: { + minAggregateSize: 2 ** 13, + minUtilizationFactor: 1, + maxAggregateSize: 2 ** 18, + }, + }, + blocks.map((b) => ({ + pieces: b.cid, + group, + })) + ) + assert.ok(handledMessageRes.ok) + assert.equal(handledMessageRes.ok?.aggregatedPieces, 0) + + // Validate queue and store + await pWaitFor( + () => context.queuedMessages.get('bufferQueue')?.length === 1 + ) + /** @type {BufferMessage} */ + // @ts-expect-error cannot infer buffer message + const message = context.queuedMessages.get('bufferQueue')?.[0] + assert.equal(message.group, group) + + const bufferGet = await context.bufferStore.get(message.pieces) + assert.ok(bufferGet.ok) + assert.ok(bufferGet.ok?.block.equals(message.pieces)) + assert.equal(bufferGet.ok?.buffer.group, group) + assert.ok(!bufferGet.ok?.buffer.aggregate) + assert.equal( + bufferGet.ok?.buffer.pieces.length, + buffers.reduce((acc, v) => { + acc += v.pieces.length + return acc + }, 0) + ) + }, + 'handles buffer queue messages successfully to queue aggregate': async ( + assert, + context + ) => { + const group = context.id.did() + const { buffers, blocks } = await getBuffers(2, group, { + length: 100, + size: 128, + }) + const totalPieces = buffers.reduce((acc, v) => { + acc += v.pieces.length + return acc + }, 0) + + // Store buffers + for (let i = 0; i < blocks.length; i++) { + const putBufferRes = await context.bufferStore.put({ + buffer: buffers[i], + block: blocks[i].cid, + }) + assert.ok(putBufferRes.ok) + } + + // Handle messages + const handledMessageRes = await AggregatorEvents.handleBufferQueueMessage( + { + ...context, + config: { + minAggregateSize: 2 ** 19, + minUtilizationFactor: 10e5, + maxAggregateSize: 2 ** 35, + }, + }, + blocks.map((b) => ({ + pieces: b.cid, + group, + })) + ) + assert.ok(handledMessageRes.ok) + assert.equal(handledMessageRes.ok?.aggregatedPieces, totalPieces) + + // Validate queue and store + await pWaitFor( + () => + context.queuedMessages.get('bufferQueue')?.length === 0 && + context.queuedMessages.get('aggregateOfferQueue')?.length === 1 + ) + /** @type {AggregateOfferMessage} */ + // @ts-expect-error cannot infer buffer message + const message = context.queuedMessages.get('aggregateOfferQueue')?.[0] + assert.equal(message.group, group) + + const bufferGet = await context.bufferStore.get(message.pieces) + assert.ok(bufferGet.ok) + assert.ok(bufferGet.ok?.block.equals(message.pieces)) + assert.equal(bufferGet.ok?.buffer.group, group) + assert.ok(message.aggregate.equals(bufferGet.ok?.buffer.aggregate)) + assert.equal(bufferGet.ok?.buffer.pieces.length, totalPieces) + }, + 'handles buffer queue messages successfully to queue aggregate and remaining buffer': + async (assert, context) => { + const group = context.id.did() + const { buffers, blocks } = await getBuffers(2, group, { + length: 10, + size: 1024, + }) + const totalPieces = buffers.reduce((acc, v) => { + acc += v.pieces.length + return acc + }, 0) + + // Store buffers + for (let i = 0; i < blocks.length; i++) { + const putBufferRes = await context.bufferStore.put({ + buffer: buffers[i], + block: blocks[i].cid, + }) + assert.ok(putBufferRes.ok) + } + + // Handle messages + const handledMessageRes = await AggregatorEvents.handleBufferQueueMessage( + { + ...context, + config: { + minAggregateSize: 2 ** 13, + minUtilizationFactor: 10, + maxAggregateSize: 2 ** 15, + }, + }, + blocks.map((b) => ({ + pieces: b.cid, + group, + })) + ) + assert.ok(handledMessageRes.ok) + + // Validate queue and store + await pWaitFor( + () => + context.queuedMessages.get('bufferQueue')?.length === 1 && + context.queuedMessages.get('aggregateOfferQueue')?.length === 1 + ) + /** @type {AggregateOfferMessage} */ + // @ts-expect-error cannot infer buffer message + const aggregateOfferMessage = context.queuedMessages.get( + 'aggregateOfferQueue' + )?.[0] + /** @type {BufferMessage} */ + // @ts-expect-error cannot infer buffer message + const bufferMessage = context.queuedMessages.get('bufferQueue')?.[0] + + const aggregateBufferGet = await context.bufferStore.get( + aggregateOfferMessage.pieces + ) + assert.ok(aggregateBufferGet.ok) + const remainingBufferGet = await context.bufferStore.get( + bufferMessage.pieces + ) + assert.ok(remainingBufferGet.ok) + + assert.equal( + aggregateBufferGet.ok?.buffer.pieces.length, + handledMessageRes.ok?.aggregatedPieces + ) + assert.equal( + (aggregateBufferGet.ok?.buffer.pieces.length || 0) + + (remainingBufferGet.ok?.buffer.pieces.length || 0), + totalPieces + ) + }, + 'handles buffer queue message errors when fails to access buffer store': + wichMockableContext( + async (assert, context) => { + const group = context.id.did() + const { blocks } = await getBuffers(2, group) + + // Handle messages + const handledMessageRes = + await AggregatorEvents.handleBufferQueueMessage( + context, + blocks.map((b) => ({ + pieces: b.cid, + group, + })) + ) + assert.ok(handledMessageRes.error) + assert.equal(handledMessageRes.error?.name, StoreOperationErrorName) + }, + async (context) => ({ + ...context, + bufferStore: + getStoreImplementations(FailingStore).aggregator.bufferStore, + }) + ), + 'handles buffer queue message errors when fails to put message in buffer queue': + wichMockableContext( + async (assert, context) => { + const group = context.id.did() + const { buffers, blocks } = await getBuffers(2, group) + + // Store buffers + for (let i = 0; i < blocks.length; i++) { + const putBufferRes = await context.bufferStore.put({ + buffer: buffers[i], + block: blocks[i].cid, + }) + assert.ok(putBufferRes.ok) + } + + // Handle messages + const handledMessageRes = + await AggregatorEvents.handleBufferQueueMessage( + { + ...context, + config: { + minAggregateSize: 2 ** 34, + minUtilizationFactor: 4, + maxAggregateSize: 2 ** 35, + }, + }, + blocks.map((b) => ({ + pieces: b.cid, + group, + })) + ) + assert.ok(handledMessageRes.error) + assert.equal(handledMessageRes.error?.name, QueueOperationErrorName) + }, + async (context) => ({ + ...context, + bufferQueue: new FailingQueue(), + }) + ), + 'handles buffer queue message errors when fails to put message in aggregate queue': + wichMockableContext( + async (assert, context) => { + const group = context.id.did() + const { buffers, blocks } = await getBuffers(2, group, { + length: 100, + size: 128, + }) + + // Store buffers + for (let i = 0; i < blocks.length; i++) { + const putBufferRes = await context.bufferStore.put({ + buffer: buffers[i], + block: blocks[i].cid, + }) + assert.ok(putBufferRes.ok) + } + + // Handle messages + const handledMessageRes = + await AggregatorEvents.handleBufferQueueMessage( + { + ...context, + config: { + minAggregateSize: 2 ** 19, + minUtilizationFactor: 10e5, + maxAggregateSize: 2 ** 35, + }, + }, + blocks.map((b) => ({ + pieces: b.cid, + group, + })) + ) + assert.ok(handledMessageRes.error) + assert.equal(handledMessageRes.error?.name, QueueOperationErrorName) + }, + async (context) => ({ + ...context, + aggregateOfferQueue: new FailingQueue(), + }) + ), + 'handles aggregate offer queue messages successfully': async ( + assert, + context + ) => { + const group = context.id.did() + const { aggregate, pieces } = await randomAggregate(100, 128) + + /** @type {Buffer} */ + const buffer = { + pieces: pieces.map((p) => ({ + piece: p.link, + insertedAt: new Date().toISOString(), + policy: 0, + })), + group, + } + const block = await CBOR.write(buffer) + + /** @type {AggregateOfferMessage} */ + const message = { + aggregate: aggregate.link, + pieces: block.cid, + group, + } + + // Handle message + const handledMessageRes = + await AggregatorEvents.handleAggregateOfferMessage(context, message) + assert.ok(handledMessageRes.ok) + + // Verify store + const hasStoredAggregate = await context.aggregateStore.get({ + aggregate: message.aggregate, + }) + assert.ok(hasStoredAggregate.ok) + assert.ok(hasStoredAggregate.ok?.aggregate.equals(aggregate.link)) + assert.ok(hasStoredAggregate.ok?.pieces.equals(block.cid)) + assert.equal(hasStoredAggregate.ok?.group, group) + assert.ok(hasStoredAggregate.ok?.insertedAt) + }, + 'handles aggregate offer queue message errors when fails to store into aggregate store': + wichMockableContext( + async (assert, context) => { + const group = context.id.did() + const { aggregate, pieces } = await randomAggregate(100, 128) + + /** @type {Buffer} */ + const buffer = { + pieces: pieces.map((p) => ({ + piece: p.link, + insertedAt: new Date().toISOString(), + policy: 0, + })), + group, + } + const block = await CBOR.write(buffer) + + /** @type {AggregateOfferMessage} */ + const message = { + aggregate: aggregate.link, + pieces: block.cid, + group, + } + + // Handle message + const handledMessageRes = + await AggregatorEvents.handleAggregateOfferMessage(context, message) + assert.ok(handledMessageRes.error) + assert.equal(handledMessageRes.error?.name, StoreOperationErrorName) + }, + async (context) => ({ + ...context, + aggregateStore: + getStoreImplementations(FailingStore).aggregator.aggregateStore, + }) + ), + 'handles aggregate insert to queue piece accept successfully': async ( + assert, + context + ) => { + const piecesLength = 100 + const group = context.id.did() + const { aggregate, pieces } = await randomAggregate(piecesLength, 128) + + /** @type {Buffer} */ + const buffer = { + pieces: pieces.map((p) => ({ + piece: p.link, + insertedAt: new Date().toISOString(), + policy: 0, + })), + group, + } + const block = await CBOR.write(buffer) + + // Put buffer record + const putBufferRes = await context.bufferStore.put({ + buffer, + block: block.cid, + }) + assert.ok(putBufferRes.ok) + + // Put aggregate record + const aggregateRecord = { + pieces: block.cid, + aggregate: aggregate.link, + group, + insertedAt: new Date().toISOString(), + } + const putAggregateRes = await context.aggregateStore.put(aggregateRecord) + assert.ok(putAggregateRes.ok) + + // Handle event + const handledAggregateInsertsRes = + await AggregatorEvents.handleAggregateInsertToPieceAcceptQueue( + context, + aggregateRecord + ) + assert.ok(handledAggregateInsertsRes.ok) + + // Validate queue and store + await pWaitFor( + () => + context.queuedMessages.get('pieceAcceptQueue')?.length === piecesLength + ) + // Validate one message + /** @type {PieceAcceptMessage} */ + // @ts-expect-error cannot infer buffer message + const message = context.queuedMessages.get('pieceAcceptQueue')?.[0] + assert.ok(message.aggregate.equals(aggregate.link)) + assert.ok(pieces.find((p) => p.link.equals(message.piece))) + assert.equal(message.group, group) + + // Verify inclusion proof + const inclusionProof = aggregate.resolveProof(message.piece) + if (!inclusionProof.ok) { + throw new Error() + } + assert.deepEqual(message.inclusion.subtree[0], inclusionProof.ok?.[0][0]) + assert.deepEqual(message.inclusion.index[0], inclusionProof.ok?.[1][0]) + + assert.deepEqual(message.inclusion.subtree[1], inclusionProof.ok?.[0][1]) + assert.deepEqual(message.inclusion.index[1], inclusionProof.ok?.[1][1]) + }, + 'handles aggregate insert event to piece accept queue errors when fails to read from buffer store': + wichMockableContext( + async (assert, context) => { + const piecesLength = 100 + const group = context.id.did() + const { aggregate, pieces } = await randomAggregate(piecesLength, 128) + + /** @type {Buffer} */ + const buffer = { + pieces: pieces.map((p) => ({ + piece: p.link, + insertedAt: new Date().toISOString(), + policy: 0, + })), + group, + } + const block = await CBOR.write(buffer) + + // Put aggregate record + const aggregateRecord = { + pieces: block.cid, + aggregate: aggregate.link, + group, + insertedAt: new Date().toISOString(), + } + const putAggregateRes = await context.aggregateStore.put( + aggregateRecord + ) + assert.ok(putAggregateRes.ok) + + // Handle event + const handledAggregateInsertsRes = + await AggregatorEvents.handleAggregateInsertToPieceAcceptQueue( + context, + aggregateRecord + ) + assert.ok(handledAggregateInsertsRes.error) + assert.equal( + handledAggregateInsertsRes.error?.name, + StoreOperationErrorName + ) + }, + async (context) => ({ + ...context, + bufferStore: + getStoreImplementations(FailingStore).aggregator.bufferStore, + }) + ), + 'handles aggregate insert event to piece accept queue errors when fails to add to piece accept queue': + wichMockableContext( + async (assert, context) => { + const piecesLength = 100 + const group = context.id.did() + const { aggregate, pieces } = await randomAggregate(piecesLength, 128) + + /** @type {Buffer} */ + const buffer = { + pieces: pieces.map((p) => ({ + piece: p.link, + insertedAt: new Date().toISOString(), + policy: 0, + })), + group, + } + const block = await CBOR.write(buffer) + // Put buffer record + const putBufferRes = await context.bufferStore.put({ + buffer, + block: block.cid, + }) + assert.ok(putBufferRes.ok) + + // Put aggregate record + const aggregateRecord = { + pieces: block.cid, + aggregate: aggregate.link, + group, + insertedAt: new Date().toISOString(), + } + const putAggregateRes = await context.aggregateStore.put( + aggregateRecord + ) + assert.ok(putAggregateRes.ok) + + // Handle event + const handledAggregateInsertsRes = + await AggregatorEvents.handleAggregateInsertToPieceAcceptQueue( + context, + aggregateRecord + ) + assert.ok(handledAggregateInsertsRes.error) + assert.equal( + handledAggregateInsertsRes.error?.name, + QueueOperationErrorName + ) + }, + async (context) => ({ + ...context, + pieceAcceptQueue: new FailingQueue(), + }) + ), + 'handles piece accept queue messages successfully': async ( + assert, + context + ) => { + const group = context.id.did() + const { aggregate, pieces } = await randomAggregate(100, 128) + const piece = pieces[0].link + + // Create inclusion proof + const inclusionProof = aggregate.resolveProof(piece) + if (!inclusionProof.ok) { + throw new Error() + } + + /** @type {PieceAcceptMessage} */ + const message = { + aggregate: aggregate.link, + piece, + group, + inclusion: { + subtree: inclusionProof.ok[0], + index: inclusionProof.ok[1], + }, + } + + // Handle message + const handledMessageRes = await AggregatorEvents.handlePieceAcceptMessage( + context, + message + ) + assert.ok(handledMessageRes.ok) + + // Verify store + const hasStoredInclusion = await context.inclusionStore.get({ + piece, + aggregate: message.aggregate, + }) + assert.ok(hasStoredInclusion.ok) + assert.ok(hasStoredInclusion.ok?.aggregate.equals(aggregate.link)) + assert.ok(hasStoredInclusion.ok?.piece.equals(piece)) + assert.equal(hasStoredInclusion.ok?.group, group) + assert.ok(hasStoredInclusion.ok?.insertedAt) + assert.deepEqual(hasStoredInclusion.ok?.inclusion, message.inclusion) + }, + 'handles piece accept message errors when fails to store on inclusion store': + wichMockableContext( + async (assert, context) => { + const group = context.id.did() + const { aggregate, pieces } = await randomAggregate(100, 128) + const piece = pieces[0].link + + // Create inclusion proof + const inclusionProof = aggregate.resolveProof(piece) + if (!inclusionProof.ok) { + throw new Error() + } + + /** @type {PieceAcceptMessage} */ + const message = { + aggregate: aggregate.link, + piece, + group, + inclusion: { + subtree: inclusionProof.ok[0], + index: inclusionProof.ok[1], + }, + } + + // Handle message + const handledMessageRes = + await AggregatorEvents.handlePieceAcceptMessage(context, message) + assert.ok(handledMessageRes.error) + assert.equal(handledMessageRes.error?.name, StoreOperationErrorName) + }, + async (context) => ({ + ...context, + inclusionStore: + getStoreImplementations(FailingStore).aggregator.inclusionStore, + }) + ), + 'handles inclusion insert to update piece store entry successfully': async ( + assert, + context + ) => { + const group = context.id.did() + const { aggregate, pieces } = await randomAggregate(100, 128) + const piece = pieces[0].link + + // Store piece + const piecePut = await context.pieceStore.put({ + piece, + group, + status: 'offered', + insertedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }) + assert.ok(piecePut.ok) + + // Create inclusion proof + const inclusionProof = aggregate.resolveProof(piece) + if (!inclusionProof.ok) { + throw new Error() + } + + // Insert inclusion + const inclusionRecord = { + aggregate: aggregate.link, + piece, + group, + inclusion: { + subtree: inclusionProof.ok[0], + index: inclusionProof.ok[1], + }, + insertedAt: new Date().toISOString(), + } + const inclusionPut = await context.inclusionStore.put(inclusionRecord) + assert.ok(inclusionPut.ok) + + // Handle insert event + const handledMessageRes = + await AggregatorEvents.handleInclusionInsertToUpdateState( + context, + inclusionRecord + ) + assert.ok(handledMessageRes.ok) + + // Verify store + const pieceGet = await context.pieceStore.get({ + piece, + group, + }) + assert.ok(pieceGet.ok) + assert.equal(pieceGet.ok?.status, 'accepted') + assert.ok(pieceGet.ok?.insertedAt) + assert.ok(pieceGet.ok?.updatedAt) + }, + 'handles inclusion insert event errors when fails to update piece store entry': + wichMockableContext( + async (assert, context) => { + const group = context.id.did() + const { aggregate, pieces } = await randomAggregate(100, 128) + const piece = pieces[0].link + + // Create inclusion proof + const inclusionProof = aggregate.resolveProof(piece) + if (!inclusionProof.ok) { + throw new Error() + } + + // Insert inclusion + const inclusionRecord = { + aggregate: aggregate.link, + piece, + group, + inclusion: { + subtree: inclusionProof.ok[0], + index: inclusionProof.ok[1], + }, + insertedAt: new Date().toISOString(), + } + const inclusionPut = await context.inclusionStore.put(inclusionRecord) + assert.ok(inclusionPut.ok) + + // Handle insert event + const handledMessageRes = + await AggregatorEvents.handleInclusionInsertToUpdateState( + context, + inclusionRecord + ) + assert.ok(handledMessageRes.error) + assert.equal(handledMessageRes.error?.name, StoreOperationErrorName) + }, + async (context) => ({ + ...context, + pieceStore: getStoreImplementations(FailingStore).aggregator.pieceStore, + }) + ), + 'handles inclusion insert to issue piece accept receipt successfully': async ( + assert, + context + ) => { + const group = context.id.did() + const { aggregate, pieces } = await randomAggregate(100, 128) + const piece = pieces[0].link + + // Create inclusion proof + const inclusionProof = aggregate.resolveProof(piece) + if (!inclusionProof.ok) { + throw new Error() + } + + // Insert inclusion + const inclusionRecord = { + aggregate: aggregate.link, + piece, + group, + inclusion: { + subtree: inclusionProof.ok[0], + index: inclusionProof.ok[1], + }, + insertedAt: new Date().toISOString(), + } + const inclusionPut = await context.inclusionStore.put(inclusionRecord) + assert.ok(inclusionPut.ok) + + // Handle insert event + const handledMessageRes = + await AggregatorEvents.handleInclusionInsertToIssuePieceAccept( + context, + inclusionRecord + ) + assert.ok(handledMessageRes.ok) + + // Verify invocation + // @ts-expect-error not typed hooks + assert.equal(context.service.piece?.accept?.callCount, 1) + assert.ok( + inclusionRecord.piece.equals( + // @ts-expect-error not typed hooks + context.service.piece?.accept?._params[0].nb.piece + ) + ) + assert.equal( + inclusionRecord.group, + // @ts-expect-error not typed hooks + context.service.piece?.accept?._params[0].nb.group + ) + }, + 'handles inclusion insert failures to invoke piece accept': + wichMockableContext( + async (assert, context) => { + const group = context.id.did() + const { aggregate, pieces } = await randomAggregate(100, 128) + const piece = pieces[0].link + + // Create inclusion proof + const inclusionProof = aggregate.resolveProof(piece) + if (!inclusionProof.ok) { + throw new Error() + } + + // Insert inclusion + const inclusionRecord = { + aggregate: aggregate.link, + piece, + group, + inclusion: { + subtree: inclusionProof.ok[0], + index: inclusionProof.ok[1], + }, + insertedAt: new Date().toISOString(), + } + const inclusionPut = await context.inclusionStore.put(inclusionRecord) + assert.ok(inclusionPut.ok) + + // Handle message + const handledMessageRes = + await AggregatorEvents.handleInclusionInsertToIssuePieceAccept( + context, + inclusionRecord + ) + assert.ok(handledMessageRes.error) + }, + async (context) => { + /** + * Mock aggregator to fail + */ + const service = mockService({ + piece: { + accept: Server.provideAdvanced({ + capability: AggregatorCaps.pieceAccept, + handler: async ({ invocation, context }) => { + return { + error: new Server.Failure(), + } + }, + }), + }, + }) + const aggregatorConnection = getConnection( + context.id, + service + ).connection + return { + ...context, + service, + aggregatorService: { + connection: aggregatorConnection, + invocationConfig: { + issuer: context.id, + with: context.id.did(), + audience: context.id, + }, + }, + } + } + ), + 'handles aggregate insert to invoke aggregate offer successfully': async ( + assert, + context + ) => { + const piecesLength = 100 + const group = context.id.did() + const { aggregate, pieces } = await randomAggregate(piecesLength, 128) + + /** @type {Buffer} */ + const buffer = { + pieces: pieces.map((p) => ({ + piece: p.link, + insertedAt: new Date().toISOString(), + policy: 0, + })), + group, + } + const blockBuffer = await CBOR.write(buffer) + const blockPieces = await CBOR.write(pieces.map((p) => p.link)) + + // Put buffer record + const putBufferRes = await context.bufferStore.put({ + buffer, + block: blockBuffer.cid, + }) + assert.ok(putBufferRes.ok) + + // Put aggregate record + const aggregateRecord = { + pieces: blockBuffer.cid, + aggregate: aggregate.link, + group, + insertedAt: new Date().toISOString(), + } + const putAggregateRes = await context.aggregateStore.put(aggregateRecord) + assert.ok(putAggregateRes.ok) + + // Handle event + const handledAggregateInsertsRes = + await AggregatorEvents.handleAggregateInsertToAggregateOffer( + context, + aggregateRecord + ) + assert.ok(handledAggregateInsertsRes.ok) + + // Verify invocation + // @ts-expect-error not typed hooks + assert.equal(context.service.aggregate?.offer?.callCount, 1) + assert.ok( + blockPieces.cid.equals( + // @ts-expect-error not typed hooks + context.service.aggregate?.offer?._params[0].nb.pieces + ) + ) + assert.ok( + aggregateRecord.aggregate.equals( + // @ts-expect-error not typed hooks + context.service.aggregate?.offer?._params[0].nb.aggregate + ) + ) + }, + 'handles aggregate insert event errors when fails to read from buffer store': + wichMockableContext( + async (assert, context) => { + const piecesLength = 100 + const group = context.id.did() + const { aggregate, pieces } = await randomAggregate(piecesLength, 128) + + /** @type {Buffer} */ + const buffer = { + pieces: pieces.map((p) => ({ + piece: p.link, + insertedAt: new Date().toISOString(), + policy: 0, + })), + group, + } + const blockBuffer = await CBOR.write(buffer) + + // Put aggregate record + const aggregateRecord = { + pieces: blockBuffer.cid, + aggregate: aggregate.link, + group, + insertedAt: new Date().toISOString(), + } + const putAggregateRes = await context.aggregateStore.put( + aggregateRecord + ) + assert.ok(putAggregateRes.ok) + + // Handle event + const handledAggregateInsertsRes = + await AggregatorEvents.handleAggregateInsertToAggregateOffer( + context, + aggregateRecord + ) + assert.ok(handledAggregateInsertsRes.error) + assert.equal( + handledAggregateInsertsRes.error?.name, + StoreOperationErrorName + ) + }, + async (context) => ({ + ...context, + bufferStore: + getStoreImplementations(FailingStore).aggregator.bufferStore, + }) + ), +} + +/** + * @param {number} length + * @param {string} group + * @param {object} [piecesOptions] + * @param {number} [piecesOptions.length] + * @param {number} [piecesOptions.size] + */ +async function getBuffers(length, group, piecesOptions = {}) { + const piecesLength = piecesOptions.length || 100 + const piecesSize = piecesOptions.size || 128 + + const pieceBatches = await Promise.all( + Array.from({ length }).map(() => randomCargo(piecesLength, piecesSize)) + ) + + const buffers = pieceBatches.map((b) => ({ + pieces: b.map((p) => ({ + piece: p.link, + insertedAt: new Date().toISOString(), + policy: /** @type {PiecePolicy} */ (0), + })), + group, + })) + + return { + buffers, + blocks: await Promise.all(buffers.map((b) => CBOR.write(b))), + } +} + +/** + * @param {API.Test} testFn + * @param {(context: TestAPI.AggregatorTestEventsContext) => Promise} mockContextFunction + */ +function wichMockableContext(testFn, mockContextFunction) { + // @ts-ignore + return async function (...args) { + const modifiedArgs = [args[0], await mockContextFunction(args[1])] + // @ts-ignore + return testFn(...modifiedArgs) + } +} diff --git a/packages/filecoin-api/test/events/dealer.js b/packages/filecoin-api/test/events/dealer.js new file mode 100644 index 000000000..b583f0ffc --- /dev/null +++ b/packages/filecoin-api/test/events/dealer.js @@ -0,0 +1,448 @@ +import { CBOR } from '@ucanto/core' +import * as Signer from '@ucanto/principal/ed25519' +import * as Server from '@ucanto/server' +import * as DealerCaps from '@web3-storage/capabilities/filecoin/dealer' +import * as DealTrackerCaps from '@web3-storage/capabilities/filecoin/deal-tracker' + +import * as API from '../../src/types.js' +import * as TestAPI from '../types.js' +import * as DealerEvents from '../../src/dealer/events.js' + +import { FailingStore } from '../context/store.js' +import { mockService } from '../context/mocks.js' +import { getConnection } from '../context/service.js' +import { getStoreImplementations } from '../context/store-implementations.js' +import { randomAggregate } from '../utils.js' +import { StoreOperationErrorName } from '../../src/errors.js' + +/** + * @typedef {import('../../src/dealer/api.js').AggregateRecord} AggregateRecord + */ + +/** + * @type {API.Tests} + */ +export const test = { + 'handles aggregate insert event successfully': async (assert, context) => { + // Generate piece for test + const { pieces, aggregate } = await randomAggregate(100, 128) + const offer = pieces.map((p) => p.link) + const piecesBlock = await CBOR.write(offer) + + const putOfferRes = await context.offerStore.put({ + key: piecesBlock.cid.toString(), + value: { + issuer: context.id.did(), + aggregate: aggregate.link, + pieces: offer, + }, + }) + assert.ok(putOfferRes.ok) + + const offerStoreGetBeforeRename = await context.offerStore.get( + piecesBlock.cid.toString() + ) + assert.ok(offerStoreGetBeforeRename.ok) + + // Put aggregate record + /** @type {AggregateRecord} */ + const aggregateRecord = { + pieces: piecesBlock.cid, + aggregate: aggregate.link, + status: 'offered', + insertedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + } + const putAggregateRes = await context.aggregateStore.put(aggregateRecord) + assert.ok(putAggregateRes.ok) + + // Handle event + const handledPieceInsertsRes = await DealerEvents.handleAggregateInsert( + context, + aggregateRecord + ) + assert.ok(handledPieceInsertsRes.ok) + + // Old name not available + const offerStoreGetAfterRename0 = await context.offerStore.get( + piecesBlock.cid.toString() + ) + assert.ok(offerStoreGetAfterRename0.error) + // New name available + const offerStoreGetAfterRename1 = await context.offerStore.get( + `${new Date( + aggregateRecord.insertedAt + ).toISOString()} ${aggregateRecord.aggregate.toString()}.json` + ) + assert.ok(offerStoreGetAfterRename1.ok) + }, + 'handles aggregate insert errors when fails to update piece store': + wichMockableContext( + async (assert, context) => { + // Generate piece for test + const { pieces, aggregate } = await randomAggregate(100, 128) + const offer = pieces.map((p) => p.link) + const piecesBlock = await CBOR.write(offer) + + // Put aggregate record + /** @type {AggregateRecord} */ + const aggregateRecord = { + pieces: piecesBlock.cid, + aggregate: aggregate.link, + status: 'offered', + insertedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + } + const putAggregateRes = await context.aggregateStore.put( + aggregateRecord + ) + assert.ok(putAggregateRes.ok) + + // Handle event + const handledPieceInsertsRes = await DealerEvents.handleAggregateInsert( + context, + aggregateRecord + ) + assert.ok(handledPieceInsertsRes.error) + assert.equal( + handledPieceInsertsRes.error?.name, + StoreOperationErrorName + ) + }, + async (context) => ({ + ...context, + offerStore: getStoreImplementations(FailingStore).dealer.offerStore, + }) + ), + 'handles aggregate update status event successfully': async ( + assert, + context + ) => { + // Generate piece for test + const { pieces, aggregate } = await randomAggregate(100, 128) + const offer = pieces.map((p) => p.link) + const piecesBlock = await CBOR.write(offer) + + // Put aggregate record + /** @type {AggregateRecord} */ + const aggregateRecord = { + pieces: piecesBlock.cid, + aggregate: aggregate.link, + status: 'offered', + insertedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + } + const putAggregateRes = await context.aggregateStore.put(aggregateRecord) + assert.ok(putAggregateRes.ok) + + // Handle event + const handledPieceInsertsRes = + await DealerEvents.handleAggregatUpdatedStatus(context, aggregateRecord) + assert.ok(handledPieceInsertsRes.ok) + + // Verify invocation + // @ts-expect-error not typed hooks + assert.equal(context.service.aggregate?.accept?.callCount, 1) + assert.ok( + // @ts-expect-error not typed hooks + context.service.aggregate?.accept?._params[0].nb.pieces.equals( + piecesBlock.cid + ) + ) + assert.ok( + // @ts-expect-error not typed hooks + context.service.aggregate?.accept?._params[0].nb.aggregate.equals( + aggregate.link + ) + ) + }, + 'handles aggregate update status event errors on dealer invocation failure': + wichMockableContext( + async (assert, context) => { + // Generate piece for test + const { pieces, aggregate } = await randomAggregate(100, 128) + const offer = pieces.map((p) => p.link) + const piecesBlock = await CBOR.write(offer) + + // Put aggregate record + /** @type {AggregateRecord} */ + const aggregateRecord = { + pieces: piecesBlock.cid, + aggregate: aggregate.link, + status: 'offered', + insertedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + } + const putAggregateRes = await context.aggregateStore.put( + aggregateRecord + ) + assert.ok(putAggregateRes.ok) + + // Handle event + const handledPieceInsertsRes = + await DealerEvents.handleAggregatUpdatedStatus( + context, + aggregateRecord + ) + assert.ok(handledPieceInsertsRes.error) + }, + async (context) => { + /** + * Mock dealer to fail + */ + const service = mockService({ + aggregate: { + accept: Server.provideAdvanced({ + capability: DealerCaps.aggregateAccept, + handler: async ({ invocation, context }) => { + return { + error: new Server.Failure(), + } + }, + }), + }, + }) + const dealerConnection = getConnection(context.id, service).connection + + return { + ...context, + service, + dealerService: { + ...context.dealerService, + connection: dealerConnection, + }, + } + } + ), + 'handles cron tick successfully by swapping state of offered aggregate': + async (assert, context) => { + // Generate piece for test + const { pieces, aggregate } = await randomAggregate(100, 128) + const offer = pieces.map((p) => p.link) + const piecesBlock = await CBOR.write(offer) + + // Put aggregate record + /** @type {AggregateRecord} */ + const aggregateRecord = { + pieces: piecesBlock.cid, + aggregate: aggregate.link, + status: 'offered', + insertedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + } + const putAggregateRes = await context.aggregateStore.put(aggregateRecord) + assert.ok(putAggregateRes.ok) + const storedDealBeforeCron = await context.aggregateStore.get({ + aggregate: aggregate.link.link(), + }) + assert.ok(storedDealBeforeCron.ok) + assert.equal(storedDealBeforeCron.ok?.status, 'offered') + + // Handle event + const handledCronTick = await DealerEvents.handleCronTick(context) + assert.ok(handledCronTick.ok) + assert.equal(handledCronTick.ok?.updatedCount, 1) + assert.equal(handledCronTick.ok?.pendingCount, 0) + + // Validate stores + // Deal as in mocked service + const deal = { + dataType: 0n, + dataSource: { + dealID: 111n, + }, + } + const storedDealAfterCron = await context.aggregateStore.get({ + aggregate: aggregate.link.link(), + deal, + }) + assert.ok(storedDealAfterCron.ok) + assert.equal(storedDealAfterCron.ok?.status, 'accepted') + assert.deepEqual(storedDealAfterCron.ok?.deal, deal) + assert.ok( + storedDealBeforeCron.ok?.updatedAt !== storedDealAfterCron.ok?.updatedAt + ) + }, + 'handles cron tick several times until deal exists': wichMockableContext( + async (assert, context) => { + // Generate piece for test + const { pieces, aggregate } = await randomAggregate(100, 128) + const offer = pieces.map((p) => p.link) + const piecesBlock = await CBOR.write(offer) + + // Put aggregate record + /** @type {AggregateRecord} */ + const aggregateRecord = { + pieces: piecesBlock.cid, + aggregate: aggregate.link, + status: 'offered', + insertedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + } + const putAggregateRes = await context.aggregateStore.put(aggregateRecord) + assert.ok(putAggregateRes.ok) + + // Handle event + const handledCronTick1 = await DealerEvents.handleCronTick(context) + assert.ok(handledCronTick1.ok) + assert.equal(handledCronTick1.ok?.updatedCount, 0) + assert.equal(handledCronTick1.ok?.pendingCount, 1) + + // Should have same state and no deal + const storedDealAfterFirstCron = await context.aggregateStore.get({ + aggregate: aggregate.link.link(), + }) + assert.ok(storedDealAfterFirstCron.ok) + assert.equal(storedDealAfterFirstCron.ok?.status, 'offered') + + // Handle event second time + const handledCronTick2 = await DealerEvents.handleCronTick(context) + assert.ok(handledCronTick2.ok) + assert.equal(handledCronTick2.ok?.updatedCount, 1) + assert.equal(handledCronTick2.ok?.pendingCount, 0) + }, + async (context) => { + let counter = 1 + + /** + * Mock deal tracker to only send deal info on second call + */ + const dealTrackerSigner = await Signer.generate() + const service = mockService({ + deal: { + info: Server.provideAdvanced({ + capability: DealTrackerCaps.dealInfo, + handler: async ({ invocation, context }) => { + const invCap = invocation.capabilities[0] + if (!invCap.nb?.piece) { + throw new Error() + } + + if (counter === 2) { + /** @type {API.UcantoInterface.OkBuilder} */ + return Server.ok({ + deals: { + 111: { + provider: 'f11111', + }, + }, + }) + } + + counter += 1 + return Server.ok({ + deals: {}, + }) + }, + }), + }, + }) + const dealTrackerConnection = getConnection( + dealTrackerSigner, + service + ).connection + + return { + ...context, + service, + dealTrackerService: { + connection: dealTrackerConnection, + invocationConfig: { + issuer: context.id, + with: context.id.did(), + audience: dealTrackerSigner, + }, + }, + } + } + ), + 'handles cron tick errors when aggregate store query fails': + wichMockableContext( + async (assert, context) => { + // Handle event + const handledCronTick = await DealerEvents.handleCronTick(context) + assert.ok(handledCronTick.error) + assert.equal(handledCronTick.error?.name, StoreOperationErrorName) + }, + async (context) => ({ + ...context, + aggregateStore: + getStoreImplementations(FailingStore).dealer.aggregateStore, + }) + ), + 'handles cron tick errors when deal tracker fails to respond': + wichMockableContext( + async (assert, context) => { + // Generate piece for test + const { pieces, aggregate } = await randomAggregate(100, 128) + const offer = pieces.map((p) => p.link) + const piecesBlock = await CBOR.write(offer) + + // Put aggregate record + /** @type {AggregateRecord} */ + const aggregateRecord = { + pieces: piecesBlock.cid, + aggregate: aggregate.link, + status: 'offered', + insertedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + } + const putAggregateRes = await context.aggregateStore.put( + aggregateRecord + ) + assert.ok(putAggregateRes.ok) + + // Handle event + const handledCronTick = await DealerEvents.handleCronTick(context) + assert.ok(handledCronTick.error) + }, + async (context) => { + /** + * Mock deal tracker to fail + */ + const dealTrackerSigner = await Signer.generate() + const service = mockService({ + deal: { + info: Server.provideAdvanced({ + capability: DealTrackerCaps.dealInfo, + handler: async ({ invocation, context }) => { + return { + error: new Server.Failure(), + } + }, + }), + }, + }) + const dealTrackerConnection = getConnection( + dealTrackerSigner, + service + ).connection + + return { + ...context, + service, + dealTrackerService: { + connection: dealTrackerConnection, + invocationConfig: { + issuer: context.id, + with: context.id.did(), + audience: dealTrackerSigner, + }, + }, + } + } + ), +} + +/** + * @param {API.Test} testFn + * @param {(context: TestAPI.DealerTestEventsContext) => Promise} mockContextFunction + */ +function wichMockableContext(testFn, mockContextFunction) { + // @ts-ignore + return async function (...args) { + const modifiedArgs = [args[0], await mockContextFunction(args[1])] + // @ts-ignore + return testFn(...modifiedArgs) + } +} diff --git a/packages/filecoin-api/test/events/storefront.js b/packages/filecoin-api/test/events/storefront.js new file mode 100644 index 000000000..9de711d01 --- /dev/null +++ b/packages/filecoin-api/test/events/storefront.js @@ -0,0 +1,496 @@ +import * as Server from '@ucanto/server' +import * as Signer from '@ucanto/principal/ed25519' +import { CBOR } from '@ucanto/core' +import * as AggregatorCaps from '@web3-storage/capabilities/filecoin/aggregator' + +import * as API from '../../src/types.js' +import * as TestAPI from '../types.js' +import * as StorefrontEvents from '../../src/storefront/events.js' +import { + StoreOperationErrorName, + UnexpectedStateErrorName, +} from '../../src/errors.js' + +import { randomCargo, randomAggregate } from '../utils.js' +import { FailingStore } from '../context/store.js' +import { mockService } from '../context/mocks.js' +import { getConnection } from '../context/service.js' +import { getStoreImplementations } from '../context/store-implementations.js' +import { createInvocationsAndReceiptsForDealDataProofChain } from '../context/receipts.js' + +/** + * @typedef {import('../../src/storefront/api.js').PieceRecord} PieceRecord + */ + +/** + * @type {API.Tests} + */ +export const test = { + 'handles filecoin submit messages successfully': async (assert, context) => { + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + + // Store piece into store + const message = { + piece: cargo.link.link(), + content: cargo.content.link(), + group: context.id.did(), + } + + // Handle message + const handledMessageRes = + await StorefrontEvents.handleFilecoinSubmitMessage(context, message) + assert.ok(handledMessageRes.ok) + + // Verify store + const hasStoredPiece = await context.pieceStore.get({ + piece: cargo.link.link(), + }) + assert.ok(hasStoredPiece.ok) + assert.equal(hasStoredPiece.ok?.status, 'submitted') + }, + 'handles filecoin submit messages deduping when stored': async ( + assert, + context + ) => { + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + + // Store piece into store + const message = { + piece: cargo.link.link(), + content: cargo.content.link(), + group: context.id.did(), + } + /** @type {PieceRecord} */ + const pieceRecord = { + ...message, + status: 'submitted', + insertedAt: new Date(Date.now() - 10).toISOString(), + updatedAt: new Date(Date.now() - 5).toISOString(), + } + const putRes = await context.pieceStore.put(pieceRecord) + assert.ok(putRes.ok) + + // Handle message + const handledMessageRes = + await StorefrontEvents.handleFilecoinSubmitMessage(context, message) + assert.ok(handledMessageRes.ok) + + // Verify store + const hasStoredPiece = await context.pieceStore.get({ + piece: cargo.link.link(), + }) + assert.ok(hasStoredPiece.ok) + assert.equal(hasStoredPiece.ok?.status, 'submitted') + assert.equal(hasStoredPiece.ok?.updatedAt, pieceRecord.updatedAt) + }, + 'handles filecoin submit messages errors when fails to access piece store': + wichMockableContext( + async (assert, context) => { + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + + // Store piece into store + const message = { + piece: cargo.link.link(), + content: cargo.content.link(), + group: context.id.did(), + } + + // Handle message + const handledMessageRes = + await StorefrontEvents.handleFilecoinSubmitMessage(context, message) + assert.ok(handledMessageRes.error) + assert.equal(handledMessageRes.error?.name, StoreOperationErrorName) + }, + async (context) => ({ + ...context, + pieceStore: getStoreImplementations(FailingStore).storefront.pieceStore, + }) + ), + 'handles piece offer messages successfully': async (assert, context) => { + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + + // Store piece into store + const message = { + piece: cargo.link.link(), + content: cargo.content.link(), + group: context.id.did(), + } + + // Handle message + const handledMessageRes = await StorefrontEvents.handlePieceOfferMessage( + context, + message + ) + assert.ok(handledMessageRes.ok) + + // Verify invocation + // @ts-expect-error not typed hooks + assert.equal(context.service.piece?.offer?.callCount, 1) + assert.equal( + // @ts-expect-error not typed hooks + context.service.piece?.offer?._params[0].nb.group, + message.group + ) + assert.ok( + // @ts-expect-error not typed hooks + message.piece.equals(context.service.piece?.offer?._params[0].nb.piece) + ) + }, + 'handles piece offer messages erroring when fails to invoke piece offer': + wichMockableContext( + async (assert, context) => { + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + + // Store piece into store + const message = { + piece: cargo.link.link(), + content: cargo.content.link(), + group: context.id.did(), + } + + // Handle message + const handledMessageRes = + await StorefrontEvents.handlePieceOfferMessage(context, message) + assert.ok(handledMessageRes.error) + }, + async (context) => { + /** + * Mock aggregator to fail + */ + const aggregatorSigner = await Signer.generate() + const service = mockService({ + piece: { + offer: Server.provideAdvanced({ + capability: AggregatorCaps.pieceOffer, + handler: async ({ invocation, context }) => { + return { + error: new Server.Failure(), + } + }, + }), + }, + }) + const aggregatorConnection = getConnection( + aggregatorSigner, + service + ).connection + return { + ...context, + service, + aggregatorService: { + connection: aggregatorConnection, + invocationConfig: { + issuer: context.id, + with: context.id.did(), + audience: aggregatorSigner, + }, + }, + } + } + ), + 'handles piece insert event successfully': async (assert, context) => { + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + + // Store piece into store + const message = { + piece: cargo.link.link(), + content: cargo.content.link(), + group: context.id.did(), + } + /** @type {PieceRecord} */ + const pieceRecord = { + ...message, + status: 'submitted', + insertedAt: new Date(Date.now() - 10).toISOString(), + updatedAt: new Date(Date.now() - 5).toISOString(), + } + + // Handle message + const handledMessageRes = await StorefrontEvents.handlePieceInsert( + context, + pieceRecord + ) + assert.ok(handledMessageRes.ok) + + // Verify invocation + // @ts-expect-error not typed hooks + assert.equal(context.service.filecoin?.submit?.callCount, 1) + assert.ok( + message.content.equals( + // @ts-expect-error not typed hooks + context.service.filecoin?.submit?._params[0].nb.content + ) + ) + assert.ok( + message.piece.equals( + // @ts-expect-error not typed hooks + context.service.filecoin?.submit?._params[0].nb.piece + ) + ) + }, + 'handles piece status update event successfully': async (assert, context) => { + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + + // Store piece into store + const message = { + piece: cargo.link.link(), + content: cargo.content.link(), + group: context.id.did(), + } + /** @type {PieceRecord} */ + const pieceRecord = { + ...message, + status: 'accepted', + insertedAt: new Date(Date.now() - 10).toISOString(), + updatedAt: new Date(Date.now() - 5).toISOString(), + } + + // Handle message + const handledMessageRes = await StorefrontEvents.handlePieceStatusUpdate( + context, + pieceRecord + ) + assert.ok(handledMessageRes.ok) + + // Verify invocation + // @ts-expect-error not typed hooks + assert.equal(context.service.filecoin?.accept?.callCount, 1) + assert.ok( + message.content.equals( + // @ts-expect-error not typed hooks + context.service.filecoin?.accept?._params[0].nb.content + ) + ) + assert.ok( + message.piece.equals( + // @ts-expect-error not typed hooks + context.service.filecoin?.accept?._params[0].nb.piece + ) + ) + }, + 'fails to handle piece status update event if unexpected state': async ( + assert, + context + ) => { + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + + // Store piece into store + const message = { + piece: cargo.link.link(), + content: cargo.content.link(), + group: context.id.did(), + } + /** @type {PieceRecord} */ + const pieceRecord = { + ...message, + status: 'submitted', + insertedAt: new Date(Date.now() - 10).toISOString(), + updatedAt: new Date(Date.now() - 5).toISOString(), + } + + // Handle message + const handledMessageRes = await StorefrontEvents.handlePieceStatusUpdate( + context, + pieceRecord + ) + assert.ok(handledMessageRes.error) + assert.equal(handledMessageRes.error?.name, UnexpectedStateErrorName) + }, + 'handles cron tick successfully to modify status': async ( + assert, + context + ) => { + const { dealer } = await getServiceContext() + const group = context.id.did() + + // Create piece and aggregate for test + const { aggregate, pieces } = await randomAggregate(10, 128) + const piece = pieces[0] + const offer = pieces.map((p) => p.link) + const piecesBlock = await CBOR.write(offer) + + // Store pieces into store + await Promise.all( + pieces.map(async (p) => { + const putRes = await context.pieceStore.put({ + piece: p.link, + content: p.content, + group: context.id.did(), + status: 'submitted', + insertedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }) + assert.ok(putRes.ok) + }) + ) + + // Create inclusion proof for test + const inclusionProof = aggregate.resolveProof(piece.link) + if (inclusionProof.error) { + throw new Error('could not compute inclusion proof') + } + + // Cron ticks with no deals or receipts still available + const handledCronTickResBeforeAnyReceipt = + await StorefrontEvents.handleCronTick(context) + assert.ok(handledCronTickResBeforeAnyReceipt.ok) + assert.equal(handledCronTickResBeforeAnyReceipt.ok?.updatedCount, 0) + assert.equal( + handledCronTickResBeforeAnyReceipt.ok?.pendingCount, + pieces.length + ) + + // Create invocations and receipts for chain into DealDataProof + const dealMetadata = { + dataType: 0n, + dataSource: { + dealID: 100n, + }, + } + + // Create invocation and receipts chain until deal + const { invocations, receipts } = + await createInvocationsAndReceiptsForDealDataProofChain({ + storefront: context.id, + aggregator: context.aggregatorId, + dealer, + aggregate: aggregate.link, + group, + piece: piece.link, + piecesBlock, + inclusionProof: { + subtree: inclusionProof.ok[0], + index: inclusionProof.ok[1], + }, + aggregateAcceptStatus: { + ...dealMetadata, + aggregate: aggregate.link, + }, + }) + + // Store all invocations and receipts, except for the very last receipt on the chain + const storedInvocationsAndReceiptsRes = await storeInvocationsAndReceipts({ + invocations, + receipts: { + pieceOfferReceipt: receipts.pieceOfferReceipt, + pieceAcceptReceipt: receipts.pieceAcceptReceipt, + aggregateOfferReceipt: receipts.aggregateOfferReceipt, + }, + taskStore: context.taskStore, + receiptStore: context.receiptStore, + }) + assert.ok(storedInvocationsAndReceiptsRes.ok) + + // Cron ticks with no deals or receipts still available + const handledCronTickResBeforeFinalReceipt = + await StorefrontEvents.handleCronTick(context) + assert.ok(handledCronTickResBeforeFinalReceipt.ok) + assert.equal(handledCronTickResBeforeFinalReceipt.ok?.updatedCount, 0) + assert.equal( + handledCronTickResBeforeFinalReceipt.ok?.pendingCount, + pieces.length + ) + + // Store all invocations and receipts, except for the very last receipt on the chain + const storeLastReceipt = await storeInvocationsAndReceipts({ + invocations: {}, + receipts: { + aggregateAcceptReceipt: receipts.aggregateAcceptReceipt, + }, + taskStore: context.taskStore, + receiptStore: context.receiptStore, + }) + assert.ok(storeLastReceipt.ok) + + // Cron ticks with one deal for the first piece + const handledCronTickResAfterFinalReceipt = + await StorefrontEvents.handleCronTick(context) + assert.ok(handledCronTickResAfterFinalReceipt.ok) + assert.equal(handledCronTickResAfterFinalReceipt.ok?.updatedCount, 1) + assert.equal( + handledCronTickResAfterFinalReceipt.ok?.pendingCount, + pieces.length - 1 + ) + }, + 'handles cron tick error attempting to find pieces to track': + wichMockableContext( + async (assert, context) => { + // Cron ticks with no deals or receipts still available + const handledCronTickResBeforeAnyReceipt = + await StorefrontEvents.handleCronTick(context) + assert.ok(handledCronTickResBeforeAnyReceipt.error) + assert.equal( + handledCronTickResBeforeAnyReceipt.error?.name, + StoreOperationErrorName + ) + }, + async (context) => ({ + ...context, + pieceStore: getStoreImplementations(FailingStore).storefront.pieceStore, + }) + ), +} + +/** + * @param {API.Test} testFn + * @param {(context: TestAPI.StorefrontTestEventsContext) => Promise} mockContextFunction + */ +function wichMockableContext(testFn, mockContextFunction) { + // @ts-ignore + return async function (...args) { + const modifiedArgs = [args[0], await mockContextFunction(args[1])] + // @ts-ignore + return testFn(...modifiedArgs) + } +} + +async function getServiceContext() { + const dealer = await Signer.generate() + + return { dealer } +} + +/** + * @param {object} context + * @param {Record} context.invocations + * @param {Record} context.receipts + * @param {API.Store} context.taskStore + * @param {API.Store} context.receiptStore + */ +async function storeInvocationsAndReceipts({ + invocations, + receipts, + taskStore, + receiptStore, +}) { + // Store invocations + const storedInvocations = await Promise.all( + Object.values(invocations).map((invocation) => { + return taskStore.put(invocation) + }) + ) + if (storedInvocations.find((si) => si.error)) { + throw new Error('failed to store test invocations') + } + // Store receipts + const storedReceipts = await Promise.all( + Object.values(receipts).map((receipt) => { + return receiptStore.put(receipt) + }) + ) + if (storedReceipts.find((si) => si.error)) { + throw new Error('failed to store test receipts') + } + + return { + ok: {}, + } +} diff --git a/packages/filecoin-api/test/lib.js b/packages/filecoin-api/test/lib.js index 0ff2d6c31..d9f015187 100644 --- a/packages/filecoin-api/test/lib.js +++ b/packages/filecoin-api/test/lib.js @@ -1,12 +1,14 @@ -import * as Aggregator from './services/aggregator.js' -import * as Dealer from './services/dealer.js' -import * as Storefront from './services/storefront.js' +import * as AggregatorService from './services/aggregator.js' +import * as DealerService from './services/dealer.js' +import * as StorefrontService from './services/storefront.js' + export * from './utils.js' export const test = { - ...Aggregator.test, - ...Dealer.test, - ...Storefront.test, + service: { + ...AggregatorService.test, + ...DealerService.test, + ...StorefrontService.test, + }, + events: {}, } - -export { Aggregator, Dealer, Storefront } diff --git a/packages/filecoin-api/test/services/aggregator.js b/packages/filecoin-api/test/services/aggregator.js index 76c04bff7..61432e077 100644 --- a/packages/filecoin-api/test/services/aggregator.js +++ b/packages/filecoin-api/test/services/aggregator.js @@ -1,75 +1,226 @@ -import { Filecoin } from '@web3-storage/capabilities' +import { Aggregator } from '@web3-storage/capabilities' +import * as DealerCaps from '@web3-storage/capabilities/filecoin/dealer' import * as Signer from '@ucanto/principal/ed25519' +import { CBOR } from '@ucanto/core' import pWaitFor from 'p-wait-for' import * as API from '../../src/types.js' +import * as AggregatorApi from '../../src/aggregator/api.js' -import { randomCargo } from '../utils.js' -import { createServer, connect } from '../../src/aggregator.js' +import { createServer, connect } from '../../src/aggregator/service.js' +import { randomAggregate, randomCargo } from '../utils.js' +import { FailingStore } from '../context/store.js' +import { FailingQueue } from '../context/queue.js' +import { getStoreImplementations } from '../context/store-implementations.js' +import { + QueueOperationErrorName, + StoreOperationErrorName, +} from '../../src/errors.js' /** - * @type {API.Tests} + * @typedef {import('@web3-storage/data-segment').PieceLink} PieceLink + * @typedef {import('@ucanto/interface').Link} Link + * @typedef {import('../../src/aggregator/api.js').PieceRecord} PieceRecord + * @typedef {import('../../src/aggregator/api.js').PieceRecordKey} PieceRecordKey + * @typedef {import('../../src/aggregator/api.js').BufferRecord} BufferRecord + * @typedef {import('../../src/aggregator/api.js').AggregateRecord} AggregateRecord + * @typedef {import('../../src/aggregator/api.js').AggregateRecordKey} AggregateRecordKey + * @typedef {import('../../src/aggregator/api.js').InclusionRecord} InclusionRecord + * @typedef {import('../../src/aggregator/api.js').InclusionRecordKey} InclusionRecordKey + */ + +/** + * @type {API.Tests} */ export const test = { - 'piece/queue inserts piece into processing queue': async ( - assert, - context - ) => { - const { storefront } = await getServiceContext() - const connection = connect({ - id: context.id, - channel: createServer(context), - }) + 'piece/offer inserts piece into piece queue if not in piece store and returns effects': + async (assert, context) => { + const { storefront } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) - // Generate piece for test - const [cargo] = await randomCargo(1, 128) - const group = 'did:web:free.web3.storage' + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + const group = 'did:web:free.web3.storage' - // storefront invocation - const pieceAddInv = Filecoin.aggregateQueue.invoke({ - issuer: storefront, - audience: connection.id, - with: storefront.did(), - nb: { + // storefront invocation + const pieceAddInv = Aggregator.pieceOffer.invoke({ + issuer: storefront, + audience: connection.id, + with: storefront.did(), + nb: { + piece: cargo.link.link(), + group, + }, + }) + + const response = await pieceAddInv.execute(connection) + if (response.out.error) { + throw new Error('invocation failed', { cause: response.out.error }) + } + assert.ok(response.out.ok) + assert.ok(response.out.ok.piece.equals(cargo.link.link())) + + // Validate effect in receipt + const fxJoin = await Aggregator.pieceAccept + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + piece: cargo.link.link(), + group, + }, + expiration: Infinity, + }) + .delegate() + + assert.ok(response.fx.join) + assert.ok(fxJoin.link().equals(response.fx.join?.link())) + + // Validate queue and store + await pWaitFor( + () => context.queuedMessages.get('pieceQueue')?.length === 1 + ) + + // Piece not yet stored + const hasStoredPiece = await context.pieceStore.get({ piece: cargo.link.link(), group, - }, - }) + }) + assert.ok(!hasStoredPiece.ok) + }, + 'piece/offer dedupes piece and returns effects without propagating message': + async (assert, context) => { + const { storefront } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) - const response = await pieceAddInv.execute(connection) - if (response.out.error) { - throw new Error('invocation failed', { cause: response.out.error }) - } - assert.ok(response.out.ok) - assert.ok(response.out.ok.piece.equals(cargo.link.link())) + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + const group = 'did:web:free.web3.storage' - // Validate effect in receipt - const fx = await Filecoin.aggregateAdd - .invoke({ - issuer: context.id, - audience: context.id, - with: context.id.did(), + // Store piece into store + const putRes = await context.pieceStore.put({ + piece: cargo.link.link(), + group: context.id.did(), + status: 'offered', + insertedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }) + assert.ok(putRes.ok) + + // storefront invocation + const pieceAddInv = Aggregator.pieceOffer.invoke({ + issuer: storefront, + audience: connection.id, + with: storefront.did(), nb: { piece: cargo.link.link(), - storefront: storefront.did(), group, }, }) - .delegate() - assert.ok(response.fx.join) - assert.ok(fx.link().equals(response.fx.join?.link())) + const response = await pieceAddInv.execute(connection) + if (response.out.error) { + throw new Error('invocation failed', { cause: response.out.error }) + } + assert.ok(response.out.ok) + assert.ok(response.out.ok.piece.equals(cargo.link.link())) + + // Validate effect in receipt + const fxJoin = await Aggregator.pieceAccept + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + piece: cargo.link.link(), + group, + }, + expiration: Infinity, + }) + .delegate() + + assert.ok(response.fx.join) + assert.ok(fxJoin.link().equals(response.fx.join?.link())) + + // Validate queue has no new message + await pWaitFor( + () => context.queuedMessages.get('pieceQueue')?.length === 0 + ) + }, + 'piece/offer fails if not able to verify piece store': wichMockableContext( + async (assert, context) => { + const { storefront } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + const group = 'did:web:free.web3.storage' - // Validate queue and store - await pWaitFor(() => context.queuedMessages.length === 1) + // storefront invocation + const pieceAddInv = Aggregator.pieceOffer.invoke({ + issuer: storefront, + audience: connection.id, + with: storefront.did(), + nb: { + piece: cargo.link.link(), + group, + }, + }) - const hasStoredPiece = await context.pieceStore.get({ - piece: cargo.link.link(), - storefront: storefront.did(), + const response = await pieceAddInv.execute(connection) + assert.ok(response.out.error) + // @ts-ignore + assert.equal(response.out.error?.name, StoreOperationErrorName) + }, + (context) => ({ + ...context, + pieceStore: getStoreImplementations(FailingStore).aggregator.pieceStore, }) - assert.ok(!hasStoredPiece.ok) - }, - 'piece/add from signer inserts piece into store and returns accepted': async ( + ), + 'piece/offer fails if not able to add to piece queue': wichMockableContext( + async (assert, context) => { + const { storefront } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + const group = 'did:web:free.web3.storage' + + // storefront invocation + const pieceAddInv = Aggregator.pieceOffer.invoke({ + issuer: storefront, + audience: connection.id, + with: storefront.did(), + nb: { + piece: cargo.link.link(), + group, + }, + }) + + const response = await pieceAddInv.execute(connection) + assert.ok(response.out.error) + // @ts-ignore + assert.equal(response.out.error?.name, QueueOperationErrorName) + }, + (context) => ({ + ...context, + pieceQueue: new FailingQueue(), + }) + ), + 'piece/accept issues receipt with data aggregation proof': async ( assert, context ) => { @@ -80,80 +231,176 @@ export const test = { }) // Generate piece for test - const [cargo] = await randomCargo(1, 128) - const group = 'did:web:free.web3.storage' + const group = storefront.did() + const { pieces, aggregate } = await randomAggregate(100, 128) + const piece = pieces[0].link + + // Store aggregate record into store + const offer = pieces.map((p) => p.link) + const piecesBlock = await CBOR.write(offer) + const aggregatePutRes = await context.aggregateStore.put({ + aggregate: aggregate.link, + pieces: piecesBlock.cid, + group, + insertedAt: new Date().toISOString(), + }) + assert.ok(aggregatePutRes.ok) + + // compute proof for piece in aggregate + const proof = aggregate.resolveProof(piece) + if (proof.error) { + throw new Error('could not compute proof') + } + + // Store inclusion record into store + const inclusionPutRes = await context.inclusionStore.put({ + piece, + aggregate: aggregate.link, + group, + inclusion: { + subtree: proof.ok[0], + index: proof.ok[1], + }, + insertedAt: new Date().toISOString(), + }) + assert.ok(inclusionPutRes.ok) - // aggregator invocation - const pieceAddInv = Filecoin.aggregateAdd.invoke({ - issuer: context.id, + // storefront invocation + const pieceAcceptInv = Aggregator.pieceAccept.invoke({ + issuer: storefront, audience: connection.id, - with: context.id.did(), + with: storefront.did(), nb: { - piece: cargo.link.link(), - storefront: storefront.did(), + piece, group, }, }) - const response = await pieceAddInv.execute(connection) + const response = await pieceAcceptInv.execute(connection) if (response.out.error) { throw new Error('invocation failed', { cause: response.out.error }) } + // Validate receipt assert.ok(response.out.ok) - assert.ok(response.out.ok.piece.equals(cargo.link.link())) - - // Validate queue and store - await pWaitFor(() => context.queuedMessages.length === 0) + assert.ok(response.out.ok.piece.equals(piece.link())) + assert.ok(response.out.ok.aggregate.equals(aggregate.link)) + assert.equal( + BigInt(response.out.ok.inclusion.subtree[0]), + BigInt(proof.ok[0][0]) + ) + assert.deepEqual(response.out.ok.inclusion.subtree[1], proof.ok[0][1]) + assert.equal( + BigInt(response.out.ok.inclusion.index[0]), + BigInt(proof.ok[1][0]) + ) + assert.deepEqual(response.out.ok.inclusion.index[1], proof.ok[1][1]) - const hasStoredPiece = await context.pieceStore.get({ - piece: cargo.link.link(), - storefront: storefront.did(), - }) - assert.ok(hasStoredPiece.ok) - assert.ok(hasStoredPiece.ok?.piece.equals(cargo.link.link())) - assert.deepEqual(hasStoredPiece.ok?.group, group) - assert.deepEqual(hasStoredPiece.ok?.storefront, storefront.did()) - }, - 'skip piece/add from signer inserts piece into store and returns rejected': - async (assert, context) => { - const { storefront } = await getServiceContext() - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // Generate piece for test - const [cargo] = await randomCargo(1, 128) - const group = 'did:web:free.web3.storage' - - // aggregator invocation - const pieceAddInv = Filecoin.aggregateAdd.invoke({ + // Validate effect in receipt + const fxJoin = await DealerCaps.aggregateOffer + .invoke({ issuer: context.id, - audience: connection.id, + audience: context.dealerId, with: context.id.did(), nb: { - piece: cargo.link.link(), - storefront: storefront.did(), - group, + aggregate: aggregate.link, + pieces: piecesBlock.cid, }, }) + .delegate() + assert.ok(response.fx.join) + assert.ok(fxJoin.link().equals(response.fx.join?.link())) + }, + 'piece/accept fails if not able to query inclusion store': + wichMockableContext( + async (assert, context) => { + const { storefront } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) - const response = await pieceAddInv.execute(connection) - if (response.out.error) { - throw new Error('invocation failed', { cause: response.out.error }) - } - assert.ok(response.out.ok) - assert.ok(response.out.ok.piece.equals(cargo.link.link())) + // Generate piece for test + const group = storefront.did() + const { pieces } = await randomAggregate(100, 128) + const piece = pieces[0].link - // Validate queue and store - await pWaitFor(() => context.queuedMessages.length === 0) + // storefront invocation + const pieceAcceptInv = Aggregator.pieceAccept.invoke({ + issuer: storefront, + audience: connection.id, + with: storefront.did(), + nb: { + piece, + group, + }, + }) - const hasStoredPiece = await context.pieceStore.get({ - piece: cargo.link.link(), - storefront: storefront.did(), + const response = await pieceAcceptInv.execute(connection) + // Validate receipt + assert.ok(response.out.error) + assert.equal(response.out.error?.name, StoreOperationErrorName) + }, + (context) => ({ + ...context, + inclusionStore: + getStoreImplementations(FailingStore).aggregator.inclusionStore, }) - assert.ok(!hasStoredPiece.ok) - }, + ), + 'piece/accept fails if not able to read from aggregate store': + wichMockableContext( + async (assert, context) => { + const { storefront } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Generate piece for test + const group = storefront.did() + const { pieces, aggregate } = await randomAggregate(100, 128) + const piece = pieces[0].link + + // compute proof for piece in aggregate + const proof = aggregate.resolveProof(piece) + if (proof.error) { + throw new Error('could not compute proof') + } + + // Store inclusion record into store + const inclusionPutRes = await context.inclusionStore.put({ + piece, + aggregate: aggregate.link, + group, + inclusion: { + subtree: proof.ok[0], + index: proof.ok[1], + }, + insertedAt: new Date().toISOString(), + }) + assert.ok(inclusionPutRes.ok) + + // storefront invocation + const pieceAcceptInv = Aggregator.pieceAccept.invoke({ + issuer: storefront, + audience: connection.id, + with: storefront.did(), + nb: { + piece, + group, + }, + }) + + const response = await pieceAcceptInv.execute(connection) + // Validate receipt + assert.ok(response.out.error) + assert.equal(response.out.error?.name, StoreOperationErrorName) + }, + (context) => ({ + ...context, + aggregateStore: + getStoreImplementations(FailingStore).aggregator.aggregateStore, + }) + ), } async function getServiceContext() { @@ -161,3 +408,16 @@ async function getServiceContext() { return { storefront } } + +/** + * @param {API.Test} testFn + * @param {(context: AggregatorApi.ServiceContext) => AggregatorApi.ServiceContext} mockContextFunction + */ +function wichMockableContext(testFn, mockContextFunction) { + // @ts-ignore + return function (...args) { + const modifiedArgs = [args[0], mockContextFunction(args[1])] + // @ts-ignore + return testFn(...modifiedArgs) + } +} diff --git a/packages/filecoin-api/test/services/deal-tracker.js b/packages/filecoin-api/test/services/deal-tracker.js new file mode 100644 index 000000000..bd059b485 --- /dev/null +++ b/packages/filecoin-api/test/services/deal-tracker.js @@ -0,0 +1,143 @@ +import { DealTracker } from '@web3-storage/capabilities' +import * as Signer from '@ucanto/principal/ed25519' + +import * as API from '../../src/types.js' +import * as DealTrackerApi from '../../src/deal-tracker/api.js' + +import { createServer, connect } from '../../src/deal-tracker/service.js' +import { randomCargo } from '../utils.js' +import { FailingStore } from '../context/store.js' +import { getStoreImplementations } from '../context/store-implementations.js' +import { StoreOperationErrorName } from '../../src/errors.js' + +/** + * @typedef {import('../../src/deal-tracker/api.js').DealRecord} DealRecord + * @typedef {import('../../src/deal-tracker/api.js').DealRecordKey} DealRecordKey + */ + +/** + * @type {API.Tests} + */ +export const test = { + 'deal/info fails to get info for non existent piece CID': async ( + assert, + context + ) => { + const { dealer } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + + // dealer invocation + const dealInfoInv = DealTracker.dealInfo.invoke({ + issuer: dealer, + audience: connection.id, + with: dealer.did(), + nb: { + piece: cargo.link.link(), + }, + }) + + const response = await dealInfoInv.execute(connection) + assert.ok(response.out) + assert.deepEqual(response.out.ok?.deals, {}) + }, + 'deal/info retrieves available deals for aggregate piece CID': async ( + assert, + context + ) => { + const { dealer } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + const dealIds = [111, 222] + await Promise.all( + dealIds.map(async (dealId) => { + const dealPutRes = await context.dealStore.put({ + piece: cargo.link, + dealId, + provider: `f0${dealId}`, + expirationEpoch: Date.now() + 10e9, + source: 'cargo.dag.haus', + insertedAt: new Date().toISOString(), + }) + + assert.ok(dealPutRes.ok) + }) + ) + + // dealer invocation + const dealInfoInv = DealTracker.dealInfo.invoke({ + issuer: dealer, + audience: connection.id, + with: dealer.did(), + nb: { + piece: cargo.link.link(), + }, + }) + + const response = await dealInfoInv.execute(connection) + assert.ok(response.out.ok) + for (const dealId of dealIds) { + assert.ok(response.out.ok?.deals[`${dealId}`]) + assert.equal(response.out.ok?.deals[`${dealId}`].provider, `f0${dealId}`) + } + }, + 'deal/info fails if not able to query deal store': wichMockableContext( + async (assert, context) => { + const { dealer } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + + // dealer invocation + const dealInfoInv = DealTracker.dealInfo.invoke({ + issuer: dealer, + audience: connection.id, + with: dealer.did(), + nb: { + piece: cargo.link.link(), + }, + }) + + const response = await dealInfoInv.execute(connection) + assert.ok(response.out.error) + assert.equal(response.out.error?.name, StoreOperationErrorName) + }, + (context) => ({ + ...context, + dealStore: getStoreImplementations(FailingStore).dealTracker.dealStore, + }) + ), +} + +async function getServiceContext() { + const dealer = await Signer.generate() + + return { dealer } +} + +/** + * @param {API.Test} testFn + * @param {(context: DealTrackerApi.ServiceContext) => DealTrackerApi.ServiceContext} mockContextFunction + */ +function wichMockableContext(testFn, mockContextFunction) { + // @ts-ignore + return function (...args) { + const modifiedArgs = [args[0], mockContextFunction(args[1])] + // @ts-ignore + return testFn(...modifiedArgs) + } +} diff --git a/packages/filecoin-api/test/services/dealer.js b/packages/filecoin-api/test/services/dealer.js index 36125d61e..e025b62d9 100644 --- a/packages/filecoin-api/test/services/dealer.js +++ b/packages/filecoin-api/test/services/dealer.js @@ -1,23 +1,28 @@ -import { Filecoin } from '@web3-storage/capabilities' +import { Dealer } from '@web3-storage/capabilities' import * as Signer from '@ucanto/principal/ed25519' -import pWaitFor from 'p-wait-for' import { CBOR } from '@ucanto/core' import * as API from '../../src/types.js' +import * as DealerApi from '../../src/dealer/api.js' +import { createServer, connect } from '../../src/dealer/service.js' import { randomAggregate } from '../utils.js' -import { createServer, connect } from '../../src/dealer.js' +import { FailingStore } from '../context/store.js' +import { getStoreImplementations } from '../context/store-implementations.js' +import { StoreOperationErrorName } from '../../src/errors.js' /** - * @type {API.Tests} + * @typedef {import('../../src/dealer/api.js').AggregateRecord} AggregateRecord + * @typedef {import('../../src/dealer/api.js').AggregateRecordKey} AggregateRecordKey + * @typedef {import('../../src/dealer/api.js').OfferDocument} OfferDocument + */ + +/** + * @type {API.Tests} */ export const test = { - 'aggregate/queue inserts piece into processing queue': async ( - assert, - context - ) => { - const { aggregator, storefront: storefrontSigner } = - await getServiceContext() + 'aggregate/offer inserts aggregate into stores': async (assert, context) => { + const { storefront } = await getServiceContext() const connection = connect({ id: context.id, channel: createServer(context), @@ -27,19 +32,15 @@ export const test = { const { pieces, aggregate } = await randomAggregate(100, 128) const offer = pieces.map((p) => p.link) const piecesBlock = await CBOR.write(offer) - const storefront = storefrontSigner.did() - const label = 'label' // aggregator invocation - const pieceAddInv = Filecoin.dealQueue.invoke({ - issuer: aggregator, + const pieceAddInv = Dealer.aggregateOffer.invoke({ + issuer: storefront, audience: connection.id, - with: aggregator.did(), + with: storefront.did(), nb: { aggregate: aggregate.link, pieces: piecesBlock.cid, - storefront, - label, }, }) pieceAddInv.attach(piecesBlock) @@ -52,7 +53,7 @@ export const test = { assert.ok(response.out.ok.aggregate?.equals(aggregate.link)) // Validate effect in receipt - const fx = await Filecoin.dealAdd + const fxJoin = await Dealer.aggregateAccept .invoke({ issuer: context.id, audience: context.id, @@ -60,124 +61,216 @@ export const test = { nb: { aggregate: aggregate.link, pieces: piecesBlock.cid, - storefront, - label, }, + expiration: Infinity, }) .delegate() assert.ok(response.fx.join) - assert.ok(fx.link().equals(response.fx.join?.link())) + assert.ok(fxJoin.link().equals(response.fx.join?.link())) - // Validate queue and store - await pWaitFor(() => context.queuedMessages.length === 1) - - const hasStoredDeal = await context.dealStore.get({ - piece: aggregate.link.link(), + // Validate stores + const storedDeal = await context.aggregateStore.get({ + aggregate: aggregate.link.link(), }) - assert.ok(!hasStoredDeal.ok) + assert.ok(storedDeal.ok) + assert.ok(storedDeal.ok?.aggregate.equals(aggregate.link.link())) + assert.ok(storedDeal.ok?.pieces.equals(piecesBlock.cid)) + assert.equal(storedDeal.ok?.status, 'offered') + assert.ok(storedDeal.ok?.insertedAt) + assert.ok(storedDeal.ok?.updatedAt) + // Still pending resolution + assert.ok(!storedDeal.ok?.deal) + + const storedOffer = await context.offerStore.get(piecesBlock.cid.toString()) + assert.ok(storedOffer.ok) + assert.ok(storedOffer.ok?.value.aggregate.equals(aggregate.link.link())) + assert.equal(storedOffer.ok?.value.issuer, storefront.did()) + assert.deepEqual( + storedOffer.ok?.value.pieces.map((p) => p.toString()), + offer.map((p) => p.toString()) + ) }, - 'aggregate/add from signer inserts piece into store and returns accepted': - async (assert, context) => { - const { storefront: storefrontSigner } = await getServiceContext() - const connection = connect({ - id: context.id, - channel: createServer(context), - }) + 'aggregate/offer fails if not able to check aggregate store': + wichMockableContext( + async (assert, context) => { + const { storefront } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) - // Generate piece for test - const { pieces, aggregate } = await randomAggregate(100, 128) - const offer = pieces.map((p) => p.link) - const piecesBlock = await CBOR.write(offer) - const storefront = storefrontSigner.did() - const label = 'label' + // Generate piece for test + const { pieces, aggregate } = await randomAggregate(100, 128) + const offer = pieces.map((p) => p.link) + const piecesBlock = await CBOR.write(offer) - await context.dealStore.put({ - aggregate: aggregate.link, - offer: aggregate.link.toString(), - storefront, - stat: 1, - insertedAt: Date.now(), - }) + // aggregator invocation + const pieceAddInv = Dealer.aggregateOffer.invoke({ + issuer: storefront, + audience: connection.id, + with: storefront.did(), + nb: { + aggregate: aggregate.link, + pieces: piecesBlock.cid, + }, + }) + pieceAddInv.attach(piecesBlock) - // aggregator invocation - const pieceAddInv = Filecoin.dealAdd.invoke({ - issuer: context.id, - audience: connection.id, - with: context.id.did(), - nb: { - aggregate: aggregate.link, - pieces: piecesBlock.cid, - storefront, - label, - }, + const response = await pieceAddInv.execute(connection) + assert.ok(response.out.error) + assert.equal(response.out.error?.name, StoreOperationErrorName) + }, + (context) => ({ + ...context, + aggregateStore: + getStoreImplementations(FailingStore).dealer.aggregateStore, }) - pieceAddInv.attach(piecesBlock) + ), + 'aggregate/offer fails if not able to put to offer store': + wichMockableContext( + async (assert, context) => { + const { storefront } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) - const response = await pieceAddInv.execute(connection) - if (response.out.error) { - throw new Error('invocation failed', { cause: response.out.error }) - } - assert.ok(response.out.ok) - assert.ok(response.out.ok.aggregate?.equals(aggregate.link)) + // Generate piece for test + const { pieces, aggregate } = await randomAggregate(100, 128) + const offer = pieces.map((p) => p.link) + const piecesBlock = await CBOR.write(offer) - // Validate queue and store - await pWaitFor(() => context.queuedMessages.length === 0) + // aggregator invocation + const pieceAddInv = Dealer.aggregateOffer.invoke({ + issuer: storefront, + audience: connection.id, + with: storefront.did(), + nb: { + aggregate: aggregate.link, + pieces: piecesBlock.cid, + }, + }) + pieceAddInv.attach(piecesBlock) - const hasStoredDeal = await context.dealStore.get({ - aggregate: aggregate.link.link(), - }) - assert.ok(hasStoredDeal.ok) - }, - 'skip aggregate/add from signer inserts piece into store and returns rejected': - async (assert, context) => { - const { storefront: storefrontSigner } = await getServiceContext() - const connection = connect({ - id: context.id, - channel: createServer(context), + const response = await pieceAddInv.execute(connection) + assert.ok(response.out.error) + assert.equal(response.out.error?.name, StoreOperationErrorName) + }, + (context) => ({ + ...context, + offerStore: getStoreImplementations(FailingStore).dealer.offerStore, }) + ), + 'aggregate/accept issues receipt with data aggregation proof': async ( + assert, + context + ) => { + const { storefront } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Generate piece for test + const { pieces, aggregate } = await randomAggregate(100, 128) + const offer = pieces.map((p) => p.link) + const piecesBlock = await CBOR.write(offer) - // Generate piece for test - const { pieces, aggregate } = await randomAggregate(100, 128) - const offer = pieces.map((p) => p.link) - const piecesBlock = await CBOR.write(offer) - const storefront = storefrontSigner.did() - const label = 'label' + // Set aggregate with deal + const deal = { + dataType: 0n, + dataSource: { + dealID: 100n, + }, + } + const putRes = await context.aggregateStore.put({ + aggregate: aggregate.link, + pieces: piecesBlock.cid, + status: 'offered', + insertedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + deal, + }) + assert.ok(putRes.ok) - // aggregator invocation - const pieceAddInv = Filecoin.dealAdd.invoke({ - issuer: context.id, - audience: connection.id, - with: context.id.did(), - nb: { - aggregate: aggregate.link, - pieces: piecesBlock.cid, - storefront, - label, - }, - }) - pieceAddInv.attach(piecesBlock) + // aggregator invocation + const pieceAddInv = Dealer.aggregateAccept.invoke({ + issuer: storefront, + audience: connection.id, + with: storefront.did(), + nb: { + aggregate: aggregate.link, + pieces: piecesBlock.cid, + }, + }) + pieceAddInv.attach(piecesBlock) - const response = await pieceAddInv.execute(connection) - if (response.out.error) { - throw new Error('invocation failed', { cause: response.out.error }) - } - assert.ok(response.out.ok) - assert.deepEqual(response.out.ok.aggregate, aggregate.link) + const response = await pieceAddInv.execute(connection) + if (response.out.error) { + throw new Error('invocation failed', { cause: response.out.error }) + } + assert.ok(response.out.ok) + assert.equal( + BigInt(response.out.ok.dataSource.dealID), + BigInt(deal.dataSource.dealID) + ) + assert.equal(BigInt(response.out.ok.dataType), BigInt(deal.dataType)) + }, + 'aggregate/accept fails if not able to read from aggregate store': + wichMockableContext( + async (assert, context) => { + const { storefront } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Generate piece for test + const { pieces, aggregate } = await randomAggregate(100, 128) + const offer = pieces.map((p) => p.link) + const piecesBlock = await CBOR.write(offer) - // Validate queue and store - await pWaitFor(() => context.queuedMessages.length === 0) + // aggregator invocation + const pieceAddInv = Dealer.aggregateAccept.invoke({ + issuer: storefront, + audience: connection.id, + with: storefront.did(), + nb: { + aggregate: aggregate.link, + pieces: piecesBlock.cid, + }, + }) + pieceAddInv.attach(piecesBlock) - const hasStoredDeal = await context.dealStore.get({ - aggregate: aggregate.link.link(), + const response = await pieceAddInv.execute(connection) + assert.ok(response.out.error) + assert.equal(response.out.error?.name, StoreOperationErrorName) + }, + (context) => ({ + ...context, + aggregateStore: + getStoreImplementations(FailingStore).dealer.aggregateStore, }) - assert.ok(!hasStoredDeal.ok) - }, + ), } async function getServiceContext() { - const aggregator = await Signer.generate() + const dealer = await Signer.generate() const storefront = await Signer.generate() - return { aggregator, storefront } + return { dealer, storefront } +} + +/** + * @param {API.Test} testFn + * @param {(context: DealerApi.ServiceContext) => DealerApi.ServiceContext} mockContextFunction + */ +function wichMockableContext(testFn, mockContextFunction) { + // @ts-ignore + return function (...args) { + const modifiedArgs = [args[0], mockContextFunction(args[1])] + // @ts-ignore + return testFn(...modifiedArgs) + } } diff --git a/packages/filecoin-api/test/services/storefront.js b/packages/filecoin-api/test/services/storefront.js index 2c9737b66..928deac62 100644 --- a/packages/filecoin-api/test/services/storefront.js +++ b/packages/filecoin-api/test/services/storefront.js @@ -1,73 +1,105 @@ -import { Filecoin } from '@web3-storage/capabilities' +import { Filecoin, Aggregator } from '@web3-storage/capabilities' +import { CBOR } from '@ucanto/core' import * as Signer from '@ucanto/principal/ed25519' import pWaitFor from 'p-wait-for' import * as API from '../../src/types.js' +import * as StorefrontApi from '../../src/storefront/api.js' -import { randomCargo } from '../utils.js' -import { createServer, connect } from '../../src/storefront.js' +import { createServer, connect } from '../../src/storefront/service.js' +import { + QueueOperationErrorName, + StoreOperationErrorName, +} from '../../src/errors.js' +import { randomCargo, randomAggregate } from '../utils.js' +import { createInvocationsAndReceiptsForDealDataProofChain } from '../context/receipts.js' +import { getStoreImplementations } from '../context/store-implementations.js' +import { FailingStore } from '../context/store.js' +import { FailingQueue } from '../context/queue.js' /** - * @type {API.Tests} + * @typedef {import('../../src/storefront/api.js').PieceRecord} PieceRecord + * @typedef {import('../../src/storefront/api.js').PieceRecordKey} PieceRecordKey */ -export const test = { - 'filecoin/queue inserts piece into verification queue': async ( - assert, - context - ) => { - const { agent } = await getServiceContext() - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - // Generate piece for test - const [cargo] = await randomCargo(1, 128) +/** + * @type {API.Tests} + */ +export const test = { + 'filecoin/offer inserts piece into submission queue if not in piece store and returns effects': + async (assert, context) => { + const { agent } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) - // agent invocation - const filecoinAddInv = Filecoin.filecoinQueue.invoke({ - issuer: agent, - audience: connection.id, - with: agent.did(), - nb: { - piece: cargo.link.link(), - content: cargo.content.link(), - }, - }) + // Generate piece for test + const [cargo] = await randomCargo(1, 128) - const response = await filecoinAddInv.execute(connection) - if (response.out.error) { - throw new Error('invocation failed', { cause: response.out.error }) - } - assert.ok(response.out.ok) - assert.ok(response.out.ok.piece.equals(cargo.link.link())) - - // Validate effect in receipt - const fx = await Filecoin.filecoinAdd - .invoke({ - issuer: context.id, - audience: context.id, - with: context.id.did(), + // agent invocation + const filecoinAddInv = Filecoin.offer.invoke({ + issuer: agent, + audience: connection.id, + with: agent.did(), nb: { piece: cargo.link.link(), content: cargo.content.link(), }, }) - .delegate() - assert.ok(response.fx.join) - assert.ok(fx.link().equals(response.fx.join?.link())) + const response = await filecoinAddInv.execute(connection) + if (response.out.error) { + throw new Error('invocation failed', { cause: response.out.error }) + } + assert.ok(response.out.ok) + assert.ok(response.out.ok.piece.equals(cargo.link.link())) - // Validate queue and store - await pWaitFor(() => context.queuedMessages.length === 1) + // Validate effects in receipt + const fxFork = await Filecoin.submit + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + piece: cargo.link.link(), + content: cargo.content.link(), + }, + expiration: Infinity, + }) + .delegate() + const fxJoin = await Filecoin.accept + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + piece: cargo.link.link(), + content: cargo.content.link(), + }, + expiration: Infinity, + }) + .delegate() - const hasStoredPiece = await context.pieceStore.get({ - piece: cargo.link.link(), - }) - assert.ok(!hasStoredPiece.ok) - }, - 'filecoin/add from signer inserts piece into store and returns accepted': + assert.ok(response.fx.join) + assert.ok(fxJoin.link().equals(response.fx.join?.link())) + assert.equal(response.fx.fork.length, 1) + assert.ok(fxFork.link().equals(response.fx.fork[0].link())) + + // Validate queue and store + await pWaitFor( + () => context.queuedMessages.get('filecoinSubmitQueue')?.length === 1 + ) + + // Piece not yet stored + const hasStoredPiece = await context.pieceStore.get({ + piece: cargo.link.link(), + }) + assert.ok(!hasStoredPiece.ok) + }, + 'filecoin/offer dedupes piece and returns effects without propagating message': async (assert, context) => { + const { agent } = await getServiceContext() const connection = connect({ id: context.id, channel: createServer(context), @@ -76,11 +108,22 @@ export const test = { // Generate piece for test const [cargo] = await randomCargo(1, 128) - // storefront invocation - const filecoinAddInv = Filecoin.filecoinAdd.invoke({ - issuer: context.id, + // Store piece into store + const putRes = await context.pieceStore.put({ + piece: cargo.link.link(), + content: cargo.content.link(), + group: context.id.did(), + status: 'submitted', + insertedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }) + assert.ok(putRes.ok) + + // agent invocation + const filecoinAddInv = Filecoin.offer.invoke({ + issuer: agent, audience: connection.id, - with: context.id.did(), + with: agent.did(), nb: { piece: cargo.link.link(), content: cargo.content.link(), @@ -94,16 +137,109 @@ export const test = { assert.ok(response.out.ok) assert.ok(response.out.ok.piece.equals(cargo.link.link())) - // Validate queue and store - await pWaitFor(() => context.queuedMessages.length === 0) + // Validate effects in receipt + const fxFork = await Filecoin.submit + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + piece: cargo.link.link(), + content: cargo.content.link(), + }, + expiration: Infinity, + }) + .delegate() + const fxJoin = await Filecoin.accept + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + piece: cargo.link.link(), + content: cargo.content.link(), + }, + expiration: Infinity, + }) + .delegate() - const hasStoredPiece = await context.pieceStore.get({ - piece: cargo.link.link(), - }) - assert.ok(hasStoredPiece.ok) + assert.ok(response.fx.join) + assert.ok(fxJoin.link().equals(response.fx.join?.link())) + assert.equal(response.fx.fork.length, 1) + assert.ok(fxFork.link().equals(response.fx.fork[0].link())) + + // Validate queue has no new message + await pWaitFor( + () => context.queuedMessages.get('filecoinSubmitQueue')?.length === 0 + ) }, - 'skip filecoin/add from signer inserts piece into store and returns rejected': + 'filecoin/offer invocation fails if fails to write to submission queue': + wichMockableContext( + async (assert, context) => { + const { agent } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + + // agent invocation + const filecoinAddInv = Filecoin.offer.invoke({ + issuer: agent, + audience: connection.id, + with: agent.did(), + nb: { + piece: cargo.link.link(), + content: cargo.content.link(), + }, + }) + + const response = await filecoinAddInv.execute(connection) + assert.ok(response.out.error) + assert.equal(response.out.error?.name, QueueOperationErrorName) + }, + (context) => ({ + ...context, + filecoinSubmitQueue: new FailingQueue(), + }) + ), + 'filecoin/offer invocation fails if fails to check piece store': + wichMockableContext( + async (assert, context) => { + const { agent } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + + // agent invocation + const filecoinAddInv = Filecoin.offer.invoke({ + issuer: agent, + audience: connection.id, + with: agent.did(), + nb: { + piece: cargo.link.link(), + content: cargo.content.link(), + }, + }) + + const response = await filecoinAddInv.execute(connection) + assert.ok(response.out.error) + assert.equal(response.out.error?.name, StoreOperationErrorName) + }, + (context) => ({ + ...context, + pieceStore: getStoreImplementations(FailingStore).storefront.pieceStore, + }) + ), + 'filecoin/submit inserts piece into piece offer queue and returns effect': async (assert, context) => { + const { agent } = await getServiceContext() const connection = connect({ id: context.id, channel: createServer(context), @@ -111,37 +247,267 @@ export const test = { // Generate piece for test const [cargo] = await randomCargo(1, 128) - - // storefront invocation - const filecoinAddInv = Filecoin.filecoinAdd.invoke({ - issuer: context.id, + const filecoinSubmitInv = Filecoin.submit.invoke({ + issuer: agent, audience: connection.id, - with: context.id.did(), + with: agent.did(), nb: { piece: cargo.link.link(), content: cargo.content.link(), }, }) - const response = await filecoinAddInv.execute(connection) + const response = await filecoinSubmitInv.execute(connection) if (response.out.error) { throw new Error('invocation failed', { cause: response.out.error }) } assert.ok(response.out.ok) assert.ok(response.out.ok.piece.equals(cargo.link.link())) - // Validate queue and store - await pWaitFor(() => context.queuedMessages.length === 0) + // Validate effects in receipt + const fxJoin = await Aggregator.pieceOffer + .invoke({ + issuer: context.id, + audience: context.aggregatorId, + with: context.id.did(), + nb: { + piece: cargo.link.link(), + group: context.id.did(), + }, + expiration: Infinity, + }) + .delegate() - const hasStoredPiece = await context.pieceStore.get({ - piece: cargo.link.link(), - }) - assert.ok(!hasStoredPiece.ok) + assert.ok(response.fx.join) + assert.ok(fxJoin.link().equals(response.fx.join?.link())) }, + 'filecoin/submit fails if fails to write to submission queue': + wichMockableContext( + async (assert, context) => { + const { agent } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + const filecoinSubmitInv = Filecoin.submit.invoke({ + issuer: agent, + audience: connection.id, + with: agent.did(), + nb: { + piece: cargo.link.link(), + content: cargo.content.link(), + }, + }) + + const response = await filecoinSubmitInv.execute(connection) + assert.ok(response.out.error) + assert.equal(response.out.error?.name, QueueOperationErrorName) + }, + (context) => ({ + ...context, + pieceOfferQueue: new FailingQueue(), + }) + ), + 'filecoin/accept issues receipt with data aggregation proof': async ( + assert, + context + ) => { + const { agent, aggregator, dealer } = await getServiceContext() + const group = context.id.did() + const connection = connect({ + id: context.id, + channel: createServer({ + ...context, + aggregatorId: aggregator, + }), + }) + + // Create piece and aggregate for test + const { aggregate, pieces } = await randomAggregate(10, 128) + const piece = pieces[0] + const offer = pieces.map((p) => p.link) + const piecesBlock = await CBOR.write(offer) + + // Store piece into store + const putRes = await context.pieceStore.put({ + piece: piece.link, + content: piece.content, + group: context.id.did(), + status: 'submitted', + insertedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }) + assert.ok(putRes.ok) + + // Create inclusion proof for test + const inclusionProof = aggregate.resolveProof(piece.link) + if (inclusionProof.error) { + throw new Error('could not compute inclusion proof') + } + + // Create invocations and receipts for chain into DealDataProof + const dealMetadata = { + dataType: 0n, + dataSource: { + dealID: 100n, + }, + } + const { invocations, receipts } = + await createInvocationsAndReceiptsForDealDataProofChain({ + storefront: context.id, + aggregator, + dealer, + aggregate: aggregate.link, + group, + piece: piece.link, + piecesBlock, + inclusionProof: { + subtree: inclusionProof.ok[0], + index: inclusionProof.ok[1], + }, + aggregateAcceptStatus: { + ...dealMetadata, + aggregate: aggregate.link, + }, + }) + + const storedInvocationsAndReceiptsRes = await storeInvocationsAndReceipts({ + invocations, + receipts, + taskStore: context.taskStore, + receiptStore: context.receiptStore, + }) + assert.ok(storedInvocationsAndReceiptsRes.ok) + + const filecoinAddInv = Filecoin.accept.invoke({ + issuer: agent, + audience: connection.id, + with: agent.did(), + nb: { + piece: piece.link.link(), + content: piece.content.link(), + }, + }) + + const response = await filecoinAddInv.execute(connection) + if (response.out.error) { + throw new Error('invocation failed', { cause: response.out.error }) + } + assert.ok(response.out.ok) + assert.deepEqual( + response.out.ok.inclusion.subtree[1], + inclusionProof.ok[0][1] + ) + assert.deepEqual( + response.out.ok.inclusion.index[1], + inclusionProof.ok[1][1] + ) + assert.deepEqual( + BigInt(response.out.ok.inclusion.subtree[0]), + BigInt(inclusionProof.ok[0][0]) + ) + assert.deepEqual( + BigInt(response.out.ok.inclusion.index[0]), + BigInt(inclusionProof.ok[1][0]) + ) + assert.deepEqual( + BigInt(response.out.ok.aux.dataType), + BigInt(dealMetadata.dataType) + ) + assert.deepEqual( + BigInt(response.out.ok.aux.dataSource.dealID), + BigInt(dealMetadata.dataSource.dealID) + ) + }, + 'filecoin/accept fails if fails to read from piece store': + wichMockableContext( + async (assert, context) => { + const { agent } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + const filecoinSubmitInv = Filecoin.accept.invoke({ + issuer: agent, + audience: connection.id, + with: agent.did(), + nb: { + piece: cargo.link.link(), + content: cargo.content.link(), + }, + }) + + const response = await filecoinSubmitInv.execute(connection) + assert.ok(response.out.error) + assert.equal(response.out.error?.name, StoreOperationErrorName) + }, + (context) => ({ + ...context, + pieceStore: getStoreImplementations(FailingStore).storefront.pieceStore, + }) + ), +} + +/** + * @param {object} context + * @param {Record} context.invocations + * @param {Record} context.receipts + * @param {API.Store} context.taskStore + * @param {API.Store} context.receiptStore + */ +async function storeInvocationsAndReceipts({ + invocations, + receipts, + taskStore, + receiptStore, +}) { + // Store invocations + const storedInvocations = await Promise.all( + Object.values(invocations).map((invocation) => { + return taskStore.put(invocation) + }) + ) + if (storedInvocations.find((si) => si.error)) { + throw new Error('failed to store test invocations') + } + // Store receipts + const storedReceipts = await Promise.all( + Object.values(receipts).map((receipt) => { + return receiptStore.put(receipt) + }) + ) + if (storedReceipts.find((si) => si.error)) { + throw new Error('failed to store test receipts') + } + + return { + ok: {}, + } } async function getServiceContext() { const agent = await Signer.generate() + const aggregator = await Signer.generate() + const dealer = await Signer.generate() - return { agent } + return { agent, dealer, aggregator } +} + +/** + * @param {API.Test} testFn + * @param {(context: StorefrontApi.ServiceContext) => StorefrontApi.ServiceContext} mockContextFunction + */ +function wichMockableContext(testFn, mockContextFunction) { + // @ts-ignore + return function (...args) { + const modifiedArgs = [args[0], mockContextFunction(args[1])] + // @ts-ignore + return testFn(...modifiedArgs) + } } diff --git a/packages/filecoin-api/test/storefront.spec.js b/packages/filecoin-api/test/storefront.spec.js index 4de831499..64924e9d1 100644 --- a/packages/filecoin-api/test/storefront.spec.js +++ b/packages/filecoin-api/test/storefront.spec.js @@ -2,57 +2,140 @@ import * as assert from 'assert' import * as Signer from '@ucanto/principal/ed25519' -import * as Storefront from './services/storefront.js' +import * as StorefrontService from './services/storefront.js' +import * as StorefrontEvents from './events/storefront.js' -import { Store } from './context/store.js' +import { getStoreImplementations } from './context/store-implementations.js' import { Queue } from './context/queue.js' -import { validateAuthorization } from './helpers/utils.js' - -describe('filecoin/*', () => { - for (const [name, test] of Object.entries(Storefront.test)) { - const define = name.startsWith('only ') - ? it.only - : name.startsWith('skip ') - ? it.skip - : it - - define(name, async () => { - const signer = await Signer.generate() - const id = signer.withDID('did:web:test.web3.storage') - - // resources - /** @type {unknown[]} */ - const queuedMessages = [] - const addQueue = new Queue({ - onMessage: (message) => queuedMessages.push(message), - }) - const pieceLookupFn = ( - /** @type {Iterable | ArrayLike} */ items, - /** @type {any} */ record - ) => { - return Array.from(items).find((i) => i.piece.equals(record.piece)) - } - const pieceStore = new Store(pieceLookupFn) - - await test( - { - equal: assert.strictEqual, - deepEqual: assert.deepStrictEqual, - ok: assert.ok, - }, - { - id, - errorReporter: { - catch(error) { - assert.fail(error) +import { getMockService, getConnection } from './context/service.js' + +describe('storefront', () => { + describe('filecoin/*', () => { + for (const [name, test] of Object.entries(StorefrontService.test)) { + const define = name.startsWith('only ') + ? it.only + : name.startsWith('skip ') + ? it.skip + : it + + define(name, async () => { + const storefrontSigner = await Signer.generate() + const aggregatorSigner = await Signer.generate() + + // resources + /** @type {Map} */ + const queuedMessages = new Map() + queuedMessages.set('filecoinSubmitQueue', []) + queuedMessages.set('pieceOfferQueue', []) + const filecoinSubmitQueue = new Queue({ + onMessage: (message) => { + const messages = queuedMessages.get('filecoinSubmitQueue') || [] + messages.push(message) + queuedMessages.set('filecoinSubmitQueue', messages) + }, + }) + const pieceOfferQueue = new Queue({ + onMessage: (message) => { + const messages = queuedMessages.get('pieceOfferQueue') || [] + messages.push(message) + queuedMessages.set('pieceOfferQueue', messages) + }, + }) + const { + storefront: { pieceStore, receiptStore, taskStore }, + } = getStoreImplementations() + + await test( + { + equal: assert.strictEqual, + deepEqual: assert.deepStrictEqual, + ok: assert.ok, + }, + { + id: storefrontSigner, + aggregatorId: aggregatorSigner, + errorReporter: { + catch(error) { + assert.fail(error) + }, }, + pieceStore, + filecoinSubmitQueue, + pieceOfferQueue, + taskStore, + receiptStore, + queuedMessages, + } + ) + }) + } + }) + + describe('events', () => { + for (const [name, test] of Object.entries(StorefrontEvents.test)) { + const define = name.startsWith('only ') + ? it.only + : name.startsWith('skip ') + ? it.skip + : it + + define(name, async () => { + const storefrontSigner = await Signer.generate() + const aggregatorSigner = await Signer.generate() + + const service = getMockService() + const storefrontConnection = getConnection( + storefrontSigner, + service + ).connection + const aggregatorConnection = getConnection( + aggregatorSigner, + service + ).connection + + // context + const { + storefront: { pieceStore, taskStore, receiptStore }, + } = getStoreImplementations() + + await test( + { + equal: assert.strictEqual, + deepEqual: assert.deepStrictEqual, + ok: assert.ok, }, - addQueue, - pieceStore, - queuedMessages, - validateAuthorization, - } - ) - }) - } + { + id: storefrontSigner, + aggregatorId: aggregatorSigner, + pieceStore, + receiptStore, + taskStore, + storefrontService: { + connection: storefrontConnection, + invocationConfig: { + issuer: storefrontSigner, + with: storefrontSigner.did(), + audience: storefrontSigner, + }, + }, + aggregatorService: { + connection: aggregatorConnection, + invocationConfig: { + issuer: storefrontSigner, + with: storefrontSigner.did(), + audience: aggregatorSigner, + }, + }, + queuedMessages: new Map(), + service, + errorReporter: { + catch(error) { + assert.fail(error) + }, + }, + } + ) + }) + } + }) }) diff --git a/packages/filecoin-api/test/types.js b/packages/filecoin-api/test/types.js new file mode 100644 index 000000000..336ce12bb --- /dev/null +++ b/packages/filecoin-api/test/types.js @@ -0,0 +1 @@ +export {} diff --git a/packages/filecoin-api/test/types.ts b/packages/filecoin-api/test/types.ts new file mode 100644 index 000000000..03f9beb6e --- /dev/null +++ b/packages/filecoin-api/test/types.ts @@ -0,0 +1,50 @@ +import type { Signer } from '@ucanto/interface' +import * as AggregatorInterface from '../src/aggregator/api.js' +import * as DealerInterface from '../src/dealer/api.js' +import * as StorefrontInterface from '../src/storefront/api.js' + +export interface AggregatorTestEventsContext + extends AggregatorInterface.PieceMessageContext, + AggregatorInterface.PieceAcceptMessageContext, + AggregatorInterface.AggregateOfferMessageContext, + AggregatorInterface.PieceInsertEventContext, + AggregatorInterface.InclusionInsertEventToUpdateState, + AggregatorInterface.InclusionInsertEventToIssuePieceAccept, + AggregatorInterface.AggregateInsertEventToAggregateOfferContext, + AggregatorInterface.AggregateInsertEventToPieceAcceptQueueContext, + AggregatorInterface.BufferMessageContext { + id: Signer + service: Partial<{ + filecoin: Partial + piece: Partial + aggregate: Partial + deal: Partial + }> +} + +export interface DealerTestEventsContext + extends DealerInterface.AggregateInsertEventContext, + DealerInterface.AggregateUpdatedStatusEventContext, + DealerInterface.CronContext { + id: Signer + service: Partial<{ + filecoin: Partial + piece: Partial + aggregate: Partial + deal: Partial + }> +} + +export interface StorefrontTestEventsContext + extends StorefrontInterface.FilecoinSubmitMessageContext, + StorefrontInterface.PieceOfferMessageContext, + StorefrontInterface.StorefrontClientContext, + StorefrontInterface.CronContext { + id: Signer + service: Partial<{ + filecoin: Partial + piece: Partial + aggregate: Partial + deal: Partial + }> +} diff --git a/packages/filecoin-client/src/dealer.js b/packages/filecoin-client/src/dealer.js index aca3d7c52..1c6919a8c 100644 --- a/packages/filecoin-client/src/dealer.js +++ b/packages/filecoin-client/src/dealer.js @@ -82,7 +82,7 @@ export async function aggregateOffer( * * @param {import('./types.js').InvocationConfig} conf - Configuration * @param {import('@web3-storage/data-segment').PieceLink} aggregate - * @param {import('@web3-storage/data-segment').PieceLink[]} pieces + * @param {import('@ucanto/interface').Link} pieces * @param {import('./types.js').RequestOptions} [options] */ export async function aggregateAccept( @@ -93,8 +93,6 @@ export async function aggregateAccept( ) { /* c8 ignore next */ const conn = options.connection ?? connection - - const block = await CBOR.write(pieces) const invocation = Dealer.aggregateAccept.invoke({ issuer, /* c8 ignore next */ @@ -102,11 +100,10 @@ export async function aggregateAccept( with: resource, nb: { aggregate, - pieces: block.cid, + pieces, }, proofs, }) - invocation.attach(block) return await invocation.execute(conn) } diff --git a/packages/filecoin-client/test/aggregator.test.js b/packages/filecoin-client/test/aggregator.test.js index b70277de0..ad833588f 100644 --- a/packages/filecoin-client/test/aggregator.test.js +++ b/packages/filecoin-client/test/aggregator.test.js @@ -77,21 +77,22 @@ describe('aggregator', () => { it('aggregator accepts a filecoin piece', async () => { const { pieces, aggregate } = await randomAggregate(100, 100) + const piece = pieces[0].link const group = 'did:web:free.web3.storage' + // compute proof for piece in aggregate + const proof = aggregate.resolveProof(piece) + if (proof.error) { + throw new Error('could not compute proof') + } + /** @type {import('@web3-storage/capabilities/types').PieceAcceptSuccess} */ const pieceAcceptResponse = { - piece: pieces[0].link, + piece, aggregate: aggregate.link, inclusion: { - subtree: { - path: [], - index: 0n, - }, - index: { - path: [], - index: 0n, - }, + subtree: proof.ok[0], + index: proof.ok[1], }, } @@ -107,7 +108,7 @@ describe('aggregator', () => { assert.strictEqual(invCap.can, AggregatorCaps.pieceAccept.can) assert.equal(invCap.with, invocation.issuer.did()) // piece link - assert.ok(invCap.nb?.piece.equals(pieces[0].link)) + assert.ok(invCap.nb?.piece.equals(piece)) // group assert.strictEqual(invCap.nb?.group, group) @@ -124,7 +125,7 @@ describe('aggregator', () => { with: aggregatorService.did(), audience: aggregatorService, }, - pieces[0].link, + piece, group, { connection: getConnection(service).connection } ) diff --git a/packages/filecoin-client/test/dealer.test.js b/packages/filecoin-client/test/dealer.test.js index 95d3b6cd5..dc809fdfa 100644 --- a/packages/filecoin-client/test/dealer.test.js +++ b/packages/filecoin-client/test/dealer.test.js @@ -86,20 +86,11 @@ describe('dealer', () => { /** @type {import('@web3-storage/capabilities/types').AggregateAcceptSuccess} */ const aggregateAcceptResponse = { - inclusion: { - subtree: { - path: [], - index: 0n, - }, - index: { - path: [], - index: 0n, - }, - }, - auxDataType: 0n, - auxDataSource: { + dataType: 0n, + dataSource: { dealID: 1138n, }, + aggregate: aggregate.link, } // Create Ucanto service @@ -118,12 +109,6 @@ describe('dealer', () => { // piece link assert.ok(invCap.nb.aggregate.equals(aggregate.link.link())) - // Validate block inline exists - const invocationBlocks = Array.from(invocation.iterateIPLDBlocks()) - assert.ok( - invocationBlocks.find((b) => b.cid.equals(piecesBlock.cid)) - ) - return Server.ok(aggregateAcceptResponse) }, }), @@ -138,12 +123,16 @@ describe('dealer', () => { audience: dealerService, }, aggregate.link.link(), - offer, + piecesBlock.cid, { connection: getConnection(service).connection } ) assert.ok(res.out.ok) - assert.deepEqual(res.out.ok, aggregateAcceptResponse) + assert.ok(res.out.ok.aggregate.equals(aggregate.link)) + assert.deepEqual( + BigInt(res.out.ok.dataSource.dealID), + BigInt(aggregateAcceptResponse.dataSource.dealID) + ) // does not include effect fx in receipt assert.ok(!res.fx.join) }) @@ -182,12 +171,6 @@ describe('dealer', () => { // piece link assert.ok(invCap.nb.aggregate.equals(aggregate.link.link())) - // Validate block inline exists - const invocationBlocks = Array.from(invocation.iterateIPLDBlocks()) - assert.ok( - invocationBlocks.find((b) => b.cid.equals(piecesBlock.cid)) - ) - return { error: aggregateAcceptResponse, } @@ -204,7 +187,7 @@ describe('dealer', () => { audience: dealerService, }, aggregate.link.link(), - offer, + piecesBlock.cid, { connection: getConnection(service).connection } ) diff --git a/packages/filecoin-client/test/storefront.test.js b/packages/filecoin-client/test/storefront.test.js index 4c075ee5d..2aad9ad88 100644 --- a/packages/filecoin-client/test/storefront.test.js +++ b/packages/filecoin-client/test/storefront.test.js @@ -10,7 +10,7 @@ import { filecoinSubmit, filecoinAccept, } from '../src/storefront.js' -import { randomCargo } from './helpers/random.js' +import { randomAggregate, randomCargo } from './helpers/random.js' import { mockService } from './helpers/mocks.js' import { serviceProvider as storefrontService } from './fixtures.js' import { validateAuthorization } from './helpers/utils.js' @@ -132,23 +132,28 @@ describe('storefront', () => { }) it('storefront accepts a filecoin piece', async () => { - const [cargo] = await randomCargo(1, 100) + const { pieces, aggregate } = await randomAggregate(100, 100) + const cargo = pieces[0] + + // compute proof for piece in aggregate + const proof = aggregate.resolveProof(cargo.link) + if (proof.error) { + throw new Error('could not compute proof') + } /** @type {import('@web3-storage/capabilities/types').FilecoinAcceptSuccess} */ const filecoinAcceptResponse = { + aggregate: aggregate.link, + piece: cargo.link, inclusion: { - subtree: { - path: [], - index: 0n, - }, - index: { - path: [], - index: 0n, - }, + subtree: proof.ok[0], + index: proof.ok[1], }, - auxDataType: 0n, - auxDataSource: { - dealID: 1138n, + aux: { + dataType: 0n, + dataSource: { + dealID: 1138n, + }, }, } @@ -188,7 +193,13 @@ describe('storefront', () => { ) assert.ok(res.out.ok) - assert.deepEqual(res.out.ok, filecoinAcceptResponse) + assert.ok(res.out.ok.aggregate.equals(aggregate.link)) + assert.ok(res.out.ok.piece.equals(cargo.link)) + assert.equal( + BigInt(res.out.ok.aux.dataSource.dealID), + BigInt(filecoinAcceptResponse.aux.dataSource.dealID) + ) + assert.deepEqual(res.out.ok.inclusion, filecoinAcceptResponse.inclusion) // does not include effect fx in receipt assert.ok(!res.fx.join) })