From 05c1ced291c5c77a9486f39e1adce8eff5377668 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 26 Jul 2023 16:15:03 +0100 Subject: [PATCH] feat: wip --- packages/upload-client/README.md | 4 + packages/upload-client/src/sharding.js | 25 +++++- packages/upload-client/src/types.ts | 8 ++ packages/w3up-client/package.json | 6 +- packages/w3up-client/src/base.js | 4 +- packages/w3up-client/src/capability/assert.js | 86 +++++++++++++++++++ packages/w3up-client/src/client.js | 7 +- packages/w3up-client/src/content-claims.js | 71 +++++++++++++++ packages/w3up-client/src/service.js | 2 + packages/w3up-client/src/types.ts | 2 + pnpm-lock.yaml | 60 ++++++++++++- 11 files changed, 267 insertions(+), 8 deletions(-) create mode 100644 packages/w3up-client/src/capability/assert.js create mode 100644 packages/w3up-client/src/content-claims.js diff --git a/packages/upload-client/README.md b/packages/upload-client/README.md index 2805d6ae8..124103c21 100644 --- a/packages/upload-client/README.md +++ b/packages/upload-client/README.md @@ -466,6 +466,10 @@ export interface CARMetadata { * Size of the CAR file in bytes. */ size: number + /** + * The CAR file data that was stored. + */ + blob(): Promise } ``` diff --git a/packages/upload-client/src/sharding.js b/packages/upload-client/src/sharding.js index 282f13d18..0081350a7 100644 --- a/packages/upload-client/src/sharding.js +++ b/packages/upload-client/src/sharding.js @@ -97,7 +97,7 @@ export class ShardStoringStream extends TransformStream { const opts = { ...options, signal: abortController.signal } const cid = await add(conf, car, opts) const { version, roots, size } = car - controller.enqueue({ version, roots, cid, size }) + controller.enqueue(new ShardMetadata(version, roots, cid, size, car)) } catch (err) { controller.error(err) abortController.abort(err) @@ -117,3 +117,26 @@ export class ShardStoringStream extends TransformStream { }) } } + +class ShardMetadata { + #blob + + /** + * @param {number} version + * @param {import('multiformats').UnknownLink[]} roots + * @param {import('./types').CARLink} cid + * @param {number} size + * @param {Blob} blob + */ + constructor (version, roots, cid, size, blob) { + this.version = version + this.roots = roots + this.cid = cid + this.size = size + this.#blob = blob + } + + async blob () { + return this.#blob + } +} diff --git a/packages/upload-client/src/types.ts b/packages/upload-client/src/types.ts index 0c06868cf..f5accd09d 100644 --- a/packages/upload-client/src/types.ts +++ b/packages/upload-client/src/types.ts @@ -175,6 +175,10 @@ export interface CARMetadata extends CARHeaderInfo { * Size of the CAR file in bytes. */ size: number + /** + * The CAR file data that was stored. + */ + blob(): Promise } export interface Retryable { @@ -254,6 +258,10 @@ export interface UploadOptions ShardingOptions, ShardStoringOptions, UploadProgressTrackable { + /** + * A function called after a DAG shard has been successfully stored by the + * service. + */ onShardStored?: (meta: CARMetadata) => void } diff --git a/packages/w3up-client/package.json b/packages/w3up-client/package.json index 36a7f637d..56f0e54fe 100644 --- a/packages/w3up-client/package.json +++ b/packages/w3up-client/package.json @@ -81,7 +81,11 @@ "@ucanto/transport": "^8.0.0", "@web3-storage/access": "workspace:^", "@web3-storage/capabilities": "workspace:^", - "@web3-storage/upload-client": "workspace:^" + "@web3-storage/content-claims": "^3.0.1", + "@web3-storage/upload-client": "workspace:^", + "cardex": "^2.3.1", + "carstream": "^1.1.0", + "p-queue": "^7.3.0" }, "devDependencies": { "@docusaurus/core": "^2.2.0", diff --git a/packages/w3up-client/src/base.js b/packages/w3up-client/src/base.js index dd337470f..170fb449b 100644 --- a/packages/w3up-client/src/base.js +++ b/packages/w3up-client/src/base.js @@ -17,10 +17,10 @@ export class Base { /** * @param {import('@web3-storage/access').AgentData} agentData * @param {object} [options] - * @param {import('./types').ServiceConf} [options.serviceConf] + * @param {Partial} [options.serviceConf] */ constructor(agentData, options = {}) { - this._serviceConf = options.serviceConf ?? serviceConf + this._serviceConf = { ...serviceConf, ...options.serviceConf } this._agent = new Agent(agentData, { servicePrincipal: this._serviceConf.access.id, // @ts-expect-error I know but it will be HTTP for the forseeable. diff --git a/packages/w3up-client/src/capability/assert.js b/packages/w3up-client/src/capability/assert.js new file mode 100644 index 000000000..f7054b9f9 --- /dev/null +++ b/packages/w3up-client/src/capability/assert.js @@ -0,0 +1,86 @@ +import { Assert } from '@web3-storage/content-claims/capability' +import { Base } from '../base.js' + +/** + * Client for interacting with the content claim `assert/*` capabilities. + */ +export class AssertClient extends Base { + /** + * Claims that a CID is available at a URL. + * + * @param {import('multiformats').UnknownLink} content - Claim subject. + * @param {URL[]} location - Location(s) the content may be found. + */ + async location(content, location) { + const conf = await this._invocationConfig([Assert.location.can]) + const locs = location.map(l => /** @type {import('@ucanto/interface').URI} */ (l.toString())) + const result = await Assert.location + .invoke({ ...conf, nb: { content, location: locs } }) + .execute(this._serviceConf.claim) + if (result.out.error) { + const cause = result.out.error + throw new Error(`failed ${Assert.location.can} invocation`, { cause }) + } + return result.out.ok + } + + /** + * Claims that a CID's graph can be read from the blocks found in parts. + * + * @param {import('multiformats').UnknownLink} content - Claim subject. + * @param {import('multiformats').Link|undefined} blocks - CIDs CID. + * @param {import('multiformats').Link[]} parts - CIDs of CAR files the content can be found within. + */ + async partition(content, blocks, parts) { + const conf = await this._invocationConfig([Assert.partition.can]) + const result = await Assert.partition + .invoke({ ...conf, nb: { content, blocks, parts } }) + .execute(this._serviceConf.claim) + if (result.out.error) { + const cause = result.out.error + throw new Error(`failed ${Assert.partition.can} invocation`, { cause }) + } + return result.out.ok + } + + /** + * Claims that a CID includes the contents claimed in another CID. + * + * @param {import('multiformats').UnknownLink} content - Claim subject. + * @param {import('multiformats').Link} includes - Contents the claim content includes. + * @param {import('multiformats').Link} [proof] - Inclusion proof. + */ + async inclusion(content, includes, proof) { + const conf = await this._invocationConfig([Assert.inclusion.can]) + const result = await Assert.inclusion + .invoke({ ...conf, nb: { content, includes, proof } }) + .execute(this._serviceConf.claim) + + if (result.out.error) { + const cause = result.out.error + throw new Error(`failed ${Assert.inclusion.can} invocation`, { cause }) + } + + return result.out.ok + } + + /** + * Claim that a CID is linked to directly or indirectly by another CID. + * + * @param {import('multiformats').UnknownLink} content - Claim subject. + * @param {import('multiformats').UnknownLink} ancestor - Ancestor content CID. + */ + async descendant(content, ancestor) { + const conf = await this._invocationConfig([Assert.descendant.can]) + const result = await Assert.descendant + .invoke({ ...conf, nb: { content, ancestor } }) + .execute(this._serviceConf.claim) + + if (result.out.error) { + const cause = result.out.error + throw new Error(`failed ${Assert.descendant.can} invocation`, { cause }) + } + + return result.out.ok + } +} diff --git a/packages/w3up-client/src/client.js b/packages/w3up-client/src/client.js index a8e716e2a..e005a33ca 100644 --- a/packages/w3up-client/src/client.js +++ b/packages/w3up-client/src/client.js @@ -14,6 +14,7 @@ import { StoreClient } from './capability/store.js' import { UploadClient } from './capability/upload.js' import { SpaceClient } from './capability/space.js' import { AccessClient } from './capability/access.js' +import { ContentClaimsBuilder } from './content-claims.js' export class Client extends Base { /** @@ -59,7 +60,11 @@ export class Client extends Base { UploadCapabilities.add.can, ]) options.connection = this._serviceConf.upload - return uploadFile(conf, file, options) + const claimsBuilder = new ContentClaimsBuilder(this.capability.store) + const root = await uploadFile(conf, file, options) + const claims = await claimsBuilder.setRoot(root).close() + // TODO: submit claims + return root } /** diff --git a/packages/w3up-client/src/content-claims.js b/packages/w3up-client/src/content-claims.js new file mode 100644 index 000000000..16b297906 --- /dev/null +++ b/packages/w3up-client/src/content-claims.js @@ -0,0 +1,71 @@ +import { MultihashIndexSortedWriter } from 'cardex/multihash-index-sorted' +import { CARReaderStream } from 'carstream' +import Queue from 'p-queue' +import { CAR } from '@web3-storage/upload-client' +import * as Link from 'multiformats/link' +import { sha256 } from 'multiformats/hashes/sha2' + +export class ContentClaimsBuilder { + #store + #queue + /** @type {import('multiformats').UnknownLink|undefined} */ + #root + /** @type {import('./types').CARLink[]} */ + #shards + + /** + * @param {import('./capability/store').StoreClient} store + */ + constructor (store) { + this.#store = store + this.#queue = new Queue() + this.#shards = [] + } + + /** + * @param {import('./types').CARLink} shard + * @param {Blob} data + */ + async addShard (shard, data) { + this.#shards.push(shard) + this.#queue.add(async () => { + const { readable, writable } = new TransformStream() + const writer = MultihashIndexSortedWriter.createWriter({ writer: writable.getWriter() }) + + const [, indexBlock] = await Promise.all([ + data.stream() + .pipeThrough(new CARReaderStream()) + .pipeTo(new WritableStream({ + async write (block) { await writer.add(block.cid, block.offset) }, + async close () { await writer.close() } + })), + (async () => { + const bytes = new Uint8Array(await new Response(readable).arrayBuffer()) + const cid = Link.create(MultihashIndexSortedWriter.codec, await sha256.digest(bytes)) + return { cid, bytes } + })() + ]) + + const car = await CAR.encode([indexBlock], indexBlock.cid) + const indexCARCID = await this.#store.add(car) + + // TODO: create inclusion claim for shard => indexBlock.cid + // TODO: create partition claim for indexBlock.cid => indexCARCID + // TODO: create relation claims for block => shard => indexBlock.cid => indexCARCID + }) + return this + } + + /** + * @param {import('multiformats').UnknownLink} root + */ + setRoot (root) { + this.#root = root + return this + } + + async close () { + await this.#queue.onIdle() + // TODO: generate partition claim for root + } +} \ No newline at end of file diff --git a/packages/w3up-client/src/service.js b/packages/w3up-client/src/service.js index bff1c2951..c4d38cc9d 100644 --- a/packages/w3up-client/src/service.js +++ b/packages/w3up-client/src/service.js @@ -1,6 +1,7 @@ import { connect } from '@ucanto/client' import { CAR, HTTP } from '@ucanto/transport' import * as DID from '@ipld/dag-ucan/did' +import { connection as claimServiceConnection } from '@web3-storage/content-claims/client' export const accessServiceURL = new URL('https://up.web3.storage') export const accessServicePrincipal = DID.parse('did:web:web3.storage') @@ -30,4 +31,5 @@ export const uploadServiceConnection = connect({ export const serviceConf = { access: accessServiceConnection, upload: uploadServiceConnection, + claim: claimServiceConnection, } diff --git a/packages/w3up-client/src/types.ts b/packages/w3up-client/src/types.ts index 3b9575230..7d7add02c 100644 --- a/packages/w3up-client/src/types.ts +++ b/packages/w3up-client/src/types.ts @@ -4,12 +4,14 @@ import { type AgentDataExport, } from '@web3-storage/access/types' import { type Service as UploadService } from '@web3-storage/upload-client/types' +import type { Service as ClaimService } from '@web3-storage/content-claims/server/service/api' import type { ConnectionView, Signer, DID } from '@ucanto/interface' import { type Client } from './client' export interface ServiceConf { access: ConnectionView upload: ConnectionView + claim: ConnectionView } export interface ClientFactoryOptions { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7d1de0fc6..2fde61d83 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1,4 +1,4 @@ -lockfileVersion: '6.1' +lockfileVersion: '6.0' settings: autoInstallPeers: true @@ -513,9 +513,21 @@ importers: '@web3-storage/capabilities': specifier: workspace:^ version: link:../capabilities + '@web3-storage/content-claims': + specifier: ^3.0.1 + version: 3.0.1 '@web3-storage/upload-client': specifier: workspace:^ version: link:../upload-client + cardex: + specifier: ^2.3.1 + version: 2.3.1 + carstream: + specifier: ^1.1.0 + version: 1.1.0 + p-queue: + specifier: ^7.3.0 + version: 7.3.0 devDependencies: '@docusaurus/core': specifier: ^2.2.0 @@ -2356,6 +2368,14 @@ packages: cborg: 1.10.2 multiformats: 11.0.2 + /@ipld/dag-cbor@9.0.3: + resolution: {integrity: sha512-A2UFccS0+sARK9xwXiVZIaWbLbPxLGP3UZOjBeOMWfDY04SXi8h1+t4rHBzOlKYF/yWNm3RbFLyclWO7hZcy4g==} + engines: {node: '>=16.0.0', npm: '>=7.0.0'} + dependencies: + cborg: 2.0.2 + multiformats: 12.0.1 + dev: false + /@ipld/dag-json@10.1.2: resolution: {integrity: sha512-z38JDQXzDW6mtU+ZfLO6/lXbJ4BEEDYY5cyW6+Nl7OpjWSV0mt57cE8LK6+krXlhxwuCnA+/sOtaXuJ3lImvfw==} engines: {node: '>=16.0.0', npm: '>=7.0.0'} @@ -3351,6 +3371,16 @@ packages: web-streams-polyfill: 3.2.1 dev: false + /@web3-storage/content-claims@3.0.1: + resolution: {integrity: sha512-7+JLmQaw4mDgs+VREtkHtk3Q0Bp1rv4Kt5YN2Qw/g73xvRja28/Uioy7iv83YHez52lAPwXFkSkGb0ifJy5PLw==} + dependencies: + '@ucanto/client': 8.0.0 + '@ucanto/server': 8.0.1 + '@ucanto/transport': 8.0.0 + carstream: 1.1.0 + multiformats: 12.0.1 + dev: false + /@web3-storage/data-segment@1.0.1: resolution: {integrity: sha512-qgVSLN/VZhNgprFJvzLTK4wGTAQYpQ9O42FrskGxlDgGjBx7ZCw4VL8mgzDVhR4MHXL/yA/bXFQtm5JST+JAZQ==} dependencies: @@ -4123,6 +4153,25 @@ packages: resolution: {integrity: sha512-2S9nK0G/mE+jasCUsMPlARhRCts1ebcp2Ji8Y8PWi4NDE1iRdLCnEPHkEfeBrGC45L4isBx5ur3IQ6yTE2mRZw==} dev: true + /cardex@2.3.1: + resolution: {integrity: sha512-850vVrRGg48z3aPuEmPvDjQBs+Bp4shSEF8Smmr412NvhCJ7ka2kBQzruoBF6FrotexGxftvmILRH3gUTZQmpQ==} + hasBin: true + dependencies: + '@ipld/car': 5.1.1 + multiformats: 11.0.2 + sade: 1.8.1 + uint8arrays: 4.0.3 + varint: 6.0.0 + dev: false + + /carstream@1.1.0: + resolution: {integrity: sha512-tbf8FOnGX1+0kOe77nm9MG53REiqQopDwzwbXYVxUcsKOAHG2KSD++qy95v1vrtRt1Q6L0Sb01it7QwJ+Yt1sQ==} + dependencies: + '@ipld/dag-cbor': 9.0.3 + multiformats: 12.0.1 + uint8arraylist: 2.4.3 + dev: false + /cborg@1.10.2: resolution: {integrity: sha512-b3tFPA9pUr2zCUiCfRd2+wok2/LBSNUMKOuRRok+WlvvAgEt/PlbgPTsZUcwCOs53IJvLgTp0eotwtosE6njug==} hasBin: true @@ -8338,7 +8387,6 @@ packages: /mri@1.2.0: resolution: {integrity: sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==} engines: {node: '>=4'} - dev: true /mrmime@1.0.1: resolution: {integrity: sha512-hzzEagAgDyoU1Q6yg5uI+AorQgdvMCur3FcKf7NhMKWsaYg+RnbTyHRa/9IlLF9rf455MOCtcqqrQQ83pPP7Uw==} @@ -10148,7 +10196,6 @@ packages: engines: {node: '>=6'} dependencies: mri: 1.2.0 - dev: true /safe-buffer@5.1.2: resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==} @@ -11191,6 +11238,13 @@ packages: dev: true optional: true + /uint8arraylist@2.4.3: + resolution: {integrity: sha512-oEVZr4/GrH87K0kjNce6z8pSCzLEPqHNLNR5sj8cJOySrTP8Vb/pMIbZKLJGhQKxm1TiZ31atNrpn820Pyqpow==} + engines: {node: '>=16.0.0', npm: '>=7.0.0'} + dependencies: + uint8arrays: 4.0.3 + dev: false + /uint8arrays@4.0.3: resolution: {integrity: sha512-b+aKlI2oTnxnfeSQWV1sMacqSNxqhtXySaH6bflvONGxF8V/fT3ZlYH7z2qgGfydsvpVo4JUgM/Ylyfl2YouCg==} engines: {node: '>=16.0.0', npm: '>=7.0.0'}