feat: publish index claim (#1487)

This PR updates the `index/add` handler to publish an [index
claim](https://github.com/w3s-project/content-claims?tab=readme-ov-file#index-claim)
for the added index.

Reading an index from the index claim allows freeway and hoverboard to
serve blocks from all of the shards without having to request a location
claim for each block.

refs storacha/blob-fetcher#10

packages/capabilities/src/index/index.js

@@ -39,7 +39,7 @@ export const add = capability({
   with: SpaceDID,
   nb: Schema.struct({
     /** Content Archive (CAR) containing the `Index`. */
-    index: Schema.link({ code: CAR.code }),
+    index: Schema.link({ code: CAR.code, version: 1 }),
   }),
   derives: (claimed, delegated) =>
     and(equalWith(claimed, delegated)) ||

packages/upload-api/package.json

@@ -201,7 +201,7 @@
     "@web3-storage/access": "workspace:^",
     "@web3-storage/blob-index": "workspace:^",
     "@web3-storage/capabilities": "workspace:^",
-    "@web3-storage/content-claims": "^5.0.0",
+    "@web3-storage/content-claims": "^5.1.0",
     "@web3-storage/did-mailto": "workspace:^",
     "@web3-storage/filecoin-api": "workspace:^",
     "multiformats": "^12.1.2",

packages/upload-api/src/index/add.js

@@ -2,6 +2,7 @@ import * as Server from '@ucanto/server'
 import { ok, error } from '@ucanto/server'
 import * as Index from '@web3-storage/capabilities/index'
 import { ShardedDAGIndex } from '@web3-storage/blob-index'
+import { Assert } from '@web3-storage/content-claims/capability'
 import { concat } from 'uint8arrays'
 import * as API from '../types.js'
 
@@ -61,13 +62,21 @@ const add = async ({ capability }, context) => {
     shardDigests.map((s) => assertAllocated(context, space, s, 'ShardNotFound'))
   )
   for (const res of shardAllocRes) {
-    if (!res.ok) return res
+    if (res.error) return res
   }
 
   // TODO: randomly validate slices in the index correspond to slices in the blob
 
-  // publish the index data to IPNI
-  return context.ipniService.publish(idxRes.ok)
+  const publishRes = await Promise.all([
+    // publish the index data to IPNI
+    context.ipniService.publish(idxRes.ok),
+    // publish a content claim for the index
+    publishIndexClaim(context, { content: idxRes.ok.content, index: idxLink }),
+  ])
+  for (const res of publishRes) {
+    if (res.error) return res
+  }
+  return ok({})
 }
 
 /**
@@ -87,3 +96,23 @@ const assertAllocated = async (context, space, digest, errorName) => {
     )
   return ok({})
 }
+
+/**
+ * @param {API.ClaimsClientContext} ctx
+ * @param {{ content: API.UnknownLink, index: API.CARLink }} params
+ */
+const publishIndexClaim = async (ctx, { content, index }) => {
+  const { invocationConfig, connection } = ctx.claimsService
+  const { issuer, audience, with: resource, proofs } = invocationConfig
+  const res = await Assert.index
+    .invoke({
+      issuer,
+      audience,
+      with: resource,
+      nb: { content, index },
+      expiration: Infinity,
+      proofs,
+    })
+    .execute(connection)
+  return res.out
+}

packages/upload-api/src/types.ts

@@ -204,13 +204,21 @@ import { StorageGetError } from './types/storage.js'
 import { AllocationsStorage, BlobsStorage, BlobAddInput } from './types/blob.js'
 export type { AllocationsStorage, BlobsStorage, BlobAddInput }
 import { IPNIService, IndexServiceContext } from './types/index.js'
+import { ClaimsClientConfig } from './types/content-claims.js'
+import { Claim } from '@web3-storage/content-claims/client/api'
 export type {
   IndexServiceContext,
   IPNIService,
   BlobRetriever,
   BlobNotFound,
   ShardedDAGIndex,
 } from './types/index.js'
+export type {
+  ClaimsInvocationConfig,
+  ClaimsClientConfig,
+  ClaimsClientContext,
+  Service as ClaimsService,
+} from './types/content-claims.js'
 
 export interface Service extends StorefrontService, W3sService {
   store: {
@@ -590,6 +598,18 @@ export interface UcantoServerTestContext
   ipniService: IPNIService & {
     query(digest: MultihashDigest): Promise<Result<Unit, RecordNotFound>>
   }
+
+  carStoreBucket: CarStoreBucket & Deactivator
+  blobsStorage: BlobsStorage & Deactivator
+  claimsService: ClaimsClientConfig & ClaimReader & Deactivator
+}
+
+export interface ClaimReader {
+  read(digest: MultihashDigest): Promise<Result<Claim[], Failure>>
+}
+
+export interface Deactivator {
+  deactivate: () => Promise<void>
 }
 
 export interface StoreTestContext {}

packages/upload-api/src/types/content-claims.ts

@@ -0,0 +1,31 @@
+import {
+  ConnectionView,
+  DID,
+  Principal,
+  Proof,
+  Signer,
+} from '@ucanto/interface'
+import { Service } from '@web3-storage/content-claims/server/service/api'
+
+export type { ConnectionView, DID, Principal, Proof, Signer }
+export type { Service }
+
+export interface ClaimsInvocationConfig {
+  /** Signing authority issuing the UCAN invocation(s). */
+  issuer: Signer
+  /** The principal delegated to in the current UCAN. */
+  audience: Principal
+  /** The resource the invocation applies to. */
+  with: DID
+  /** Proof(s) the issuer has the capability to perform the action. */
+  proofs?: Proof[]
+}
+
+export interface ClaimsClientConfig {
+  invocationConfig: ClaimsInvocationConfig
+  connection: ConnectionView<Service>
+}
+
+export interface ClaimsClientContext {
+  claimsService: ClaimsClientConfig
+}

packages/upload-api/src/types/index.ts

@@ -2,8 +2,9 @@ import { MultihashDigest } from 'multiformats'
 import { Failure, Result, Unit } from '@ucanto/interface'
 import { ShardedDAGIndex } from '@web3-storage/blob-index/types'
 import { AllocationsStorage } from './blob.js'
+import { ClaimsClientContext } from './content-claims.js'
 
-export type { ShardedDAGIndex }
+export type { ShardedDAGIndex, ClaimsClientContext }
 
 /**
  * Service that allows publishing a set of multihashes to IPNI for a
@@ -26,7 +27,7 @@ export interface BlobRetriever {
   ): Promise<Result<ReadableStream<Uint8Array>, BlobNotFound>>
 }
 
-export interface IndexServiceContext {
+export interface IndexServiceContext extends ClaimsClientContext {
   allocationsStorage: AllocationsStorage
   blobRetriever: BlobRetriever
   ipniService: IPNIService

packages/upload-api/test/external-service/content-claims.js

@@ -0,0 +1,131 @@
+import * as API from '../../src/types.js'
+import { connect } from '@ucanto/client'
+import { ed25519 } from '@ucanto/principal'
+import { CAR, HTTP } from '@ucanto/transport'
+import { Assert } from '@web3-storage/content-claims/capability'
+import * as Client from '@web3-storage/content-claims/client'
+import * as Server from '@web3-storage/content-claims/server'
+import { DigestMap } from '@web3-storage/blob-index'
+
+/**
+ * @param {object} params
+ * @param {API.Signer} params.serviceSigner
+ * @param {API.Transport.Channel<API.ClaimsService>} params.channel
+ * @returns {Promise<API.ClaimsClientConfig>}
+ */
+export const create = async ({ serviceSigner, channel }) => {
+  const agent = await ed25519.generate()
+  const proofs = [
+    await Assert.assert.delegate({
+      issuer: serviceSigner,
+      with: serviceSigner.did(),
+      audience: agent,
+    }),
+  ]
+  return {
+    invocationConfig: {
+      issuer: agent,
+      with: serviceSigner.did(),
+      audience: serviceSigner,
+      proofs,
+    },
+    connection: connect({
+      id: serviceSigner,
+      codec: CAR.outbound,
+      channel,
+    }),
+  }
+}
+
+/**
+ * @param {{ http?: import('node:http') }} [options]
+ * @returns {Promise<API.ClaimsClientConfig & API.ClaimReader & API.Deactivator>}
+ */
+export const activate = async ({ http } = {}) => {
+  const serviceSigner = await ed25519.generate()
+
+  const claimStore = new ClaimStorage()
+  /** @param {API.MultihashDigest} content */
+  const read = async (content) => {
+    /** @type {import('@web3-storage/content-claims/client/api').Claim[]} */
+    const claims = []
+    await Server.walkClaims(
+      { claimFetcher: claimStore },
+      content,
+      new Set()
+    ).pipeTo(
+      new WritableStream({
+        async write(block) {
+          const claim = await Client.decode(block.bytes)
+          claims.push(claim)
+        },
+      })
+    )
+    return { ok: claims }
+  }
+
+  const server = Server.createServer({
+    id: serviceSigner,
+    codec: CAR.inbound,
+    claimStore,
+    validateAuthorization: () => ({ ok: {} }),
+  })
+
+  if (!http) {
+    const conf = await create({ serviceSigner, channel: server })
+    return Object.assign(conf, { read, deactivate: async () => {} })
+  }
+
+  const httpServer = http.createServer(async (req, res) => {
+    const chunks = []
+    for await (const chunk of req) {
+      chunks.push(chunk)
+    }
+
+    const { headers, body } = await server.request({
+      // @ts-expect-error
+      headers: req.headers,
+      body: new Uint8Array(await new Blob(chunks).arrayBuffer()),
+    })
+
+    res.writeHead(200, headers)
+    res.write(body)
+    res.end()
+  })
+  await new Promise((resolve) => httpServer.listen(resolve))
+  // @ts-expect-error
+  const { port } = httpServer.address()
+  const serviceURL = new URL(`http://127.0.0.1:${port}`)
+
+  const channel = HTTP.open({ url: serviceURL, method: 'POST' })
+  const conf = await create({ serviceSigner, channel })
+  return Object.assign(conf, {
+    read,
+    deactivate: () =>
+      new Promise((resolve, reject) => {
+        httpServer.closeAllConnections()
+        httpServer.close((err) => {
+          err ? reject(err) : resolve(undefined)
+        })
+      }),
+  })
+}
+
+class ClaimStorage {
+  constructor() {
+    /** @type {Map<API.MultihashDigest, import('@web3-storage/content-claims/server/api').Claim[]>} */
+    this.data = new DigestMap()
+  }
+
+  /** @param {import('@web3-storage/content-claims/server/api').Claim} claim */
+  async put(claim) {
+    const claims = this.data.get(claim.content) ?? []
+    claims.push(claim)
+    this.data.set(claim.content, claims)
+  }
+
+  /** @param {API.MultihashDigest} content */
+  async get(content) {
+    return this.data.get(content) ?? []
+  }
+}

packages/upload-api/test/external-service/index.js

@@ -1,5 +1,11 @@
-import { IPNIService } from './ipni-service.js'
+import { IPNIService } from './ipni.js'
+import * as ClaimsService from './content-claims.js'
 
-export const getExternalServiceImplementations = async () => ({
+/**
+ * @param {object} [options]
+ * @param {import('node:http')} [options.http]
+ */
+export const getExternalServiceImplementations = async (options) => ({
   ipniService: new IPNIService(),
+  claimsService: await ClaimsService.activate(options),
 })

packages/upload-api/test/handlers/index.js

@@ -171,4 +171,78 @@ export const test = {
     assert.ok(receipt.out.error)
     assert.equal(receipt.out.error?.name, 'ShardNotFound')
   },
+  'index/add should publish index claim': async (assert, context) => {
+    const { proof, spaceDid } = await registerSpace(alice, context)
+    const contentCAR = await randomCAR(32)
+    const contentCARBytes = new Uint8Array(await contentCAR.arrayBuffer())
+
+    const connection = connect({
+      id: context.id,
+      channel: createServer(context),
+    })
+
+    // upload the content CAR to the space
+    await uploadBlob(
+      context,
+      {
+        connection,
+        issuer: alice,
+        audience: context.id,
+        with: spaceDid,
+        proofs: [proof],
+      },
+      {
+        cid: contentCAR.cid,
+        bytes: contentCARBytes,
+      }
+    )
+
+    const index = await fromShardArchives(contentCAR.roots[0], [
+      contentCARBytes,
+    ])
+    const indexCAR = Result.unwrap(await index.archive())
+    const indexLink = await CAR.link(indexCAR)
+
+    // upload the index CAR to the space
+    await uploadBlob(
+      context,
+      {
+        connection,
+        issuer: alice,
+        audience: context.id,
+        with: spaceDid,
+        proofs: [proof],
+      },
+      {
+        cid: indexLink,
+        bytes: indexCAR,
+      }
+    )
+
+    const indexAdd = IndexCapabilities.add.invoke({
+      issuer: alice,
+      audience: context.id,
+      with: spaceDid,
+      nb: { index: indexLink },
+      proofs: [proof],
+    })
+    const receipt = await indexAdd.execute(connection)
+    Result.try(receipt.out)
+
+    // ensure an index claim exists for the content root
+    const claims = Result.unwrap(
+      await context.claimsService.read(contentCAR.roots[0].multihash)
+    )
+
+    let found = false
+    for (const c of claims) {
+      if (
+        c.type === 'assert/index' &&
+        c.index.toString() === indexLink.toString()
+      ) {
+        found = true
+      }
+    }
+    assert.ok(found, 'did not found index claim')
+  },
 }