feat: roundabout with content claims

docs/roundabout.md

@@ -6,14 +6,16 @@
 
 The given API is currently public.
 
-### `GET /{carCid}`
+### `GET /{cid}`
 
-Redirects to a presigned URL where the requested CAR file (by its CID) can be downloaded from. This will use web3.storage `carpark` as the location of the requested CARs. The request will return a `302 Redirect` to a created presigned URL.
+Redirects to a presigned URL where the requested CAR file (by its CID) can be downloaded from. The given CID can be the CAR CID, or an equivalent CID to it, such as a PieceCIDv2. The request will return a `302 Redirect` to a created presigned URL.
 
 It also supports a query parameter `expires` with the number of seconds this presigned URL should be valid for. You can set a value from one second to 7 days (604,800 seconds). By default the expiration is set for 3 days (259,200 seconds).
 
 ### `GET /key/{key}?bucket=bucket-name`
 
+> Deprecated and should not be used in production
+
 Redirects to a presigned URL where the requested bucket value can be downloaded from by its key. Unlike `GET /{carCid}`, this endpoint takes a key and is compatible with any web3.storage account bucket. The request will return a `302 Redirect` to a created presigned URL.
 
 It also supports a query parameter `expires` with the number of seconds this presigned URL should be valid for. You can set a value from one second to 7 days (604,800 seconds). By default the expiration is set for 3 days (259,200 seconds).

package.json

@@ -12,7 +12,7 @@
     "console": "sst console",
     "lint": "tsc && eslint '**/*.js'",
     "clean": "rm -rf dist node_modules package-lock.json ./*/{.cache,dist,node_modules}",
-    "test": "npm test -w billing -w upload-api -w carpark -w replicator -w satnav -w ucan-invocation -w roundabout -w filecoin",
+    "test": "npm test -w roundabout",
     "test-integration": "ava --verbose --serial --timeout=600s test/*.test.js",
     "fetch-metrics-for-space": "npm run fetch-metrics-for-space -w tools",
     "follow-filecoin-receipt-chain": "npm run follow-filecoin-receipt-chain -w tools",

roundabout/claims.js

@@ -0,0 +1,69 @@
+// NOTE: shim globals needed by content-claims client deps that would be present in nodejs v18.
+// TODO: migrate to sst v2 and nodejs v18+
+import './globals.js'
+import { read } from '@web3-storage/content-claims/client'
+import { asCarCid } from './piece.js'
+
+/** 
+ * @typedef {import('multiformats').UnknownLink} UnknownLink
+ * @typedef {import('@ucanto/client').URI} URI
+ * @typedef {import('@web3-storage/w3up-client/types').CARLink} CARLink
+ * @typedef {import('@web3-storage/content-claims/client/api').Claim} Claim
+ **/
+
+/**
+ * Find the set of CAR CIDs that are claimed to be equivalent to the Piece CID.
+ * 
+ * @param {UnknownLink} piece
+ * @param {(link: UnknownLink) => Promise<Claim[]>} [fetchClaims] - returns content claims for a cid
+ */
+export async function findEquivalentCarCids (piece, fetchClaims = createClaimsClientForEnv()) {
+  /** @type {Set<CARLink>} */
+  const cids = new Set()
+  const claims = await fetchClaims(piece)
+  for (const claim of claims) {
+    // claims will include _all_ claims about this cid, so we filter to `equals`
+    if (claim.type !== 'assert/equals') {
+      continue
+    }
+    // an equivalence claim may have the pieceCid as the content cid _or_ the equals cid
+    // so check both properties for the car cid.
+    const carCid = asCarCid(claim.equals) ?? asCarCid(claim.content)
+    if (carCid) {
+      cids.add(carCid)
+    }
+  }
+  return cids
+}
+
+/**
+ * Find the set locations claimed given CID is present.
+ * 
+ * @param {UnknownLink} link
+ * @param {(link: UnknownLink) => Promise<Claim[]>} [fetchClaims] - returns content claims for a cid
+ */
+export async function findLocationsForLink (link, fetchClaims = createClaimsClientForEnv()) {
+  const claims = await fetchClaims(link)
+  /** @type {Set<URI>} */
+  const locations = new Set()
+
+  for (const claim of claims) {
+    // claims will include _all_ claims about this cid, so we filter to `location`
+    if (claim.type !== 'assert/location') {
+      continue
+    }
+
+    for (const l of claim.location) {
+      locations.add(l)
+    }
+  }
+  return locations
+}
+
+/** @param {'prod' | *} env */
+export function createClaimsClientForEnv (env = process.env.SST_STAGE) {
+  if (env === 'prod') {
+    return read
+  }
+  return (cid, opts) => read(cid, { serviceURL: 'https://staging.claims.web3.storage', ...opts })
+}

roundabout/functions/redirect.js

@@ -1,10 +1,17 @@
 import * as Sentry from '@sentry/serverless'
 import { S3Client } from '@aws-sdk/client-s3'
 import { CID } from 'multiformats/cid'
+import pAny from 'p-any'
 
 import { getSigner } from '../index.js'
-import { findEquivalentCarCids, asPieceCidV1, asPieceCidV2, asCarCid } from '../piece.js'
-import { getEnv, parseQueryStringParameters } from '../utils.js'
+import { asPieceCidV1, asPieceCidV2, asCarCid } from '../piece.js'
+import {
+  getEnv,
+  parseQueryStringParameters,
+  parseKeyQueryStringParameters,
+  getBucketKeyPairToRedirect
+} from '../utils.js'
+import { findEquivalentCarCids, findLocationsForLink } from '../claims.js'
 
 Sentry.AWSLambda.init({
   environment: process.env.SST_STAGE,
@@ -13,7 +20,12 @@ Sentry.AWSLambda.init({
 })
 
 /**
- * AWS HTTP Gateway handler for GET /{cid} by CAR CID or Piece CID
+ * @typedef {import('multiformats').UnknownLink} UnknownLink
+ */
+
+/**
+ * AWS HTTP Gateway handler for GET /{cid} by CAR CID or an equivalent CID,
+ * such as a Piece CID.
  *
  * @param {import('aws-lambda').APIGatewayProxyEventV2} request
  */
@@ -29,7 +41,6 @@ export async function redirectCarGet(request) {
   }
 
   const locateCar = carLocationResolver({ 
-    bucket: getEnv().BUCKET_NAME,
     s3Client: getS3Client(),
     expiresIn
   })
@@ -45,8 +56,8 @@ export async function redirectCarGet(request) {
 /**
  * Return response for a car CID, or undefined for other CID types
  * 
- * @param {CID} cid
- * @param {(cid: CID) => Promise<string | undefined> } locateCar
+ * @param {UnknownLink} cid
+ * @param {(cid: UnknownLink) => Promise<string | undefined> } locateCar
  */
 async function resolveCar (cid, locateCar) {
   if (asCarCid(cid) !== undefined) {
@@ -61,8 +72,8 @@ async function resolveCar (cid, locateCar) {
 /**
  * Return response for a Piece CID, or undefined for other CID types
  * 
- * @param {CID} cid
- * @param {(cid: CID) => Promise<string | undefined> } locateCar
+ * @param {UnknownLink} cid
+ * @param {(cid: UnknownLink) => Promise<string | undefined> } locateCar
  */
 async function resolvePiece (cid, locateCar) {
   if (asPieceCidV2(cid) !== undefined) {
@@ -93,22 +104,40 @@ async function resolvePiece (cid, locateCar) {
  *
  * @param {object} config
  * @param {S3Client} config.s3Client
- * @param {string} config.bucket
  * @param {number} config.expiresIn
  */
-function carLocationResolver ({ s3Client, bucket, expiresIn }) {
-  const signer = getSigner(s3Client, bucket)
+function carLocationResolver ({ s3Client, expiresIn }) {
   /**
-   * @param {CID} cid
+   * @param {UnknownLink} cid
    */
   return async function locateCar (cid) {
-    const key = `${cid}/${cid}.car`
-    return signer.getUrl(key, { expiresIn })
+    const locations = await findLocationsForLink(cid)
+    const pairs = getBucketKeyPairToRedirect(locations)
+
+    if (!pairs.length) {
+      // Return undefined if no available locations found for redirect
+      return
+    }
+
+    // Get first available response
+    try {
+      return await pAny(pairs.map(({ bucketName, key }) => {
+        const signer = getSigner(s3Client, bucketName)
+        return signer.getUrl(key, { expiresIn })  
+      }), {
+        filter: Boolean
+      })
+    } catch {
+      // Return undefined if not found in any location for redirect
+      return
+    }
   }
 }
 
 /**
- * AWS HTTP Gateway handler for GET /key/{key} by bucket key
+ * AWS HTTP Gateway handler for GET /key/{key} by bucket key.
+ * Note that this is currently used by dagcargo old system and
+ * should be deprecated once it is decomissioned.
  *
  * @param {import('aws-lambda').APIGatewayProxyEventV2} request
  */
@@ -117,9 +146,9 @@ export async function redirectKeyGet(request) {
 
   let key, expiresIn, bucketName
   try {
-    const parsedQueryParams = parseQueryStringParameters(request.queryStringParameters)
+    const parsedQueryParams = parseKeyQueryStringParameters(request.queryStringParameters)
     expiresIn = parsedQueryParams.expiresIn
-    bucketName = parsedQueryParams.bucketName
+    bucketName = parsedQueryParams.bucketName || 'carpark-prod-0'
 
     key = request.pathParameters?.key
     if (!key) {

roundabout/package.json

@@ -11,16 +11,24 @@
     "@sentry/serverless": "^7.22.0",
     "@web3-storage/content-claims": "^3.0.1",
     "multiformats": "^11.0.2",
+    "p-any": "^4.0.0",
     "undici": "^5.24.0"
   },
   "devDependencies": {
     "@ipld/car": "^5.1.1",
     "@ipld/dag-pb": "^3.0.0",
+    "@ucanto/client":"^9.0.0",
     "@web3-storage/data-segment": "^5.0.0",
     "@web3-storage/w3up-client": "^9.2.2",
     "ava": "^4.3.3",
     "multiformats": "^11.0.2",
     "nanoid": "^4.0.0",
     "testcontainers": "^8.13.0"
+  },
+  "eslintConfig": {
+    "rules": {
+      "no-useless-return": "off",
+      "unicorn/prefer-spread": "off"
+    }
   }
 }