feat: roundabout with content claims

docs/roundabout.md

@@ -6,14 +6,16 @@
 
 The given API is currently public.
 
-### `GET /{carCid}`
+### `GET /{cid}`
 
-Redirects to a presigned URL where the requested CAR file (by its CID) can be downloaded from. This will use web3.storage `carpark` as the location of the requested CARs. The request will return a `302 Redirect` to a created presigned URL.
+Redirects to a presigned URL where the requested CAR file (by its CID) can be downloaded from. The given CID can be the CAR CID, or an equivalent CID to it, such as a PieceCIDv2. The request will return a `302 Redirect` to a created presigned URL.
 
 It also supports a query parameter `expires` with the number of seconds this presigned URL should be valid for. You can set a value from one second to 7 days (604,800 seconds). By default the expiration is set for 3 days (259,200 seconds).
 
 ### `GET /key/{key}?bucket=bucket-name`
 
+> Deprecated and should not be used in production
+
 Redirects to a presigned URL where the requested bucket value can be downloaded from by its key. Unlike `GET /{carCid}`, this endpoint takes a key and is compatible with any web3.storage account bucket. The request will return a `302 Redirect` to a created presigned URL.
 
 It also supports a query parameter `expires` with the number of seconds this presigned URL should be valid for. You can set a value from one second to 7 days (604,800 seconds). By default the expiration is set for 3 days (259,200 seconds).

roundabout/claims.js

@@ -0,0 +1,69 @@
+// NOTE: shim globals needed by content-claims client deps that would be present in nodejs v18.
+// TODO: migrate to sst v2 and nodejs v18+
+import './globals.js'
+import { read } from '@web3-storage/content-claims/client'
+import { asCarCid } from './piece.js'
+
+/** 
+ * @typedef {import('multiformats').UnknownLink} UnknownLink
+ * @typedef {import('@ucanto/client').URI} URI
+ * @typedef {import('@web3-storage/w3up-client/types').CARLink} CARLink
+ * @typedef {import('@web3-storage/content-claims/client/api').Claim} Claim
+ **/
+
+/**
+ * Find the set of CAR CIDs that are claimed to be equivalent to the Piece CID.
+ * 
+ * @param {UnknownLink} piece
+ * @param {(link: UnknownLink) => Promise<Claim[]>} [fetchClaims] - returns content claims for a cid
+ */
+export async function findEquivalentCarCids (piece, fetchClaims = createClaimsClientForEnv()) {
+  /** @type {Set<CARLink>} */
+  const cids = new Set()
+  const claims = await fetchClaims(piece)
+  for (const claim of claims) {
+    // claims will include _all_ claims about this cid, so we filter to `equals`
+    if (claim.type !== 'assert/equals') {
+      continue
+    }
+    // an equivalence claim may have the pieceCid as the content cid _or_ the equals cid
+    // so check both properties for the car cid.
+    const carCid = asCarCid(claim.equals) ?? asCarCid(claim.content)
+    if (carCid) {
+      cids.add(carCid)
+    }
+  }
+  return cids
+}
+
+/**
+ * Find the set locations claimed given CID is present.
+ * 
+ * @param {UnknownLink} link
+ * @param {(link: UnknownLink) => Promise<Claim[]>} [fetchClaims] - returns content claims for a cid
+ */
+export async function findLocationsForLink (link, fetchClaims = createClaimsClientForEnv()) {
+  const claims = await fetchClaims(link)
+  /** @type {Set<URI>} */
+  const locations = new Set()
+
+  for (const claim of claims) {
+    // claims will include _all_ claims about this cid, so we filter to `location`
+    if (claim.type !== 'assert/location') {
+      continue
+    }
+
+    for (const l of claim.location) {
+      locations.add(l)
+    }
+  }
+  return locations
+}
+
+/** @param {'prod' | *} env */
+export function createClaimsClientForEnv (env = process.env.SST_STAGE) {
+  if (env === 'prod') {
+    return read
+  }
+  return (cid, opts) => read(cid, { serviceURL: 'https://staging.claims.web3.storage', ...opts })
+}

roundabout/functions/redirect.js

@@ -2,9 +2,18 @@ import * as Sentry from '@sentry/serverless'
 import { S3Client } from '@aws-sdk/client-s3'
 import { CID } from 'multiformats/cid'
 
-import { getSigner } from '../index.js'
-import { findEquivalentCarCids, asPieceCidV1, asPieceCidV2, asCarCid } from '../piece.js'
-import { getEnv, parseQueryStringParameters } from '../utils.js'
+import {
+  getSigner,
+  carLocationResolver,
+  resolveCar,
+  resolvePiece,
+  redirectTo
+} from '../index.js'
+import {
+  getEnv,
+  parseQueryStringParameters,
+  parseKeyQueryStringParameters,
+} from '../utils.js'
 
 Sentry.AWSLambda.init({
   environment: process.env.SST_STAGE,
@@ -13,7 +22,8 @@ Sentry.AWSLambda.init({
 })
 
 /**
- * AWS HTTP Gateway handler for GET /{cid} by CAR CID or Piece CID
+ * AWS HTTP Gateway handler for GET /{cid} by CAR CID or an equivalent CID,
+ * such as a Piece CID.
  *
  * @param {import('aws-lambda').APIGatewayProxyEventV2} request
  */
@@ -29,7 +39,6 @@ export async function redirectCarGet(request) {
   }
 
   const locateCar = carLocationResolver({ 
-    bucket: getEnv().BUCKET_NAME,
     s3Client: getS3Client(),
     expiresIn
   })
@@ -43,72 +52,9 @@ export async function redirectCarGet(request) {
 }
 
 /**
- * Return response for a car CID, or undefined for other CID types
- * 
- * @param {CID} cid
- * @param {(cid: CID) => Promise<string | undefined> } locateCar
- */
-async function resolveCar (cid, locateCar) {
-  if (asCarCid(cid) !== undefined) {
-    const url = await locateCar(cid)
-    if (url) {
-      return redirectTo(url)
-    }
-    return { statusCode: 404, body: 'CAR Not found'}
-  }
-}
-
-/**
- * Return response for a Piece CID, or undefined for other CID types
- * 
- * @param {CID} cid
- * @param {(cid: CID) => Promise<string | undefined> } locateCar
- */
-async function resolvePiece (cid, locateCar) {
-  if (asPieceCidV2(cid) !== undefined) {
-    const cars = await findEquivalentCarCids(cid)
-    if (cars.size === 0) {
-      return { statusCode: 404, body: 'No equivalent CAR CID for Piece CID found' }
-    }
-    for (const cid of cars) {
-      const url = await locateCar(cid)
-      if (url) {
-        return redirectTo(url)
-      }
-    }
-    return { statusCode: 404, body: 'No CARs found for Piece CID' }
-  }
-
-  if (asPieceCidV1(cid) !== undefined) {
-    return {
-      statusCode: 415,
-      body: 'v1 Piece CIDs are not supported yet. Please provide a V2 Piece CID. https://github.com/filecoin-project/FIPs/blob/master/FRCs/frc-0069.md'
-    }
-  }
-}
-
-/**
- * Creates a helper function that returns signed bucket url for a car cid, 
- * or undefined if the CAR does not exist in the bucket.
- *
- * @param {object} config
- * @param {S3Client} config.s3Client
- * @param {string} config.bucket
- * @param {number} config.expiresIn
- */
-function carLocationResolver ({ s3Client, bucket, expiresIn }) {
-  const signer = getSigner(s3Client, bucket)
-  /**
-   * @param {CID} cid
-   */
-  return async function locateCar (cid) {
-    const key = `${cid}/${cid}.car`
-    return signer.getUrl(key, { expiresIn })
-  }
-}
-
-/**
- * AWS HTTP Gateway handler for GET /key/{key} by bucket key
+ * AWS HTTP Gateway handler for GET /key/{key} by bucket key.
+ * Note that this is currently used by dagcargo old system and
+ * should be deprecated once it is decomissioned.
  *
  * @param {import('aws-lambda').APIGatewayProxyEventV2} request
  */
@@ -117,9 +63,9 @@ export async function redirectKeyGet(request) {
 
   let key, expiresIn, bucketName
   try {
-    const parsedQueryParams = parseQueryStringParameters(request.queryStringParameters)
+    const parsedQueryParams = parseKeyQueryStringParameters(request.queryStringParameters)
     expiresIn = parsedQueryParams.expiresIn
-    bucketName = parsedQueryParams.bucketName
+    bucketName = parsedQueryParams.bucketName || 'carpark-prod-0'
 
     key = request.pathParameters?.key
     if (!key) {
@@ -153,18 +99,6 @@ function toLambdaResponse(signedUrl) {
   return redirectTo(signedUrl)
 }
 
-/**
- * @param {string} url
- */
-function redirectTo (url) {
-  return {
-    statusCode: 302,
-    headers: {
-      Location: url
-    }
-  }
-}
-
 function getS3Client(){
   const {
     BUCKET_ENDPOINT,