feat: roundabout with content claims

docs/roundabout.md

@@ -6,14 +6,16 @@
 
 The given API is currently public.
 
-### `GET /{carCid}`
+### `GET /{cid}`
 
-Redirects to a presigned URL where the requested CAR file (by its CID) can be downloaded from. This will use web3.storage `carpark` as the location of the requested CARs. The request will return a `302 Redirect` to a created presigned URL.
+Redirects to a presigned URL where the requested CAR file (by its CID) can be downloaded from. The given CID can be the CAR CID, or an equivalent CID to it, such as a PieceCIDv2. The request will return a `302 Redirect` to a created presigned URL.
 
 It also supports a query parameter `expires` with the number of seconds this presigned URL should be valid for. You can set a value from one second to 7 days (604,800 seconds). By default the expiration is set for 3 days (259,200 seconds).
 
 ### `GET /key/{key}?bucket=bucket-name`
 
+> Deprecated and should not be used in production
+
 Redirects to a presigned URL where the requested bucket value can be downloaded from by its key. Unlike `GET /{carCid}`, this endpoint takes a key and is compatible with any web3.storage account bucket. The request will return a `302 Redirect` to a created presigned URL.
 
 It also supports a query parameter `expires` with the number of seconds this presigned URL should be valid for. You can set a value from one second to 7 days (604,800 seconds). By default the expiration is set for 3 days (259,200 seconds).

roundabout/claims.js

@@ -0,0 +1,69 @@
+// NOTE: shim globals needed by content-claims client deps that would be present in nodejs v18.
+// TODO: migrate to sst v2 and nodejs v18+
+import './globals.js'
+import { read } from '@web3-storage/content-claims/client'
+import { asCarCid } from './piece.js'
+
+/** 
+ * @typedef {import('multiformats').UnknownLink} UnknownLink
+ * @typedef {import('@ucanto/client').URI} URI
+ * @typedef {import('@web3-storage/w3up-client/types').CARLink} CARLink
+ * @typedef {import('@web3-storage/content-claims/client/api').Claim} Claim
+ **/
+
+/**
+ * Find the set locations claimed given CID is present.
+ * 
+ * @param {CARLink} link
+ * @param {(link: UnknownLink) => Promise<Claim[]>} [fetchClaims] - returns content claims for a cid
+ */
+export async function findLocationsForCar (link, fetchClaims = createClaimsClientForEnv()) {
+  const claims = await fetchClaims(link)
+  /** @type {Set<URI>} */
+  const locations = new Set()
+
+  for (const claim of claims) {
+    // claims will include _all_ claims about this cid, so we filter to `location`
+    if (claim.type !== 'assert/location') {
+      continue
+    }
+
+    for (const l of claim.location) {
+      locations.add(l)
+    }
+  }
+  return locations
+}
+
+/**
+ * Find the set of CAR CIDs that are claimed to be equivalent to the Piece CID.
+ * 
+ * @param {UnknownLink} piece
+ * @param {(link: UnknownLink) => Promise<Claim[]>} [fetchClaims] - returns content claims for a cid
+ */
+export async function findEquivalentCarCids (piece, fetchClaims = createClaimsClientForEnv()) {
+  /** @type {Set<CARLink>} */
+  const cids = new Set()
+  const claims = await fetchClaims(piece)
+  for (const claim of claims) {
+    // claims will include _all_ claims about this cid, so we filter to `equals`
+    if (claim.type !== 'assert/equals') {
+      continue
+    }
+    // an equivalence claim may have the pieceCid as the content cid _or_ the equals cid
+    // so check both properties for the car cid.
+    const carCid = asCarCid(claim.equals) ?? asCarCid(claim.content)
+    if (carCid) {
+      cids.add(carCid)
+    }
+  }
+  return cids
+}
+
+/** @param {'prod' | *} env */
+export function createClaimsClientForEnv (env = process.env.SST_STAGE) {
+  if (env === 'prod') {
+    return read
+  }
+  return (cid, opts) => read(cid, { serviceURL: 'https://staging.claims.web3.storage', ...opts })
+}

roundabout/functions/redirect.js

@@ -3,8 +3,9 @@ import { S3Client } from '@aws-sdk/client-s3'
 import { CID } from 'multiformats/cid'
 
 import { getSigner } from '../index.js'
-import { findEquivalentCarCids, asPieceCidV1, asPieceCidV2, asCarCid } from '../piece.js'
-import { getEnv, parseQueryStringParameters } from '../utils.js'
+import { asPieceCidV1, asPieceCidV2, asCarCid } from '../piece.js'
+import { getEnv, parseQueryStringParameters, parseKeyQueryStringParameters } from '../utils.js'
+import { findEquivalentCarCids, findLocationsForCar } from '../claims.js'
 
 Sentry.AWSLambda.init({
   environment: process.env.SST_STAGE,
@@ -13,7 +14,12 @@ Sentry.AWSLambda.init({
 })
 
 /**
- * AWS HTTP Gateway handler for GET /{cid} by CAR CID or Piece CID
+ * @typedef {import('multiformats').UnknownLink} UnknownLink
+ */
+
+/**
+ * AWS HTTP Gateway handler for GET /{cid} by CAR CID or an equivalent CID,
+ * such as a Piece CID.
  *
  * @param {import('aws-lambda').APIGatewayProxyEventV2} request
  */
@@ -29,7 +35,6 @@ export async function redirectCarGet(request) {
   }
 
   const locateCar = carLocationResolver({ 
-    bucket: getEnv().BUCKET_NAME,
     s3Client: getS3Client(),
     expiresIn
   })
@@ -45,12 +50,13 @@ export async function redirectCarGet(request) {
 /**
  * Return response for a car CID, or undefined for other CID types
  * 
- * @param {CID} cid
- * @param {(cid: CID) => Promise<string | undefined> } locateCar
+ * @param {UnknownLink} cid
+ * @param {(cid: UnknownLink, bucketName: string) => Promise<string | undefined> } locateCar
  */
 async function resolveCar (cid, locateCar) {
   if (asCarCid(cid) !== undefined) {
-    const url = await locateCar(cid)
+    const bucketName = 'bucket-name'
+    const url = await locateCar(cid, bucketName)
     if (url) {
       return redirectTo(url)
     }
@@ -61,8 +67,8 @@ async function resolveCar (cid, locateCar) {
 /**
  * Return response for a Piece CID, or undefined for other CID types
  * 
- * @param {CID} cid
- * @param {(cid: CID) => Promise<string | undefined> } locateCar
+ * @param {UnknownLink} cid
+ * @param {(cid: UnknownLink, bucketName: string) => Promise<string | undefined> } locateCar
  */
 async function resolvePiece (cid, locateCar) {
   if (asPieceCidV2(cid) !== undefined) {
@@ -71,7 +77,8 @@ async function resolvePiece (cid, locateCar) {
       return { statusCode: 404, body: 'No equivalent CAR CID for Piece CID found' }
     }
     for (const cid of cars) {
-      const url = await locateCar(cid)
+      const bucketName = 'bucket-name'
+      const url = await locateCar(cid, bucketName)
       if (url) {
         return redirectTo(url)
       }
@@ -93,22 +100,24 @@ async function resolvePiece (cid, locateCar) {
  *
  * @param {object} config
  * @param {S3Client} config.s3Client
- * @param {string} config.bucket
  * @param {number} config.expiresIn
  */
-function carLocationResolver ({ s3Client, bucket, expiresIn }) {
-  const signer = getSigner(s3Client, bucket)
+function carLocationResolver ({ s3Client, expiresIn }) {
   /**
-   * @param {CID} cid
+   * @param {UnknownLink} cid
+   * @param {string} bucket
    */
-  return async function locateCar (cid) {
+  return async function locateCar (cid, bucket) {
+    const signer = getSigner(s3Client, bucket)
     const key = `${cid}/${cid}.car`
     return signer.getUrl(key, { expiresIn })
   }
 }
 
 /**
- * AWS HTTP Gateway handler for GET /key/{key} by bucket key
+ * AWS HTTP Gateway handler for GET /key/{key} by bucket key.
+ * Note that this is currently used by dagcargo old system and
+ * should be deprecated once it is decomissioned.
  *
  * @param {import('aws-lambda').APIGatewayProxyEventV2} request
  */
@@ -117,9 +126,9 @@ export async function redirectKeyGet(request) {
 
   let key, expiresIn, bucketName
   try {
-    const parsedQueryParams = parseQueryStringParameters(request.queryStringParameters)
+    const parsedQueryParams = parseKeyQueryStringParameters(request.queryStringParameters)
     expiresIn = parsedQueryParams.expiresIn
-    bucketName = parsedQueryParams.bucketName
+    bucketName = parsedQueryParams.bucketName || 'carpark-prod-0'
 
     key = request.pathParameters?.key
     if (!key) {

roundabout/package.json

@@ -16,6 +16,7 @@
   "devDependencies": {
     "@ipld/car": "^5.1.1",
     "@ipld/dag-pb": "^3.0.0",
+    "@ucanto/client":"^9.0.0",
     "@web3-storage/data-segment": "^5.0.0",
     "@web3-storage/w3up-client": "^9.2.2",
     "ava": "^4.3.3",

roundabout/piece.js

@@ -1,8 +1,3 @@
-// NOTE: shim globals needed by content-claims client deps that would be present in nodejs v18.
-// TODO: migrate to sst v2 and nodejs v18+
-import './globals.js'
-
-import { read } from '@web3-storage/content-claims/client'
 import * as Raw from 'multiformats/codecs/raw'
 
 /** https://github.com/multiformats/multicodec/blob/master/table.csv#L140 */
@@ -18,15 +13,15 @@ export const PIECE_V1_MULTIHASH = 0x10_12
 export const PIECE_V2_MULTIHASH = 0x10_11
 
 /** 
- * @typedef {import('multiformats/cid').Link} Link
+ * @typedef {import('multiformats').UnknownLink} UnknownLink
  * @typedef {import('@web3-storage/w3up-client/types').CARLink} CARLink
  * @typedef {import('@web3-storage/content-claims/client/api').Claim} Claim
  **/
 
 /**
  * Return the cid if it is a Piece CID or undefined if not
  *
- * @param {Link} cid
+ * @param {UnknownLink} cid
  */
 export function asPieceCidV2 (cid) {
   if (cid.multihash.code === PIECE_V2_MULTIHASH && cid.code === Raw.code) {
@@ -37,7 +32,7 @@ export function asPieceCidV2 (cid) {
 /**
  * Return the cid if it is a v1 Piece CID or undefined if not
  *
- * @param {Link} cid
+ * @param {UnknownLink} cid
  */
 export function asPieceCidV1 (cid) {
   if (cid.multihash.code === PIECE_V1_MULTIHASH && cid.code === PIECE_V1_CODE) {
@@ -48,43 +43,12 @@ export function asPieceCidV1 (cid) {
 /**
  * Return the cid if it is a CAR CID or undefined if not
  *
- * @param {Link} cid
+ * @param {UnknownLink} cid
+ * @returns {CARLink | undefined}
  */
 export function asCarCid(cid) {
   if (cid.code === CAR_CODE) {
+    // @ts-expect-error types fail to understand this is CAR Link
     return cid
   }
 }
-
-/**
- * Find the set of CAR CIDs that are claimed to be equivalent to the Piece CID.
- * 
- * @param {Link} piece
- * @param {(Link) => Promise<Claim[]>} [fetchClaims] - returns content claims for a cid
- */
-export async function findEquivalentCarCids (piece, fetchClaims = createClaimsClientForEnv()) {
-  /** @type {Set<CARLink>} */
-  const cids = new Set()
-  const claims = await fetchClaims(piece)
-  for (const claim of claims) {
-    // claims will include _all_ claims about this cid, so we filter to `equals`
-    if (claim.type !== 'assert/equals') {
-      continue
-    }
-    // an equivalence claim may have the pieceCid as the content cid _or_ the equals cid
-    // so check both properties for the car cid.
-    const carCid = asCarCid(claim.equals) ?? asCarCid(claim.content)
-    if (carCid) {
-      cids.add(carCid)
-    }
-  }
-  return cids
-}
-
-/** @param {'prod' | *} env */
-export function createClaimsClientForEnv (env = process.env.SST_STAGE) {
-  if (env === 'prod') {
-    return read
-  }
-  return (cid, opts) => read(cid, { serviceURL: 'https://staging.claims.web3.storage', ...opts })
-}

roundabout/utils.js

@@ -3,7 +3,7 @@ export const MAX_EXPIRES_IN = 3 * 24 * 60 * 60 // 7 days in seconds
 export const MIN_EXPIRES_IN = 1
 export const DEFAULT_EXPIRES_IN = 3 * 24 * 60 * 60 // 3 days in seconds by default
 
-export const VALID_BUCKETS = ['dagcargo']
+export const VALID_BUCKETS_BY_KEY = ['dagcargo']
 
 /**
  * @param {import('aws-lambda').APIGatewayProxyEventPathParameters | undefined} queryStringParameters
@@ -16,9 +16,25 @@ export function parseQueryStringParameters (queryStringParameters) {
     throw new Error(`Bad request with not acceptable expires parameter: ${queryStringParameters?.expires}`)
   }
 
+  return {
+    expiresIn
+  }
+}
+
+/**
+ * @param {import('aws-lambda').APIGatewayProxyEventPathParameters | undefined} queryStringParameters
+ */
+export function parseKeyQueryStringParameters (queryStringParameters) {
+  const expiresIn = queryStringParameters?.expires ?
+    parseInt(queryStringParameters?.expires) : DEFAULT_EXPIRES_IN
+
+  if (expiresIn > MAX_EXPIRES_IN || expiresIn < MIN_EXPIRES_IN) {
+    throw new Error(`Bad request with not acceptable expires parameter: ${queryStringParameters?.expires}`)
+  }
+
   const bucketName = queryStringParameters?.bucket
 
-  if (bucketName && !VALID_BUCKETS.includes(bucketName)) {
+  if (bucketName && !VALID_BUCKETS_BY_KEY.includes(bucketName)) {
     throw new Error(`Bad requested with not acceptable bucket: ${bucketName}`)
   }
 

stacks/roundabout-stack.js

@@ -25,7 +25,6 @@ export function RoundaboutStack({ stack, app }) {
         environment: {
           BUCKET_ENDPOINT: process.env.R2_ENDPOINT ?? '',
           BUCKET_REGION: process.env.R2_REGION ?? '',
-          BUCKET_NAME: process.env.R2_CARPARK_BUCKET_NAME ?? '',
           BUCKET_ACCESS_KEY_ID: process.env.R2_ACCESS_KEY_ID ?? '',
           BUCKET_SECRET_ACCESS_KEY: process.env.R2_SECRET_ACCESS_KEY ?? '',
         }