Skip to content

Commit

Permalink
feat: roundabout with content claims
Browse files Browse the repository at this point in the history
  • Loading branch information
vasco-santos committed Dec 29, 2023
1 parent 9e93982 commit 679f4fa
Show file tree
Hide file tree
Showing 13 changed files with 568 additions and 211 deletions.
6 changes: 4 additions & 2 deletions docs/roundabout.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,16 @@

The given API is currently public.

### `GET /{carCid}`
### `GET /{cid}`

Redirects to a presigned URL where the requested CAR file (by its CID) can be downloaded from. This will use web3.storage `carpark` as the location of the requested CARs. The request will return a `302 Redirect` to a created presigned URL.
Redirects to a presigned URL where the requested CAR file (by its CID) can be downloaded from. The given CID can be the CAR CID, or an equivalent CID to it, such as a PieceCIDv2. The request will return a `302 Redirect` to a created presigned URL.

It also supports a query parameter `expires` with the number of seconds this presigned URL should be valid for. You can set a value from one second to 7 days (604,800 seconds). By default the expiration is set for 3 days (259,200 seconds).

### `GET /key/{key}?bucket=bucket-name`

> Deprecated and should not be used in production
Redirects to a presigned URL where the requested bucket value can be downloaded from by its key. Unlike `GET /{carCid}`, this endpoint takes a key and is compatible with any web3.storage account bucket. The request will return a `302 Redirect` to a created presigned URL.

It also supports a query parameter `expires` with the number of seconds this presigned URL should be valid for. You can set a value from one second to 7 days (604,800 seconds). By default the expiration is set for 3 days (259,200 seconds).
Expand Down
44 changes: 40 additions & 4 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

69 changes: 69 additions & 0 deletions roundabout/claims.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
// NOTE: shim globals needed by content-claims client deps that would be present in nodejs v18.
// TODO: migrate to sst v2 and nodejs v18+
import './globals.js'
import { read } from '@web3-storage/content-claims/client'
import { asCarCid } from './piece.js'

/**
* @typedef {import('multiformats').UnknownLink} UnknownLink
* @typedef {import('@ucanto/client').URI} URI
* @typedef {import('@web3-storage/w3up-client/types').CARLink} CARLink
* @typedef {import('@web3-storage/content-claims/client/api').Claim} Claim
**/

/**
* Find the set of CAR CIDs that are claimed to be equivalent to the Piece CID.
*
* @param {UnknownLink} piece
* @param {(link: UnknownLink) => Promise<Claim[]>} [fetchClaims] - returns content claims for a cid
*/
export async function findEquivalentCarCids (piece, fetchClaims = createClaimsClientForEnv()) {
/** @type {Set<CARLink>} */
const cids = new Set()
const claims = await fetchClaims(piece)
for (const claim of claims) {
// claims will include _all_ claims about this cid, so we filter to `equals`
if (claim.type !== 'assert/equals') {
continue
}
// an equivalence claim may have the pieceCid as the content cid _or_ the equals cid
// so check both properties for the car cid.
const carCid = asCarCid(claim.equals) ?? asCarCid(claim.content)
if (carCid) {
cids.add(carCid)
}
}
return cids
}

/**
* Find the set locations claimed given CID is present.
*
* @param {UnknownLink} link
* @param {(link: UnknownLink) => Promise<Claim[]>} [fetchClaims] - returns content claims for a cid
*/
export async function findLocationsForLink (link, fetchClaims = createClaimsClientForEnv()) {
const claims = await fetchClaims(link)
/** @type {Set<URI>} */
const locations = new Set()

for (const claim of claims) {
// claims will include _all_ claims about this cid, so we filter to `location`
if (claim.type !== 'assert/location') {
continue
}

for (const l of claim.location) {
locations.add(l)
}
}
return locations
}

/** @param {'prod' | *} env */
export function createClaimsClientForEnv (env = process.env.SST_STAGE) {
if (env === 'prod') {
return read
}
return (cid, opts) => read(cid, { serviceURL: 'https://staging.claims.web3.storage', ...opts })
}
104 changes: 19 additions & 85 deletions roundabout/functions/redirect.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,18 @@ import * as Sentry from '@sentry/serverless'
import { S3Client } from '@aws-sdk/client-s3'
import { CID } from 'multiformats/cid'

import { getSigner } from '../index.js'
import { findEquivalentCarCids, asPieceCidV1, asPieceCidV2, asCarCid } from '../piece.js'
import { getEnv, parseQueryStringParameters } from '../utils.js'
import {
getSigner,
carLocationResolver,
resolveCar,
resolvePiece,
redirectTo
} from '../index.js'
import {
getEnv,
parseQueryStringParameters,
parseKeyQueryStringParameters,
} from '../utils.js'

Sentry.AWSLambda.init({
environment: process.env.SST_STAGE,
Expand All @@ -13,7 +22,8 @@ Sentry.AWSLambda.init({
})

/**
* AWS HTTP Gateway handler for GET /{cid} by CAR CID or Piece CID
* AWS HTTP Gateway handler for GET /{cid} by CAR CID or an equivalent CID,
* such as a Piece CID.
*
* @param {import('aws-lambda').APIGatewayProxyEventV2} request
*/
Expand All @@ -29,7 +39,6 @@ export async function redirectCarGet(request) {
}

const locateCar = carLocationResolver({
bucket: getEnv().BUCKET_NAME,
s3Client: getS3Client(),
expiresIn
})
Expand All @@ -43,72 +52,9 @@ export async function redirectCarGet(request) {
}

/**
* Return response for a car CID, or undefined for other CID types
*
* @param {CID} cid
* @param {(cid: CID) => Promise<string | undefined> } locateCar
*/
async function resolveCar (cid, locateCar) {
if (asCarCid(cid) !== undefined) {
const url = await locateCar(cid)
if (url) {
return redirectTo(url)
}
return { statusCode: 404, body: 'CAR Not found'}
}
}

/**
* Return response for a Piece CID, or undefined for other CID types
*
* @param {CID} cid
* @param {(cid: CID) => Promise<string | undefined> } locateCar
*/
async function resolvePiece (cid, locateCar) {
if (asPieceCidV2(cid) !== undefined) {
const cars = await findEquivalentCarCids(cid)
if (cars.size === 0) {
return { statusCode: 404, body: 'No equivalent CAR CID for Piece CID found' }
}
for (const cid of cars) {
const url = await locateCar(cid)
if (url) {
return redirectTo(url)
}
}
return { statusCode: 404, body: 'No CARs found for Piece CID' }
}

if (asPieceCidV1(cid) !== undefined) {
return {
statusCode: 415,
body: 'v1 Piece CIDs are not supported yet. Please provide a V2 Piece CID. https://github.com/filecoin-project/FIPs/blob/master/FRCs/frc-0069.md'
}
}
}

/**
* Creates a helper function that returns signed bucket url for a car cid,
* or undefined if the CAR does not exist in the bucket.
*
* @param {object} config
* @param {S3Client} config.s3Client
* @param {string} config.bucket
* @param {number} config.expiresIn
*/
function carLocationResolver ({ s3Client, bucket, expiresIn }) {
const signer = getSigner(s3Client, bucket)
/**
* @param {CID} cid
*/
return async function locateCar (cid) {
const key = `${cid}/${cid}.car`
return signer.getUrl(key, { expiresIn })
}
}

/**
* AWS HTTP Gateway handler for GET /key/{key} by bucket key
* AWS HTTP Gateway handler for GET /key/{key} by bucket key.
* Note that this is currently used by dagcargo old system and
* should be deprecated once it is decomissioned.
*
* @param {import('aws-lambda').APIGatewayProxyEventV2} request
*/
Expand All @@ -117,9 +63,9 @@ export async function redirectKeyGet(request) {

let key, expiresIn, bucketName
try {
const parsedQueryParams = parseQueryStringParameters(request.queryStringParameters)
const parsedQueryParams = parseKeyQueryStringParameters(request.queryStringParameters)
expiresIn = parsedQueryParams.expiresIn
bucketName = parsedQueryParams.bucketName
bucketName = parsedQueryParams.bucketName || 'carpark-prod-0'

key = request.pathParameters?.key
if (!key) {
Expand Down Expand Up @@ -153,18 +99,6 @@ function toLambdaResponse(signedUrl) {
return redirectTo(signedUrl)
}

/**
* @param {string} url
*/
function redirectTo (url) {
return {
statusCode: 302,
headers: {
Location: url
}
}
}

function getS3Client(){
const {
BUCKET_ENDPOINT,
Expand Down
Loading

0 comments on commit 679f4fa

Please sign in to comment.