-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* WIP: server blobs over http * Add basic test * add magic bytes dep * implement initial mime type support * updates to hyper* types declarations * fix mime guessing * add more tests * maybe fix CI issue * attempt 2 at fixing CI * Revert "attempt 2 at fixing CI" This reverts commit 9383baf. Just to figure out the original issue and go from there... * adjust export of SUPPORTED_BLOB_VARIANTS * implement fastify plugin * Avoid unused import * Fix magicbytes type & avoid Array.from(buf) * Set .mvnrc * fix Hyperdrive.entry return type * fix Hyperdrive.get return type * fix Hyperdrive.getBlobs return type * extract fixtures setup in tests * remove use of solo * update fastify import * move fixtures constants * fix return type of Hyperblobs.get * fix return type of BlobStore.entry * add error code paths * update populateStore test helper * create separate test helper for creating server * WIP: add test for non-replicated blob 404 test passes but not for the expected reason * remove console log * mark non-replicated blob test as todo * specify fastify as dev dep for tests * remove plain http implementation * add blob-server to exports field * Revert "remove plain http implementation" This reverts commit c3aae56. * replace plain http implementation with fastify * fix non-replicated 404 test * fix jsdoc import * add createEntryReadStream and getEntryBlob to BlobStore * fix test names * update types for plugin opts * runtime check for blobStore plugin opt * remove type annotation for params json schema def * add projectId route param * add route creation util for tests * add projectId param to fastify validation schema * remove commented line * chore: use typebox for fastify param * validate blobId without casting type * 404 if projectId not found * DRY test setup * make #getDrive() a private method * Add comment as future reminder * stricter metadata and rename mimeType Don't see a reason to allow any metadata at this time * Add test for invalid variant-type combo Checked coverage and this was not being tested * fix tests for mimeType handling * Add test for throw on invalid option * Add logger option to testenv * Add random fixture to test application/octet-stream The branch for returning this header for an unrecognized mimeType was not being tested by existing fixtures --------- Co-authored-by: Andrew Chou <[email protected]>
- Loading branch information
1 parent
16bef6c
commit 8e64abd
Showing
14 changed files
with
1,171 additions
and
77 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| 18 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,129 @@ | ||
| // @ts-check | ||
| import fp from 'fastify-plugin' | ||
| import { filetypemime } from 'magic-bytes.js' | ||
| import { Type as T } from '@sinclair/typebox' | ||
|
|
||
| import { SUPPORTED_BLOB_VARIANTS } from '../blob-store/index.js' | ||
|
|
||
| export default fp(blobServerPlugin, { | ||
| fastify: '4.x', | ||
| name: 'mapeo-blob-server', | ||
| }) | ||
|
|
||
| /** @typedef {import('../types').BlobId} BlobId */ | ||
|
|
||
| /** | ||
| * @typedef {Object} BlobServerPluginOpts | ||
| * | ||
| * @property {(projectId: string) => import('../blob-store/index.js').BlobStore} getBlobStore | ||
| */ | ||
|
|
||
| const BLOB_TYPES = /** @type {BlobId['type'][]} */ ( | ||
| Object.keys(SUPPORTED_BLOB_VARIANTS) | ||
| ) | ||
| const BLOB_VARIANTS = [ | ||
| ...new Set(Object.values(SUPPORTED_BLOB_VARIANTS).flat()), | ||
| ] | ||
| const HEX_REGEX_32_BYTES = '^[0-9a-fA-F]{64}$' | ||
| const HEX_STRING_32_BYTES = T.String({ pattern: HEX_REGEX_32_BYTES }) | ||
|
|
||
| const PARAMS_JSON_SCHEMA = T.Object({ | ||
| projectId: HEX_STRING_32_BYTES, | ||
| driveId: HEX_STRING_32_BYTES, | ||
| type: T.Union( | ||
| BLOB_TYPES.map((type) => { | ||
| return T.Literal(type) | ||
| }) | ||
| ), | ||
| variant: T.Union( | ||
| BLOB_VARIANTS.map((variant) => { | ||
| return T.Literal(variant) | ||
| }) | ||
| ), | ||
| name: T.String(), | ||
| }) | ||
|
|
||
| /** @type {import('fastify').FastifyPluginAsync<import('fastify').RegisterOptions & BlobServerPluginOpts>} */ | ||
| async function blobServerPlugin(fastify, options) { | ||
| if (!options.getBlobStore) throw new Error('Missing getBlobStore') | ||
|
|
||
| // We call register here so that the `prefix` option can work if desired | ||
| // https://fastify.dev/docs/latest/Reference/Routes#route-prefixing-and-fastify-plugin | ||
| fastify.register(routes, options) | ||
| } | ||
|
|
||
| /** @type {import('fastify').FastifyPluginAsync<Omit<BlobServerPluginOpts, 'prefix'>, import('fastify').RawServerDefault, import('@fastify/type-provider-typebox').TypeBoxTypeProvider>} */ | ||
| async function routes(fastify, options) { | ||
| const { getBlobStore } = options | ||
|
|
||
| fastify.get( | ||
| '/:projectId/:driveId/:type/:variant/:name', | ||
| { schema: { params: PARAMS_JSON_SCHEMA } }, | ||
| async (request, reply) => { | ||
| const { projectId, ...blobId } = request.params | ||
|
|
||
| if (!isValidBlobId(blobId)) { | ||
| reply.code(400) | ||
| throw new Error( | ||
| `Unsupported variant "${blobId.variant}" for ${blobId.type}` | ||
| ) | ||
| } | ||
| const { driveId } = blobId | ||
|
|
||
| let blobStore | ||
| try { | ||
| blobStore = getBlobStore(projectId) | ||
| } catch (e) { | ||
| reply.code(404) | ||
| throw e | ||
| } | ||
|
|
||
| const entry = await blobStore.entry(blobId, { wait: false }) | ||
|
|
||
| if (!entry) { | ||
| reply.code(404) | ||
| throw new Error('Entry not found') | ||
| } | ||
|
|
||
| const { metadata } = entry.value | ||
|
|
||
| const blobStream = await blobStore.createEntryReadStream(driveId, entry) | ||
|
|
||
| // Extract the 'mimeType' property of the metadata and use it for the response header if found | ||
| if ( | ||
| metadata && | ||
| 'mimeType' in metadata && | ||
| typeof metadata.mimeType === 'string' | ||
| ) { | ||
| reply.header('Content-Type', metadata.mimeType) | ||
| } else { | ||
| // Attempt to guess the MIME type based on the blob contents | ||
| const blobSlice = await blobStore.getEntryBlob(driveId, entry, { | ||
| length: 20, | ||
| }) | ||
|
|
||
| if (!blobSlice) { | ||
| reply.code(404) | ||
| throw new Error('Blob not found') | ||
| } | ||
|
|
||
| const [guessedMime] = filetypemime(blobSlice) | ||
|
|
||
| reply.header('Content-Type', guessedMime || 'application/octet-stream') | ||
| } | ||
|
|
||
| return reply.send(blobStream) | ||
| } | ||
| ) | ||
| } | ||
|
|
||
| /** | ||
| * @param {Omit<BlobId, 'variant'> & { variant: BlobId['variant'] }} maybeBlobId | ||
| * @returns {maybeBlobId is BlobId} | ||
| */ | ||
| function isValidBlobId(maybeBlobId) { | ||
| const { type, variant } = maybeBlobId | ||
| /** @type {readonly BlobId['variant'][]} */ | ||
| const validVariants = SUPPORTED_BLOB_VARIANTS[type] | ||
| return validVariants.includes(variant) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| import fastify from 'fastify' | ||
|
|
||
| import BlobServerPlugin from './fastify-plugin.js' | ||
|
|
||
| /** | ||
| * @param {object} opts | ||
| * @param {import('fastify').FastifyServerOptions['logger']} opts.logger | ||
| * @param {import('../blob-store/index.js').BlobStore} opts.blobStore | ||
| * @param {import('./fastify-plugin.js').BlobServerPluginOpts['getBlobStore']} opts.getBlobStore | ||
| * @param {import('fastify').RegisterOptions['prefix']} opts.prefix | ||
| * @param {string} opts.projectId Temporary option to enable `getBlobStore` option. Will be removed when multiproject support in Mapeo class is implemented. | ||
| * | ||
| */ | ||
| export function createBlobServer({ logger, blobStore, prefix, projectId }) { | ||
| const server = fastify({ logger }) | ||
| server.register(BlobServerPlugin, { | ||
| getBlobStore: (projId) => { | ||
| // Temporary measure until multiprojects is implemented in Mapeo class | ||
| if (projectId !== projId) throw new Error('Project ID does not match') | ||
| return blobStore | ||
| }, | ||
| prefix, | ||
| }) | ||
| return server | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.