From 15b06f8016129981fb3c4c628bf4ab002c6f2ab8 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Thu, 21 Mar 2024 16:42:30 -0700 Subject: [PATCH 1/2] feat: proof of concept of w3up-on-WASM This is a proof-of-concept of running w3up on a WASM runtime. Zinnia is "a sandboxed and resource-limited runtime for distributed workers" that provides an easy way to run w3up on a WASM-based execution platform. This proof-of-concept shows that it is possible to run w3up in "a sandboxed and resource-limited runtime" which may be useful if we'd like to enable many different entities to run w3up instances. --- packages/zinniup/README.md | 30 ++++ packages/zinniup/package.json | 35 +++++ packages/zinniup/rollup.config.js | 11 ++ packages/zinniup/src/main.js | 17 +++ pnpm-lock.yaml | 232 +++++++++++++++++++++++++++++- 5 files changed, 320 insertions(+), 5 deletions(-) create mode 100644 packages/zinniup/README.md create mode 100644 packages/zinniup/package.json create mode 100644 packages/zinniup/rollup.config.js create mode 100644 packages/zinniup/src/main.js diff --git a/packages/zinniup/README.md b/packages/zinniup/README.md new file mode 100644 index 000000000..71193bd36 --- /dev/null +++ b/packages/zinniup/README.md @@ -0,0 +1,30 @@ +# ZinniUP + +w3up blooms + +## What is this? + +This is a proof-of-concept of running w3up on a WASM runtime. + +Zinnia is "a sandboxed and resource-limited runtime for distributed workers" that +provides an easy way to run w3up on a WASM-based execution platform. + +This proof-of-concept shows that it is possible to run w3up in "a sandboxed +and resource-limited runtime" which may be useful if we'd like to enable many different +entities to run w3up instances. + +## Installation + +First, `pnpm install` in the same directory as this README. + +Next, follow Zinnia's instructions to install the `zinnia` CLI: + +https://github.com/filecoin-station/zinnia/blob/main/cli/README.md#installation + +Finally, run `pnpm build` in the same directory as this README. + +## Running + +If you added `zinnia` to your shell `PATH`, `pnpm start` should run the +rollup-bundled script at `dist/main.js` which, as of 3/21/2024, will create +a w3up server, print a trivial log line, and exit. \ No newline at end of file diff --git a/packages/zinniup/package.json b/packages/zinniup/package.json new file mode 100644 index 000000000..273b8f959 --- /dev/null +++ b/packages/zinniup/package.json @@ -0,0 +1,35 @@ +{ + "name": "@web3-storage/zinniup", + "description": "The upload api for web3.storage, on Zinnia!", + "version": "0.0.0", + "type": "module", + "main": "./src/main.js", + "homepage": "https://web3.storage", + "repository": { + "type": "git", + "url": "https://github.com/web3-storage/w3up.git", + "directory": "packages/zinniup" + }, + "files": [ + "src", + "test", + "dist/**/*.d.ts", + "dist/**/*.d.ts.map" + ], + "scripts": { + "build": "rollup -c rollup.config.js", + "start": "zinnia run dist/main.js" + }, + "dependencies": { + "@ucanto/principal": "^9.0.0", + "@web3-storage/upload-api": "workspace:^" + }, + "devDependencies": { + "@rollup/plugin-commonjs": "^25.0.7", + "@rollup/plugin-node-resolve": "^15.2.3", + "rollup": "^4.13.0" + }, + "engines": { + "node": ">=16.15" + } +} diff --git a/packages/zinniup/rollup.config.js b/packages/zinniup/rollup.config.js new file mode 100644 index 000000000..4edcff682 --- /dev/null +++ b/packages/zinniup/rollup.config.js @@ -0,0 +1,11 @@ +import { nodeResolve } from '@rollup/plugin-node-resolve'; +import commonjs from '@rollup/plugin-commonjs'; + +export default { + input: 'src/main.js', + output: { + dir: 'dist', + format: 'es' + }, + plugins: [nodeResolve({ browser: true }), commonjs()] +}; \ No newline at end of file diff --git a/packages/zinniup/src/main.js b/packages/zinniup/src/main.js new file mode 100644 index 000000000..eee81ee25 --- /dev/null +++ b/packages/zinniup/src/main.js @@ -0,0 +1,17 @@ +import * as UploadAPI from '@web3-storage/upload-api' +import * as ed25519 from '@ucanto/principal/ed25519' + +const servicePrincipal = await ed25519.generate() +/** + * TODO: remove this any type and add the appropriate context variables to make this work! + * @type {any} + */ +const context = {} + +const server = UploadAPI.createServer({ + ...context, + id: servicePrincipal, + maxUploadSize: 10000 +}) + +console.log(`started server ${server}`) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3ed85d28f..46d8b9e8a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1,4 +1,4 @@ -lockfileVersion: '6.1' +lockfileVersion: '6.0' settings: autoInstallPeers: true @@ -620,6 +620,25 @@ importers: specifier: ^5.2.2 version: 5.2.2 + packages/zinniup: + dependencies: + '@ucanto/principal': + specifier: ^9.0.0 + version: 9.0.0 + '@web3-storage/upload-api': + specifier: workspace:^ + version: link:../upload-api + devDependencies: + '@rollup/plugin-commonjs': + specifier: ^25.0.7 + version: 25.0.7(rollup@4.13.0) + '@rollup/plugin-node-resolve': + specifier: ^15.2.3 + version: 15.2.3(rollup@4.13.0) + rollup: + specifier: ^4.13.0 + version: 4.13.0 + packages: /@aashutoshrathi/word-wrap@1.2.6: @@ -3538,6 +3557,161 @@ packages: /@protobufjs/utf8@1.1.0: resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==} + /@rollup/plugin-commonjs@25.0.7(rollup@4.13.0): + resolution: {integrity: sha512-nEvcR+LRjEjsaSsc4x3XZfCCvZIaSMenZu/OiwOKGN2UhQpAYI7ru7czFvyWbErlpoGjnSX3D5Ch5FcMA3kRWQ==} + engines: {node: '>=14.0.0'} + peerDependencies: + rollup: ^2.68.0||^3.0.0||^4.0.0 + peerDependenciesMeta: + rollup: + optional: true + dependencies: + '@rollup/pluginutils': 5.1.0(rollup@4.13.0) + commondir: 1.0.1 + estree-walker: 2.0.2 + glob: 8.1.0 + is-reference: 1.2.1 + magic-string: 0.30.5 + rollup: 4.13.0 + dev: true + + /@rollup/plugin-node-resolve@15.2.3(rollup@4.13.0): + resolution: {integrity: sha512-j/lym8nf5E21LwBT4Df1VD6hRO2L2iwUeUmP7litikRsVp1H6NWx20NEp0Y7su+7XGc476GnXXc4kFeZNGmaSQ==} + engines: {node: '>=14.0.0'} + peerDependencies: + rollup: ^2.78.0||^3.0.0||^4.0.0 + peerDependenciesMeta: + rollup: + optional: true + dependencies: + '@rollup/pluginutils': 5.1.0(rollup@4.13.0) + '@types/resolve': 1.20.2 + deepmerge: 4.3.1 + is-builtin-module: 3.2.1 + is-module: 1.0.0 + resolve: 1.22.8 + rollup: 4.13.0 + dev: true + + /@rollup/pluginutils@5.1.0(rollup@4.13.0): + resolution: {integrity: sha512-XTIWOPPcpvyKI6L1NHo0lFlCyznUEyPmPY1mc3KpPVDYulHSTvyeLNVW00QTLIAFNhR3kYnJTQHeGqU4M3n09g==} + engines: {node: '>=14.0.0'} + peerDependencies: + rollup: ^1.20.0||^2.0.0||^3.0.0||^4.0.0 + peerDependenciesMeta: + rollup: + optional: true + dependencies: + '@types/estree': 1.0.5 + estree-walker: 2.0.2 + picomatch: 2.3.1 + rollup: 4.13.0 + dev: true + + /@rollup/rollup-android-arm-eabi@4.13.0: + resolution: {integrity: sha512-5ZYPOuaAqEH/W3gYsRkxQATBW3Ii1MfaT4EQstTnLKViLi2gLSQmlmtTpGucNP3sXEpOiI5tdGhjdE111ekyEg==} + cpu: [arm] + os: [android] + requiresBuild: true + dev: true + optional: true + + /@rollup/rollup-android-arm64@4.13.0: + resolution: {integrity: sha512-BSbaCmn8ZadK3UAQdlauSvtaJjhlDEjS5hEVVIN3A4bbl3X+otyf/kOJV08bYiRxfejP3DXFzO2jz3G20107+Q==} + cpu: [arm64] + os: [android] + requiresBuild: true + dev: true + optional: true + + /@rollup/rollup-darwin-arm64@4.13.0: + resolution: {integrity: sha512-Ovf2evVaP6sW5Ut0GHyUSOqA6tVKfrTHddtmxGQc1CTQa1Cw3/KMCDEEICZBbyppcwnhMwcDce9ZRxdWRpVd6g==} + cpu: [arm64] + os: [darwin] + requiresBuild: true + dev: true + optional: true + + /@rollup/rollup-darwin-x64@4.13.0: + resolution: {integrity: sha512-U+Jcxm89UTK592vZ2J9st9ajRv/hrwHdnvyuJpa5A2ngGSVHypigidkQJP+YiGL6JODiUeMzkqQzbCG3At81Gg==} + cpu: [x64] + os: [darwin] + requiresBuild: true + dev: true + optional: true + + /@rollup/rollup-linux-arm-gnueabihf@4.13.0: + resolution: {integrity: sha512-8wZidaUJUTIR5T4vRS22VkSMOVooG0F4N+JSwQXWSRiC6yfEsFMLTYRFHvby5mFFuExHa/yAp9juSphQQJAijQ==} + cpu: [arm] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@rollup/rollup-linux-arm64-gnu@4.13.0: + resolution: {integrity: sha512-Iu0Kno1vrD7zHQDxOmvweqLkAzjxEVqNhUIXBsZ8hu8Oak7/5VTPrxOEZXYC1nmrBVJp0ZcL2E7lSuuOVaE3+w==} + cpu: [arm64] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@rollup/rollup-linux-arm64-musl@4.13.0: + resolution: {integrity: sha512-C31QrW47llgVyrRjIwiOwsHFcaIwmkKi3PCroQY5aVq4H0A5v/vVVAtFsI1nfBngtoRpeREvZOkIhmRwUKkAdw==} + cpu: [arm64] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@rollup/rollup-linux-riscv64-gnu@4.13.0: + resolution: {integrity: sha512-Oq90dtMHvthFOPMl7pt7KmxzX7E71AfyIhh+cPhLY9oko97Zf2C9tt/XJD4RgxhaGeAraAXDtqxvKE1y/j35lA==} + cpu: [riscv64] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@rollup/rollup-linux-x64-gnu@4.13.0: + resolution: {integrity: sha512-yUD/8wMffnTKuiIsl6xU+4IA8UNhQ/f1sAnQebmE/lyQ8abjsVyDkyRkWop0kdMhKMprpNIhPmYlCxgHrPoXoA==} + cpu: [x64] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@rollup/rollup-linux-x64-musl@4.13.0: + resolution: {integrity: sha512-9RyNqoFNdF0vu/qqX63fKotBh43fJQeYC98hCaf89DYQpv+xu0D8QFSOS0biA7cGuqJFOc1bJ+m2rhhsKcw1hw==} + cpu: [x64] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@rollup/rollup-win32-arm64-msvc@4.13.0: + resolution: {integrity: sha512-46ue8ymtm/5PUU6pCvjlic0z82qWkxv54GTJZgHrQUuZnVH+tvvSP0LsozIDsCBFO4VjJ13N68wqrKSeScUKdA==} + cpu: [arm64] + os: [win32] + requiresBuild: true + dev: true + optional: true + + /@rollup/rollup-win32-ia32-msvc@4.13.0: + resolution: {integrity: sha512-P5/MqLdLSlqxbeuJ3YDeX37srC8mCflSyTrUsgbU1c/U9j6l2g2GiIdYaGD9QjdMQPMSgYm7hgg0551wHyIluw==} + cpu: [ia32] + os: [win32] + requiresBuild: true + dev: true + optional: true + + /@rollup/rollup-win32-x64-msvc@4.13.0: + resolution: {integrity: sha512-UKXUQNbO3DOhzLRwHSpa0HnhhCgNODvfoPWv2FCXme8N/ANFfhIPMGuOT+QuKd16+B5yxZ0HdpNlqPvTMS1qfw==} + cpu: [x64] + os: [win32] + requiresBuild: true + dev: true + optional: true + /@scure/base@1.1.3: resolution: {integrity: sha512-/+SgoRjLq7Xlf0CWuLHq2LUZeL/w65kfzAPG5NH9pcmBhs+nunQTn4gvdwgMTIXnt9b2C/1SeL2XiysZEyIC9Q==} dev: false @@ -4031,6 +4205,10 @@ packages: csstype: 3.1.2 dev: true + /@types/resolve@1.20.2: + resolution: {integrity: sha512-60BCwRFOZCQhDncwQdxxeOEEkbc5dIMccYLwbxsS4TUNeVECQ/pBJ0j09mrHOl/JJvpRPGwO9SvE4nR2Nb/a4Q==} + dev: true + /@types/retry@0.12.0: resolution: {integrity: sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==} dev: true @@ -5101,7 +5279,6 @@ packages: /builtin-modules@3.3.0: resolution: {integrity: sha512-zhaCDicdLuWN5UbN5IMnFqNMhNfo919sH85y2/ea+5Yg9TsTkeZxpL+JLbp6cgYFS4sRLp3YV4S6yDuqVWHYOw==} engines: {node: '>=6'} - dev: false /builtins@5.0.1: resolution: {integrity: sha512-qwVpFEHNfhYJIzNRBvd2C1kyo6jz3ZSMPyyuR47OPdiKWlbYnZNyDWuyR175qDnAJLiCo5fBBqPb3RiXgWlkOQ==} @@ -5497,6 +5674,10 @@ packages: resolution: {integrity: sha512-QE33hToZseCH3jS0qN96O/bSh3kaw/h+Tq7ngyY9eWDUnTlTNUyqfqvCXioLe5Na5jFsL78ra/wuBU4iuEgd4w==} dev: true + /commondir@1.0.1: + resolution: {integrity: sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==} + dev: true + /compressible@2.0.18: resolution: {integrity: sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==} engines: {node: '>= 0.6'} @@ -6626,7 +6807,6 @@ packages: /estree-walker@2.0.2: resolution: {integrity: sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==} - dev: false /estree-walker@3.0.3: resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} @@ -7216,6 +7396,17 @@ packages: once: 1.4.0 path-is-absolute: 1.0.1 + /glob@8.1.0: + resolution: {integrity: sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==} + engines: {node: '>=12'} + dependencies: + fs.realpath: 1.0.0 + inflight: 1.0.6 + inherits: 2.0.4 + minimatch: 5.0.1 + once: 1.4.0 + dev: true + /global-dirs@3.0.1: resolution: {integrity: sha512-NBcGGFbBA9s1VzD41QXDG+3++t9Mn5t1FpLdhESY6oKY4gYTFpX4wO3sqGUa0Srjtbfj3szX0RnemmrVRUdULA==} engines: {node: '>=10'} @@ -7998,7 +8189,6 @@ packages: engines: {node: '>=6'} dependencies: builtin-modules: 3.3.0 - dev: false /is-callable@1.2.7: resolution: {integrity: sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==} @@ -8086,6 +8276,10 @@ packages: engines: {node: '>=12'} dev: true + /is-module@1.0.0: + resolution: {integrity: sha512-51ypPSPCoTEIN9dy5Oy+h4pShgJmPCygKfyRCISBI+JoWT/2oJvK8QPxmwv7b/p239jXrm9M1mlQbyKJ5A152g==} + dev: true + /is-nan@1.3.2: resolution: {integrity: sha512-E+zBKpQ2t6MEo1VsonYmluk9NxGrbzpeeLC2xIViuO2EjU2xsXsBPwTr3Ykv9l08UYEVEdWeRZNouaZqF6RN0w==} engines: {node: '>= 0.4'} @@ -8160,6 +8354,12 @@ packages: engines: {node: '>=0.10.0'} dev: true + /is-reference@1.2.1: + resolution: {integrity: sha512-U82MsXXiFIrjCK4otLT+o2NA2Cd2g5MLoOVXUZjIOhLurrRxpEXzI8O0KZHr3IjLvlAH1kTPYSuqer5T9ZVBKQ==} + dependencies: + '@types/estree': 1.0.5 + dev: true + /is-reference@3.0.2: resolution: {integrity: sha512-v3rht/LgVcsdZa3O2Nqs+NMowLOxeOm7Ay9+/ARQ2F+qEoANRcqrjAZKGN0v8ymUetZGgkp26LTnGT7H0Qo9Pg==} dependencies: @@ -8765,7 +8965,6 @@ packages: engines: {node: '>=12'} dependencies: '@jridgewell/sourcemap-codec': 1.4.15 - dev: false /make-dir@4.0.0: resolution: {integrity: sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==} @@ -11439,6 +11638,29 @@ packages: dependencies: glob: 7.2.3 + /rollup@4.13.0: + resolution: {integrity: sha512-3YegKemjoQnYKmsBlOHfMLVPPA5xLkQ8MHLLSw/fBrFaVkEayL51DilPpNNLq1exr98F2B1TzrV0FUlN3gWRPg==} + engines: {node: '>=18.0.0', npm: '>=8.0.0'} + hasBin: true + dependencies: + '@types/estree': 1.0.5 + optionalDependencies: + '@rollup/rollup-android-arm-eabi': 4.13.0 + '@rollup/rollup-android-arm64': 4.13.0 + '@rollup/rollup-darwin-arm64': 4.13.0 + '@rollup/rollup-darwin-x64': 4.13.0 + '@rollup/rollup-linux-arm-gnueabihf': 4.13.0 + '@rollup/rollup-linux-arm64-gnu': 4.13.0 + '@rollup/rollup-linux-arm64-musl': 4.13.0 + '@rollup/rollup-linux-riscv64-gnu': 4.13.0 + '@rollup/rollup-linux-x64-gnu': 4.13.0 + '@rollup/rollup-linux-x64-musl': 4.13.0 + '@rollup/rollup-win32-arm64-msvc': 4.13.0 + '@rollup/rollup-win32-ia32-msvc': 4.13.0 + '@rollup/rollup-win32-x64-msvc': 4.13.0 + fsevents: 2.3.3 + dev: true + /rtl-detect@1.1.2: resolution: {integrity: sha512-PGMBq03+TTG/p/cRB7HCLKJ1MgDIi07+QU1faSjiYRfmY5UsAttV9Hs08jDAHVwcOwmVLcSJkpwyfXszVjWfIQ==} dev: true From f61ecde78abed1ec4b3354e9e891c98d49057688 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Fri, 22 Mar 2024 11:02:32 -0700 Subject: [PATCH 2/2] fix: tweak README for accuracy --- packages/zinniup/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/zinniup/README.md b/packages/zinniup/README.md index 71193bd36..bc01f455c 100644 --- a/packages/zinniup/README.md +++ b/packages/zinniup/README.md @@ -4,10 +4,10 @@ w3up blooms ## What is this? -This is a proof-of-concept of running w3up on a WASM runtime. +This is a proof-of-concept of running w3up on a WASM-adjacent runtime. Zinnia is "a sandboxed and resource-limited runtime for distributed workers" that -provides an easy way to run w3up on a WASM-based execution platform. +provides an easy way to run w3up on an execution platform that supports WASM. This proof-of-concept shows that it is possible to run w3up in "a sandboxed and resource-limited runtime" which may be useful if we'd like to enable many different