diff --git a/pkg/fs/.npmignore b/pkg/fs/.npmignore new file mode 120000 index 00000000..b4359f69 --- /dev/null +++ b/pkg/fs/.npmignore @@ -0,0 +1 @@ +../../.npmignore \ No newline at end of file diff --git a/pkg/fs/package.json b/pkg/fs/package.json new file mode 100644 index 00000000..342368b3 --- /dev/null +++ b/pkg/fs/package.json @@ -0,0 +1,39 @@ +{ + "name": "@slangroom/fs", + "version": "1.0.0", + "dependencies": { + "@slangroom/core": "workspace:*", + "axios": "^1.5.1", + "extract-zip": "^2.0.1" + }, + "repository": "https://github.com/dyne/slangroom", + "license": "AGPL-3.0-only", + "type": "module", + "main": "./build/cjs/src/index.js", + "types": "./build/cjs/src/index.d.ts", + "exports": { + ".": { + "import": { + "types": "./build/esm/src/index.d.ts", + "default": "./build/esm/src/index.js" + }, + "require": { + "types": "./build/cjs/src/index.d.ts", + "default": "./build/cjs/src/index.js" + } + }, + "./*": { + "import": { + "types": "./build/esm/src/*.d.ts", + "default": "./build/esm/src/*.js" + }, + "require": { + "types": "./build/cjs/src/*.d.ts", + "default": "./build/cjs/src/*.js" + } + } + }, + "publishConfig": { + "access": "public" + } +} diff --git a/pkg/fs/src/index.ts b/pkg/fs/src/index.ts new file mode 100644 index 00000000..8afa28e5 --- /dev/null +++ b/pkg/fs/src/index.ts @@ -0,0 +1 @@ +export * from '@slangroom/fs/plugins'; diff --git a/pkg/fs/src/plugins.ts b/pkg/fs/src/plugins.ts new file mode 100644 index 00000000..f5155ef9 --- /dev/null +++ b/pkg/fs/src/plugins.ts @@ -0,0 +1,142 @@ +import type { PluginContext, PluginResult } from '@slangroom/core'; +import * as path from 'node:path'; +import * as fs from 'node:fs/promises'; +import * as os from 'node:os'; +import axios from 'axios'; +import extractZip from 'extract-zip'; + +export const SandboxDir = process.env['FILES_DIR']; +if (!SandboxDir) throw new Error('$FILE_DIR must be provided'); + +const resolveDirPath = (unsafe: string) => { + const normalized = path.normalize(unsafe); + // `/` and `..` prevent directory traversal + const doesDirectoryTraversal = normalized.startsWith('/') || normalized.startsWith('..'); + // Unlike `resolveFilepath`, we allow `.` to be used here, obviously. + if (doesDirectoryTraversal) return { error: `dirpath is unsafe: ${unsafe}` }; + return { dirpath: path.dirname(path.join(SandboxDir, normalized)) }; +}; + +const resolveFilepath = (unsafe: string) => { + const normalized = path.normalize(unsafe); + // `/` and `..` prevent directory traversal + const doesDirectoryTraversal = normalized.startsWith('/') || normalized.startsWith('..'); + // `.` ensures that `foo/bar` or `./foo` is valid, while `.` isn't + // (that is, no "real" filepath is provided) + const DoesntProvideFile = normalized.startsWith('.'); + if (doesDirectoryTraversal || DoesntProvideFile) + return { error: `filepath is unsafe: ${unsafe}` }; + return { filepath: path.join(SandboxDir, normalized) }; +}; + +const readFile = async (safePath: string) => { + const str = await fs.readFile(safePath, 'utf8'); + return JSON.parse(str); +}; + +/** + * @internal + */ +export const executeDownloadExtract = async (ctx: PluginContext): Promise => { + const zipUrl = ctx.fetchConnect()[0]; + const unsafe = ctx.fetch('path'); + if (typeof unsafe !== 'string') return ctx.fail('path must be string'); + + const { dirpath: dirPath, error } = resolveDirPath(unsafe); + if (!dirPath) return ctx.fail(error); + await fs.mkdir(dirPath, { recursive: true }); + + try { + const resp = await axios.get(zipUrl, { responseType: 'arraybuffer' }); + const tempdir = await fs.mkdtemp(path.join(os.tmpdir(), 'slangroom-')); + const tempfile = path.join(tempdir, 'downloaded'); + await fs.writeFile(tempfile, resp.data); + await extractZip(tempfile, { dir: dirPath }); + await fs.rm(tempdir, { recursive: true }); + return ctx.pass('yes'); + } catch (e) { + if (e instanceof Error) return ctx.fail(e.message); + return ctx.fail(`unknown error: ${e}`); + } +}; + +/** + * @internal + */ +export const executeReadFileContent = async (ctx: PluginContext): Promise => { + const unsafe = ctx.fetch('path'); + if (typeof unsafe !== 'string') return ctx.fail('path must be string'); + + const { filepath, error } = resolveFilepath(unsafe); + if (!filepath) return ctx.fail(error); + + return ctx.pass(await readFile(unsafe)); +}; + +/** + * @internal + */ +export const executeStoreInFile = async (ctx: PluginContext): Promise => { + // TODO: should `ctx.fetch('content')` return a JsonableObject? + const content = JSON.stringify(ctx.fetch('content')); + const unsafe = ctx.fetch('path'); + if (typeof unsafe !== 'string') return ctx.fail('path must be string'); + + const { filepath, error } = resolveFilepath(unsafe); + if (!filepath) return ctx.fail(error); + + await fs.mkdir(path.dirname(filepath), { recursive: true }); + await fs.writeFile(filepath, content); + + return ctx.fail('TODO'); +}; + +/** + * @internal + */ +export const executeListDirectoryContent = async (ctx: PluginContext): Promise => { + const unsafe = ctx.fetch('path'); + if (typeof unsafe !== 'string') return ctx.fail('path must be string'); + + const { dirpath, error } = resolveDirPath(unsafe); + if (!dirpath) return ctx.fail(error); + + const filepaths = (await fs.readdir(dirpath)).map((f) => path.join(dirpath, f)); + const stats = await Promise.all(filepaths.map((f) => fs.stat(f))); + const result = stats.map((stat, i) => { + const filepath = filepaths[i] as string; + return { + name: filepath, + mode: stat.mode.toString(8), + dev: stat.dev, + nlink: stat.nlink, + uid: stat.uid, + gid: stat.gid, + size: stat.size, + blksize: stat.blksize, + blocks: stat.blocks, + atime: stat.atime.toISOString(), + mtime: stat.mtime.toISOString(), + ctime: stat.ctime.toISOString(), + birthtime: stat.birthtime.toISOString(), + }; + }); + return ctx.pass(result); +}; + +const fsPlugin = async (ctx: PluginContext): Promise => { + switch (ctx.phrase) { + case 'download extract': + return await executeDownloadExtract(ctx); + case 'read file content': + return await executeReadFileContent(ctx); + case 'store in file': + return await executeStoreInFile(ctx); + case 'list directory content': + return await executeListDirectoryContent(ctx); + default: + return ctx.fail('no math'); + } +}; + +export const fsPlugins = new Set([fsPlugin]); diff --git a/pkg/fs/test/make-ava-happy.ts b/pkg/fs/test/make-ava-happy.ts new file mode 100644 index 00000000..aaa68a69 --- /dev/null +++ b/pkg/fs/test/make-ava-happy.ts @@ -0,0 +1,3 @@ +import test from 'ava'; + +test('ava is happy', (t) => t.true(true)); diff --git a/pkg/fs/tsconfig.json b/pkg/fs/tsconfig.json new file mode 120000 index 00000000..fd0e4743 --- /dev/null +++ b/pkg/fs/tsconfig.json @@ -0,0 +1 @@ +../../tsconfig.json \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0e4c2293..3c882b03 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -75,6 +75,18 @@ importers: specifier: ^3.10.0 version: 3.10.0 + pkg/fs: + dependencies: + '@slangroom/core': + specifier: workspace:* + version: link:../core + axios: + specifier: ^1.5.1 + version: 1.5.1 + extract-zip: + specifier: ^2.0.1 + version: 2.0.1 + pkg/http: dependencies: '@slangroom/core': @@ -1062,7 +1074,6 @@ packages: /@types/node@20.3.1: resolution: {integrity: sha512-EhcH/wvidPy1WeML3TtYFGR83UzjxeWRen9V402T8aUGYsCHOmfoisV3ZSg03gAFIbLq8TnWOJ0f4cALtnSEUg==} - dev: true /@types/normalize-package-data@2.4.2: resolution: {integrity: sha512-lqa4UEhhv/2sjjIQgjX8B+RBjj47eo0mzGasklVJ78UKGQY1r0VpB9XHDaZZO9qzEFDdy4MrXLuEaSmPrPSe/A==} @@ -1072,6 +1083,14 @@ packages: resolution: {integrity: sha512-G8hZ6XJiHnuhQKR7ZmysCeJWE08o8T0AXtk5darsCaTVsYZhhgUrq53jizaR2FvsoeCwJhlmwTjkXBY5Pn/ZHw==} dev: true + /@types/yauzl@2.10.1: + resolution: {integrity: sha512-CHzgNU3qYBnp/O4S3yv2tXPlvMTq0YWSTVg2/JYLqWZGHwwgJGAwd00poay/11asPq8wLFwHzubyInqHIFmmiw==} + requiresBuild: true + dependencies: + '@types/node': 20.3.1 + dev: false + optional: true + /@typescript-eslint/eslint-plugin@5.59.11(@typescript-eslint/parser@5.59.11)(eslint@8.43.0)(typescript@4.9.5): resolution: {integrity: sha512-XxuOfTkCUiOSyBWIvHlUraLw/JT/6Io1365RO6ZuI88STKMavJZPNMU0lFcUTeQXEhHiv64CbxYxBNoDVSmghg==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} @@ -1540,6 +1559,10 @@ packages: fill-range: 7.0.1 dev: true + /buffer-crc32@0.2.13: + resolution: {integrity: sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==} + dev: false + /buffer-from@1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} dev: true @@ -2078,7 +2101,6 @@ packages: optional: true dependencies: ms: 2.1.2 - dev: true /decamelize-keys@1.1.1: resolution: {integrity: sha512-WiPxgEirIV0/eIOMcnFBA3/IJZAZqKnwAwWyvvdi4lsr1WCN22nhdf/3db3DoZcUjTV2SqfzIwNyp6y2xs3nmg==} @@ -2195,6 +2217,12 @@ packages: dev: true optional: true + /end-of-stream@1.4.4: + resolution: {integrity: sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==} + dependencies: + once: 1.4.0 + dev: false + /env-paths@2.2.1: resolution: {integrity: sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==} engines: {node: '>=6'} @@ -2429,6 +2457,20 @@ packages: tmp: 0.0.33 dev: true + /extract-zip@2.0.1: + resolution: {integrity: sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==} + engines: {node: '>= 10.17.0'} + hasBin: true + dependencies: + debug: 4.3.4 + get-stream: 5.2.0 + yauzl: 2.10.0 + optionalDependencies: + '@types/yauzl': 2.10.1 + transitivePeerDependencies: + - supports-color + dev: false + /fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} dev: true @@ -2473,6 +2515,12 @@ packages: reusify: 1.0.4 dev: true + /fd-slicer@1.1.0: + resolution: {integrity: sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==} + dependencies: + pend: 1.2.0 + dev: false + /fetch-blob@3.2.0: resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} engines: {node: ^12.20 || >= 14.13} @@ -2671,6 +2719,13 @@ packages: yargs: 16.2.0 dev: true + /get-stream@5.2.0: + resolution: {integrity: sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==} + engines: {node: '>=8'} + dependencies: + pump: 3.0.0 + dev: false + /get-stream@8.0.1: resolution: {integrity: sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==} engines: {node: '>=16'} @@ -3717,7 +3772,6 @@ packages: /ms@2.1.2: resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} - dev: true /ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} @@ -3957,7 +4011,6 @@ packages: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} dependencies: wrappy: 1.0.2 - dev: true /onetime@5.1.2: resolution: {integrity: sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==} @@ -4269,6 +4322,10 @@ packages: engines: {node: '>=8'} dev: true + /pend@1.2.0: + resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==} + dev: false + /picomatch@2.3.1: resolution: {integrity: sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==} engines: {node: '>=8.6'} @@ -4393,6 +4450,13 @@ packages: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} dev: false + /pump@3.0.0: + resolution: {integrity: sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==} + dependencies: + end-of-stream: 1.4.4 + once: 1.4.0 + dev: false + /punycode@2.3.0: resolution: {integrity: sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==} engines: {node: '>=6'} @@ -5323,7 +5387,6 @@ packages: /wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} - dev: true /write-file-atomic@3.0.3: resolution: {integrity: sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q==} @@ -5413,6 +5476,13 @@ packages: yargs-parser: 21.1.1 dev: true + /yauzl@2.10.0: + resolution: {integrity: sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==} + dependencies: + buffer-crc32: 0.2.13 + fd-slicer: 1.1.0 + dev: false + /yn@3.1.1: resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==} engines: {node: '>=6'}