From 92bd1fa6965da50152205d5d5c09cde93b85c509 Mon Sep 17 00:00:00 2001 From: Steven Roussey Date: Fri, 31 Jan 2025 06:27:30 -0800 Subject: [PATCH 1/4] clean: no-worker package ai --- packages/ai/package.json | 12 +++--------- packages/ai/src/worker.ts | 0 packages/ai/tsconfig.json | 2 +- 3 files changed, 4 insertions(+), 10 deletions(-) delete mode 100644 packages/ai/src/worker.ts diff --git a/packages/ai/package.json b/packages/ai/package.json index 40c274e..ccb4f02 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -5,13 +5,11 @@ "description": "Ellmers is a tool for building and running DAG pipelines of AI tasks.", "scripts": { "watch": "concurrently -c 'auto' 'bun:watch-*'", - "watch-browser": "bun build --watch --no-clear-screen --target=browser --sourcemap=external --external ellmers-core --outdir ./dist ./src/index.ts", - "watch-worker": "bun build --watch --no-clear-screen --target=bun --sourcemap=external --external ellmers-core --outdir ./dist ./src/worker.ts", + "watch-code": "bun build --watch --no-clear-screen --target=browser --sourcemap=external --external ellmers-core --outdir ./dist ./src/index.ts", "watch-types": "tsc --watch --preserveWatchOutput", - "build": "bun run build-clean && bun run build-types && bun run build-browser && bun run build-worker", + "build": "bun run build-clean && bun run build-types && bun run build-code", "build-clean": "rm -fr dist/* tsconfig.tsbuildinfo", - "build-browser": "bun build --target=browser --sourcemap=external --external ellmers-core --outdir ./dist ./src/index.ts", - "build-worker": "bun build --target=bun --sourcemap=external --external ellmers-core --outdir ./dist ./src/worker.ts", + "build-code": "bun build --target=browser --sourcemap=external --external ellmers-core --outdir ./dist ./src/index.ts", "build-types": "tsc", "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0", "test": "bun test" @@ -22,10 +20,6 @@ ".": { "import": "./dist/index.js", "types": "./dist/index.d.ts" - }, - "./worker": { - "import": "./dist/worker.js", - "types": "./dist/worker.d.ts" } }, "files": [ diff --git a/packages/ai/src/worker.ts b/packages/ai/src/worker.ts deleted file mode 100644 index e69de29..0000000 diff --git a/packages/ai/tsconfig.json b/packages/ai/tsconfig.json index e948941..3fa377b 100644 --- a/packages/ai/tsconfig.json +++ b/packages/ai/tsconfig.json @@ -1,7 +1,7 @@ { "extends": "../../tsconfig.json", "include": ["src/**/*"], - "files": ["src/index.ts", "src/worker.ts"], + "files": ["src/index.ts"], "exclude": ["**/*.test.ts", "dist"], "compilerOptions": { "composite": true, From d5c8294cbd0a5f76d325caada551be08a4d09956 Mon Sep 17 00:00:00 2001 From: Steven Roussey Date: Fri, 31 Jan 2025 06:31:04 -0800 Subject: [PATCH 2/4] clean: package core --- packages/core/package.json | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/packages/core/package.json b/packages/core/package.json index 9021527..66e8747 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -5,15 +5,11 @@ "description": "Ellmers is a tool for building and running DAG pipelines of AI tasks.", "scripts": { "watch": "concurrently -c 'auto' 'bun:watch-*'", - "watch-browser": "bun build --watch --no-clear-screen --target=browser --sourcemap=external --outdir ./dist ./src/index.ts", - "no-watch-node": "bun build --watch --no-clear-screen --target=node --sourcemap=external --outdir ./dist ./src/index.ts --entry-naming node.js", - "no-watch-bun": "bun build --watch --no-clear-screen --target=bun --sourcemap=external --outdir ./dist ./src/index.ts --entry-naming bun.js", + "watch-code": "bun build --watch --no-clear-screen --target=browser --sourcemap=external --outdir ./dist ./src/index.ts", "watch-types": "tsc --watch --preserveWatchOutput", - "build": "bun run build-clean && bun run build-types && bun run build-browser", + "build": "bun run build-clean && bun run build-types && bun run build-code", "build-clean": "rm -fr dist/* tsconfig.tsbuildinfo", - "build-browser": "bun build --target=browser --sourcemap=external --outdir ./dist ./src/index.ts", - "build-node": "bun build --target=node --sourcemap=external --outdir ./dist ./src/index.ts --entry-naming node.js", - "build-bun": "bun build --target=bun --sourcemap=external --outdir ./dist ./src/index.ts --entry-naming bun.js", + "build-code": "bun build --target=browser --sourcemap=external --outdir ./dist ./src/index.ts", "build-types": "tsc", "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0", "test": "bun test" From ba5ff537213bbe7e4d199e6607d94b0cd719ccda Mon Sep 17 00:00:00 2001 From: Steven Roussey Date: Fri, 31 Jan 2025 07:03:31 -0800 Subject: [PATCH 3/4] clean: simplify ellmers-ai-provider --- examples/web/src/App.tsx | 4 +- packages/ai-provider/package.json | 46 ++++--------------- .../hf-transformers/{browser.ts => index.ts} | 0 .../provider/HuggingFaceLocal_TaskRun.ts | 2 +- .../ai-provider/src/hf-transformers/server.ts | 8 ---- .../ai-provider/src/hf-transformers/worker.ts | 0 .../tf-mediapipe/bindings/registerTasks.ts | 2 +- .../src/tf-mediapipe/{browser.ts => index.ts} | 0 .../ai-provider/src/tf-mediapipe/server.ts | 8 ---- .../ai-provider/src/tf-mediapipe/worker.ts | 0 packages/ai-provider/tsconfig.json | 9 +--- .../test/InMemoryModelRepository.test.ts | 2 +- .../sqlite/test/SqliteModelRepository.test.ts | 2 +- packages/test/src/index.ts | 4 +- .../test/src/sample/MediaPipeModelSamples.ts | 2 +- packages/test/src/sample/ONNXModelSamples.ts | 2 +- 16 files changed, 21 insertions(+), 70 deletions(-) rename packages/ai-provider/src/hf-transformers/{browser.ts => index.ts} (100%) delete mode 100644 packages/ai-provider/src/hf-transformers/server.ts delete mode 100644 packages/ai-provider/src/hf-transformers/worker.ts rename packages/ai-provider/src/tf-mediapipe/{browser.ts => index.ts} (100%) delete mode 100644 packages/ai-provider/src/tf-mediapipe/server.ts delete mode 100644 packages/ai-provider/src/tf-mediapipe/worker.ts diff --git a/examples/web/src/App.tsx b/examples/web/src/App.tsx index 4926e02..11c8c4d 100644 --- a/examples/web/src/App.tsx +++ b/examples/web/src/App.tsx @@ -24,11 +24,11 @@ import { getProviderRegistry } from "ellmers-ai"; import { LOCAL_ONNX_TRANSFORMERJS, registerHuggingfaceLocalTasks, -} from "ellmers-ai-provider/hf-transformers/browser"; +} from "ellmers-ai-provider/hf-transformers"; import { MEDIA_PIPE_TFJS_MODEL, registerMediaPipeTfJsLocalTasks, -} from "ellmers-ai-provider/tf-mediapipe/browser"; +} from "ellmers-ai-provider/tf-mediapipe"; import "ellmers-task"; import "ellmers-test"; import { registerMediaPipeTfJsLocalModels } from "ellmers-test"; diff --git a/packages/ai-provider/package.json b/packages/ai-provider/package.json index 6742c65..6ddded4 100644 --- a/packages/ai-provider/package.json +++ b/packages/ai-provider/package.json @@ -5,51 +5,25 @@ "description": "Ellmers is a tool for building and running DAG pipelines of AI tasks.", "scripts": { "watch": "concurrently -c 'auto' 'bun:watch-*'", - "watch-hf-transformers-browser": "bun build --watch --no-clear-screen --target=browser --sourcemap=external --external @huggingface/transformers --external ellmers-core --external ellmers-ai --outdir ./dist/hf-transformers ./src/hf-transformers/browser.ts", - "watch-hf-transformers-server": "bun build --watch --no-clear-screen --target=node --sourcemap=external --external @huggingface/transformers --external ellmers-core --external ellmers-ai --outdir ./dist/hf-transformers ./src/hf-transformers/server.ts", - "watch-hf-transformers-worker": "bun build --watch --no-clear-screen --target=bun --sourcemap=external --external @huggingface/transformers --external ellmers-core --external ellmers-ai --outdir ./dist/hf-transformers ./src/hf-transformers/worker.ts", - "watch-tf-mediapipe-browser": "bun build --watch --no-clear-screen --target=browser --sourcemap=external --external @mediapipe/tasks-text --external ellmers-core --external ellmers-ai --outdir ./dist/tf-mediapipe ./src/tf-mediapipe/browser.ts", - "watch-tf-mediapipe-server": "bun build --watch --no-clear-screen --target=node --sourcemap=external --external @mediapipe/tasks-text --external ellmers-core --external ellmers-ai --outdir ./dist/tf-mediapipe ./src/tf-mediapipe/server.ts", - "watch-tf-mediapipe-worker": "bun build --watch --no-clear-screen --target=bun --sourcemap=external --external @mediapipe/tasks-text --external ellmers-core --external ellmers-ai --outdir ./dist/tf-mediapipe ./src/tf-mediapipe/worker.ts", + "watch-hf-transformers": "bun build --watch --no-clear-screen --sourcemap=external --external @huggingface/transformers --external ellmers-core --external ellmers-ai --outdir ./dist/hf-transformers ./src/hf-transformers/index.ts", + "watch-tf-mediapipe": "bun build --watch --no-clear-screen --sourcemap=external --external @mediapipe/tasks-text --external ellmers-core --external ellmers-ai --outdir ./dist/tf-mediapipe ./src/tf-mediapipe/index.ts", "watch-types": "tsc --watch --preserveWatchOutput", "build": "bun run build-clean && bun run build-types && bun run build-hf-transformers && bun run build-tf-mediapipe", "build-clean": "rm -fr dist/* tsconfig.tsbuildinfo", - "build-hf-transformers": "bun run build-hf-transformers-browser && bun run build-hf-transformers-server && bun run build-hf-transformers-worker", - "build-hf-transformers-browser": "bun build --target=browser --sourcemap=external --external @huggingface/transformers --external ellmers-core --external ellmers-ai --outdir ./dist/hf-transformers ./src/hf-transformers/browser.ts", - "build-hf-transformers-server": "bun build --target=node --sourcemap=external --external @huggingface/transformers --external ellmers-core --external ellmers-ai --outdir ./dist/hf-transformers ./src/hf-transformers/server.ts", - "build-hf-transformers-worker": "bun build --target=bun --sourcemap=external --external @huggingface/transformers --external ellmers-core --external ellmers-ai --outdir ./dist/hf-transformers ./src/hf-transformers/worker.ts", - "build-tf-mediapipe": "bun run build-tf-mediapipe-browser && bun run build-tf-mediapipe-server && bun run build-tf-mediapipe-worker", - "build-tf-mediapipe-browser": "bun build --target=browser --sourcemap=external --external @mediapipe/tasks-text --external ellmers-core --external ellmers-ai --outdir ./dist/tf-mediapipe ./src/tf-mediapipe/browser.ts", - "build-tf-mediapipe-server": "bun build --target=node --sourcemap=external --external @mediapipe/tasks-text --external ellmers-core --external ellmers-ai --outdir ./dist/tf-mediapipe ./src/tf-mediapipe/server.ts", - "build-tf-mediapipe-worker": "bun build --target=bun --sourcemap=external --external @mediapipe/tasks-text --external ellmers-core --external ellmers-ai --outdir ./dist/tf-mediapipe ./src/tf-mediapipe/worker.ts", + "build-hf-transformers": "bun build --sourcemap=external --external @huggingface/transformers --external ellmers-core --external ellmers-ai --outdir ./dist/hf-transformers ./src/hf-transformers/index.ts", + "build-tf-mediapipe": "bun build --sourcemap=external --external @mediapipe/tasks-text --external ellmers-core --external ellmers-ai --outdir ./dist/tf-mediapipe ./src/tf-mediapipe/index.ts", "build-types": "tsc", "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0", "test": "bun test" }, "exports": { - "./hf-transformers/browser": { - "import": "./dist/hf-transformers/browser.js", - "types": "./dist/hf-transformers/browser.d.ts" + "./hf-transformers": { + "import": "./dist/hf-transformers/index.js", + "types": "./dist/hf-transformers/index.d.ts" }, - "./hf-transformers/server": { - "import": "./dist/hf-transformers/server.js", - "types": "./dist/hf-transformers/server.d.ts" - }, - "./hf-transformers/worker": { - "import": "./dist/hf-transformers/worker.js", - "types": "./dist/hf-transformers/worker.d.ts" - }, - "./tf-mediapipe/browser": { - "import": "./dist/tf-mediapipe/browser.js", - "types": "./dist/tf-mediapipe/browser.d.ts" - }, - "./tf-mediapipe/server": { - "import": "./dist/tf-mediapipe/server.js", - "types": "./dist/tf-mediapipe/server.d.ts" - }, - "./tf-mediapipe/worker": { - "import": "./dist/tf-mediapipe/worker.js", - "types": "./dist/tf-mediapipe/worker.d.ts" + "./tf-mediapipe": { + "import": "./dist/tf-mediapipe/index.js", + "types": "./dist/tf-mediapipe/index.d.ts" } }, "files": [ diff --git a/packages/ai-provider/src/hf-transformers/browser.ts b/packages/ai-provider/src/hf-transformers/index.ts similarity index 100% rename from packages/ai-provider/src/hf-transformers/browser.ts rename to packages/ai-provider/src/hf-transformers/index.ts diff --git a/packages/ai-provider/src/hf-transformers/provider/HuggingFaceLocal_TaskRun.ts b/packages/ai-provider/src/hf-transformers/provider/HuggingFaceLocal_TaskRun.ts index 41b1c4b..1853dec 100644 --- a/packages/ai-provider/src/hf-transformers/provider/HuggingFaceLocal_TaskRun.ts +++ b/packages/ai-provider/src/hf-transformers/provider/HuggingFaceLocal_TaskRun.ts @@ -47,7 +47,7 @@ import type { TextTranslationTaskOutput, Model, } from "ellmers-ai"; -import { QUANTIZATION_DATA_TYPES } from "../browser"; +import { QUANTIZATION_DATA_TYPES } from ".."; env.cacheDir = "./.cache"; diff --git a/packages/ai-provider/src/hf-transformers/server.ts b/packages/ai-provider/src/hf-transformers/server.ts deleted file mode 100644 index 094a65b..0000000 --- a/packages/ai-provider/src/hf-transformers/server.ts +++ /dev/null @@ -1,8 +0,0 @@ -// ******************************************************************************* -// * ELLMERS: Embedding Large Language Model Experiential Retrieval Service * -// * * -// * Copyright Steven Roussey * -// * Licensed under the Apache License, Version 2.0 (the "License"); * -// ******************************************************************************* - -export * from "./browser"; diff --git a/packages/ai-provider/src/hf-transformers/worker.ts b/packages/ai-provider/src/hf-transformers/worker.ts deleted file mode 100644 index e69de29..0000000 diff --git a/packages/ai-provider/src/tf-mediapipe/bindings/registerTasks.ts b/packages/ai-provider/src/tf-mediapipe/bindings/registerTasks.ts index 06cdf81..042d78b 100644 --- a/packages/ai-provider/src/tf-mediapipe/bindings/registerTasks.ts +++ b/packages/ai-provider/src/tf-mediapipe/bindings/registerTasks.ts @@ -4,7 +4,7 @@ import { MediaPipeTfJsLocal_Download, MediaPipeTfJsLocal_Embedding, } from "../provider/MediaPipeLocalTaskRun"; -import { MEDIA_PIPE_TFJS_MODEL } from "../browser"; +import { MEDIA_PIPE_TFJS_MODEL } from ".."; export const registerMediaPipeTfJsLocalTasks = () => { const ProviderRegistry = getProviderRegistry(); diff --git a/packages/ai-provider/src/tf-mediapipe/browser.ts b/packages/ai-provider/src/tf-mediapipe/index.ts similarity index 100% rename from packages/ai-provider/src/tf-mediapipe/browser.ts rename to packages/ai-provider/src/tf-mediapipe/index.ts diff --git a/packages/ai-provider/src/tf-mediapipe/server.ts b/packages/ai-provider/src/tf-mediapipe/server.ts deleted file mode 100644 index 094a65b..0000000 --- a/packages/ai-provider/src/tf-mediapipe/server.ts +++ /dev/null @@ -1,8 +0,0 @@ -// ******************************************************************************* -// * ELLMERS: Embedding Large Language Model Experiential Retrieval Service * -// * * -// * Copyright Steven Roussey * -// * Licensed under the Apache License, Version 2.0 (the "License"); * -// ******************************************************************************* - -export * from "./browser"; diff --git a/packages/ai-provider/src/tf-mediapipe/worker.ts b/packages/ai-provider/src/tf-mediapipe/worker.ts deleted file mode 100644 index e69de29..0000000 diff --git a/packages/ai-provider/tsconfig.json b/packages/ai-provider/tsconfig.json index cc4669d..39093d4 100644 --- a/packages/ai-provider/tsconfig.json +++ b/packages/ai-provider/tsconfig.json @@ -1,14 +1,7 @@ { "extends": "../../tsconfig.json", "include": ["src/**/*"], - "files": [ - "src/hf-transformers/browser.ts", - "src/hf-transformers/server.ts", - "src/hf-transformers/worker.ts", - "src/tf-mediapipe/browser.ts", - "src/tf-mediapipe/server.ts", - "src/tf-mediapipe/worker.ts" - ], + "files": ["src/hf-transformers/index.ts", "src/tf-mediapipe/index.ts"], "exclude": ["**/*.test.ts", "dist"], "compilerOptions": { "composite": true, diff --git a/packages/storage/src/browser/inmemory/test/InMemoryModelRepository.test.ts b/packages/storage/src/browser/inmemory/test/InMemoryModelRepository.test.ts index 496bc95..0cfb396 100644 --- a/packages/storage/src/browser/inmemory/test/InMemoryModelRepository.test.ts +++ b/packages/storage/src/browser/inmemory/test/InMemoryModelRepository.test.ts @@ -8,7 +8,7 @@ import { describe, expect, it, beforeEach } from "bun:test"; import { setGlobalModelRepository, getGlobalModelRepository } from "ellmers-ai"; import { InMemoryModelRepository } from "../InMemoryModelRepository"; -import { LOCAL_ONNX_TRANSFORMERJS } from "ellmers-ai-provider/hf-transformers/server"; +import { LOCAL_ONNX_TRANSFORMERJS } from "ellmers-ai-provider/hf-transformers"; describe("InMemoryModelRepository", () => { it("store and find model by task", async () => { diff --git a/packages/storage/src/bun/sqlite/test/SqliteModelRepository.test.ts b/packages/storage/src/bun/sqlite/test/SqliteModelRepository.test.ts index 74edc6c..ae77a94 100644 --- a/packages/storage/src/bun/sqlite/test/SqliteModelRepository.test.ts +++ b/packages/storage/src/bun/sqlite/test/SqliteModelRepository.test.ts @@ -8,7 +8,7 @@ import { describe, expect, it, beforeEach } from "bun:test"; import { setGlobalModelRepository, getGlobalModelRepository } from "ellmers-ai"; import { SqliteModelRepository } from "../SqliteModelRepository"; -import { LOCAL_ONNX_TRANSFORMERJS } from "ellmers-ai-provider/hf-transformers/server"; +import { LOCAL_ONNX_TRANSFORMERJS } from "ellmers-ai-provider/hf-transformers"; describe("SqliteModelRepository", () => { it("store and find model by task", async () => { diff --git a/packages/test/src/index.ts b/packages/test/src/index.ts index 27d9f56..50ec1c5 100644 --- a/packages/test/src/index.ts +++ b/packages/test/src/index.ts @@ -2,11 +2,11 @@ import { getProviderRegistry } from "ellmers-ai"; import { LOCAL_ONNX_TRANSFORMERJS, registerHuggingfaceLocalTasks, -} from "ellmers-ai-provider/hf-transformers/browser"; +} from "ellmers-ai-provider/hf-transformers"; import { MEDIA_PIPE_TFJS_MODEL, registerMediaPipeTfJsLocalTasks, -} from "ellmers-ai-provider/tf-mediapipe/browser"; +} from "../../ai-provider/dist/tf-mediapipe"; import { ConcurrencyLimiter, TaskInput, TaskOutput } from "ellmers-core"; import { InMemoryJobQueue } from "ellmers-storage/inmemory"; diff --git a/packages/test/src/sample/MediaPipeModelSamples.ts b/packages/test/src/sample/MediaPipeModelSamples.ts index f15ae35..3640dee 100644 --- a/packages/test/src/sample/MediaPipeModelSamples.ts +++ b/packages/test/src/sample/MediaPipeModelSamples.ts @@ -1,4 +1,4 @@ -import { MEDIA_PIPE_TFJS_MODEL } from "ellmers-ai-provider/tf-mediapipe/browser"; +import { MEDIA_PIPE_TFJS_MODEL } from "../../../ai-provider/dist/tf-mediapipe"; import { getGlobalModelRepository, Model } from "ellmers-ai"; async function addMediaPipeModel(info: Partial, tasks: string[]) { diff --git a/packages/test/src/sample/ONNXModelSamples.ts b/packages/test/src/sample/ONNXModelSamples.ts index cc127d0..d8831a9 100644 --- a/packages/test/src/sample/ONNXModelSamples.ts +++ b/packages/test/src/sample/ONNXModelSamples.ts @@ -1,7 +1,7 @@ import { LOCAL_ONNX_TRANSFORMERJS, QUANTIZATION_DATA_TYPES, -} from "ellmers-ai-provider/hf-transformers/browser"; +} from "ellmers-ai-provider/hf-transformers"; import { getGlobalModelRepository, Model } from "ellmers-ai"; async function addONNXModel(info: Partial, tasks: string[]) { From 30b3a18dcbb7b466b764fdabe6cb63259fa94775 Mon Sep 17 00:00:00 2001 From: Steven Roussey Date: Fri, 31 Jan 2025 07:06:24 -0800 Subject: [PATCH 4/4] clean: nit --- .vscode/settings.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.vscode/settings.json b/.vscode/settings.json index 6adefc4..687f24c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -10,6 +10,7 @@ "files.exclude": { // "**/dist": true, // "**/build": true, + "**/storybook-static": true, "**/*.tsbuildinfo": true, "**/node_modules": true, "**/dist": true