Merge pull request #20 from sroussey/no-worker-cleanup

Cleanup some old idea
sroussey · Jan 31, 2025 · 015c416 · 015c416
2 parents 015bd97 + 30b3a18
commit 015c416
Show file tree

Hide file tree

Showing 21 changed files with 29 additions and 87 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -10,6 +10,7 @@
   "files.exclude": {
     // "**/dist": true,
     // "**/build": true,
+    "**/storybook-static": true,
     "**/*.tsbuildinfo": true,
     "**/node_modules": true,
     "**/dist": true

diff --git a/examples/web/src/App.tsx b/examples/web/src/App.tsx
@@ -24,11 +24,11 @@ import { getProviderRegistry } from "ellmers-ai";
 import {
   LOCAL_ONNX_TRANSFORMERJS,
   registerHuggingfaceLocalTasks,
-} from "ellmers-ai-provider/hf-transformers/browser";
+} from "ellmers-ai-provider/hf-transformers";
 import {
   MEDIA_PIPE_TFJS_MODEL,
   registerMediaPipeTfJsLocalTasks,
-} from "ellmers-ai-provider/tf-mediapipe/browser";
+} from "ellmers-ai-provider/tf-mediapipe";
 import "ellmers-task";
 import "ellmers-test";
 import { registerMediaPipeTfJsLocalModels } from "ellmers-test";

diff --git a/packages/ai-provider/package.json b/packages/ai-provider/package.json
@@ -5,51 +5,25 @@
   "description": "Ellmers is a tool for building and running DAG pipelines of AI tasks.",
   "scripts": {
     "watch": "concurrently -c 'auto' 'bun:watch-*'",
-    "watch-hf-transformers-browser": "bun build --watch --no-clear-screen --target=browser --sourcemap=external --external @huggingface/transformers --external ellmers-core --external ellmers-ai --outdir ./dist/hf-transformers ./src/hf-transformers/browser.ts",
-    "watch-hf-transformers-server": "bun build --watch --no-clear-screen --target=node --sourcemap=external --external @huggingface/transformers --external ellmers-core --external ellmers-ai --outdir ./dist/hf-transformers ./src/hf-transformers/server.ts",
-    "watch-hf-transformers-worker": "bun build --watch --no-clear-screen --target=bun --sourcemap=external --external @huggingface/transformers --external ellmers-core --external ellmers-ai --outdir ./dist/hf-transformers ./src/hf-transformers/worker.ts",
-    "watch-tf-mediapipe-browser": "bun build --watch --no-clear-screen --target=browser --sourcemap=external --external @mediapipe/tasks-text --external ellmers-core --external ellmers-ai --outdir ./dist/tf-mediapipe ./src/tf-mediapipe/browser.ts",
-    "watch-tf-mediapipe-server": "bun build --watch --no-clear-screen --target=node --sourcemap=external --external @mediapipe/tasks-text --external ellmers-core --external ellmers-ai --outdir ./dist/tf-mediapipe ./src/tf-mediapipe/server.ts",
-    "watch-tf-mediapipe-worker": "bun build --watch --no-clear-screen --target=bun --sourcemap=external --external @mediapipe/tasks-text --external ellmers-core --external ellmers-ai --outdir ./dist/tf-mediapipe ./src/tf-mediapipe/worker.ts",
+    "watch-hf-transformers": "bun build --watch --no-clear-screen --sourcemap=external --external @huggingface/transformers --external ellmers-core --external ellmers-ai --outdir ./dist/hf-transformers ./src/hf-transformers/index.ts",
+    "watch-tf-mediapipe": "bun build --watch --no-clear-screen --sourcemap=external --external @mediapipe/tasks-text --external ellmers-core --external ellmers-ai --outdir ./dist/tf-mediapipe ./src/tf-mediapipe/index.ts",
     "watch-types": "tsc --watch --preserveWatchOutput",
     "build": "bun run build-clean && bun run build-types && bun run build-hf-transformers && bun run build-tf-mediapipe",
     "build-clean": "rm -fr dist/* tsconfig.tsbuildinfo",
-    "build-hf-transformers": "bun run build-hf-transformers-browser && bun run build-hf-transformers-server && bun run build-hf-transformers-worker",
-    "build-hf-transformers-browser": "bun build --target=browser --sourcemap=external --external @huggingface/transformers --external ellmers-core --external ellmers-ai --outdir ./dist/hf-transformers ./src/hf-transformers/browser.ts",
-    "build-hf-transformers-server": "bun build --target=node --sourcemap=external --external @huggingface/transformers --external ellmers-core --external ellmers-ai --outdir ./dist/hf-transformers ./src/hf-transformers/server.ts",
-    "build-hf-transformers-worker": "bun build --target=bun --sourcemap=external --external @huggingface/transformers --external ellmers-core --external ellmers-ai --outdir ./dist/hf-transformers ./src/hf-transformers/worker.ts",
-    "build-tf-mediapipe": "bun run build-tf-mediapipe-browser && bun run build-tf-mediapipe-server && bun run build-tf-mediapipe-worker",
-    "build-tf-mediapipe-browser": "bun build --target=browser --sourcemap=external --external @mediapipe/tasks-text --external ellmers-core --external ellmers-ai --outdir ./dist/tf-mediapipe ./src/tf-mediapipe/browser.ts",
-    "build-tf-mediapipe-server": "bun build --target=node --sourcemap=external --external @mediapipe/tasks-text --external ellmers-core --external ellmers-ai --outdir ./dist/tf-mediapipe ./src/tf-mediapipe/server.ts",
-    "build-tf-mediapipe-worker": "bun build --target=bun --sourcemap=external --external @mediapipe/tasks-text --external ellmers-core --external ellmers-ai --outdir ./dist/tf-mediapipe ./src/tf-mediapipe/worker.ts",
+    "build-hf-transformers": "bun build --sourcemap=external --external @huggingface/transformers --external ellmers-core --external ellmers-ai --outdir ./dist/hf-transformers ./src/hf-transformers/index.ts",
+    "build-tf-mediapipe": "bun build --sourcemap=external --external @mediapipe/tasks-text --external ellmers-core --external ellmers-ai --outdir ./dist/tf-mediapipe ./src/tf-mediapipe/index.ts",
     "build-types": "tsc",
     "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
     "test": "bun test"
   },
   "exports": {
-    "./hf-transformers/browser": {
-      "import": "./dist/hf-transformers/browser.js",
-      "types": "./dist/hf-transformers/browser.d.ts"
+    "./hf-transformers": {
+      "import": "./dist/hf-transformers/index.js",
+      "types": "./dist/hf-transformers/index.d.ts"
     },
-    "./hf-transformers/server": {
-      "import": "./dist/hf-transformers/server.js",
-      "types": "./dist/hf-transformers/server.d.ts"
-    },
-    "./hf-transformers/worker": {
-      "import": "./dist/hf-transformers/worker.js",
-      "types": "./dist/hf-transformers/worker.d.ts"
-    },
-    "./tf-mediapipe/browser": {
-      "import": "./dist/tf-mediapipe/browser.js",
-      "types": "./dist/tf-mediapipe/browser.d.ts"
-    },
-    "./tf-mediapipe/server": {
-      "import": "./dist/tf-mediapipe/server.js",
-      "types": "./dist/tf-mediapipe/server.d.ts"
-    },
-    "./tf-mediapipe/worker": {
-      "import": "./dist/tf-mediapipe/worker.js",
-      "types": "./dist/tf-mediapipe/worker.d.ts"
+    "./tf-mediapipe": {
+      "import": "./dist/tf-mediapipe/index.js",
+      "types": "./dist/tf-mediapipe/index.d.ts"
     }
   },
   "files": [

diff --git a/...i-provider/src/hf-transformers/browser.ts → .../ai-provider/src/hf-transformers/index.ts b/...i-provider/src/hf-transformers/browser.ts → .../ai-provider/src/hf-transformers/index.ts
diff --git a/packages/ai-provider/src/hf-transformers/provider/HuggingFaceLocal_TaskRun.ts b/packages/ai-provider/src/hf-transformers/provider/HuggingFaceLocal_TaskRun.ts
@@ -47,7 +47,7 @@ import type {
   TextTranslationTaskOutput,
   Model,
 } from "ellmers-ai";
-import { QUANTIZATION_DATA_TYPES } from "../browser";
+import { QUANTIZATION_DATA_TYPES } from "..";
 
 env.cacheDir = "./.cache";
 

diff --git a/packages/ai-provider/src/hf-transformers/server.ts b/packages/ai-provider/src/hf-transformers/server.ts
diff --git a/packages/ai-provider/src/hf-transformers/worker.ts b/packages/ai-provider/src/hf-transformers/worker.ts
diff --git a/packages/ai-provider/src/tf-mediapipe/bindings/registerTasks.ts b/packages/ai-provider/src/tf-mediapipe/bindings/registerTasks.ts
@@ -4,7 +4,7 @@ import {
   MediaPipeTfJsLocal_Download,
   MediaPipeTfJsLocal_Embedding,
 } from "../provider/MediaPipeLocalTaskRun";
-import { MEDIA_PIPE_TFJS_MODEL } from "../browser";
+import { MEDIA_PIPE_TFJS_MODEL } from "..";
 
 export const registerMediaPipeTfJsLocalTasks = () => {
   const ProviderRegistry = getProviderRegistry();

diff --git a/...s/ai-provider/src/tf-mediapipe/browser.ts → ...ges/ai-provider/src/tf-mediapipe/index.ts b/...s/ai-provider/src/tf-mediapipe/browser.ts → ...ges/ai-provider/src/tf-mediapipe/index.ts
diff --git a/packages/ai-provider/src/tf-mediapipe/server.ts b/packages/ai-provider/src/tf-mediapipe/server.ts
diff --git a/packages/ai-provider/src/tf-mediapipe/worker.ts b/packages/ai-provider/src/tf-mediapipe/worker.ts
diff --git a/packages/ai-provider/tsconfig.json b/packages/ai-provider/tsconfig.json
@@ -1,14 +1,7 @@
 {
   "extends": "../../tsconfig.json",
   "include": ["src/**/*"],
-  "files": [
-    "src/hf-transformers/browser.ts",
-    "src/hf-transformers/server.ts",
-    "src/hf-transformers/worker.ts",
-    "src/tf-mediapipe/browser.ts",
-    "src/tf-mediapipe/server.ts",
-    "src/tf-mediapipe/worker.ts"
-  ],
+  "files": ["src/hf-transformers/index.ts", "src/tf-mediapipe/index.ts"],
   "exclude": ["**/*.test.ts", "dist"],
   "compilerOptions": {
     "composite": true,

diff --git a/packages/ai/package.json b/packages/ai/package.json
@@ -5,13 +5,11 @@
   "description": "Ellmers is a tool for building and running DAG pipelines of AI tasks.",
   "scripts": {
     "watch": "concurrently -c 'auto' 'bun:watch-*'",
-    "watch-browser": "bun build --watch --no-clear-screen --target=browser --sourcemap=external --external ellmers-core --outdir ./dist ./src/index.ts",
-    "watch-worker": "bun build --watch --no-clear-screen --target=bun --sourcemap=external --external ellmers-core --outdir ./dist ./src/worker.ts",
+    "watch-code": "bun build --watch --no-clear-screen --target=browser --sourcemap=external --external ellmers-core --outdir ./dist ./src/index.ts",
     "watch-types": "tsc --watch --preserveWatchOutput",
-    "build": "bun run build-clean && bun run build-types && bun run build-browser && bun run build-worker",
+    "build": "bun run build-clean && bun run build-types && bun run build-code",
     "build-clean": "rm -fr dist/* tsconfig.tsbuildinfo",
-    "build-browser": "bun build --target=browser --sourcemap=external --external ellmers-core --outdir ./dist ./src/index.ts",
-    "build-worker": "bun build --target=bun --sourcemap=external --external ellmers-core --outdir ./dist ./src/worker.ts",
+    "build-code": "bun build --target=browser --sourcemap=external --external ellmers-core --outdir ./dist ./src/index.ts",
     "build-types": "tsc",
     "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
     "test": "bun test"
@@ -22,10 +20,6 @@
     ".": {
       "import": "./dist/index.js",
       "types": "./dist/index.d.ts"
-    },
-    "./worker": {
-      "import": "./dist/worker.js",
-      "types": "./dist/worker.d.ts"
     }
   },
   "files": [

diff --git a/packages/ai/src/worker.ts b/packages/ai/src/worker.ts
diff --git a/packages/ai/tsconfig.json b/packages/ai/tsconfig.json
@@ -1,7 +1,7 @@
 {
   "extends": "../../tsconfig.json",
   "include": ["src/**/*"],
-  "files": ["src/index.ts", "src/worker.ts"],
+  "files": ["src/index.ts"],
   "exclude": ["**/*.test.ts", "dist"],
   "compilerOptions": {
     "composite": true,

diff --git a/packages/core/package.json b/packages/core/package.json
@@ -5,15 +5,11 @@
   "description": "Ellmers is a tool for building and running DAG pipelines of AI tasks.",
   "scripts": {
     "watch": "concurrently -c 'auto' 'bun:watch-*'",
-    "watch-browser": "bun build --watch --no-clear-screen --target=browser --sourcemap=external --outdir ./dist ./src/index.ts",
-    "no-watch-node": "bun build --watch --no-clear-screen --target=node --sourcemap=external --outdir ./dist ./src/index.ts --entry-naming node.js",
-    "no-watch-bun": "bun build --watch --no-clear-screen --target=bun --sourcemap=external --outdir ./dist ./src/index.ts --entry-naming bun.js",
+    "watch-code": "bun build --watch --no-clear-screen --target=browser --sourcemap=external --outdir ./dist ./src/index.ts",
     "watch-types": "tsc --watch --preserveWatchOutput",
-    "build": "bun run build-clean && bun run build-types && bun run build-browser",
+    "build": "bun run build-clean && bun run build-types && bun run build-code",
     "build-clean": "rm -fr dist/* tsconfig.tsbuildinfo",
-    "build-browser": "bun build --target=browser --sourcemap=external --outdir ./dist ./src/index.ts",
-    "build-node": "bun build --target=node --sourcemap=external --outdir ./dist ./src/index.ts --entry-naming node.js",
-    "build-bun": "bun build --target=bun --sourcemap=external --outdir ./dist ./src/index.ts --entry-naming bun.js",
+    "build-code": "bun build --target=browser --sourcemap=external --outdir ./dist ./src/index.ts",
     "build-types": "tsc",
     "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
     "test": "bun test"

diff --git a/packages/storage/src/browser/inmemory/test/InMemoryModelRepository.test.ts b/packages/storage/src/browser/inmemory/test/InMemoryModelRepository.test.ts
@@ -8,7 +8,7 @@
 import { describe, expect, it, beforeEach } from "bun:test";
 import { setGlobalModelRepository, getGlobalModelRepository } from "ellmers-ai";
 import { InMemoryModelRepository } from "../InMemoryModelRepository";
-import { LOCAL_ONNX_TRANSFORMERJS } from "ellmers-ai-provider/hf-transformers/server";
+import { LOCAL_ONNX_TRANSFORMERJS } from "ellmers-ai-provider/hf-transformers";
 
 describe("InMemoryModelRepository", () => {
   it("store and find model by task", async () => {

diff --git a/packages/storage/src/bun/sqlite/test/SqliteModelRepository.test.ts b/packages/storage/src/bun/sqlite/test/SqliteModelRepository.test.ts
@@ -8,7 +8,7 @@
 import { describe, expect, it, beforeEach } from "bun:test";
 import { setGlobalModelRepository, getGlobalModelRepository } from "ellmers-ai";
 import { SqliteModelRepository } from "../SqliteModelRepository";
-import { LOCAL_ONNX_TRANSFORMERJS } from "ellmers-ai-provider/hf-transformers/server";
+import { LOCAL_ONNX_TRANSFORMERJS } from "ellmers-ai-provider/hf-transformers";
 
 describe("SqliteModelRepository", () => {
   it("store and find model by task", async () => {

diff --git a/packages/test/src/index.ts b/packages/test/src/index.ts
@@ -2,11 +2,11 @@ import { getProviderRegistry } from "ellmers-ai";
 import {
   LOCAL_ONNX_TRANSFORMERJS,
   registerHuggingfaceLocalTasks,
-} from "ellmers-ai-provider/hf-transformers/browser";
+} from "ellmers-ai-provider/hf-transformers";
 import {
   MEDIA_PIPE_TFJS_MODEL,
   registerMediaPipeTfJsLocalTasks,
-} from "ellmers-ai-provider/tf-mediapipe/browser";
+} from "../../ai-provider/dist/tf-mediapipe";
 import { ConcurrencyLimiter, TaskInput, TaskOutput } from "ellmers-core";
 import { InMemoryJobQueue } from "ellmers-storage/inmemory";
 

diff --git a/packages/test/src/sample/MediaPipeModelSamples.ts b/packages/test/src/sample/MediaPipeModelSamples.ts
@@ -1,4 +1,4 @@
-import { MEDIA_PIPE_TFJS_MODEL } from "ellmers-ai-provider/tf-mediapipe/browser";
+import { MEDIA_PIPE_TFJS_MODEL } from "../../../ai-provider/dist/tf-mediapipe";
 import { getGlobalModelRepository, Model } from "ellmers-ai";
 
 async function addMediaPipeModel(info: Partial<Model>, tasks: string[]) {

diff --git a/packages/test/src/sample/ONNXModelSamples.ts b/packages/test/src/sample/ONNXModelSamples.ts
@@ -1,7 +1,7 @@
 import {
   LOCAL_ONNX_TRANSFORMERJS,
   QUANTIZATION_DATA_TYPES,
-} from "ellmers-ai-provider/hf-transformers/browser";
+} from "ellmers-ai-provider/hf-transformers";
 import { getGlobalModelRepository, Model } from "ellmers-ai";
 
 async function addONNXModel(info: Partial<Model>, tasks: string[]) {