diff --git a/.changeset/cool-cows-lick.md b/.changeset/cool-cows-lick.md new file mode 100644 index 0000000000..4751bf2180 --- /dev/null +++ b/.changeset/cool-cows-lick.md @@ -0,0 +1,8 @@ +--- +"@llamaindex/env": patch +"llamaindex": patch +"@llamaindex/clip": patch +"@llamaindex/huggingface": patch +--- + +chore: move multi-model into single sub module diff --git a/packages/env/multi-model/package.json b/packages/env/multi-model/package.json new file mode 100644 index 0000000000..35136ddebd --- /dev/null +++ b/packages/env/multi-model/package.json @@ -0,0 +1,12 @@ +{ + "type": "module", + "main": "./dist/index.cjs", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + "browser": "./dist/index.browser.js", + "edge-light": "./dist/index.edge-light.js", + "workerd": "./dist/index.workerd.js" + }, + "private": true +} diff --git a/packages/env/package.json b/packages/env/package.json index 16d43caf02..c11889f799 100644 --- a/packages/env/package.json +++ b/packages/env/package.json @@ -28,7 +28,7 @@ "types": "./dist/index.d.ts", "import": "./dist/index.js", "require": "./dist/index.cjs", - "default": "./dist/index.cjs" + "default": "./dist/index.js" }, "workerd": { "types": "./dist/index.workerd.d.ts", @@ -50,9 +50,36 @@ "types": "./dist/index.d.cts", "default": "./dist/index.cjs" } + }, + "./multi-model": { + "workerd": { + "types": "./multi-model/dist/multi-model.workerd.d.ts", + "default": "./multi-model/dist/multi-model.workerd.js" + }, + "edge-light": { + "types": "./multi-model/dist/multi-model.edge-light.d.ts", + "default": "./multi-model/dist/multi-model.edge-light.js" + }, + "browser": { + "types": "./multi-model/dist/multi-model.browser.d.ts", + "default": "./multi-model/dist/multi-model.browser.js" + }, + "import": { + "types": "./multi-model/dist/multi-model.d.ts", + "default": "./multi-model/dist/multi-model.js" + }, + "require": { + "types": "./multi-model/dist/multi-model.d.cts", + "default": "./multi-model/dist/multi-model.cjs" + }, + "default": { + "types": "./multi-model/dist/multi-model.d.ts", + "default": "./multi-model/dist/multi-model.js" + } } }, "files": [ + "multi-model", "dist", "CHANGELOG.md", "!**/*.tsbuildinfo" diff --git a/packages/env/src/index.browser.ts b/packages/env/src/index.browser.ts index cd1747ffe0..9d55f99edd 100644 --- a/packages/env/src/index.browser.ts +++ b/packages/env/src/index.browser.ts @@ -6,12 +6,6 @@ import "./global-check.js"; export { consoleLogger, emptyLogger, type Logger } from "./logger/index.js"; -export { - loadTransformers, - setTransformers, - type LoadTransformerEvent, - type OnLoad, -} from "./multi-model/index.browser.js"; export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/js.js"; export { NotSupportCurrentRuntimeClass } from "./utils/shared.js"; export * from "./web-polyfill.js"; diff --git a/packages/env/src/index.edge-light.ts b/packages/env/src/index.edge-light.ts index 7facc4e44e..f7e49cd319 100644 --- a/packages/env/src/index.edge-light.ts +++ b/packages/env/src/index.edge-light.ts @@ -5,12 +5,6 @@ */ import "./global-check.js"; export { consoleLogger, emptyLogger, type Logger } from "./logger/index.js"; -export { - loadTransformers, - setTransformers, - type LoadTransformerEvent, - type OnLoad, -} from "./multi-model/index.non-nodejs.js"; export * from "./node-polyfill.js"; export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/js.js"; export { NotSupportCurrentRuntimeClass } from "./utils/shared.js"; diff --git a/packages/env/src/index.ts b/packages/env/src/index.ts index a1233b171b..cb7e77e64f 100644 --- a/packages/env/src/index.ts +++ b/packages/env/src/index.ts @@ -35,12 +35,6 @@ export function createSHA256(): SHA256 { } export { consoleLogger, emptyLogger, type Logger } from "./logger/index.js"; -export { - loadTransformers, - setTransformers, - type LoadTransformerEvent, - type OnLoad, -} from "./multi-model/index.js"; export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/node.js"; export { AsyncLocalStorage, diff --git a/packages/env/src/index.workerd.ts b/packages/env/src/index.workerd.ts index 171fb7d374..1eacb2860c 100644 --- a/packages/env/src/index.workerd.ts +++ b/packages/env/src/index.workerd.ts @@ -16,10 +16,4 @@ export function getEnv(name: string): string | undefined { } export { consoleLogger, emptyLogger, type Logger } from "./logger/index.js"; -export { - loadTransformers, - setTransformers, - type LoadTransformerEvent, - type OnLoad, -} from "./multi-model/index.non-nodejs.js"; export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/js.js"; diff --git a/packages/env/src/multi-model/index.browser.ts b/packages/env/src/internal/multi-model/browser.ts similarity index 100% rename from packages/env/src/multi-model/index.browser.ts rename to packages/env/src/internal/multi-model/browser.ts diff --git a/packages/env/src/multi-model/index.ts b/packages/env/src/internal/multi-model/node.ts similarity index 100% rename from packages/env/src/multi-model/index.ts rename to packages/env/src/internal/multi-model/node.ts diff --git a/packages/env/src/multi-model/index.non-nodejs.ts b/packages/env/src/internal/multi-model/non-node.ts similarity index 100% rename from packages/env/src/multi-model/index.non-nodejs.ts rename to packages/env/src/internal/multi-model/non-node.ts diff --git a/packages/env/src/multi-model/shared.ts b/packages/env/src/internal/multi-model/shared.ts similarity index 100% rename from packages/env/src/multi-model/shared.ts rename to packages/env/src/internal/multi-model/shared.ts diff --git a/packages/env/src/multi-model.browser.ts b/packages/env/src/multi-model.browser.ts new file mode 100644 index 0000000000..d37b0793ec --- /dev/null +++ b/packages/env/src/multi-model.browser.ts @@ -0,0 +1,6 @@ +export { + loadTransformers, + setTransformers, + type LoadTransformerEvent, + type OnLoad, +} from "./internal/multi-model/browser.js"; diff --git a/packages/env/src/multi-model.edge-light.ts b/packages/env/src/multi-model.edge-light.ts new file mode 100644 index 0000000000..b38c78b7f1 --- /dev/null +++ b/packages/env/src/multi-model.edge-light.ts @@ -0,0 +1,6 @@ +export { + loadTransformers, + setTransformers, + type LoadTransformerEvent, + type OnLoad, +} from "./internal/multi-model/non-node.js"; diff --git a/packages/env/src/multi-model.ts b/packages/env/src/multi-model.ts new file mode 100644 index 0000000000..c0a9dbb989 --- /dev/null +++ b/packages/env/src/multi-model.ts @@ -0,0 +1,6 @@ +export { + loadTransformers, + setTransformers, + type LoadTransformerEvent, + type OnLoad, +} from "./internal/multi-model/node.js"; diff --git a/packages/env/src/multi-model.workerd.ts b/packages/env/src/multi-model.workerd.ts new file mode 100644 index 0000000000..b38c78b7f1 --- /dev/null +++ b/packages/env/src/multi-model.workerd.ts @@ -0,0 +1,6 @@ +export { + loadTransformers, + setTransformers, + type LoadTransformerEvent, + type OnLoad, +} from "./internal/multi-model/non-node.js"; diff --git a/packages/llamaindex/e2e/node/embedding/clip.e2e.ts b/packages/llamaindex/e2e/node/embedding/clip.e2e.ts index d0cf2e7072..e21c9fe16c 100644 --- a/packages/llamaindex/e2e/node/embedding/clip.e2e.ts +++ b/packages/llamaindex/e2e/node/embedding/clip.e2e.ts @@ -1,5 +1,5 @@ -import type { LoadTransformerEvent } from "@llamaindex/env"; -import { setTransformers } from "@llamaindex/env"; +import type { LoadTransformerEvent } from "@llamaindex/env/multi-model"; +import { setTransformers } from "@llamaindex/env/multi-model"; import { ClipEmbedding, ImageNode, Settings } from "llamaindex"; import assert from "node:assert"; import { type Mock, test } from "node:test"; diff --git a/packages/providers/clip/src/embedding.ts b/packages/providers/clip/src/embedding.ts index cf29eff213..4889ef6978 100644 --- a/packages/providers/clip/src/embedding.ts +++ b/packages/providers/clip/src/embedding.ts @@ -2,7 +2,10 @@ import { MultiModalEmbedding } from "@llamaindex/core/embeddings"; import type { ImageType } from "@llamaindex/core/schema"; // only import type, to avoid bundling error import { Settings } from "@llamaindex/core/global"; -import { type LoadTransformerEvent, loadTransformers } from "@llamaindex/env"; +import { + type LoadTransformerEvent, + loadTransformers, +} from "@llamaindex/env/multi-model"; import type { CLIPTextModelWithProjection, CLIPVisionModelWithProjection, diff --git a/packages/providers/huggingface/src/embedding.ts b/packages/providers/huggingface/src/embedding.ts index 4cf8aa157c..3e741b36f5 100644 --- a/packages/providers/huggingface/src/embedding.ts +++ b/packages/providers/huggingface/src/embedding.ts @@ -1,6 +1,9 @@ import { BaseEmbedding } from "@llamaindex/core/embeddings"; import { Settings } from "@llamaindex/core/global"; -import { type LoadTransformerEvent, loadTransformers } from "@llamaindex/env"; +import { + type LoadTransformerEvent, + loadTransformers, +} from "@llamaindex/env/multi-model"; import type { pipeline } from "@xenova/transformers"; import { HuggingFaceEmbeddingModelType } from "./shared"; diff --git a/packages/providers/huggingface/src/llm.ts b/packages/providers/huggingface/src/llm.ts index 82d8e3c322..4a5b7ca858 100644 --- a/packages/providers/huggingface/src/llm.ts +++ b/packages/providers/huggingface/src/llm.ts @@ -9,7 +9,7 @@ import { type LLMChatParamsStreaming, type LLMMetadata, } from "@llamaindex/core/llms"; -import { loadTransformers } from "@llamaindex/env"; +import { loadTransformers } from "@llamaindex/env/multi-model"; import type { PreTrainedModel, PreTrainedTokenizer,