From c3747d092a227b2f7fcf34758301269d7ae84519 Mon Sep 17 00:00:00 2001 From: Alex Yang Date: Thu, 9 May 2024 02:29:11 -0500 Subject: [PATCH] feat: add nextjs plugin for `llamaindex` (#824) --- .changeset/warm-papayas-wait.md | 10 + README.md | 11 + .../e2e/examples/nextjs-agent/next.config.mjs | 4 +- .../nextjs-edge-runtime/next.config.mjs | 4 +- .../src/app/page.module.css | 232 ------------------ .../nextjs-edge-runtime/src/app/page.tsx | 96 +------- .../nextjs-edge-runtime/src/utils/llm.ts | 16 +- .../nextjs-edge-runtime/tsconfig.json | 1 + packages/core/src/embeddings/ClipEmbedding.ts | 41 ++-- .../src/embeddings/HuggingFaceEmbedding.ts | 3 +- .../core/src/internal/deps/transformers.ts | 15 ++ packages/core/src/next.ts | 36 +++ 12 files changed, 122 insertions(+), 347 deletions(-) create mode 100644 .changeset/warm-papayas-wait.md delete mode 100644 packages/core/e2e/examples/nextjs-edge-runtime/src/app/page.module.css create mode 100644 packages/core/src/internal/deps/transformers.ts create mode 100644 packages/core/src/next.ts diff --git a/.changeset/warm-papayas-wait.md b/.changeset/warm-papayas-wait.md new file mode 100644 index 0000000000..d8bd813b19 --- /dev/null +++ b/.changeset/warm-papayas-wait.md @@ -0,0 +1,10 @@ +--- +"llamaindex": patch +"@llamaindex/core-e2e": patch +"@llamaindex/next-agent-test": patch +"@llamaindex/nextjs-edge-runtime-test": patch +--- + +fix: import `@xenova/transformers` + +For now, if you use llamaindex in next.js, you need to add a plugin from `llamaindex/next` to ensure some module resolutions are correct. diff --git a/README.md b/README.md index b00b4ff832..f01298d843 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,17 @@ node --import tsx ./main.ts ### Next.js +First, you will need to add a llamaindex plugin to your Next.js project. + +```js +// next.config.js +const withLlamaIndex = require("llamaindex/next"); + +module.exports = withLlamaIndex({ + // your next.js config +}); +``` + You can combine `ai` with `llamaindex` in Next.js with RSC (React Server Components). ```tsx diff --git a/packages/core/e2e/examples/nextjs-agent/next.config.mjs b/packages/core/e2e/examples/nextjs-agent/next.config.mjs index 4678774e6d..894e9a1dc4 100644 --- a/packages/core/e2e/examples/nextjs-agent/next.config.mjs +++ b/packages/core/e2e/examples/nextjs-agent/next.config.mjs @@ -1,4 +1,6 @@ /** @type {import('next').NextConfig} */ const nextConfig = {}; -export default nextConfig; +import withLlamaIndex from "llamaindex/next"; + +export default withLlamaIndex(nextConfig); diff --git a/packages/core/e2e/examples/nextjs-edge-runtime/next.config.mjs b/packages/core/e2e/examples/nextjs-edge-runtime/next.config.mjs index 4678774e6d..894e9a1dc4 100644 --- a/packages/core/e2e/examples/nextjs-edge-runtime/next.config.mjs +++ b/packages/core/e2e/examples/nextjs-edge-runtime/next.config.mjs @@ -1,4 +1,6 @@ /** @type {import('next').NextConfig} */ const nextConfig = {}; -export default nextConfig; +import withLlamaIndex from "llamaindex/next"; + +export default withLlamaIndex(nextConfig); diff --git a/packages/core/e2e/examples/nextjs-edge-runtime/src/app/page.module.css b/packages/core/e2e/examples/nextjs-edge-runtime/src/app/page.module.css deleted file mode 100644 index d979f776c7..0000000000 --- a/packages/core/e2e/examples/nextjs-edge-runtime/src/app/page.module.css +++ /dev/null @@ -1,232 +0,0 @@ -.main { - display: flex; - flex-direction: column; - justify-content: space-between; - align-items: center; - padding: 6rem; - min-height: 100vh; -} - -.description { - display: inherit; - justify-content: inherit; - align-items: inherit; - font-size: 0.85rem; - max-width: var(--max-width); - width: 100%; - z-index: 2; - font-family: var(--font-mono); -} - -.description a { - display: flex; - justify-content: center; - align-items: center; - gap: 0.5rem; -} - -.description p { - position: relative; - margin: 0; - padding: 1rem; - background-color: rgba(var(--callout-rgb), 0.5); - border: 1px solid rgba(var(--callout-border-rgb), 0.3); - border-radius: var(--border-radius); -} - -.code { - font-weight: 700; - font-family: var(--font-mono); -} - -.grid { - display: grid; - grid-template-columns: repeat(4, minmax(25%, auto)); - max-width: 100%; - width: var(--max-width); -} - -.card { - padding: 1rem 1.2rem; - border-radius: var(--border-radius); - background: rgba(var(--card-rgb), 0); - border: 1px solid rgba(var(--card-border-rgb), 0); - transition: - background 200ms, - border 200ms; -} - -.card span { - display: inline-block; - transition: transform 200ms; -} - -.card h2 { - font-weight: 600; - margin-bottom: 0.7rem; -} - -.card p { - margin: 0; - opacity: 0.6; - font-size: 0.9rem; - line-height: 1.5; - max-width: 30ch; - text-wrap: balance; -} - -.center { - display: flex; - justify-content: center; - align-items: center; - position: relative; - padding: 4rem 0; -} - -.center::before { - background: var(--secondary-glow); - border-radius: 50%; - width: 480px; - height: 360px; - margin-left: -400px; -} - -.center::after { - background: var(--primary-glow); - width: 240px; - height: 180px; - z-index: -1; -} - -.center::before, -.center::after { - content: ""; - left: 50%; - position: absolute; - filter: blur(45px); - transform: translateZ(0); -} - -.logo { - position: relative; -} -/* Enable hover only on non-touch devices */ -@media (hover: hover) and (pointer: fine) { - .card:hover { - background: rgba(var(--card-rgb), 0.1); - border: 1px solid rgba(var(--card-border-rgb), 0.15); - } - - .card:hover span { - transform: translateX(4px); - } -} - -@media (prefers-reduced-motion) { - .card:hover span { - transform: none; - } -} - -/* Mobile */ -@media (max-width: 700px) { - .content { - padding: 4rem; - } - - .grid { - grid-template-columns: 1fr; - margin-bottom: 120px; - max-width: 320px; - text-align: center; - } - - .card { - padding: 1rem 2.5rem; - } - - .card h2 { - margin-bottom: 0.5rem; - } - - .center { - padding: 8rem 0 6rem; - } - - .center::before { - transform: none; - height: 300px; - } - - .description { - font-size: 0.8rem; - } - - .description a { - padding: 1rem; - } - - .description p, - .description div { - display: flex; - justify-content: center; - position: fixed; - width: 100%; - } - - .description p { - align-items: center; - inset: 0 0 auto; - padding: 2rem 1rem 1.4rem; - border-radius: 0; - border: none; - border-bottom: 1px solid rgba(var(--callout-border-rgb), 0.25); - background: linear-gradient( - to bottom, - rgba(var(--background-start-rgb), 1), - rgba(var(--callout-rgb), 0.5) - ); - background-clip: padding-box; - backdrop-filter: blur(24px); - } - - .description div { - align-items: flex-end; - pointer-events: none; - inset: auto 0 0; - padding: 2rem; - height: 200px; - background: linear-gradient( - to bottom, - transparent 0%, - rgb(var(--background-end-rgb)) 40% - ); - z-index: 1; - } -} - -/* Tablet and Smaller Desktop */ -@media (min-width: 701px) and (max-width: 1120px) { - .grid { - grid-template-columns: repeat(2, 50%); - } -} - -@media (prefers-color-scheme: dark) { - .vercelLogo { - filter: invert(1); - } - - .logo { - filter: invert(1) drop-shadow(0 0 0.3rem #ffffff70); - } -} - -@keyframes rotate { - from { - transform: rotate(360deg); - } - to { - transform: rotate(0deg); - } -} diff --git a/packages/core/e2e/examples/nextjs-edge-runtime/src/app/page.tsx b/packages/core/e2e/examples/nextjs-edge-runtime/src/app/page.tsx index 2642896211..6765b53440 100644 --- a/packages/core/e2e/examples/nextjs-edge-runtime/src/app/page.tsx +++ b/packages/core/e2e/examples/nextjs-edge-runtime/src/app/page.tsx @@ -1,98 +1,20 @@ -import Image from "next/image"; -import "../utils/llm"; -import styles from "./page.module.css"; +import { tokenizerResultPromise } from "@/utils/llm"; +import { use } from "react"; export const runtime = "edge"; export default function Home() { + const result = use(tokenizerResultPromise); return ( -
-
-

- Get started by editing  - src/app/page.tsx -

+
+
+

Next.js Edge Runtime

- - By{" "} - Vercel Logo - + {result.map((value, index) => ( + {value} + ))}
- -
- Next.js Logo -
- -
- -

- Docs -> -

-

Find in-depth information about Next.js features and API.

- - - -

- Learn -> -

-

Learn about Next.js in an interactive course with quizzes!

- - - -

- Templates -> -

-

Explore starter templates for Next.js.

- - - -

- Deploy -> -

-

- Instantly deploy your Next.js site to a shareable URL with Vercel. -

- -
); } diff --git a/packages/core/e2e/examples/nextjs-edge-runtime/src/utils/llm.ts b/packages/core/e2e/examples/nextjs-edge-runtime/src/utils/llm.ts index 29635d9c68..0396636617 100644 --- a/packages/core/e2e/examples/nextjs-edge-runtime/src/utils/llm.ts +++ b/packages/core/e2e/examples/nextjs-edge-runtime/src/utils/llm.ts @@ -1,9 +1,23 @@ -"use server"; // test runtime import "llamaindex"; +import { ClipEmbedding } from "llamaindex/embeddings/ClipEmbedding"; import "llamaindex/readers/SimpleDirectoryReader"; // @ts-expect-error if (typeof EdgeRuntime !== "string") { throw new Error("Expected run in EdgeRuntime"); } + +export const tokenizerResultPromise = new Promise( + (resolve, reject) => { + const embedding = new ClipEmbedding(); + //#region make sure @xenova/transformers is working in edge runtime + embedding + .getTokenizer() + .then((tokenizer) => { + resolve(tokenizer.encode("hello world")); + }) + .catch(reject); + //#endregion + }, +); diff --git a/packages/core/e2e/examples/nextjs-edge-runtime/tsconfig.json b/packages/core/e2e/examples/nextjs-edge-runtime/tsconfig.json index b5e6c4d437..0f4b49ede3 100644 --- a/packages/core/e2e/examples/nextjs-edge-runtime/tsconfig.json +++ b/packages/core/e2e/examples/nextjs-edge-runtime/tsconfig.json @@ -1,5 +1,6 @@ { "compilerOptions": { + "target": "ESNext", "lib": ["dom", "dom.iterable", "esnext"], "outDir": "./dist", "allowJs": true, diff --git a/packages/core/src/embeddings/ClipEmbedding.ts b/packages/core/src/embeddings/ClipEmbedding.ts index 5cd6f1bc39..f4f1082916 100644 --- a/packages/core/src/embeddings/ClipEmbedding.ts +++ b/packages/core/src/embeddings/ClipEmbedding.ts @@ -1,12 +1,17 @@ import _ from "lodash"; import type { ImageType } from "../Node.js"; +import { lazyLoadTransformers } from "../internal/deps/transformers.js"; import { MultiModalEmbedding } from "./MultiModalEmbedding.js"; +// only import type, to avoid bundling error +import type { + CLIPTextModelWithProjection, + CLIPVisionModelWithProjection, + PreTrainedTokenizer, + Processor, +} from "@xenova/transformers"; async function readImage(input: ImageType) { - const { RawImage } = await import( - /* webpackIgnore: true */ - "@xenova/transformers" - ); + const { RawImage } = await lazyLoadTransformers(); if (input instanceof Blob) { return await RawImage.fromBlob(input); } else if (_.isString(input) || input instanceof URL) { @@ -25,39 +30,30 @@ export class ClipEmbedding extends MultiModalEmbedding { modelType: ClipEmbeddingModelType = ClipEmbeddingModelType.XENOVA_CLIP_VIT_BASE_PATCH16; - private tokenizer: any; - private processor: any; - private visionModel: any; - private textModel: any; + private tokenizer: PreTrainedTokenizer | null = null; + private processor: Processor | null = null; + private visionModel: CLIPVisionModelWithProjection | null = null; + private textModel: CLIPTextModelWithProjection | null = null; async getTokenizer() { + const { AutoTokenizer } = await lazyLoadTransformers(); if (!this.tokenizer) { - const { AutoTokenizer } = await import( - /* webpackIgnore: true */ - "@xenova/transformers" - ); this.tokenizer = await AutoTokenizer.from_pretrained(this.modelType); } return this.tokenizer; } async getProcessor() { + const { AutoProcessor } = await lazyLoadTransformers(); if (!this.processor) { - const { AutoProcessor } = await import( - /* webpackIgnore: true */ - "@xenova/transformers" - ); this.processor = await AutoProcessor.from_pretrained(this.modelType); } return this.processor; } async getVisionModel() { + const { CLIPVisionModelWithProjection } = await lazyLoadTransformers(); if (!this.visionModel) { - const { CLIPVisionModelWithProjection } = await import( - /* webpackIgnore: true */ - "@xenova/transformers" - ); this.visionModel = await CLIPVisionModelWithProjection.from_pretrained( this.modelType, ); @@ -67,11 +63,8 @@ export class ClipEmbedding extends MultiModalEmbedding { } async getTextModel() { + const { CLIPTextModelWithProjection } = await lazyLoadTransformers(); if (!this.textModel) { - const { CLIPTextModelWithProjection } = await import( - /* webpackIgnore: true */ - "@xenova/transformers" - ); this.textModel = await CLIPTextModelWithProjection.from_pretrained( this.modelType, ); diff --git a/packages/core/src/embeddings/HuggingFaceEmbedding.ts b/packages/core/src/embeddings/HuggingFaceEmbedding.ts index 73e6cb2e37..d889aaeeb1 100644 --- a/packages/core/src/embeddings/HuggingFaceEmbedding.ts +++ b/packages/core/src/embeddings/HuggingFaceEmbedding.ts @@ -1,3 +1,4 @@ +import { lazyLoadTransformers } from "../internal/deps/transformers.js"; import { BaseEmbedding } from "./types.js"; export enum HuggingFaceEmbeddingModelType { @@ -31,7 +32,7 @@ export class HuggingFaceEmbedding extends BaseEmbedding { async getExtractor() { if (!this.extractor) { - const { pipeline } = await import("@xenova/transformers"); + const { pipeline } = await lazyLoadTransformers(); this.extractor = await pipeline("feature-extraction", this.modelType, { quantized: this.quantized, }); diff --git a/packages/core/src/internal/deps/transformers.ts b/packages/core/src/internal/deps/transformers.ts new file mode 100644 index 0000000000..96ebceaa47 --- /dev/null +++ b/packages/core/src/internal/deps/transformers.ts @@ -0,0 +1,15 @@ +let transformer: typeof import("@xenova/transformers") | null = null; + +export async function lazyLoadTransformers() { + if (!transformer) { + transformer = await import("@xenova/transformers"); + } + + // @ts-expect-error + if (typeof EdgeRuntime === "string") { + // there is no local file system in the edge runtime + transformer.env.allowLocalModels = false; + } + // fixme: handle cloudflare workers case here? + return transformer; +} diff --git a/packages/core/src/next.ts b/packages/core/src/next.ts new file mode 100644 index 0000000000..8097a0ea73 --- /dev/null +++ b/packages/core/src/next.ts @@ -0,0 +1,36 @@ +/** + * This is a Next.js configuration file that is used to customize the build process. + * + * @example + * ```js + * // next.config.js + * const withLlamaIndex = require("llamaindex/next") + * + * module.exports = withLlamaIndex({ + * // Your Next.js configuration + * }) + * ``` + * + * This is only for Next.js projects, do not export this function on top-level. + * + * @module + */ +export default function withLlamaIndex(config: any) { + const userWebpack = config.webpack; + //#region hack for `@xenova/transformers` + // Ignore node-specific modules when bundling for the browser + // See https://webpack.js.org/configuration/resolve/#resolvealias + config.webpack = function (webpackConfig: any) { + if (userWebpack) { + webpackConfig = userWebpack(webpackConfig); + } + webpackConfig.resolve.alias = { + ...webpackConfig.resolve.alias, + sharp$: false, + "onnxruntime-node$": false, + }; + return webpackConfig; + }; + //#endregion + return config; +}