merge

.changeset/funny-dancers-listen.md

@@ -0,0 +1,6 @@
+---
+"@llamaindex/cloud": patch
+"llamaindex": patch
+---
+
+fix(cloud): do not detect file type in llama parse

.github/workflows/preview.yml

@@ -25,4 +25,4 @@ jobs:
         run: pnpm run build
 
       - name: Pre Release
-        run: pnpx pkg-pr-new publish ./packages/*
+        run: pnpx pkg-pr-new publish ./packages/* ./packages/llm/*

apps/docs/CHANGELOG.md

@@ -1,5 +1,98 @@
 # docs
 
+## 0.0.92
+
+### Patch Changes
+
+- Updated dependencies [1364e8e]
+- Updated dependencies [3b7736f]
+- Updated dependencies [96fc69c]
+  - [email protected]
+  - @llamaindex/examples@0.0.9
+
+## 0.0.91
+
+### Patch Changes
+
+- Updated dependencies [5729bd9]
+  - [email protected]
+
+## 0.0.90
+
+### Patch Changes
+
+- Updated dependencies [6f75306]
+- Updated dependencies [94cb4ad]
+  - [email protected]
+
+## 0.0.89
+
+### Patch Changes
+
+- Updated dependencies [6a9a7b1]
+  - [email protected]
+
+## 0.0.88
+
+### Patch Changes
+
+- Updated dependencies [62cba52]
+- Updated dependencies [d265e96]
+- Updated dependencies [d30bbf7]
+- Updated dependencies [53fd00a]
+  - [email protected]
+
+## 0.0.87
+
+### Patch Changes
+
+- Updated dependencies [5f67820]
+- Updated dependencies [fe08d04]
+  - [email protected]
+
+## 0.0.86
+
+### Patch Changes
+
+- Updated dependencies [ee697fb]
+  - [email protected]
+
+## 0.0.85
+
+### Patch Changes
+
+- Updated dependencies [63e9846]
+- Updated dependencies [6f3a31c]
+  - [email protected]
+
+## 0.0.84
+
+### Patch Changes
+
+- Updated dependencies [2a82413]
+  - [email protected]
+
+## 0.0.83
+
+### Patch Changes
+
+- [email protected]
+
+## 0.0.82
+
+### Patch Changes
+
+- [email protected]
+
+## 0.0.81
+
+### Patch Changes
+
+- Updated dependencies [f7b4e94]
+- Updated dependencies [78037a6]
+- Updated dependencies [1d9e3b1]
+  - [email protected]
+
 ## 0.0.80
 
 ### Patch Changes

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docs",
-  "version": "0.0.80",
+  "version": "0.0.92",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",

examples/CHANGELOG.md

@@ -1,5 +1,16 @@
 # examples
 
+## 0.0.9
+
+### Patch Changes
+
+- Updated dependencies [1364e8e]
+- Updated dependencies [96fc69c]
+- Updated dependencies [3b7736f]
+- Updated dependencies [96fc69c]
+  - [email protected]
+  - @llamaindex/core@0.3.0
+
 ## 0.0.8
 
 ### Patch Changes

examples/agent/large_toolcall_with_gpt4o.ts

@@ -13,7 +13,7 @@ import { FunctionTool, OpenAI, ToolCallOptions } from "llamaindex";
   }
 })();
 
-async function callLLM(init: Partial<OpenAI>) {
+async function callLLM(init: { model: string }) {
   const csvData =
     "Country,Average Height (cm)\nNetherlands,156\nDenmark,158\nNorway,160";
 

examples/astradb/example.ts

@@ -1,6 +1,7 @@
 import {
   AstraDBVectorStore,
   Document,
+  MetadataFilters,
   storageContextFromDefaults,
   VectorStoreIndex,
 } from "llamaindex";
@@ -42,8 +43,10 @@ async function main() {
     const index = await VectorStoreIndex.fromDocuments(docs, {
       storageContext: ctx,
     });
-
-    const queryEngine = index.asQueryEngine();
+    const preFilters: MetadataFilters = {
+      filters: [{ key: "id", operator: "in", value: [123, 789] }],
+    }; // try changing the filters to see the different results
+    const queryEngine = index.asQueryEngine({ preFilters });
     const response = await queryEngine.query({
       query: "Describe AstraDB.",
     });

examples/chromadb/preFilters.ts

@@ -1,57 +1,83 @@
 import {
   ChromaVectorStore,
   Document,
+  MetadataFilters,
   VectorStoreIndex,
   storageContextFromDefaults,
 } from "llamaindex";
 
-const collectionName = "dog_colors";
+const collectionName = "dogs_with_color";
 
 async function main() {
   try {
-    const docs = [
-      new Document({
-        text: "The dog is brown",
-        metadata: {
-          dogId: "1",
-        },
-      }),
-      new Document({
-        text: "The dog is red",
-        metadata: {
-          dogId: "2",
-        },
-      }),
-    ];
-
-    console.log("Creating ChromaDB vector store");
     const chromaVS = new ChromaVectorStore({ collectionName });
-    const ctx = await storageContextFromDefaults({ vectorStore: chromaVS });
-
-    console.log("Embedding documents and adding to index");
-    const index = await VectorStoreIndex.fromDocuments(docs, {
-      storageContext: ctx,
-    });
-
-    console.log("Querying index");
-    const queryEngine = index.asQueryEngine({
-      preFilters: {
-        filters: [
-          {
-            key: "dogId",
-            value: "2",
-            operator: "==",
-          },
-        ],
-      },
-    });
-    const response = await queryEngine.query({
-      query: "What is the color of the dog?",
-    });
-    console.log(response.toString());
+    const index = await VectorStoreIndex.fromVectorStore(chromaVS);
+
+    const queryFn = async (filters?: MetadataFilters) => {
+      console.log("\nQuerying dogs by filters: ", JSON.stringify(filters));
+      const query = "List all colors of dogs";
+      const queryEngine = index.asQueryEngine({
+        preFilters: filters,
+        similarityTopK: 3,
+      });
+      const response = await queryEngine.query({ query });
+      console.log(response.toString());
+    };
+
+    await queryFn(); // red, brown, yellow
+    await queryFn({ filters: [{ key: "dogId", value: "1", operator: "==" }] }); // brown
+    await queryFn({ filters: [{ key: "dogId", value: "1", operator: "!=" }] }); // red, yellow
+    await queryFn({
+      filters: [
+        { key: "dogId", value: "1", operator: "==" },
+        { key: "dogId", value: "3", operator: "==" },
+      ],
+      condition: "or",
+    }); // brown, yellow
+    await queryFn({
+      filters: [{ key: "dogId", value: ["1", "2"], operator: "in" }],
+    }); // red, brown
   } catch (e) {
     console.error(e);
   }
 }
 
-void main();
+async function generate() {
+  const docs = [
+    new Document({
+      id_: "doc1",
+      text: "The dog is brown",
+      metadata: {
+        dogId: "1",
+      },
+    }),
+    new Document({
+      id_: "doc2",
+      text: "The dog is red",
+      metadata: {
+        dogId: "2",
+      },
+    }),
+    new Document({
+      id_: "doc3",
+      text: "The dog is yellow",
+      metadata: {
+        dogId: "3",
+      },
+    }),
+  ];
+
+  console.log("Creating ChromaDB vector store");
+  const chromaVS = new ChromaVectorStore({ collectionName });
+  const ctx = await storageContextFromDefaults({ vectorStore: chromaVS });
+
+  console.log("Embedding documents and adding to index");
+  await VectorStoreIndex.fromDocuments(docs, {
+    storageContext: ctx,
+  });
+}
+
+(async () => {
+  await generate();
+  await main();
+})();

examples/metadata-filter/preFilters.ts

@@ -0,0 +1,51 @@
+import {
+  Document,
+  MetadataFilters,
+  Settings,
+  SimpleDocumentStore,
+  VectorStoreIndex,
+  storageContextFromDefaults,
+} from "llamaindex";
+
+async function getDataSource() {
+  const docs = [
+    new Document({ text: "The dog is brown", metadata: { dogId: "1" } }),
+    new Document({ text: "The dog is yellow", metadata: { dogId: "2" } }),
+  ];
+  const storageContext = await storageContextFromDefaults({
+    persistDir: "./cache",
+  });
+  const numberOfDocs = Object.keys(
+    (storageContext.docStore as SimpleDocumentStore).toDict(),
+  ).length;
+  if (numberOfDocs === 0) {
+    return await VectorStoreIndex.fromDocuments(docs, { storageContext });
+  }
+  return await VectorStoreIndex.init({
+    storageContext,
+  });
+}
+
+Settings.callbackManager.on("retrieve-end", (event) => {
+  const { nodes, query } = event.detail;
+  console.log(`${query.query} - Number of retrieved nodes:`, nodes.length);
+});
+
+async function main() {
+  const index = await getDataSource();
+  const filters: MetadataFilters = {
+    filters: [{ key: "dogId", value: "2", operator: "==" }],
+  };
+
+  const retriever = index.asRetriever({ similarityTopK: 3, filters });
+  const queryEngine = index.asQueryEngine({
+    similarityTopK: 3,
+    preFilters: filters,
+  });
+
+  console.log("Retriever and query engine should only retrieve 1 node:");
+  await retriever.retrieve({ query: "Retriever: get dog" });
+  await queryEngine.query({ query: "QueryEngine: get dog" });
+}
+
+void main();

examples/package.json

@@ -1,14 +1,14 @@
 {
   "name": "@llamaindex/examples",
   "private": true,
-  "version": "0.0.8",
+  "version": "0.0.9",
   "dependencies": {
     "@aws-crypto/sha256-js": "^5.2.0",
     "@azure/identity": "^4.4.1",
     "@datastax/astra-db-ts": "^1.4.1",
-    "@llamaindex/core": "^0.2.0",
-    "@llamaindex/ollama": "workspace:^0.0.3",
-    "@llamaindex/openai": "workspace:^0.1.10",
+    "@llamaindex/core": "^0.3.0",
+    "@llamaindex/ollama": "^0.0.3",
+    "@llamaindex/openai": "^0.1.10",
     "@notionhq/client": "^2.2.15",
     "@pinecone-database/pinecone": "^3.0.2",
     "@vercel/postgres": "^0.10.0",
@@ -17,7 +17,7 @@
     "commander": "^12.1.0",
     "dotenv": "^16.4.5",
     "js-tiktoken": "^1.0.14",
-    "llamaindex": "^0.6.0",
+    "llamaindex": "^0.7.0",
     "mongodb": "^6.7.0",
     "pathe": "^1.1.2",
     "postgres": "^3.4.4"