diff --git a/.changeset/clever-doors-invite.md b/.changeset/clever-doors-invite.md new file mode 100644 index 0000000000..d92bc6c6ed --- /dev/null +++ b/.changeset/clever-doors-invite.md @@ -0,0 +1,5 @@ +--- +"llamaindex": patch +--- + +feat: experimental package + json query engine diff --git a/package.json b/package.json index fe6981ed3d..c3157dc037 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "release": "pnpm run build:release && changeset publish", "new-llamaindex": "pnpm run build:release && changeset version --ignore create-llama", "new-create-llama": "pnpm run build:release && changeset version --ignore llamaindex --ignore @llamaindex/core-test", - "new-snapshots": "pnpm run build:release && changeset version --snapshot" + "new-experimental": "pnpm run build:release && changeset version --ignore create-llama" }, "devDependencies": { "@changesets/cli": "^2.27.1", diff --git a/packages/core/src/agent/types.ts b/packages/core/src/agent/types.ts index e277d7af0c..185c26b43f 100644 --- a/packages/core/src/agent/types.ts +++ b/packages/core/src/agent/types.ts @@ -3,6 +3,7 @@ import type { ChatEngineAgentParams, StreamingAgentChatResponse, } from "../engines/chat/index.js"; + import type { QueryEngineParamsNonStreaming } from "../types.js"; export interface AgentWorker { diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 6fb9b90d3e..a4c0c8f105 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -30,3 +30,4 @@ export * from "./selectors/index.js"; export * from "./storage/index.js"; export * from "./synthesizers/index.js"; export * from "./tools/index.js"; +export * from "./types.js"; diff --git a/packages/experimental/.cjs.swcrc b/packages/experimental/.cjs.swcrc new file mode 100644 index 0000000000..4329f46564 --- /dev/null +++ b/packages/experimental/.cjs.swcrc @@ -0,0 +1,12 @@ +{ + "jsc": { + "parser": { + "syntax": "typescript" + }, + "target": "esnext" + }, + "module": { + "type": "commonjs", + "ignoreDynamic": true + } +} diff --git a/packages/experimental/.swcrc b/packages/experimental/.swcrc new file mode 100644 index 0000000000..60e144e720 --- /dev/null +++ b/packages/experimental/.swcrc @@ -0,0 +1,8 @@ +{ + "jsc": { + "parser": { + "syntax": "typescript" + }, + "target": "esnext" + } +} diff --git a/packages/experimental/CHANGELOG.md b/packages/experimental/CHANGELOG.md new file mode 100644 index 0000000000..804da5a3fb --- /dev/null +++ b/packages/experimental/CHANGELOG.md @@ -0,0 +1 @@ +# @llamaindex/experimental diff --git a/packages/experimental/examples/jsonQueryEngine.ts b/packages/experimental/examples/jsonQueryEngine.ts new file mode 100644 index 0000000000..5390932586 --- /dev/null +++ b/packages/experimental/examples/jsonQueryEngine.ts @@ -0,0 +1,119 @@ +import { JSONQueryEngine } from "@llamaindex/experimental"; + +import { OpenAI, serviceContextFromDefaults } from "llamaindex"; + +const jsonValue = { + blogPosts: [ + { + id: 1, + title: "First blog post", + content: "This is my first blog post", + }, + { + id: 2, + title: "Second blog post", + content: "This is my second blog post", + }, + ], + comments: [ + { + id: 1, + content: "Nice post!", + username: "jerry", + blogPostId: 1, + }, + { + id: 2, + content: "Interesting thoughts", + username: "simon", + blogPostId: 2, + }, + { + id: 3, + content: "Loved reading this!", + username: "simon", + blogPostId: 2, + }, + ], +}; + +const jsonSchema = { + type: "object", + properties: { + blogPosts: { + type: "array", + items: { + type: "object", + properties: { + id: { + type: "number", + }, + title: { + type: "string", + }, + content: { + type: "string", + }, + }, + required: ["id", "title", "content"], + }, + }, + comments: { + type: "array", + items: { + type: "object", + properties: { + id: { + type: "number", + }, + content: { + type: "string", + }, + username: { + type: "string", + }, + blogPostId: { + type: "number", + }, + }, + required: ["id", "content", "username", "blogPostId"], + }, + }, + }, + required: ["blogPosts", "comments"], +}; + +async function main() { + const llm = new OpenAI({ model: "gpt-4" }); + + const serviceContext = serviceContextFromDefaults({ + llm, + }); + + const jsonQueryEngine = new JSONQueryEngine({ + jsonValue, + jsonSchema, + serviceContext, + }); + + const rawQueryEngine = new JSONQueryEngine({ + jsonValue, + jsonSchema, + serviceContext, + synthesizeResponse: false, + }); + + const response = await jsonQueryEngine.query({ + query: "give to me the comment with id 1", + }); + + const rawResponse = await rawQueryEngine.query({ + query: "give me all simon comments", + }); + + console.log({ response }); + + console.log({ rawResponse }); +} + +main(); diff --git a/packages/experimental/examples/package.json b/packages/experimental/examples/package.json new file mode 100644 index 0000000000..a14c9335a8 --- /dev/null +++ b/packages/experimental/examples/package.json @@ -0,0 +1,17 @@ +{ + "name": "examples", + "private": true, + "version": "0.0.3", + "dependencies": { + "@llamaindex/experimental": "latest", + "llamaindex": "workspace:*" + }, + "devDependencies": { + "@types/node": "^18.19.10", + "ts-node": "^10.9.2", + "typescript": "^5.3.3" + }, + "scripts": { + "lint": "eslint ." + } +} diff --git a/packages/experimental/jsr.json b/packages/experimental/jsr.json new file mode 100644 index 0000000000..f037d28bea --- /dev/null +++ b/packages/experimental/jsr.json @@ -0,0 +1,8 @@ +{ + "name": "@llamaindex/experimental", + "version": "0.0.5", + "exports": { + ".": "./src/index.ts", + "./type": "./src/type.ts" + } +} diff --git a/packages/experimental/package.json b/packages/experimental/package.json new file mode 100644 index 0000000000..05bf4631ff --- /dev/null +++ b/packages/experimental/package.json @@ -0,0 +1,71 @@ +{ + "name": "@llamaindex/experimental", + "description": "Experimental package for LlamaIndexTS", + "version": "0.0.2", + "type": "module", + "types": "dist/type/index.d.ts", + "main": "dist/cjs/index.js", + "exports": { + ".": { + "workerd": { + "types": "./dist/type/index.d.ts", + "default": "./dist/index.polyfill.js" + }, + "edge-light": { + "types": "./dist/type/index.d.ts", + "default": "./dist/index.polyfill.js" + }, + "import": { + "types": "./dist/type/index.d.ts", + "default": "./dist/index.js" + }, + "require": { + "types": "./dist/type/index.d.ts", + "default": "./dist/cjs/index.js" + } + }, + "./*": { + "import": { + "types": "./dist/type/*.d.ts", + "default": "./dist/*.js" + }, + "require": { + "types": "./dist/type/*.d.ts", + "default": "./dist/cjs/*.js" + } + } + }, + "files": [ + "dist", + "CHANGELOG.md" + ], + "repository": { + "type": "git", + "url": "https://github.com/run-llama/LlamaIndexTS.git", + "directory": "packages/experimental" + }, + "scripts": { + "lint": "eslint .", + "build": "rm -rf ./dist && pnpm run build:esm && pnpm run build:cjs && pnpm run build:type", + "build:esm": "swc src -d dist --strip-leading-paths --config-file ../../.swcrc", + "build:cjs": "swc src -d dist/cjs --strip-leading-paths --config-file ../../.cjs.swcrc", + "build:type": "tsc -p tsconfig.json", + "postbuild": "node -e \"require('fs').writeFileSync('./dist/cjs/package.json', JSON.stringify({ type: 'commonjs' }))\"", + "dev": "concurrently \"pnpm run build:esm --watch\" \"pnpm run build:cjs --watch\" \"pnpm run build:type --watch\"" + }, + "devDependencies": { + "@aws-crypto/sha256-js": "^5.2.0", + "@swc/cli": "^0.3.9", + "@swc/core": "^1.4.2", + "@types/jsonpath": "^0.2.4", + "concurrently": "^8.2.2", + "pathe": "^1.1.2" + }, + "dependencies": { + "@types/lodash": "^4.14.202", + "@types/node": "^20.11.20", + "jsonpath": "^1.1.1", + "llamaindex": "workspace:*", + "lodash": "^4.17.21" + } +} diff --git a/packages/experimental/src/engines/query/JSONQueryEngine.ts b/packages/experimental/src/engines/query/JSONQueryEngine.ts new file mode 100644 index 0000000000..070a389fc4 --- /dev/null +++ b/packages/experimental/src/engines/query/JSONQueryEngine.ts @@ -0,0 +1,209 @@ +import jsonpath from "jsonpath"; + +import { Response } from "llamaindex"; + +import { serviceContextFromDefaults, type ServiceContext } from "llamaindex"; + +import type { + BaseQueryEngine, + QueryEngineParamsNonStreaming, + QueryEngineParamsStreaming, +} from "llamaindex"; + +import { + defaultJsonPathPrompt, + defaultResponseSynthesizePrompt, + type JSONPathPrompt, + type ResponseSynthesisPrompt, +} from "./prompt.js"; + +export type JSONSchemaType = Record; + +function removeExtraQuotes(expr: string) { + let startIndex = 0; + let endIndex = expr.length; + + // Trim the leading backticks and single quotes + while ( + startIndex < endIndex && + (expr[startIndex] === "`" || expr[startIndex] === "'") + ) { + startIndex++; + } + + // Trim the trailing backticks and single quotes + while ( + endIndex > startIndex && + (expr[endIndex - 1] === "`" || expr[endIndex - 1] === "'") + ) { + endIndex--; + } + + // Return the trimmed substring + return expr.substring(startIndex, endIndex); +} + +export const defaultOutputProcessor = async ({ + llmOutput, + jsonValue, +}: { + llmOutput: string; + jsonValue: JSONSchemaType; +}): Promise[]> => { + const expressions = llmOutput + .split(",") + .map((expr) => removeExtraQuotes(expr.trim())); + + const results: Record[] = []; + + for (const expression of expressions) { + // get the key for example content from $.content + const key = expression.split(".").pop(); + + try { + const datums = jsonpath.query(jsonValue, expression); + + if (!key) throw new Error(`Invalid JSON Path: ${expression}`); + + for (const datum of datums) { + // in case there is a filter like [?(@.username=='simon')] without a key ie: $..comments[?(@.username=='simon').content] + if (key.includes("==")) { + results.push(datum); + continue; + } + + results.push({ + [key]: datum, + }); + } + } catch (err) { + throw new Error(`Invalid JSON Path: ${expression}`); + } + } + + return results; +}; + +type OutputProcessor = typeof defaultOutputProcessor; + +/** + * A JSON query engine that uses JSONPath to query a JSON object. + */ +export class JSONQueryEngine implements BaseQueryEngine { + jsonValue: JSONSchemaType; + jsonSchema: JSONSchemaType; + serviceContext: ServiceContext; + outputProcessor: OutputProcessor; + verbose: boolean; + jsonPathPrompt: JSONPathPrompt; + synthesizeResponse: boolean; + responseSynthesisPrompt: ResponseSynthesisPrompt; + + constructor(init: { + jsonValue: JSONSchemaType; + jsonSchema: JSONSchemaType; + serviceContext?: ServiceContext; + jsonPathPrompt?: JSONPathPrompt; + outputProcessor?: OutputProcessor; + synthesizeResponse?: boolean; + responseSynthesisPrompt?: ResponseSynthesisPrompt; + verbose?: boolean; + }) { + this.jsonValue = init.jsonValue; + this.jsonSchema = init.jsonSchema; + this.serviceContext = init.serviceContext ?? serviceContextFromDefaults({}); + this.jsonPathPrompt = init.jsonPathPrompt ?? defaultJsonPathPrompt; + this.outputProcessor = init.outputProcessor ?? defaultOutputProcessor; + this.verbose = init.verbose ?? false; + this.synthesizeResponse = init.synthesizeResponse ?? true; + this.responseSynthesisPrompt = + init.responseSynthesisPrompt ?? defaultResponseSynthesizePrompt; + } + + getPrompts(): Record { + return { + jsonPathPrompt: this.jsonPathPrompt, + responseSynthesisPrompt: this.responseSynthesisPrompt, + }; + } + + updatePrompts(prompts: { + jsonPathPrompt?: JSONPathPrompt; + responseSynthesisPrompt?: ResponseSynthesisPrompt; + }): void { + if (prompts.jsonPathPrompt) { + this.jsonPathPrompt = prompts.jsonPathPrompt; + } + if (prompts.responseSynthesisPrompt) { + this.responseSynthesisPrompt = prompts.responseSynthesisPrompt; + } + } + + getPromptModules(): Record { + return {}; + } + + getSchemaContext(): string { + return JSON.stringify(this.jsonSchema); + } + + query(params: QueryEngineParamsStreaming): Promise>; + query(params: QueryEngineParamsNonStreaming): Promise; + async query( + params: QueryEngineParamsStreaming | QueryEngineParamsNonStreaming, + ): Promise> { + const { query, stream } = params; + + if (stream) { + throw new Error("Streaming is not supported"); + } + + const schema = this.getSchemaContext(); + + const jsonPathResponseStr = await this.serviceContext.llm.complete({ + prompt: this.jsonPathPrompt({ query, schema }), + }); + + if (this.verbose) { + console.log( + `> JSONPath Instructions:\n\`\`\`\n${jsonPathResponseStr}\n\`\`\`\n`, + ); + } + + const jsonPathOutput = await this.outputProcessor({ + llmOutput: jsonPathResponseStr.text, + jsonValue: this.jsonValue, + }); + + if (this.verbose) { + console.log(`> JSONPath Output: ${jsonPathOutput}\n`); + } + + let responseStr; + + if (this.synthesizeResponse) { + responseStr = await this.serviceContext.llm.complete({ + prompt: this.responseSynthesisPrompt({ + query, + jsonSchema: schema, + jsonPath: jsonPathResponseStr.text, + jsonPathValue: JSON.stringify(jsonPathOutput), + }), + }); + + responseStr = responseStr.text; + } else { + responseStr = JSON.stringify(jsonPathOutput); + } + + const responseMetadata = { + jsonPathResponseStr, + }; + + const response = new Response(responseStr, []); + + response.metadata = responseMetadata; + + return response; + } +} diff --git a/packages/experimental/src/engines/query/index.ts b/packages/experimental/src/engines/query/index.ts new file mode 100644 index 0000000000..9187f0735a --- /dev/null +++ b/packages/experimental/src/engines/query/index.ts @@ -0,0 +1 @@ +export * from "./JSONQueryEngine.js"; diff --git a/packages/experimental/src/engines/query/prompt.ts b/packages/experimental/src/engines/query/prompt.ts new file mode 100644 index 0000000000..0b38c3f9f0 --- /dev/null +++ b/packages/experimental/src/engines/query/prompt.ts @@ -0,0 +1,36 @@ +export const defaultJsonPathPrompt = ({ + query, + schema, +}: { + query: string; + schema: string; +}) => ` +We have provided a JSON schema below: +${schema} +Given a task, respond with a JSON Path query that can retrieve data from a JSON value that matches the schema. +Task: ${query} +JSONPath: +`; + +export type JSONPathPrompt = typeof defaultJsonPathPrompt; + +export const defaultResponseSynthesizePrompt = ({ + query, + jsonSchema, + jsonPath, + jsonPathValue, +}: { + query: string; + jsonSchema: string; + jsonPath: string; + jsonPathValue: string; +}) => ` +Given a query, synthesize a response to satisfy the query using the JSON results. Only include details that are relevant to the query. If you don't know the answer, then say that. +JSON Schema: ${jsonSchema} +JSON Path: ${jsonPath} +Value at path: ${jsonPathValue} +Query: ${query} +Response: +`; + +export type ResponseSynthesisPrompt = typeof defaultResponseSynthesizePrompt; diff --git a/packages/experimental/src/index.ts b/packages/experimental/src/index.ts new file mode 100644 index 0000000000..10e6ce4d1f --- /dev/null +++ b/packages/experimental/src/index.ts @@ -0,0 +1 @@ +export * from "./engines/query/index.js"; diff --git a/packages/experimental/tsconfig.json b/packages/experimental/tsconfig.json new file mode 100644 index 0000000000..b3bb538267 --- /dev/null +++ b/packages/experimental/tsconfig.json @@ -0,0 +1,19 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "rootDir": "./src", + "outDir": "./dist/type", + "tsBuildInfoFile": "./dist/.tsbuildinfo", + "emitDeclarationOnly": true, + "module": "node16", + "moduleResolution": "node16", + "types": ["node"] + }, + "include": ["./src"], + "exclude": ["node_modules"], + "references": [ + { + "path": "../core/tsconfig.json" + } + ] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f8780ce008..d412d43e87 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -421,6 +421,43 @@ importers: specifier: ^13.5.6 version: 13.5.6(react-dom@18.2.0)(react@18.2.0) + packages/experimental: + dependencies: + '@types/lodash': + specifier: ^4.14.202 + version: 4.14.202 + '@types/node': + specifier: ^20.11.20 + version: 20.11.20 + jsonpath: + specifier: ^1.1.1 + version: 1.1.1 + llamaindex: + specifier: workspace:* + version: link:../core + lodash: + specifier: ^4.17.21 + version: 4.17.21 + devDependencies: + '@aws-crypto/sha256-js': + specifier: ^5.2.0 + version: 5.2.0 + '@swc/cli': + specifier: ^0.3.9 + version: 0.3.9(@swc/core@1.4.2) + '@swc/core': + specifier: ^1.4.2 + version: 1.4.2 + '@types/jsonpath': + specifier: ^0.2.4 + version: 0.2.4 + concurrently: + specifier: ^8.2.2 + version: 8.2.2 + pathe: + specifier: ^1.1.2 + version: 1.1.2 + packages/tsconfig: {} packages: @@ -2630,7 +2667,7 @@ packages: peerDependencies: react: '*' dependencies: - '@types/react': 18.2.48 + '@types/react': 18.2.61 prop-types: 15.8.1 react: 18.2.0 @@ -4139,19 +4176,19 @@ packages: resolution: {integrity: sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==} dependencies: '@types/connect': 3.4.38 - '@types/node': 20.11.17 + '@types/node': 20.11.20 /@types/bonjour@3.5.13: resolution: {integrity: sha512-z9fJ5Im06zvUL548KvYNecEVlA7cVDkGUi6kZusb04mpyEFKCIZJvloCcmpmLaIahDpOQGHaHmG6imtPMmPXGQ==} dependencies: - '@types/node': 20.11.17 + '@types/node': 20.11.20 /@types/cacheable-request@6.0.3: resolution: {integrity: sha512-IQ3EbTzGxIigb1I3qPZc1rWJnH0BmSKv5QYTalEwweFvyBDLSAe24zP0le/hyi7ecGfZVlIVAg4BZqb8WBwKqw==} dependencies: '@types/http-cache-semantics': 4.0.4 '@types/keyv': 3.1.4 - '@types/node': 20.11.17 + '@types/node': 20.11.20 '@types/responselike': 1.0.3 dev: true @@ -4163,7 +4200,7 @@ packages: resolution: {integrity: sha512-n6Cr2xS1h4uAulPRdlw6Jl6s1oG8KrVilPN2yUITEs+K48EzMJJ3W1xy8K5eWuFvjp3R74AOIGSmp2UfBJ8HFw==} dependencies: '@types/express-serve-static-core': 4.17.42 - '@types/node': 20.11.17 + '@types/node': 20.11.20 /@types/connect@3.4.38: resolution: {integrity: sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==} @@ -4173,7 +4210,7 @@ packages: /@types/cross-spawn@6.0.0: resolution: {integrity: sha512-evp2ZGsFw9YKprDbg8ySgC9NA15g3YgiI8ANkGmKKvvi0P2aDGYLPxQIC5qfeKNUOe3TjABVGuah6omPRpIYhg==} dependencies: - '@types/node': 20.11.17 + '@types/node': 20.11.20 dev: true /@types/debug@4.1.12: @@ -4204,7 +4241,7 @@ packages: /@types/express-serve-static-core@4.17.42: resolution: {integrity: sha512-ckM3jm2bf/MfB3+spLPWYPUH573plBFwpOhqQ2WottxYV85j1HQFlxmnTq57X1yHY9awZPig06hL/cLMgNWHIQ==} dependencies: - '@types/node': 20.11.17 + '@types/node': 20.11.20 '@types/qs': 6.9.12 '@types/range-parser': 1.2.7 '@types/send': 0.17.4 @@ -4241,7 +4278,7 @@ packages: /@types/http-proxy@1.17.14: resolution: {integrity: sha512-SSrD0c1OQzlFX7pGu1eXxSEjemej64aaNPRhhVYUGqXh0BtldAAx37MG8btcumvpgKyZp1F5Gn3JkktdxiFv6w==} dependencies: - '@types/node': 20.11.17 + '@types/node': 20.11.20 /@types/istanbul-lib-coverage@2.0.6: resolution: {integrity: sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==} @@ -4265,10 +4302,14 @@ packages: /@types/json5@0.0.29: resolution: {integrity: sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==} + /@types/jsonpath@0.2.4: + resolution: {integrity: sha512-K3hxB8Blw0qgW6ExKgMbXQv2UPZBoE2GqLpVY+yr7nMD2Pq86lsuIzyAaiQ7eMqFL5B6di6pxSkogLJEyEHoGA==} + dev: true + /@types/keyv@3.1.4: resolution: {integrity: sha512-BQ5aZNSCpj7D6K2ksrRCTmKRLEpnPvWDiLPfoGyhZ++8YtiK9d/3DBKPJgry359X/P1PfruyYwvnvwFjuEiEIg==} dependencies: - '@types/node': 20.11.17 + '@types/node': 20.11.20 dev: true /@types/lodash-es@4.17.12: @@ -4309,14 +4350,14 @@ packages: /@types/node-fetch@2.6.9: resolution: {integrity: sha512-bQVlnMLFJ2d35DkPNjEPmd9ueO/rh5EiaZt2bhqiSarPjZIuIV6bPQVqcrEyvNo+AfTrRGVazle1tl597w3gfA==} dependencies: - '@types/node': 20.11.17 + '@types/node': 20.11.20 form-data: 4.0.0 dev: false /@types/node-forge@1.3.11: resolution: {integrity: sha512-FQx220y22OKNTqaByeBGqHWYz4cl94tpcxeFdvBo3wjG6XPBuZ0BNgNZRV5J5TFmmcsJ4IzsLkmGRiQbnYsBEQ==} dependencies: - '@types/node': 20.11.17 + '@types/node': 20.11.20 /@types/node@12.20.55: resolution: {integrity: sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ==} @@ -4344,11 +4385,6 @@ packages: undici-types: 5.26.5 dev: true - /@types/node@20.11.17: - resolution: {integrity: sha512-QmgQZGWu1Yw9TDyAP9ZzpFJKynYNeOvwMJmaxABfieQoVoiVOS6MN1WSpqpRcbeA5+RW82kraAVxCCJg+780Qw==} - dependencies: - undici-types: 5.26.5 - /@types/node@20.11.20: resolution: {integrity: sha512-7/rR21OS+fq8IyHTgtLkDK949uzsa6n8BkziAKtPVpugIkO6D+/ooXMvzXxDnZrmtXVfjb1bKQafYpb8s89LOg==} dependencies: @@ -4367,7 +4403,7 @@ packages: /@types/papaparse@5.3.14: resolution: {integrity: sha512-LxJ4iEFcpqc6METwp9f6BV6VVc43m6MfH0VqFosHvrUgfXiFe6ww7R3itkOQ+TCK6Y+Iv/+RnnvtRZnkc5Kc9g==} dependencies: - '@types/node': 20.11.17 + '@types/node': 20.11.20 dev: false /@types/parse-json@4.0.2: @@ -4376,7 +4412,7 @@ packages: /@types/pg@8.11.0: resolution: {integrity: sha512-sDAlRiBNthGjNFfvt0k6mtotoVYVQ63pA8R4EMWka7crawSR60waVYR0HAgmPRs/e2YaeJTD/43OoZ3PFw80pw==} dependencies: - '@types/node': 20.11.17 + '@types/node': 20.11.20 pg-protocol: 1.6.0 pg-types: 4.0.1 dev: false @@ -4448,7 +4484,7 @@ packages: /@types/responselike@1.0.3: resolution: {integrity: sha512-H/+L+UkTV33uf49PH5pCAUBVPNj2nDBXTN+qS1dOwyyg24l3CcicicCA7ca+HMvJBZcFgl5r8e+RR6elsb4Lyw==} dependencies: - '@types/node': 20.11.17 + '@types/node': 20.11.20 dev: true /@types/retry@0.12.0: @@ -4461,7 +4497,7 @@ packages: /@types/sax@1.2.7: resolution: {integrity: sha512-rO73L89PJxeYM3s3pPPjiPgVVcymqU490g0YO5n5By0k2Erzj6tay/4lr1CHAAU4JyOWd1rpQ8bCf6cZfHU96A==} dependencies: - '@types/node': 20.11.17 + '@types/node': 20.11.20 dev: true /@types/scheduler@0.16.4: @@ -4490,17 +4526,17 @@ packages: dependencies: '@types/http-errors': 2.0.4 '@types/mime': 3.0.4 - '@types/node': 20.11.17 + '@types/node': 20.11.20 /@types/sockjs@0.3.36: resolution: {integrity: sha512-MK9V6NzAS1+Ud7JV9lJLFqW85VbC9dq3LmwZCuBe4wBDgKC0Kj/jd8Xl+nSviU+Qc3+m7umHHyHg//2KSa0a0Q==} dependencies: - '@types/node': 20.11.17 + '@types/node': 20.11.20 /@types/tar@6.1.5: resolution: {integrity: sha512-qm2I/RlZij5RofuY7vohTpYNaYcrSQlN2MyjucQc7ZweDwaEWkdN/EeNh6e9zjK6uEm6PwjdMXkcj05BxZdX1Q==} dependencies: - '@types/node': 20.11.17 + '@types/node': 20.11.20 minipass: 4.2.8 dev: true @@ -4531,7 +4567,7 @@ packages: /@types/ws@8.5.10: resolution: {integrity: sha512-vmQSUcfalpIq0R9q7uTo2lXs6eGIpt9wtnLdMv9LVpIjCA/+ufZRozlVoVelIYixx1ugCBKDhn89vnsEGOCx9A==} dependencies: - '@types/node': 20.11.17 + '@types/node': 20.11.20 /@types/yargs-parser@21.0.3: resolution: {integrity: sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==} @@ -7245,6 +7281,19 @@ packages: resolution: {integrity: sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==} engines: {node: '>=12'} + /escodegen@1.14.3: + resolution: {integrity: sha512-qFcX0XJkdg+PB3xjZZG/wKSuT1PnQWx57+TVSjIMmILd2yC/6ByYElPwJnslDsuWuSAp4AwJGumarAAmJch5Kw==} + engines: {node: '>=4.0'} + hasBin: true + dependencies: + esprima: 4.0.1 + estraverse: 4.3.0 + esutils: 2.0.3 + optionator: 0.8.3 + optionalDependencies: + source-map: 0.6.1 + dev: false + /escodegen@2.1.0: resolution: {integrity: sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==} engines: {node: '>=6.0'} @@ -7613,6 +7662,12 @@ packages: acorn-jsx: 5.3.2(acorn@8.11.3) eslint-visitor-keys: 3.4.3 + /esprima@1.2.2: + resolution: {integrity: sha512-+JpPZam9w5DuJ3Q67SqsMGtiHKENSMRVoxvArfJZK01/BfLEObtZ6orJa/MtoGNR/rfMgp5837T41PAmTwAv/A==} + engines: {node: '>=0.4.0'} + hasBin: true + dev: false + /esprima@4.0.1: resolution: {integrity: sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==} engines: {node: '>=4'} @@ -7695,7 +7750,7 @@ packages: resolution: {integrity: sha512-EzV94NYKoO09GLXGjXj9JIlXijVck4ONSr5wiCWDvhsvj5jxSrzTmRU/9C1DyB6uToszLs8aifA6NQ7lEQdvFw==} engines: {node: '>= 0.8'} dependencies: - '@types/node': 20.11.17 + '@types/node': 20.11.20 require-like: 0.1.2 /event-target-shim@5.0.1: @@ -9453,7 +9508,7 @@ packages: engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} dependencies: '@jest/types': 29.6.3 - '@types/node': 20.11.17 + '@types/node': 20.11.20 chalk: 4.1.2 ci-info: 3.9.0 graceful-fs: 4.2.11 @@ -9463,7 +9518,7 @@ packages: resolution: {integrity: sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==} engines: {node: '>= 10.13.0'} dependencies: - '@types/node': 20.11.17 + '@types/node': 20.11.20 merge-stream: 2.0.0 supports-color: 8.1.1 @@ -9471,7 +9526,7 @@ packages: resolution: {integrity: sha512-eIz2msL/EzL9UFTFFx7jBTkeZfku0yUAyZZZmJ93H2TYEiroIx2PQjEXcwYtYl8zXCxb+PAmA2hLIt/6ZEkPHw==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} dependencies: - '@types/node': 20.11.17 + '@types/node': 20.11.20 jest-util: 29.7.0 merge-stream: 2.0.0 supports-color: 8.1.1 @@ -9585,6 +9640,14 @@ packages: optionalDependencies: graceful-fs: 4.2.11 + /jsonpath@1.1.1: + resolution: {integrity: sha512-l6Cg7jRpixfbgoWgkrl77dgEj8RPvND0wMH6TwQmi9Qs4TFfS9u5cUFnbeKTwj5ga5Y3BTGGNI28k117LJ009w==} + dependencies: + esprima: 1.2.2 + static-eval: 2.0.2 + underscore: 1.12.1 + dev: false + /jsx-ast-utils@3.3.3: resolution: {integrity: sha512-fYQHZTZ8jSfmWZ0iyzfwiU4WDX4HpHbMCZ3gPlWYiCl3BoeOTsqKBqnTVfH2rYT7eP5c3sVbeSPHnnJOaTrWiw==} engines: {node: '>=4.0'} @@ -9661,6 +9724,14 @@ packages: resolution: {integrity: sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==} engines: {node: '>=6'} + /levn@0.3.0: + resolution: {integrity: sha512-0OO4y2iOHix2W6ujICbKIaEQXvFQHue65vUG3pb5EUomzPI90z9hsA1VsO/dbIIpC53J8gxM9Q4Oho0jrCM/yA==} + engines: {node: '>= 0.8.0'} + dependencies: + prelude-ls: 1.1.2 + type-check: 0.3.2 + dev: false + /levn@0.4.1: resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==} engines: {node: '>= 0.8.0'} @@ -11261,6 +11332,18 @@ packages: resolution: {integrity: sha512-pkEqbDyl8ou5cpq+VsnQbe/WlEy5qS7xPzMS1U55OCG9KPvwFD46zDbxQIj3egJSFc3D+XhYOPUzz49zQAVy7A==} dev: false + /optionator@0.8.3: + resolution: {integrity: sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==} + engines: {node: '>= 0.8.0'} + dependencies: + deep-is: 0.1.4 + fast-levenshtein: 2.0.6 + levn: 0.3.0 + prelude-ls: 1.1.2 + type-check: 0.3.2 + word-wrap: 1.2.5 + dev: false + /optionator@0.9.3: resolution: {integrity: sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg==} engines: {node: '>= 0.8.0'} @@ -12285,6 +12368,11 @@ packages: which-pm: 2.0.0 dev: true + /prelude-ls@1.1.2: + resolution: {integrity: sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w==} + engines: {node: '>= 0.8.0'} + dev: false + /prelude-ls@1.2.1: resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} engines: {node: '>= 0.8.0'} @@ -13751,6 +13839,12 @@ packages: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} dev: true + /static-eval@2.0.2: + resolution: {integrity: sha512-N/D219Hcr2bPjLxPiV+TQE++Tsmrady7TqAJugLy7Xk1EumfDWS/f5dtBbkRCGE7wKKXuYockQoj8Rm2/pVKyg==} + dependencies: + escodegen: 1.14.3 + dev: false + /statuses@1.5.0: resolution: {integrity: sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==} engines: {node: '>= 0.6'} @@ -14552,6 +14646,13 @@ packages: turbo-windows-arm64: 1.12.3 dev: true + /type-check@0.3.2: + resolution: {integrity: sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==} + engines: {node: '>= 0.8.0'} + dependencies: + prelude-ls: 1.1.2 + dev: false + /type-check@0.4.0: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} engines: {node: '>= 0.8.0'} @@ -14706,6 +14807,10 @@ packages: has-symbols: 1.0.3 which-boxed-primitive: 1.0.2 + /underscore@1.12.1: + resolution: {integrity: sha512-hEQt0+ZLDVUMhebKxL4x1BTtDY7bavVofhZ9KZ4aI26X9SRaE+Y3m83XUL1UP2jn8ynjndwCCpEHdUG+9pP1Tw==} + dev: false + /underscore@1.13.6: resolution: {integrity: sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==} dev: false @@ -15474,6 +15579,11 @@ packages: winston-transport: 4.6.0 dev: false + /word-wrap@1.2.5: + resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} + engines: {node: '>=0.10.0'} + dev: false + /wordwrap@1.0.0: resolution: {integrity: sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==} dev: true diff --git a/tsconfig.json b/tsconfig.json index ee2dbb5316..d95d2781d5 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -16,7 +16,9 @@ "llamaindex": ["./packages/core/src/index.ts"], "llamaindex/*": ["./packages/core/src/*.ts"], "@llamaindex/env": ["./packages/env/src/index.ts"], - "@llamaindex/env/*": ["./packages/env/src/*.ts"] + "@llamaindex/env/*": ["./packages/env/src/*.ts"], + "@llamaindex/experimental": ["./packages/experimental/src/index.ts"], + "@llamaindex/experimental/*": ["./packages/experimental/src/*.ts"] } }, "files": [], @@ -44,6 +46,9 @@ }, { "path": "./examples/readers" + }, + { + "path": "./packages/experimental/tsconfig.json" } ] }