From bff9494d3217c36278428b475c9ace42b0891949 Mon Sep 17 00:00:00 2001 From: n4ze3m Date: Sat, 13 Apr 2024 17:02:47 +0530 Subject: [PATCH 01/16] Update dependencies and fix TypeScript strict mode errors --- app/ui/src/hooks/useMessage.tsx | 1 - server/package.json | 17 +- .../api/v1/bot/playground/chat.handler.ts | 1 - server/src/handlers/api/v1/yt/get.handler.ts | 2 +- server/src/loader/youtube-transcript.ts | 83 ++- server/src/plugins/bull.ts | 2 +- server/src/plugins/jwt.ts | 93 ++-- server/src/utils/store.ts | 2 +- server/src/utils/youtube.ts | 2 +- server/tsconfig.json | 1 + server/yarn.lock | 477 ++++++++++++------ 11 files changed, 434 insertions(+), 247 deletions(-) diff --git a/app/ui/src/hooks/useMessage.tsx b/app/ui/src/hooks/useMessage.tsx index ef8368ee..494919be 100644 --- a/app/ui/src/hooks/useMessage.tsx +++ b/app/ui/src/hooks/useMessage.tsx @@ -170,7 +170,6 @@ export const useMessage = () => { setHistory(responseData.history); setMessages(newMessage); setIsProcessing(false); - reader.releaseLock(); break; } } diff --git a/server/package.json b/server/package.json index b234801b..cfcb5cd0 100644 --- a/server/package.json +++ b/server/package.json @@ -29,7 +29,7 @@ "@fastify/autoload": "^5.0.0", "@fastify/cookie": "^9.1.0", "@fastify/cors": "^8.3.0", - "@fastify/jwt": "^7.0.0", + "@fastify/jwt": "^8.0.0", "@fastify/multipart": "^7.6.0", "@fastify/sensible": "^5.0.0", "@fastify/session": "^10.5.0", @@ -62,10 +62,10 @@ "copyfiles": "^2.4.1", "d3-dsv": "2", "discord.js": "^14.11.0", - "fastify": "^4.0.0", - "fastify-cli": "^5.7.1", - "fastify-plugin": "^4.0.0", - "fastify-raw-body": "4.0.0", + "fastify": "^4.26.2", + "fastify-cli": "6.1.1", + "fastify-plugin": "4.0.0", + "fastify-raw-body": "^4.3.0", "fastify-sse-v2": "^3.1.1", "google-auth-library": "^8.8.0", "grammy": "^1.16.2", @@ -87,13 +87,14 @@ }, "devDependencies": { "@types/bcryptjs": "^2.4.2", - "@types/node": "^18.0.0", + "@types/node": "20.4.4", "@types/pubsub-js": "^1.8.3", "@types/tap": "^15.0.5", - "fastify-tsconfig": "^1.0.1", + "c8": "9.0.0", + "fastify-tsconfig": "2.0.0", "prisma": "^5.9.1", "tap": "^16.1.0", - "typescript": "^4.5.4" + "typescript": "5.2.2" }, "resolutions": { "@langchain/core": "0.1.43" diff --git a/server/src/handlers/api/v1/bot/playground/chat.handler.ts b/server/src/handlers/api/v1/bot/playground/chat.handler.ts index b65c74c0..18837717 100644 --- a/server/src/handlers/api/v1/bot/playground/chat.handler.ts +++ b/server/src/handlers/api/v1/bot/playground/chat.handler.ts @@ -510,7 +510,6 @@ export const chatRequestStreamHandler = async ( }), }, }); - reply.sse({ event: "result", id: "", diff --git a/server/src/handlers/api/v1/yt/get.handler.ts b/server/src/handlers/api/v1/yt/get.handler.ts index 4871fbff..9b0bcc90 100644 --- a/server/src/handlers/api/v1/yt/get.handler.ts +++ b/server/src/handlers/api/v1/yt/get.handler.ts @@ -1,8 +1,8 @@ import { FastifyRequest, FastifyReply } from "fastify"; import { GetYoutubeTranscript } from "./types"; -import { YtTranscript } from "yt-transcript" const getYoutubeTranscript = async (url: string) => { + const { YtTranscript } = await import("yt-transcript"); const transcript = new YtTranscript({ url }); const data = await transcript.listAllTranscripts(); return data; diff --git a/server/src/loader/youtube-transcript.ts b/server/src/loader/youtube-transcript.ts index 03eb607a..e49337d1 100644 --- a/server/src/loader/youtube-transcript.ts +++ b/server/src/loader/youtube-transcript.ts @@ -1,51 +1,48 @@ import { BaseDocumentLoader } from "langchain/document_loaders/base"; import { Document } from "langchain/document"; -import { YtTranscript } from "yt-transcript"; export interface YoutubeTranscriptParams { - url: string; - language_code: string; + url: string; + language_code: string; } export class DialoqbaseYoutubeTranscript - extends BaseDocumentLoader - implements YoutubeTranscriptParams { - language_code: string; - url: string; - - constructor({ language_code, url }: YoutubeTranscriptParams) { - super(); - this.language_code = language_code; - this.url = url - } - - - - async load(): Promise>[]> { - const ytTranscript = new YtTranscript({ - url: this.url, - }) - - const script = await ytTranscript.getTranscript(this.language_code) - - if (!script) throw new Error("No script found") - - - let text = "" - - script.forEach((item) => { - text += item.text + " " - }) - - - return [ - { - metadata: { - source: this.url, - audio: { chunks: script } - }, - pageContent: text - } - ] - } + extends BaseDocumentLoader + implements YoutubeTranscriptParams +{ + language_code: string; + url: string; + + constructor({ language_code, url }: YoutubeTranscriptParams) { + super(); + this.language_code = language_code; + this.url = url; + } + + async load(): Promise>[]> { + const { YtTranscript } = await import("yt-transcript"); + + const ytTranscript = new YtTranscript({ + url: this.url, + }); + + const script = await ytTranscript.getTranscript(this.language_code); + + if (!script) throw new Error("No script found"); + + let text = ""; + + script.forEach((item) => { + text += item.text + " "; + }); + + return [ + { + metadata: { + source: this.url, + }, + pageContent: text, + }, + ]; + } } diff --git a/server/src/plugins/bull.ts b/server/src/plugins/bull.ts index f1c325d9..8c2c3631 100644 --- a/server/src/plugins/bull.ts +++ b/server/src/plugins/bull.ts @@ -1,7 +1,7 @@ import fp from "fastify-plugin"; import { FastifyPluginAsync } from "fastify"; -import * as Queue from "bull"; +import Queue from "bull"; import { join } from "path"; declare module "fastify" { diff --git a/server/src/plugins/jwt.ts b/server/src/plugins/jwt.ts index 50bb3f6c..4b290697 100644 --- a/server/src/plugins/jwt.ts +++ b/server/src/plugins/jwt.ts @@ -11,7 +11,6 @@ declare module "@fastify/jwt" { } } - export default fp(async (fastify, opts) => { fastify.register(fastifyJwt, { secret: process.env.DB_SECRET_KEY!, @@ -20,60 +19,60 @@ export default fp(async (fastify, opts) => { }, }); - fastify.decorate( - "authenticate", - async function (request: FastifyRequest, reply: FastifyReply) { - try { - const token = request.headers.authorization; - if (token && token.startsWith("db_")) { - const apiKey = await fastify.prisma.userApiKey.findFirst({ - where: { - api_key: token, - }, - include: { - User: true, - } - }); - - if (!apiKey) { - return reply.status(401).send({ - message: "Unauthorized", - }); - } - request.user = { - user_id: apiKey.User.user_id, - username: apiKey.User.username, - is_admin: apiKey.User.isAdministrator, - }; - } else { - - await request.jwtVerify(); - const { user_id } = request.user; + async function authenticate(request: FastifyRequest, reply: FastifyReply) { + try { + const token = request.headers.authorization; + if (token && token.startsWith("db_")) { + const apiKey = await fastify.prisma.userApiKey.findFirst({ + where: { + api_key: token, + }, + include: { + User: true, + }, + }); - const user = await fastify.prisma.user.findUnique({ - where: { - user_id, - }, + if (!apiKey) { + return reply.status(401).send({ + message: "Unauthorized", }); + } + request.user = { + user_id: apiKey.User.user_id, + username: apiKey.User.username, + is_admin: apiKey.User.isAdministrator, + }; + } else { + await request.jwtVerify(); + const { user_id } = request.user; + + const user = await fastify.prisma.user.findUnique({ + where: { + user_id, + }, + }); - if (!user) { - throw new Error("User not found"); - } - request.user = { - user_id: user.user_id, - username: user.username, - is_admin: user.isAdministrator, - }; + if (!user) { + throw new Error("User not found"); } - } catch (err) { - reply.send(err); + request.user = { + user_id: user.user_id, + username: user.username, + is_admin: user.isAdministrator, + }; } - }, - ); + } catch (err) { + reply.send(err); + } + } + fastify.decorate("authenticate", authenticate); }); declare module "fastify" { export interface FastifyInstance { - authenticate(): Promise; + authenticate( + request: FastifyRequest, + reply: FastifyReply + ): Promise; } } diff --git a/server/src/utils/store.ts b/server/src/utils/store.ts index e0ef283e..7bcff0d1 100644 --- a/server/src/utils/store.ts +++ b/server/src/utils/store.ts @@ -18,7 +18,7 @@ interface SearchEmbeddingsResponse { export class DialoqbaseVectorStore extends VectorStore { botId: string; sourceId: string | null; - embeddings: Embeddings; + declare embeddings: Embeddings; constructor(embeddings: Embeddings, args: DialoqbaseLibArgs) { super(embeddings, args); diff --git a/server/src/utils/youtube.ts b/server/src/utils/youtube.ts index 3d1f5641..e8db7af9 100644 --- a/server/src/utils/youtube.ts +++ b/server/src/utils/youtube.ts @@ -1,4 +1,4 @@ -import * as ytdl from "ytdl-core"; +import ytdl from "ytdl-core"; export function isYouTubeUrl(url: string): boolean { const youtubeRegex = /^(https?:\/\/)?(www\.)?(youtube\.com|youtu\.be)\/.+$/; diff --git a/server/tsconfig.json b/server/tsconfig.json index ae655f89..7de7c456 100644 --- a/server/tsconfig.json +++ b/server/tsconfig.json @@ -5,6 +5,7 @@ "sourceMap": true, "allowSyntheticDefaultImports": true, "checkJs": false, + "strict": false }, "include": ["src/**/*.ts"], "exclude": [ diff --git a/server/yarn.lock b/server/yarn.lock index 9e2140b4..03874cdc 100644 --- a/server/yarn.lock +++ b/server/yarn.lock @@ -290,6 +290,11 @@ "@babel/helper-validator-identifier" "^7.19.1" to-fast-properties "^2.0.0" +"@bcoe/v8-coverage@^0.2.3": + version "0.2.3" + resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" + integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== + "@cspotcode/source-map-support@^0.8.0": version "0.8.1" resolved "https://registry.yarnpkg.com/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz#00629c35a688e05a88b1cda684fb9d5e73f000a1" @@ -410,6 +415,11 @@ resolved "https://registry.yarnpkg.com/@fastify/error/-/error-3.2.1.tgz#5d1cb182a8bb9b103c556b0da35fd77f319fc15e" integrity sha512-scZVbcpPNWw/yyFmzzO7cf1daTeJp53spN2n7dBTHZd+cV7791fcWJCPP1Tfhdbre+8vDiCyQyqqXfQnYMntYQ== +"@fastify/error@^3.3.0", "@fastify/error@^3.4.0": + version "3.4.1" + resolved "https://registry.yarnpkg.com/@fastify/error/-/error-3.4.1.tgz#b14bb4cac3dd4ec614becbc643d1511331a6425c" + integrity sha512-wWSvph+29GR783IhmvdwWnN4bUxTD01Vm5Xad4i7i1VuAOItLvbPAb69sb0IQ2N57yprvhNIwAP5B6xfKTmjmQ== + "@fastify/fast-json-stringify-compiler@^4.3.0": version "4.3.0" resolved "https://registry.yarnpkg.com/@fastify/fast-json-stringify-compiler/-/fast-json-stringify-compiler-4.3.0.tgz#5df89fa4d1592cbb8780f78998355feb471646d5" @@ -417,17 +427,24 @@ dependencies: fast-json-stringify "^5.7.0" -"@fastify/jwt@^7.0.0": - version "7.0.0" - resolved "https://registry.yarnpkg.com/@fastify/jwt/-/jwt-7.0.0.tgz#b3ebfb762bcc59b98fd595ed94900262454b03cf" - integrity sha512-y8n7qhBb/U+qWRUJzjZ+SJckv9wmOrA1eC/lM34SnOopt7VlK3hdfox7T3iuurAWVb8HjS9GxmZx+zFZO+Vb5A== +"@fastify/jwt@^8.0.0": + version "8.0.0" + resolved "https://registry.yarnpkg.com/@fastify/jwt/-/jwt-8.0.0.tgz#66aed823457ccc90579c5ce680b004db01053827" + integrity sha512-pJHjmZaokteZFMbsVVt7pbyJpbDogTnpl/aD7eR3vLOPgfktp4k4gUM6cd7RtjL/Ol1qDwL5mup+vdNlZI3K0Q== dependencies: "@fastify/error" "^3.0.0" "@lukeed/ms" "^2.0.0" - fast-jwt "^3.0.0" + fast-jwt "^3.3.2" fastify-plugin "^4.0.0" steed "^1.1.3" +"@fastify/merge-json-schemas@^0.1.0": + version "0.1.1" + resolved "https://registry.yarnpkg.com/@fastify/merge-json-schemas/-/merge-json-schemas-0.1.1.tgz#3551857b8a17a24e8c799e9f51795edb07baa0bc" + integrity sha512-fERDVz7topgNjtXsJTTW1JKLy0rhuLRcquYqNR9rF7OcVpCa2OVW49ZPDIhaRRCaUuvVxI+N416xUoF76HNSXA== + dependencies: + fast-deep-equal "^3.1.3" + "@fastify/multipart@^7.6.0": version "7.6.0" resolved "https://registry.yarnpkg.com/@fastify/multipart/-/multipart-7.6.0.tgz#8b7baf00945509aac54d249424768c7c520ee741" @@ -622,7 +639,7 @@ js-yaml "^3.13.1" resolve-from "^5.0.0" -"@istanbuljs/schema@^0.1.2": +"@istanbuljs/schema@^0.1.2", "@istanbuljs/schema@^0.1.3": version "0.1.3" resolved "https://registry.yarnpkg.com/@istanbuljs/schema/-/schema-0.1.3.tgz#e45e384e4b8ec16bce2fd903af78450f6bf7ec98" integrity sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA== @@ -646,6 +663,11 @@ resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.1.tgz#c08679063f279615a3326583ba3a90d1d82cc721" integrity sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA== +"@jridgewell/resolve-uri@^3.1.0": + version "3.1.2" + resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz#7a0ee601f60f99a20c7c7c5ff0c80388c1189bd6" + integrity sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw== + "@jridgewell/set-array@^1.0.1": version "1.1.2" resolved "https://registry.yarnpkg.com/@jridgewell/set-array/-/set-array-1.1.2.tgz#7c6cf998d6d20b914c0a55a91ae928ff25965e72" @@ -656,7 +678,7 @@ resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz#add4c98d341472a289190b424efbdb096991bb24" integrity sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw== -"@jridgewell/sourcemap-codec@^1.4.10": +"@jridgewell/sourcemap-codec@^1.4.10", "@jridgewell/sourcemap-codec@^1.4.14": version "1.4.15" resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz#d7c6e6755c78567a951e04ab52ef0fd26de59f32" integrity sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg== @@ -669,6 +691,14 @@ "@jridgewell/resolve-uri" "^3.0.3" "@jridgewell/sourcemap-codec" "^1.4.10" +"@jridgewell/trace-mapping@^0.3.12": + version "0.3.25" + resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz#15f190e98895f3fc23276ee14bc76b675c2e50f0" + integrity sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ== + dependencies: + "@jridgewell/resolve-uri" "^3.1.0" + "@jridgewell/sourcemap-codec" "^1.4.14" + "@jridgewell/trace-mapping@^0.3.17", "@jridgewell/trace-mapping@^0.3.9": version "0.3.18" resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.18.tgz#25783b2086daf6ff1dcb53c9249ae480e4dd4cd6" @@ -764,7 +794,12 @@ zod "^3.22.4" zod-to-json-schema "^3.22.3" -"@lukeed/ms@^2.0.0", "@lukeed/ms@^2.0.1": +"@lukeed/ms@^2.0.0": + version "2.0.2" + resolved "https://registry.yarnpkg.com/@lukeed/ms/-/ms-2.0.2.tgz#07f09e59a74c52f4d88c6db5c1054e819538e2a8" + integrity sha512-9I2Zn6+NJLfaGoz9jN3lpwDgAYvfGeNYdbAIjJOqzs4Tpc+VU3Jqq4IofSUBKajiDS8k9fZIg18/z13mpk1bsA== + +"@lukeed/ms@^2.0.1": version "2.0.1" resolved "https://registry.yarnpkg.com/@lukeed/ms/-/ms-2.0.1.tgz#3c2bbc258affd9cc0e0cc7828477383c73afa6ee" integrity sha512-Xs/4RZltsAL7pkvaNStUQt7netTkyxrS0K+RILcVr3TRMS/ToOg4I6uNfhB9SlGsnWBym4U+EaXq0f0cEMNkHA== @@ -1180,6 +1215,11 @@ dependencies: "@types/node" "*" +"@types/istanbul-lib-coverage@^2.0.1": + version "2.0.6" + resolved "https://registry.yarnpkg.com/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz#7739c232a1fee9b4d3ce8985f314c0c6d33549d7" + integrity sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w== + "@types/jsonwebtoken@^8.3.7": version "8.5.9" resolved "https://registry.yarnpkg.com/@types/jsonwebtoken/-/jsonwebtoken-8.5.9.tgz#2c064ecb0b3128d837d2764aa0b117b0ff6e4586" @@ -1222,6 +1262,11 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-20.2.5.tgz#26d295f3570323b2837d322180dfbf1ba156fefb" integrity sha512-JJulVEQXmiY9Px5axXHeYGLSjhkZEnD+MDPDGbCbIAbMslkKwmygtZFy1X6s/075Yo94sf8GuSlFfPzysQrWZQ== +"@types/node@20.4.4": + version "20.4.4" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.4.4.tgz#c79c7cc22c9d0e97a7944954c9e663bcbd92b0cb" + integrity sha512-CukZhumInROvLq3+b5gLev+vgpsIqC2D0deQr/yS1WnxvmYLlJXZpaQrQiseMY+6xusl79E04UjWoqyr+t1/Ew== + "@types/node@>=12", "@types/node@>=12.0.0": version "20.4.2" resolved "https://registry.yarnpkg.com/@types/node/-/node-20.4.2.tgz#129cc9ae69f93824f92fac653eebfb4812ab4af9" @@ -1237,11 +1282,6 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-20.3.1.tgz#e8a83f1aa8b649377bb1fb5d7bac5cb90e784dfe" integrity sha512-EhcH/wvidPy1WeML3TtYFGR83UzjxeWRen9V402T8aUGYsCHOmfoisV3ZSg03gAFIbLq8TnWOJ0f4cALtnSEUg== -"@types/node@^18.0.0": - version "18.16.16" - resolved "https://registry.yarnpkg.com/@types/node/-/node-18.16.16.tgz#3b64862856c7874ccf7439e6bab872d245c86d8e" - integrity sha512-NpaM49IGQQAUlBhHMF82QH80J08os4ZmyF9MkpCzWAGuOHqE4gTEbhzd7L3l5LmWuZ6E0OiC1FweQ4tsiW35+g== - "@types/node@^18.11.18": version "18.19.22" resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.22.tgz#f622f92514b897e6b09903e97c16a0db8e94689f" @@ -1485,6 +1525,13 @@ ajv-formats@^2.1.1: dependencies: ajv "^8.0.0" +ajv-formats@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/ajv-formats/-/ajv-formats-3.0.1.tgz#3d5dc762bca17679c3c2ea7e90ad6b7532309578" + integrity sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ== + dependencies: + ajv "^8.0.0" + ajv@^8.0.0, ajv@^8.10.0, ajv@^8.11.0: version "8.12.0" resolved "https://registry.yarnpkg.com/ajv/-/ajv-8.12.0.tgz#d1a0527323e22f53562c567c00991577dfbe19d1" @@ -1650,14 +1697,15 @@ available-typed-arrays@^1.0.5: resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz#92f95616501069d07d10edb2fc37d3e1c65123b7" integrity sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw== -avvio@^8.2.0: - version "8.2.1" - resolved "https://registry.yarnpkg.com/avvio/-/avvio-8.2.1.tgz#b5a482729847abb84d5aadce06511c04a0a62f82" - integrity sha512-TAlMYvOuwGyLK3PfBb5WKBXZmXz2fVCgv23d6zZFdle/q3gPjmxBaeuC0pY0Dzs5PWMSgfqqEZkrye19GlDTgw== +avvio@^8.3.0: + version "8.3.0" + resolved "https://registry.yarnpkg.com/avvio/-/avvio-8.3.0.tgz#1e019433d935730b814978a583eefac41a65082f" + integrity sha512-VBVH0jubFr9LdFASy/vNtm5giTrnbVquWBhT0fyizuNK2rQ7e7ONU2plZQWUNqtE1EmxFEb+kbSkFRkstiaS9Q== dependencies: + "@fastify/error" "^3.3.0" archy "^1.0.0" debug "^4.0.0" - fastq "^1.6.1" + fastq "^1.17.1" axios@^0.27.2: version "0.27.2" @@ -1858,6 +1906,23 @@ bytes@3.1.2: resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.2.tgz#8b0beeb98605adf1b128fa4386403c009e0221a5" integrity sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg== +c8@9.0.0: + version "9.0.0" + resolved "https://registry.yarnpkg.com/c8/-/c8-9.0.0.tgz#96f135754bbfc9fceb3c118970dcfd4a158de59d" + integrity sha512-nFJhU2Cz6Frh2awk3IW7wwk3wx27/U2v8ojQCHGc1GWTCHS6aMu4lal327/ZnnYj7oSThGF1X3qUP1yzAJBcOQ== + dependencies: + "@bcoe/v8-coverage" "^0.2.3" + "@istanbuljs/schema" "^0.1.3" + find-up "^5.0.0" + foreground-child "^3.1.1" + istanbul-lib-coverage "^3.2.0" + istanbul-lib-report "^3.0.1" + istanbul-reports "^3.1.6" + test-exclude "^6.0.0" + v8-to-istanbul "^9.0.0" + yargs "^17.7.2" + yargs-parser "^21.1.1" + cacheable-lookup@^5.0.3: version "5.0.4" resolved "https://registry.yarnpkg.com/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz#5a6b865b2c44357be3d5ebc2a467b032719a7005" @@ -2000,7 +2065,7 @@ cheerio@1.0.0-rc.12: parse5 "^7.0.0" parse5-htmlparser2-tree-adapter "^7.0.0" -chokidar@^3.3.0, chokidar@^3.5.2: +chokidar@^3.3.0: version "3.5.3" resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.3.tgz#1cf37c8707b932bd1af1ae22c0432e2acd1903bd" integrity sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw== @@ -2015,6 +2080,21 @@ chokidar@^3.3.0, chokidar@^3.5.2: optionalDependencies: fsevents "~2.3.2" +chokidar@^3.5.2: + version "3.6.0" + resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.6.0.tgz#197c6cc669ef2a8dc5e7b4d97ee4e092c3eb0d5b" + integrity sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw== + dependencies: + anymatch "~3.1.2" + braces "~3.0.2" + glob-parent "~5.1.2" + is-binary-path "~2.1.0" + is-glob "~4.0.1" + normalize-path "~3.0.0" + readdirp "~3.6.0" + optionalDependencies: + fsevents "~2.3.2" + chownr@^1.1.1: version "1.1.4" resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b" @@ -2090,9 +2170,9 @@ clone-response@^1.0.2: mimic-response "^1.0.0" close-with-grace@^1.1.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/close-with-grace/-/close-with-grace-1.2.0.tgz#9af82cc62b40125125e4c772e4dbe3cd8c3ff494" - integrity sha512-Xga0jyAb4fX98u5pZAgqlbqHP8cHuy5M3Wto0k0L/36aP2C25Cjp51XfPw3Hz7dNC2L2/hF/PK/KJhO275L+VA== + version "1.3.0" + resolved "https://registry.yarnpkg.com/close-with-grace/-/close-with-grace-1.3.0.tgz#ff0f5889ad076f79c9d4308db75cc827027366b8" + integrity sha512-lvm0rmLIR5bNz4CRKW6YvCfn9Wg5Wb9A8PJ3Bb+hjyikgC1RO1W3J4z9rBXQYw97mAte7dNSQI8BmUsxdlXQyw== cluster-key-slot@^1.1.0: version "1.1.2" @@ -2241,6 +2321,11 @@ convert-source-map@^1.7.0: resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-1.9.0.tgz#7faae62353fb4213366d0ca98358d22e8368b05f" integrity sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A== +convert-source-map@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-2.0.0.tgz#4b560f649fc4e918dd0ab75cf4961e8bc882d82a" + integrity sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg== + convert-to-spaces@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/convert-to-spaces/-/convert-to-spaces-1.0.2.tgz#7e3e48bbe6d997b1417ddca2868204b4d3d85715" @@ -2256,6 +2341,11 @@ cookie@0.5.0, cookie@^0.5.0: resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.5.0.tgz#d1f5d71adec6558c58f389987c366aa47e994f8b" integrity sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw== +cookie@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.6.0.tgz#2798b04b071b0ecbff0dbb62a505a8efa4e19051" + integrity sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw== + copyfiles@^2.4.1: version "2.4.1" resolved "https://registry.yarnpkg.com/copyfiles/-/copyfiles-2.4.1.tgz#d2dcff60aaad1015f09d0b66e7f0f1c5cd3c5da5" @@ -2536,9 +2626,9 @@ domutils@^3.0.1: domhandler "^5.0.3" dotenv@^16.0.0: - version "16.1.4" - resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.1.4.tgz#67ac1a10cd9c25f5ba604e4e08bc77c0ebe0ca8c" - integrity sha512-m55RtE8AsPeJBpOIFKihEmqUcoVncQIwo7x9U8ZwLEZw9ZpXboz2c+rvog+jUaJvVrZ5kBOeYQBX5+8Aa/OZQw== + version "16.4.5" + resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.4.5.tgz#cdd3b3b604cb327e286b4762e13502f717cb099f" + integrity sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg== duck@^0.1.12: version "0.1.12" @@ -2820,15 +2910,15 @@ extend@^3.0.2: resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== -fast-content-type-parse@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/fast-content-type-parse/-/fast-content-type-parse-1.0.0.tgz#cddce00df7d7efb3727d375a598e4904bfcb751c" - integrity sha512-Xbc4XcysUXcsP5aHUU7Nq3OwvHq97C+WnbkeIefpeYLX+ryzFJlU6OStFJhs6Ol0LkUGpcK+wL0JwfM+FCU5IA== +fast-content-type-parse@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/fast-content-type-parse/-/fast-content-type-parse-1.1.0.tgz#4087162bf5af3294d4726ff29b334f72e3a1092c" + integrity sha512-fBHHqSTFLVnR61C+gltJuE5GkVQMV0S2nqUO8TJ+5Z3qAKG8vAx4FKai1s5jq/inV1+sREynIWSuQ6HgoSXpDQ== fast-copy@^3.0.0: - version "3.0.1" - resolved "https://registry.yarnpkg.com/fast-copy/-/fast-copy-3.0.1.tgz#9e89ef498b8c04c1cd76b33b8e14271658a732aa" - integrity sha512-Knr7NOtK3HWRYGtHoJrjkaWepqT8thIVGAwt0p0aUs1zqkAzXZV4vo9fFNwyb5fcqK1GKYFYxldQdIDVKhUAfA== + version "3.0.2" + resolved "https://registry.yarnpkg.com/fast-copy/-/fast-copy-3.0.2.tgz#59c68f59ccbcac82050ba992e0d5c389097c9d35" + integrity sha512-dl0O9Vhju8IrcLndv2eU4ldt1ftXMqqfgN4H1cpmGV7P6jeB9FwpN9a2c8DPGE1Ys88rNUJVYDHq73CGAGOPfQ== fast-decode-uri-component@^1.0.1: version "1.0.1" @@ -2845,23 +2935,25 @@ fast-fifo@^1.0.0: resolved "https://registry.yarnpkg.com/fast-fifo/-/fast-fifo-1.3.0.tgz#03e381bcbfb29932d7c3afde6e15e83e05ab4d8b" integrity sha512-IgfweLvEpwyA4WgiQe9Nx6VV2QkML2NkvZnk1oKnIzXgXdWxuhF7zw4DvLTPZJn6PIUneiAXPF24QmoEqHTjyw== -fast-json-stringify@^5.7.0: - version "5.7.0" - resolved "https://registry.yarnpkg.com/fast-json-stringify/-/fast-json-stringify-5.7.0.tgz#b0a04c848fdeb6ecd83440c71a4db35067023bed" - integrity sha512-sBVPTgnAZseLu1Qgj6lUbQ0HfjFhZWXAmpZ5AaSGkyLh5gAXBga/uPJjQPHpDFjC9adWIpdOcCLSDTgrZ7snoQ== +fast-json-stringify@^5.7.0, fast-json-stringify@^5.8.0: + version "5.14.1" + resolved "https://registry.yarnpkg.com/fast-json-stringify/-/fast-json-stringify-5.14.1.tgz#3b1aa5a823e4dd5414ec079d32f51e33dd887766" + integrity sha512-J1Grbf0oSXV3lKsBf3itz1AvRk43qVrx3Ac10sNvi3LZaz1by4oDdYKFrJycPhS8+Gb7y8rgV/Jqw1UZVjyNvw== dependencies: - "@fastify/deepmerge" "^1.0.0" + "@fastify/merge-json-schemas" "^0.1.0" ajv "^8.10.0" - ajv-formats "^2.1.1" + ajv-formats "^3.0.1" fast-deep-equal "^3.1.3" fast-uri "^2.1.0" + json-schema-ref-resolver "^1.0.1" rfdc "^1.2.0" -fast-jwt@^3.0.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/fast-jwt/-/fast-jwt-3.1.0.tgz#3d835181bddeb9764989b0d9f839d51017ddb4a7" - integrity sha512-3xTjNdKul+YJMuknYrxiJfXL9Qg4TuqQu0SSPSCs89XikJWgnD4Zey5YREWOsnjB2R042SkJsDt1dE1AZ5XwgA== +fast-jwt@^3.3.2: + version "3.3.3" + resolved "https://registry.yarnpkg.com/fast-jwt/-/fast-jwt-3.3.3.tgz#08687e016e43937ff5dbdb7bcb5d49990e5743ba" + integrity sha512-oS3P8bRI24oPLJUePt2OgF64FBQib5TlgHLFQxYNoHYEEZe0gU3cKjJAVqpB5XKV/zjxmq4Hzbk3fgfW/wRz8Q== dependencies: + "@lukeed/ms" "^2.0.1" asn1.js "^5.4.1" ecdsa-sig-formatter "^1.0.11" mnemonist "^0.39.5" @@ -2874,9 +2966,9 @@ fast-querystring@^1.0.0: fast-decode-uri-component "^1.0.1" fast-redact@^3.1.1: - version "3.2.0" - resolved "https://registry.yarnpkg.com/fast-redact/-/fast-redact-3.2.0.tgz#b1e2d39bc731376d28bde844454fa23e26919987" - integrity sha512-zaTadChr+NekyzallAMXATXLOR8MNx3zqpZ0MUF2aGf4EathnG0f32VLODNlY8IuGY3HoRO2L6/6fSzNsLaHIw== + version "3.5.0" + resolved "https://registry.yarnpkg.com/fast-redact/-/fast-redact-3.5.0.tgz#e9ea02f7e57d0cd8438180083e93077e496285e4" + integrity sha512-dwsoQlS7h9hMeYUq1W++23NDcBLV4KqONnITDV9DjfS3q1SgDGVrBdvvTLUotWtPSD7asWDV9/CmsZPy8Hf70A== fast-safe-stringify@^2.1.1: version "2.1.1" @@ -2889,9 +2981,9 @@ fast-text-encoding@^1.0.0: integrity sha512-VhXlQgj9ioXCqGstD37E/HBeqEGV/qOD/kmbVG8h5xKBYvM1L3lR1Zn4555cQ8GkYbJa8aJSipLPndE1k6zK2w== fast-uri@^2.0.0, fast-uri@^2.1.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/fast-uri/-/fast-uri-2.2.0.tgz#519a0f849bef714aad10e9753d69d8f758f7445a" - integrity sha512-cIusKBIt/R/oI6z/1nyfe2FvGKVTohVRfvkOhvx0nCEW+xf5NoCXjAHcWp93uOUBchzYcsvPlrapAdX1uW+YGg== + version "2.3.0" + resolved "https://registry.yarnpkg.com/fast-uri/-/fast-uri-2.3.0.tgz#bdae493942483d299e7285dcb4627767d42e2793" + integrity sha512-eel5UKGn369gGEWOqBShmFJWfq/xSJvsgDzgLYC845GneayWvXBf0lJCBn5qTABfewy1ZDPoaR5OZCP+kssfuw== fast-xml-parser@^4.3.5: version "4.3.5" @@ -2907,10 +2999,10 @@ fastfall@^1.5.0: dependencies: reusify "^1.0.0" -fastify-cli@^5.7.1: - version "5.7.1" - resolved "https://registry.yarnpkg.com/fastify-cli/-/fastify-cli-5.7.1.tgz#2563718df61f91c9d3a4a5416d91d96454f2a862" - integrity sha512-0FgQux1TK+zsRxVLDCF4F+x2ilHDeaJB2TouHIHWRS0+cM6aC+bBSqHva/myrlZs/1lQCAe294DvZkAuJyHySw== +fastify-cli@6.1.1: + version "6.1.1" + resolved "https://registry.yarnpkg.com/fastify-cli/-/fastify-cli-6.1.1.tgz#08a291bb21a2b61cf6b681e5cf7d1f5c10f2ac3e" + integrity sha512-3TinVuHxcEOhwjRLoE9rTeH1IXa73B5fS21amNyTgF2kOnC72sHzp1p6y2RsbPES1NGf2A7+4S11jbfdTOchVQ== dependencies: "@fastify/deepmerge" "^1.2.0" chalk "^4.1.2" @@ -2918,34 +3010,34 @@ fastify-cli@^5.7.1: close-with-grace "^1.1.0" commist "^3.0.0" dotenv "^16.0.0" - fastify "^4.0.0" + fastify "^4.26.1" fastify-plugin "^4.0.0" generify "^4.0.0" help-me "^4.0.1" is-docker "^2.0.0" make-promises-safe "^5.1.0" - pino-pretty "^9.0.0" + pino-pretty "^10.1.0" pkg-up "^3.1.0" resolve-from "^5.0.0" semver "^7.3.5" yargs-parser "^21.1.1" -fastify-plugin@^3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/fastify-plugin/-/fastify-plugin-3.0.1.tgz#79e84c29f401020f38b524f59f2402103fd21ed2" - integrity sha512-qKcDXmuZadJqdTm6vlCqioEbyewF60b/0LOFCcYN1B6BIZGlYJumWWOYs70SFYLDAH4YqdE1cxH/RKMG7rFxgA== +fastify-plugin@4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/fastify-plugin/-/fastify-plugin-4.0.0.tgz#65c2f538a8d869d59c7364a56565687e78e32390" + integrity sha512-ZJcXPPcqkj7HFDYqbsCuOIAgIZ/sd2b+OnBxNGyxAcUDUJfIpxp4t23CwxO2E7LZpqUrIliA4TnjxTXG8mLoqw== fastify-plugin@^4.0.0, fastify-plugin@^4.3.0: version "4.5.0" resolved "https://registry.yarnpkg.com/fastify-plugin/-/fastify-plugin-4.5.0.tgz#8b853923a0bba6ab6921bb8f35b81224e6988d91" integrity sha512-79ak0JxddO0utAXAQ5ccKhvs6vX2MGyHHMMsmZkBANrq3hXc1CHzvNPHOcvTsVMEPl5I+NT+RO4YKMGehOfSIg== -fastify-raw-body@4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/fastify-raw-body/-/fastify-raw-body-4.0.0.tgz#4ea259e23f009c24f9a4344cbba4a8d83e4eb339" - integrity sha512-ZoIIdRzuv1mYuUY8nLK/oOyHvrsOt1KUHedmfxwaz4z2iRwSx8SyqKLavJj3haetcz3n4faNGNpykVyHx2f7yg== +fastify-raw-body@^4.3.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/fastify-raw-body/-/fastify-raw-body-4.3.0.tgz#f56db8f143503cc947b77497e66695f7e886be89" + integrity sha512-F4o8ZIMVx4YoxGfwrZys6wyjl40gF3Yv6AWWRy62ozFAyZBSS831/uyyCAqKYw3tR73g180ryG98yih6To1PUQ== dependencies: - fastify-plugin "^3.0.1" + fastify-plugin "^4.0.0" raw-body "^2.5.1" secure-json-parse "^2.4.0" @@ -2958,32 +3050,32 @@ fastify-sse-v2@^3.1.1: it-pushable "^1.4.2" it-to-stream "^1.0.0" -fastify-tsconfig@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/fastify-tsconfig/-/fastify-tsconfig-1.0.1.tgz#ec90e45fa8aa72b81178757bd5940c7875037639" - integrity sha512-BXkTG3JYcjJb3xX5R5FcE9ciscV/h7YtmnkiSaNAONd1g6ooMSN/4GWfhA8hnS6SRZFYBBxsn8719Mj9lbCOtA== +fastify-tsconfig@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/fastify-tsconfig/-/fastify-tsconfig-2.0.0.tgz#960367cdb368562948d446dd14f67bed45a92085" + integrity sha512-pvYwdtbZUJr/aTD7ZE0rGlvtYpx7IThHKVLBoqCKmT3FJpwm23XA2+PDmq8ZzfqqG4ajpyrHd5bkIixcIFjPhQ== -fastify@^4.0.0: - version "4.17.0" - resolved "https://registry.yarnpkg.com/fastify/-/fastify-4.17.0.tgz#b2c8245e572edef0b02a167d2d411a3c8a46d01a" - integrity sha512-tzuY1tgWJo2Y6qEKwmLhFvACUmr68Io2pqP/sDKU71KRM6A6R3DrCDqLGqANbeLZcKUfdfY58ut35CGqemcTgg== +fastify@^4.26.1, fastify@^4.26.2: + version "4.26.2" + resolved "https://registry.yarnpkg.com/fastify/-/fastify-4.26.2.tgz#9389595c46e9f4648de5bf8175e750bf32fed5a1" + integrity sha512-90pjTuPGrfVKtdpLeLzND5nyC4woXZN5VadiNQCicj/iJU4viNHKhsAnb7jmv1vu2IzkLXyBiCzdWuzeXgQ5Ug== dependencies: "@fastify/ajv-compiler" "^3.5.0" - "@fastify/error" "^3.0.0" + "@fastify/error" "^3.4.0" "@fastify/fast-json-stringify-compiler" "^4.3.0" abstract-logging "^2.0.1" - avvio "^8.2.0" - fast-content-type-parse "^1.0.0" - fast-json-stringify "^5.7.0" - find-my-way "^7.6.0" - light-my-request "^5.6.1" - pino "^8.5.0" - process-warning "^2.0.0" + avvio "^8.3.0" + fast-content-type-parse "^1.1.0" + fast-json-stringify "^5.8.0" + find-my-way "^8.0.0" + light-my-request "^5.11.0" + pino "^8.17.0" + process-warning "^3.0.0" proxy-addr "^2.0.7" rfdc "^1.3.0" - secure-json-parse "^2.5.0" - semver "^7.3.7" - tiny-lru "^11.0.1" + secure-json-parse "^2.7.0" + semver "^7.5.4" + toad-cache "^3.3.0" fastparallel@^2.2.0: version "2.4.1" @@ -2993,10 +3085,10 @@ fastparallel@^2.2.0: reusify "^1.0.4" xtend "^4.0.2" -fastq@^1.3.0, fastq@^1.6.1: - version "1.15.0" - resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.15.0.tgz#d04d07c6a2a68fe4599fea8d2e103a937fae6b3a" - integrity sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw== +fastq@^1.17.1, fastq@^1.3.0: + version "1.17.1" + resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.17.1.tgz#2a523f07a4e7b1e81a42b91b8bf2254107753b47" + integrity sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w== dependencies: reusify "^1.0.4" @@ -3046,10 +3138,10 @@ find-cache-dir@^3.2.0: make-dir "^3.0.2" pkg-dir "^4.1.0" -find-my-way@^7.6.0: - version "7.6.2" - resolved "https://registry.yarnpkg.com/find-my-way/-/find-my-way-7.6.2.tgz#4dd40200d3536aeef5c7342b10028e04cf79146c" - integrity sha512-0OjHn1b1nCX3eVbm9ByeEHiscPYiHLfhei1wOUU9qffQkk98wE0Lo8VrVYfSGMgnSnDh86DxedduAnBf4nwUEw== +find-my-way@^8.0.0: + version "8.1.0" + resolved "https://registry.yarnpkg.com/find-my-way/-/find-my-way-8.1.0.tgz#cc05e8e4b145322299d0de0a839b5be528c2083e" + integrity sha512-41QwjCGcVTODUmLLqTMeoHeiozbMXYMAE1CKFiDyi9zVZ2Vjh0yz3MF0WQZoIb+cmzP/XlbFjlF2NtJmvZHznA== dependencies: fast-deep-equal "^3.1.3" fast-querystring "^1.0.0" @@ -3070,6 +3162,14 @@ find-up@^4.0.0, find-up@^4.1.0: locate-path "^5.0.0" path-exists "^4.0.0" +find-up@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/find-up/-/find-up-5.0.0.tgz#4c92819ecb7083561e4f4a240a86be5198f536fc" + integrity sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng== + dependencies: + locate-path "^6.0.0" + path-exists "^4.0.0" + findit@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/findit/-/findit-2.0.0.tgz#6509f0126af4c178551cfa99394e032e13a4d56e" @@ -3115,6 +3215,14 @@ foreground-child@^2.0.0: cross-spawn "^7.0.0" signal-exit "^3.0.2" +foreground-child@^3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/foreground-child/-/foreground-child-3.1.1.tgz#1d173e776d75d2772fed08efe4a0de1ea1b12d0d" + integrity sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg== + dependencies: + cross-spawn "^7.0.0" + signal-exit "^4.0.1" + form-data-encoder@1.7.2: version "1.7.2" resolved "https://registry.yarnpkg.com/form-data-encoder/-/form-data-encoder-1.7.2.tgz#1f1ae3dccf58ed4690b86d87e4f57c654fbab040" @@ -3565,6 +3673,11 @@ help-me@^4.0.1: glob "^8.0.0" readable-stream "^3.6.0" +help-me@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/help-me/-/help-me-5.0.0.tgz#b1ebe63b967b74060027c2ac61f9be12d354a6f6" + integrity sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg== + hexoid@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/hexoid/-/hexoid-1.0.0.tgz#ad10c6573fb907de23d9ec63a711267d9dc9bc18" @@ -4032,6 +4145,15 @@ istanbul-lib-report@^3.0.0: make-dir "^3.0.0" supports-color "^7.1.0" +istanbul-lib-report@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz#908305bac9a5bd175ac6a74489eafd0fc2445a7d" + integrity sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw== + dependencies: + istanbul-lib-coverage "^3.0.0" + make-dir "^4.0.0" + supports-color "^7.1.0" + istanbul-lib-source-maps@^4.0.0: version "4.0.1" resolved "https://registry.yarnpkg.com/istanbul-lib-source-maps/-/istanbul-lib-source-maps-4.0.1.tgz#895f3a709fcfba34c6de5a42939022f3e4358551" @@ -4049,6 +4171,14 @@ istanbul-reports@^3.0.2: html-escaper "^2.0.0" istanbul-lib-report "^3.0.0" +istanbul-reports@^3.1.6: + version "3.1.7" + resolved "https://registry.yarnpkg.com/istanbul-reports/-/istanbul-reports-3.1.7.tgz#daed12b9e1dca518e15c056e1e537e741280fa0b" + integrity sha512-BewmUXImeuRk2YY0PVbxgKAysvhRPUQE0h5QRM++nVWyubKGV0l8qQ5op8+B2DOmwSe63Jivj0BjkPQVf8fP5g== + dependencies: + html-escaper "^2.0.0" + istanbul-lib-report "^3.0.0" + it-pushable@^1.4.2: version "1.4.2" resolved "https://registry.yarnpkg.com/it-pushable/-/it-pushable-1.4.2.tgz#fb127a53ec99b35a3a455a775abc85ab193c220b" @@ -4142,6 +4272,13 @@ json-buffer@3.0.1: resolved "https://registry.yarnpkg.com/json-buffer/-/json-buffer-3.0.1.tgz#9338802a30d3b6605fbe0613e094008ca8c05a13" integrity sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ== +json-schema-ref-resolver@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/json-schema-ref-resolver/-/json-schema-ref-resolver-1.0.1.tgz#6586f483b76254784fc1d2120f717bdc9f0a99bf" + integrity sha512-EJAj1pgHc1hxF6vo2Z3s69fMjO1INq6eGHXZ8Z6wCQeldCuwxGK9Sxf4/cScGn3FZubCVUehfWtcDM/PLteCQw== + dependencies: + fast-deep-equal "^3.1.3" + json-schema-resolver@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/json-schema-resolver/-/json-schema-resolver-2.0.0.tgz#d17fdf53560e6bc9af084b930fee27f6ce4a03b6" @@ -4298,13 +4435,13 @@ lie@~3.3.0: dependencies: immediate "~3.0.5" -light-my-request@^5.6.1: - version "5.9.2" - resolved "https://registry.yarnpkg.com/light-my-request/-/light-my-request-5.9.2.tgz#d9890292076156520b8fcbff1c0addfb6e94ab02" - integrity sha512-2+umCAZndzA/YeuoEPunRpSJTKNUVfOmmwg/TjaeTENT7HOY2hXc3W4jkffV5hnTxS2eOQkuafif1BJotySF1w== +light-my-request@^5.11.0: + version "5.13.0" + resolved "https://registry.yarnpkg.com/light-my-request/-/light-my-request-5.13.0.tgz#b29905e55e8605b77fee2a946e17b219bca35113" + integrity sha512-9IjUN9ZyCS9pTG+KqTDEQo68Sui2lHsYBrfMyVUTTZ3XhH8PMZq7xO94Kr+eP9dhi/kcKsx4N41p2IXEBil1pQ== dependencies: - cookie "^0.5.0" - process-warning "^2.0.0" + cookie "^0.6.0" + process-warning "^3.0.0" set-cookie-parser "^2.4.1" locate-path@^3.0.0: @@ -4322,6 +4459,13 @@ locate-path@^5.0.0: dependencies: p-locate "^4.1.0" +locate-path@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-6.0.0.tgz#55321eb309febbc59c4801d931a72452a681d286" + integrity sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw== + dependencies: + p-locate "^5.0.0" + lodash.camelcase@^4.3.0: version "4.3.0" resolved "https://registry.yarnpkg.com/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz#b28aa6288a2b9fc651035c7711f65ab6190331a6" @@ -4422,6 +4566,13 @@ make-dir@^3.0.0, make-dir@^3.0.2, make-dir@^3.1.0: dependencies: semver "^6.0.0" +make-dir@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-4.0.0.tgz#c3c2307a771277cd9638305f915c29ae741b614e" + integrity sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw== + dependencies: + semver "^7.5.3" + make-error@^1.1.1: version "1.3.6" resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.6.tgz#2eb2e37ea9b67c4891f684a1394799af484cf7a2" @@ -4616,13 +4767,20 @@ ml-tree-similarity@^1.0.0: binary-search "^1.3.5" num-sort "^2.0.0" -mnemonist@0.39.5, mnemonist@^0.39.5: +mnemonist@0.39.5: version "0.39.5" resolved "https://registry.yarnpkg.com/mnemonist/-/mnemonist-0.39.5.tgz#5850d9b30d1b2bc57cc8787e5caa40f6c3420477" integrity sha512-FPUtkhtJ0efmEFGpU14x7jGbTB+s18LrzRL2KgoWz9YvcY3cPomz8tih01GbHwnGk/OmkOKfqd/RAQoc8Lm7DQ== dependencies: obliterator "^2.0.1" +mnemonist@^0.39.5: + version "0.39.8" + resolved "https://registry.yarnpkg.com/mnemonist/-/mnemonist-0.39.8.tgz#9078cd8386081afd986cca34b52b5d84ea7a4d38" + integrity sha512-vyWo2K3fjrUw8YeeZ1zF0fy6Mu59RHokURlld8ymdUPjMlD9EC9ov1/YPqTgqRvUN9nTr3Gqfz29LYAmu0PHPQ== + dependencies: + obliterator "^2.0.1" + ms@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" @@ -4865,9 +5023,9 @@ obliterator@^2.0.1: integrity sha512-lgHwxlxV1qIg1Eap7LgIeoBWIMFibOjbrYPIPJZcI1mmGAI2m3lNYpK12Y+GBdPQ0U1hRwSord7GIaawz962qQ== on-exit-leak-free@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/on-exit-leak-free/-/on-exit-leak-free-2.1.0.tgz#5c703c968f7e7f851885f6459bf8a8a57edc9cc4" - integrity sha512-VuCaZZAjReZ3vUwgOB8LxAosIurDiAW0s13rI1YwmaP++jvcxP77AWoQvenZebpCA2m8WC1/EosPYPMjnRAp/w== + version "2.1.2" + resolved "https://registry.yarnpkg.com/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz#fed195c9ebddb7d9e4c3842f93f281ac8dadd3b8" + integrity sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA== on-finished@2.4.1: version "2.4.1" @@ -4998,7 +5156,7 @@ p-limit@^2.0.0, p-limit@^2.2.0: dependencies: p-try "^2.0.0" -p-limit@^3.1.0: +p-limit@^3.0.2, p-limit@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-3.1.0.tgz#e1daccbe78d0d1388ca18c64fea38e3e57e3706b" integrity sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ== @@ -5019,6 +5177,13 @@ p-locate@^4.1.0: dependencies: p-limit "^2.2.0" +p-locate@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-5.0.0.tgz#83c8315c6785005e3bd021839411c9e110e6d834" + integrity sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw== + dependencies: + p-limit "^3.0.2" + p-map@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/p-map/-/p-map-3.0.0.tgz#d704d9af8a2ba684e2600d9a215983d4141a979d" @@ -5178,24 +5343,24 @@ picomatch@^2.0.4, picomatch@^2.2.1: resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42" integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA== -pino-abstract-transport@^1.0.0, pino-abstract-transport@v1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/pino-abstract-transport/-/pino-abstract-transport-1.0.0.tgz#cc0d6955fffcadb91b7b49ef220a6cc111d48bb3" - integrity sha512-c7vo5OpW4wIS42hUVcT5REsL8ZljsUfBjqV/e2sFxmFEFZiq1XLUp5EYLtuDH6PEHq9W1egWqRbnLUP5FuZmOA== +pino-abstract-transport@^1.0.0, pino-abstract-transport@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/pino-abstract-transport/-/pino-abstract-transport-1.1.0.tgz#083d98f966262164504afb989bccd05f665937a8" + integrity sha512-lsleG3/2a/JIWUtf9Q5gUNErBqwIu1tUKTT3dUzaf5DySw9ra1wcqKjJjLX1VTY64Wk1eEOYsVGSaGfCK85ekA== dependencies: readable-stream "^4.0.0" split2 "^4.0.0" -pino-pretty@^9.0.0: - version "9.4.0" - resolved "https://registry.yarnpkg.com/pino-pretty/-/pino-pretty-9.4.0.tgz#fc4026e83c87272cbdfb7afed121770e6000940c" - integrity sha512-NIudkNLxnl7MGj1XkvsqVyRgo6meFP82ECXF2PlOI+9ghmbGuBUUqKJ7IZPIxpJw4vhhSva0IuiDSAuGh6TV9g== +pino-pretty@^10.1.0: + version "10.3.1" + resolved "https://registry.yarnpkg.com/pino-pretty/-/pino-pretty-10.3.1.tgz#e3285a5265211ac6c7cd5988f9e65bf3371a0ca9" + integrity sha512-az8JbIYeN/1iLj2t0jR9DV48/LQ3RC6hZPpapKPkb84Q+yTidMCpgWxIT3N0flnBDilyBQ1luWNpOeJptjdp/g== dependencies: colorette "^2.0.7" dateformat "^4.6.3" fast-copy "^3.0.0" fast-safe-stringify "^2.1.1" - help-me "^4.0.1" + help-me "^5.0.0" joycon "^3.1.1" minimist "^1.2.6" on-exit-leak-free "^2.1.0" @@ -5207,25 +5372,25 @@ pino-pretty@^9.0.0: strip-json-comments "^3.1.1" pino-std-serializers@^6.0.0: - version "6.2.1" - resolved "https://registry.yarnpkg.com/pino-std-serializers/-/pino-std-serializers-6.2.1.tgz#369f4ae2a19eb6d769ddf2c88a2164b76879a284" - integrity sha512-wHuWB+CvSVb2XqXM0W/WOYUkVSPbiJb9S5fNB7TBhd8s892Xq910bRxwHtC4l71hgztObTjXL6ZheZXFjhDrDQ== + version "6.2.2" + resolved "https://registry.yarnpkg.com/pino-std-serializers/-/pino-std-serializers-6.2.2.tgz#d9a9b5f2b9a402486a5fc4db0a737570a860aab3" + integrity sha512-cHjPPsE+vhj/tnhCy/wiMh3M3z3h/j15zHQX+S9GkTBgqJuTuJzYJ4gUyACLhDaJ7kk9ba9iRDmbH2tJU03OiA== -pino@^8.5.0: - version "8.14.1" - resolved "https://registry.yarnpkg.com/pino/-/pino-8.14.1.tgz#bb38dcda8b500dd90c1193b6c9171eb777a47ac8" - integrity sha512-8LYNv7BKWXSfS+k6oEc6occy5La+q2sPwU3q2ljTX5AZk7v+5kND2o5W794FyRaqha6DJajmkNRsWtPpFyMUdw== +pino@^8.17.0: + version "8.20.0" + resolved "https://registry.yarnpkg.com/pino/-/pino-8.20.0.tgz#ccfc6fef37b165e006b923834131632a8c4f036b" + integrity sha512-uhIfMj5TVp+WynVASaVEJFTncTUe4dHBq6CWplu/vBgvGHhvBvQfxz+vcOrnnBQdORH3izaGEurLfNlq3YxdFQ== dependencies: atomic-sleep "^1.0.0" fast-redact "^3.1.1" on-exit-leak-free "^2.1.0" - pino-abstract-transport v1.0.0 + pino-abstract-transport "^1.1.0" pino-std-serializers "^6.0.0" - process-warning "^2.0.0" + process-warning "^3.0.0" quick-format-unescaped "^4.0.3" real-require "^0.2.0" safe-stable-stringify "^2.3.1" - sonic-boom "^3.1.0" + sonic-boom "^3.7.0" thread-stream "^2.0.0" pkg-dir@^4.1.0: @@ -5291,10 +5456,10 @@ process-on-spawn@^1.0.0: dependencies: fromentries "^1.2.0" -process-warning@^2.0.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/process-warning/-/process-warning-2.2.0.tgz#008ec76b579820a8e5c35d81960525ca64feb626" - integrity sha512-/1WZ8+VQjR6avWOgHeEPd7SDQmFQ1B5mC1eRXsCm5TarlNmx/wCsa5GEaxGm05BORRtyG/Ex/3xq3TuRvq57qg== +process-warning@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/process-warning/-/process-warning-3.0.0.tgz#96e5b88884187a1dce6f5c3166d611132058710b" + integrity sha512-mqn0kFRl0EoqhnL0GQ0veqFHyIN1yig9RHh/InzORTUiZHFRAur+aMtRkELNwGs9aNwKS6tg/An4NYBPGwvtzQ== process@^0.11.10: version "0.11.10" @@ -5679,11 +5844,16 @@ reusify@^1.0.0, reusify@^1.0.4: resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76" integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw== -rfdc@^1.1.4, rfdc@^1.2.0, rfdc@^1.3.0: +rfdc@^1.1.4, rfdc@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/rfdc/-/rfdc-1.3.0.tgz#d0b7c441ab2720d05dc4cf26e01c89631d9da08b" integrity sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA== +rfdc@^1.2.0: + version "1.3.1" + resolved "https://registry.yarnpkg.com/rfdc/-/rfdc-1.3.1.tgz#2b6d4df52dffe8bb346992a10ea9451f24373a8f" + integrity sha512-r5a3l5HzYlIC68TpmYKlxWjmOP6wiPJ1vWv2HeLhNsRZMrCkxeqxiHlQ21oXmQ4F3SiryXBHhAD7JZqvOJjFmg== + rimraf@^3.0.0, rimraf@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a" @@ -5752,7 +5922,7 @@ scheduler@^0.20.2: loose-envify "^1.1.0" object-assign "^4.1.1" -secure-json-parse@^2.4.0, secure-json-parse@^2.5.0: +secure-json-parse@^2.4.0, secure-json-parse@^2.7.0: version "2.7.0" resolved "https://registry.yarnpkg.com/secure-json-parse/-/secure-json-parse-2.7.0.tgz#5a5f9cd6ae47df23dba3151edd06855d47e09862" integrity sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw== @@ -5774,7 +5944,7 @@ semver@^6.0.0, semver@^6.3.0: resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== -semver@^7.3.2, semver@^7.3.5, semver@^7.3.7: +semver@^7.3.2, semver@^7.3.5: version "7.5.1" resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.1.tgz#c90c4d631cf74720e46b21c1d37ea07edfab91ec" integrity sha512-Wvss5ivl8TMRZXXESstBA4uR5iXgEN/VC5/sOcuXdVLzcdkz4HWetIoRfG5gb5X+ij/G9rw9YoGn3QoQ8OCSpw== @@ -5795,6 +5965,13 @@ semver@^7.5.0: dependencies: lru-cache "^6.0.0" +semver@^7.5.3, semver@^7.5.4: + version "7.6.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.0.tgz#1a46a4db4bffcccd97b743b5005c8325f23d4e2d" + integrity sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg== + dependencies: + lru-cache "^6.0.0" + send@0.18.0: version "0.18.0" resolved "https://registry.yarnpkg.com/send/-/send-0.18.0.tgz#670167cc654b05f5aa4a767f9113bb371bc706be" @@ -5889,6 +6066,11 @@ signal-exit@^3.0.0, signal-exit@^3.0.2, signal-exit@^3.0.4, signal-exit@^3.0.6: resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.7.tgz#a9a1767f8af84155114eaabd73f99273c8f59ad9" integrity sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ== +signal-exit@^4.0.1: + version "4.1.0" + resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-4.1.0.tgz#952188c1cbd546070e2dd20d0f41c0ae0530cb04" + integrity sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw== + simple-concat@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/simple-concat/-/simple-concat-1.0.1.tgz#f46976082ba35c2263f1c8ab5edfe26c41c9552f" @@ -5938,10 +6120,10 @@ slice-ansi@^3.0.0: astral-regex "^2.0.0" is-fullwidth-code-point "^3.0.0" -sonic-boom@^3.0.0, sonic-boom@^3.1.0: - version "3.3.0" - resolved "https://registry.yarnpkg.com/sonic-boom/-/sonic-boom-3.3.0.tgz#cffab6dafee3b2bcb88d08d589394198bee1838c" - integrity sha512-LYxp34KlZ1a2Jb8ZQgFCK3niIHzibdwtwNUWKg0qQRzsDoJ3Gfgkf8KdBTFU3SkejDEIlWwnSnpVdOZIhFMl/g== +sonic-boom@^3.0.0, sonic-boom@^3.7.0: + version "3.8.1" + resolved "https://registry.yarnpkg.com/sonic-boom/-/sonic-boom-3.8.1.tgz#d5ba8c4e26d6176c9a1d14d549d9ff579a163422" + integrity sha512-y4Z8LCDBuum+PBP3lSV7RHrXscqksve/bi0as7mhwVnBW+/wUqKT/2Kb7um8yqcFy0duYbbPxzt89Zy2nOCaxg== dependencies: atomic-sleep "^1.0.0" @@ -6293,9 +6475,9 @@ text-decoding@^1.0.0: integrity sha512-/0TJD42KDnVwKmDK6jj3xP7E2MG7SHAOG4tyTgyUCRPdHwvkquYNLEQltmdMa3owq3TkddCVcTsoctJI8VQNKA== thread-stream@^2.0.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/thread-stream/-/thread-stream-2.3.0.tgz#4fc07fb39eff32ae7bad803cb7dd9598349fed33" - integrity sha512-kaDqm1DET9pp3NXwR8382WHbnpXnRkN9xGN9dQt3B2+dmXiW8X1SOwmFOxAErEQ47ObhZ96J6yhZNXuyCOL7KA== + version "2.4.1" + resolved "https://registry.yarnpkg.com/thread-stream/-/thread-stream-2.4.1.tgz#6d588b14f0546e59d3f306614f044bc01ce43351" + integrity sha512-d/Ex2iWd1whipbT681JmTINKw0ZwOUBZm7+Gjs64DHuX34mmw8vJL2bFAaNacaW72zYiTJxSHi5abUuOi5nsfg== dependencies: real-require "^0.2.0" @@ -6307,11 +6489,6 @@ through2@^2.0.1: readable-stream "~2.3.6" xtend "~4.0.1" -tiny-lru@^11.0.1: - version "11.0.1" - resolved "https://registry.yarnpkg.com/tiny-lru/-/tiny-lru-11.0.1.tgz#629d6ddd88bd03c0929722680167f1feadf576f2" - integrity sha512-iNgFugVuQgBKrqeO/mpiTTgmBsTP0WL6yeuLfLs/Ctf0pI/ixGqIRm8sDCwMcXGe9WWvt2sGXI5mNqZbValmJg== - tmpl@1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.5.tgz#8683e0b902bb9c20c4f726e3c0b69f36518c07cc" @@ -6329,6 +6506,11 @@ to-regex-range@^5.0.1: dependencies: is-number "^7.0.0" +toad-cache@^3.3.0: + version "3.7.0" + resolved "https://registry.yarnpkg.com/toad-cache/-/toad-cache-3.7.0.tgz#b9b63304ea7c45ec34d91f1d2fa513517025c441" + integrity sha512-/m8M+2BJUpoJdgAHoG+baCwBT+tf2VraSfkBgl0Y00qIWt41DJ8R5B8nsEw0I58YwF5IZH6z24/2TobDKnqSWw== + toidentifier@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.1.tgz#3be34321a88a820ed1bd80dfaa33e479fbb8dd35" @@ -6472,10 +6654,10 @@ typedarray-to-buffer@^3.1.5: dependencies: is-typedarray "^1.0.0" -typescript@^4.5.4: - version "4.9.5" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.9.5.tgz#095979f9bcc0d09da324d58d03ce8f8374cbe65a" - integrity sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g== +typescript@5.2.2: + version "5.2.2" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.2.2.tgz#5ebb5e5a5b75f085f22bc3f8460fba308310fa78" + integrity sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w== unbox-primitive@^1.0.2: version "1.0.2" @@ -6578,6 +6760,15 @@ v8-compile-cache-lib@^3.0.1: resolved "https://registry.yarnpkg.com/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf" integrity sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg== +v8-to-istanbul@^9.0.0: + version "9.2.0" + resolved "https://registry.yarnpkg.com/v8-to-istanbul/-/v8-to-istanbul-9.2.0.tgz#2ed7644a245cddd83d4e087b9b33b3e62dfd10ad" + integrity sha512-/EH/sDgxU2eGxajKdwLCDmQ4FWq+kpi3uCmBGpw1xJtnAxEjlD8j8PEiGWpCIMIs3ciNAgH0d3TTJiUkYzyZjA== + dependencies: + "@jridgewell/trace-mapping" "^0.3.12" + "@types/istanbul-lib-coverage" "^2.0.1" + convert-source-map "^2.0.0" + vary@^1.1.2, vary@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc" From 8214052878d37ad7e8844895736e8572175fa65f Mon Sep 17 00:00:00 2001 From: n4ze3m Date: Wed, 17 Apr 2024 23:48:26 +0530 Subject: [PATCH 02/16] migrated to bullmq --- server/package.json | 3 +- server/src/app.ts | 30 ++ .../handlers/api/v1/bot/bot/api.handler.ts | 354 +++++++++--------- .../handlers/api/v1/bot/bot/post.handler.ts | 6 +- .../handlers/api/v1/bot/bot/upload.handler.ts | 8 +- server/src/plugins/bull.ts | 24 +- server/src/queue/index.ts | 8 +- server/yarn.lock | 52 +++ 8 files changed, 289 insertions(+), 196 deletions(-) diff --git a/server/package.json b/server/package.json index cfcb5cd0..80c9fdbd 100644 --- a/server/package.json +++ b/server/package.json @@ -56,6 +56,7 @@ "axios": "^1.4.0", "bcryptjs": "^2.4.3", "bull": "^4.10.4", + "bullmq": "^5.7.1", "cheerio": "1.0.0-rc.12", "cohere-ai": "^6.2.1", "concurrently": "^7.0.0", @@ -71,7 +72,7 @@ "grammy": "^1.16.2", "html-to-text": "^9.0.5", "ignore": "^5.2.4", - "ioredis": "^5.3.2", + "ioredis": "^5.4.1", "langchain": "^0.1.25", "mammoth": "^1.6.0", "pdf-parse": "^1.1.1", diff --git a/server/src/app.ts b/server/src/app.ts index 24b56490..7bf351fe 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -10,6 +10,8 @@ import fastifySession from "@fastify/session"; import { getSessionSecret, isCookieSecure } from "./utils/session"; import swagger from "@fastify/swagger"; import swaggerUi from "@fastify/swagger-ui"; +import { pathToFileURL } from "url"; +import { Worker } from "bullmq"; declare module "fastify" { interface Session { is_bot_allowed: boolean; @@ -82,5 +84,33 @@ const app: FastifyPluginAsync = async ( }); }; +const redis_url = process.env.DB_REDIS_URL || process.env.REDIS_URL; +if (!redis_url) { + throw new Error("Redis url is not defined"); +} +const username = redis_url.split(":")[1].replace("//", ""); +const password = redis_url.split(":")[2].split("@")[0]; +const host = redis_url.split("@")[1].split(":")[0]; +const port = parseInt(redis_url.split(":")[3]); +const path = join(__dirname, "./queue/index.js"); +const workerUrl = pathToFileURL(path); +const concurrency = parseInt(process.env.DB_QUEUE_CONCURRENCY || "1"); +const workerThreads = process.env.DB_QUEUE_THREADS || "false"; +const worker = new Worker("vector", workerUrl, { + connection: { + host, + port, + password, + username, + }, + concurrency, + useWorkerThreads: workerThreads === "true", +}); + +process.on("SIGINT", async () => { + await worker.close(); + process.exit(); +}); + export default app; export { app, options }; diff --git a/server/src/handlers/api/v1/bot/bot/api.handler.ts b/server/src/handlers/api/v1/bot/bot/api.handler.ts index 91dd842b..a32309b6 100644 --- a/server/src/handlers/api/v1/bot/bot/api.handler.ts +++ b/server/src/handlers/api/v1/bot/bot/api.handler.ts @@ -1,209 +1,217 @@ import { FastifyReply, FastifyRequest } from "fastify"; import { - HELPFUL_ASSISTANT_WITH_CONTEXT_PROMPT, QUESTION_GENERATOR_PROMPT, + HELPFUL_ASSISTANT_WITH_CONTEXT_PROMPT, + QUESTION_GENERATOR_PROMPT, } from "../../../../../utils/prompts"; import { AddNewSourceBulkById, CreateBotAPIRequest } from "./types"; import { getSettings } from "../../../../../utils/common"; -import { apiKeyValidaton, apiKeyValidatonMessage } from "../../../../../utils/validate"; -import { adjectives, animals, colors, uniqueNamesGenerator } from "unique-names-generator"; +import { + apiKeyValidaton, + apiKeyValidatonMessage, +} from "../../../../../utils/validate"; +import { + adjectives, + animals, + colors, + uniqueNamesGenerator, +} from "unique-names-generator"; import { validateDataSource } from "../../../../../utils/datasource-validation"; export const createBotAPIHandler = async ( - request: FastifyRequest, - reply: FastifyReply + request: FastifyRequest, + reply: FastifyReply ) => { - const { - name: nameFromRequest, - embedding, - model, - question_generator_prompt, - system_prompt, - temperature - } = request.body; + const { + name: nameFromRequest, + embedding, + model, + question_generator_prompt, + system_prompt, + temperature, + } = request.body; + + const prisma = request.server.prisma; + + // only non-admin users are affected by this settings + const settings = await getSettings(prisma); + const user = request.user; + const isBotCreatingAllowed = settings?.allowUserToCreateBots; + if (!user.is_admin && !isBotCreatingAllowed) { + return reply.status(400).send({ + message: "Bot creation is disabled by admin", + }); + } - const prisma = request.server.prisma; + const totalBotsUserCreated = await prisma.bot.count({ + where: { + user_id: request.user.user_id, + }, + }); - // only non-admin users are affected by this settings - const settings = await getSettings(prisma); - const user = request.user; - const isBotCreatingAllowed = settings?.allowUserToCreateBots; - if (!user.is_admin && !isBotCreatingAllowed) { - return reply.status(400).send({ - message: "Bot creation is disabled by admin", - }); - } + const maxBotsAllowed = settings?.noOfBotsPerUser || 10; - const totalBotsUserCreated = await prisma.bot.count({ - where: { - user_id: request.user.user_id, - }, + if (!user.is_admin && totalBotsUserCreated >= maxBotsAllowed) { + return reply.status(400).send({ + message: `Reach maximum limit of ${maxBotsAllowed} bots per user`, }); + } + const modelInfo = await prisma.dialoqbaseModels.findFirst({ + where: { + hide: false, + deleted: false, + OR: [ + { + model_id: model, + }, + { + model_id: `${model}-dbase`, + }, + ], + }, + }); - const maxBotsAllowed = settings?.noOfBotsPerUser || 10; + if (!modelInfo) { + return reply.status(400).send({ + message: "Chat Model not found", + }); + } - if (!user.is_admin && totalBotsUserCreated >= maxBotsAllowed) { - return reply.status(400).send({ - message: `Reach maximum limit of ${maxBotsAllowed} bots per user`, - }); - } - const modelInfo = await prisma.dialoqbaseModels.findFirst({ - where: { - hide: false, - deleted: false, - OR: [ - { - model_id: model - }, - { - model_id: `${model}-dbase` - } - ] + const embeddingInfo = await prisma.dialoqbaseModels.findFirst({ + where: { + OR: [ + { + model_id: embedding, }, + { + model_id: `dialoqbase_eb_${embedding}`, + }, + ], + hide: false, + deleted: false, + }, + }); + + if (!embeddingInfo) { + return reply.status(400).send({ + message: "Embedding Model not found", }); + } - if (!modelInfo) { - return reply.status(400).send({ - message: "Chat Model not found", - }); - } + const isEmbeddingsValid = apiKeyValidaton( + `${embeddingInfo.model_provider}`.toLowerCase() + ); - const embeddingInfo = await prisma.dialoqbaseModels.findFirst({ - where: { - OR: [ - { - model_id: embedding, - }, - { - model_id: `dialoqbase_eb_${embedding}` - } - ], - hide: false, - deleted: false, - }, + if (!isEmbeddingsValid) { + return reply.status(400).send({ + message: apiKeyValidatonMessage(embedding), }); + } - if (!embeddingInfo) { - return reply.status(400).send({ - message: "Embedding Model not found", - }); - } + const isAPIKeyAddedForProvider = apiKeyValidaton( + `${modelInfo.model_provider}`.toLowerCase() + ); + + if (!isAPIKeyAddedForProvider) { + return reply.status(400).send({ + message: apiKeyValidatonMessage(modelInfo.model_provider || ""), + }); + } + + const shortName = uniqueNamesGenerator({ + dictionaries: [adjectives, animals, colors], + length: 2, + }); + + const name = nameFromRequest || shortName; + + const isStreamingAvilable = modelInfo.stream_available; + + const bot = await prisma.bot.create({ + data: { + name, + embedding: embeddingInfo.model_id, + model: modelInfo.model_id, + provider: modelInfo.model_provider || "", + streaming: isStreamingAvilable, + user_id: request.user.user_id, + temperature: temperature || 0.7, + qaPrompt: system_prompt || HELPFUL_ASSISTANT_WITH_CONTEXT_PROMPT, + questionGeneratorPrompt: + question_generator_prompt || QUESTION_GENERATOR_PROMPT, + }, + }); + + return { + id: bot.id, + }; +}; - const isEmbeddingsValid = apiKeyValidaton( - `${embeddingInfo.model_provider}`.toLowerCase() - ); +export const addNewSourceByIdBulkHandler = async ( + request: FastifyRequest, + reply: FastifyReply +) => { + try { + const prisma = request.server.prisma; + const id = request.params.id; + + const bot = await prisma.bot.findFirst({ + where: { + id, + user_id: request.user.user_id, + }, + include: { + source: true, + }, + }); - if (!isEmbeddingsValid) { - return reply.status(400).send({ - message: apiKeyValidatonMessage(embedding), - }); + if (!bot) { + return reply.status(404).send({ + message: "Bot not found", + }); } - const isAPIKeyAddedForProvider = apiKeyValidaton( - `${modelInfo.model_provider}`.toLowerCase() - ); + const data = request.body; - if (!isAPIKeyAddedForProvider) { - return reply.status(400).send({ - message: apiKeyValidatonMessage(modelInfo.model_provider || ""), - }); - } + const queueSource: any[] = []; - const shortName = uniqueNamesGenerator({ - dictionaries: [adjectives, animals, colors], - length: 2, - }); + const isOk = validateDataSource(data); - const name = nameFromRequest || shortName; + if (isOk.length > 0) { + return reply.status(400).send({ + message: isOk.join(", "), + }); + } - const isStreamingAvilable = modelInfo.stream_available; + for (const source of data) { + const { content, type } = source; - const bot = await prisma.bot.create({ + const botSource = await prisma.botSource.create({ data: { - name, - embedding: embeddingInfo.model_id, - model: modelInfo.model_id, - provider: modelInfo.model_provider || "", - streaming: isStreamingAvilable, - user_id: request.user.user_id, - temperature: temperature || 0.7, - qaPrompt: system_prompt || HELPFUL_ASSISTANT_WITH_CONTEXT_PROMPT, - questionGeneratorPrompt: question_generator_prompt || QUESTION_GENERATOR_PROMPT, + content, + type, + botId: bot.id, + options: source.options, }, - }); + }); + + queueSource.push({ + ...botSource, + embedding: bot.embedding, + maxDepth: source.maxDepth, + maxLinks: source.maxLinks, + options: source.options, + }); + } + await request.server.queue.add('process', queueSource); return { - id: bot.id, + success: true, + source_ids: queueSource.map((source) => source.id), }; + } catch (error) { + console.log(error); + return reply.status(500).send({ + message: "Internal Server Error", + }); + } }; - - -export const addNewSourceByIdBulkHandler = async ( - request: FastifyRequest, - reply: FastifyReply -) => { - try { - const prisma = request.server.prisma; - const id = request.params.id; - - const bot = await prisma.bot.findFirst({ - where: { - id, - user_id: request.user.user_id, - }, - include: { - source: true, - }, - }); - - if (!bot) { - return reply.status(404).send({ - message: "Bot not found", - }); - } - - const data = request.body; - - const queueSource: any[] = []; - - const isOk = validateDataSource(data); - - if (isOk.length > 0) { - return reply.status(400).send({ - message: isOk.join(", ") - }); - } - - for (const source of data) { - - const { content, type } = source; - - const botSource = await prisma.botSource.create({ - data: { - content, - type, - botId: bot.id, - options: source.options, - }, - }); - - queueSource.push({ - ...botSource, - embedding: bot.embedding, - maxDepth: source.maxDepth, - maxLinks: source.maxLinks, - options: source.options, - }); - } - await request.server.queue.add(queueSource); - - return { - success: true, - source_ids: queueSource.map((source) => source.id), - }; - } catch (error) { - console.log(error); - return reply.status(500).send({ - message: "Internal Server Error", - }); - } -}; \ No newline at end of file diff --git a/server/src/handlers/api/v1/bot/bot/post.handler.ts b/server/src/handlers/api/v1/bot/bot/post.handler.ts index f6bd13f5..5717ea8b 100644 --- a/server/src/handlers/api/v1/bot/bot/post.handler.ts +++ b/server/src/handlers/api/v1/bot/bot/post.handler.ts @@ -133,7 +133,7 @@ export const createBotHandler = async ( }, }); - await request.server.queue.add([ + await request.server.queue.add("process", [ { ...botSource, embedding, @@ -211,7 +211,7 @@ export const addNewSourceByIdHandler = async ( }); } - await request.server.queue.add([ + await request.server.queue.add('process',[ { ...botSource, embedding: bot.embedding, @@ -275,7 +275,7 @@ export const refreshSourceByIdHandler = async ( sourceId: source_id, }, }); - await request.server.queue.add([ + await request.server.queue.add('process',[ { ...botSource, embedding: bot.embedding, diff --git a/server/src/handlers/api/v1/bot/bot/upload.handler.ts b/server/src/handlers/api/v1/bot/bot/upload.handler.ts index 1b97d190..816ed107 100644 --- a/server/src/handlers/api/v1/bot/bot/upload.handler.ts +++ b/server/src/handlers/api/v1/bot/bot/upload.handler.ts @@ -138,7 +138,7 @@ export const createBotFileHandler = async ( }, }); - await request.server.queue.add([ + await request.server.queue.add("process", [ { ...botSource, embedding: bot.embedding, @@ -210,7 +210,7 @@ export const addNewSourceFileByIdHandler = async ( }); } - await request.server.queue.add([ + await request.server.queue.add("process", [ { ...botSource, embedding: bot.embedding, @@ -262,7 +262,6 @@ export const addNewSourceFileByIdBulkHandler = async ( await fs.promises.mkdir("./uploads", { recursive: true }); await pump(file.file, fs.createWriteStream(path)); - const botSource = await prisma.botSource.create({ data: { content: file.filename, @@ -276,10 +275,9 @@ export const addNewSourceFileByIdBulkHandler = async ( ...botSource, embedding: bot.embedding, }); - } - await request.server.queue.add(queueSource); + await request.server.queue.add("process", queueSource); return { source_ids: queueSource.map((source) => source.id), diff --git a/server/src/plugins/bull.ts b/server/src/plugins/bull.ts index 8c2c3631..3966e490 100644 --- a/server/src/plugins/bull.ts +++ b/server/src/plugins/bull.ts @@ -1,12 +1,9 @@ import fp from "fastify-plugin"; import { FastifyPluginAsync } from "fastify"; - -import Queue from "bull"; -import { join } from "path"; - +import { Queue } from "bullmq"; declare module "fastify" { interface FastifyInstance { - queue: Queue.Queue; + queue: Queue; } } @@ -15,12 +12,19 @@ const bullPlugin: FastifyPluginAsync = fp(async (server, options) => { if (!redis_url) { throw new Error("Redis url is not defined"); } - const queue = new Queue("vector", redis_url, {}); + const username = redis_url.split(":")[1].replace("//", ""); + const password = redis_url.split(":")[2].split("@")[0]; + const host = redis_url.split("@")[1].split(":")[0]; + const port = parseInt(redis_url.split(":")[3]); - await queue.isReady(); - const path = join(__dirname, "../queue/index.js"); - const concurrency = parseInt(process.env.DB_QUEUE_CONCURRENCY || "1"); - queue.process(concurrency, path); + const queue = new Queue("vector", { + connection: { + host, + port, + password, + username, + }, + }); server.decorate("queue", queue); diff --git a/server/src/queue/index.ts b/server/src/queue/index.ts index b5f4ff90..0de1f98a 100644 --- a/server/src/queue/index.ts +++ b/server/src/queue/index.ts @@ -1,4 +1,3 @@ -import { DoneCallback, Job } from "bull"; import { PrismaClient } from "@prisma/client"; import { QSource } from "./type"; import { pdfQueueController } from "./controllers/pdf.controller"; @@ -14,10 +13,11 @@ import { videoQueueController } from "./controllers/video.controller"; import { youtubeQueueController } from "./controllers/youtube.controller"; import { restQueueController } from "./controllers/rest.controller"; import { sitemapQueueController } from "./controllers/sitemap.controller"; +import { SandboxedJob } from "bullmq"; const prisma = new PrismaClient(); -export default async function queueHandler(job: Job, done: DoneCallback) { +export default async function queueHandler(job: SandboxedJob) { const data = job.data as QSource[]; await prisma.$connect(); console.log("Processing queue"); @@ -88,7 +88,6 @@ export default async function queueHandler(job: Job, done: DoneCallback) { }, }); - done(); await prisma.$disconnect(); } catch (e) { console.log(e); @@ -102,10 +101,11 @@ export default async function queueHandler(job: Job, done: DoneCallback) { }, }); await prisma.$disconnect(); - done(); } } } catch (e) { console.log(e); } + + return Promise.resolve(); } diff --git a/server/yarn.lock b/server/yarn.lock index 03874cdc..039f02f0 100644 --- a/server/yarn.lock +++ b/server/yarn.lock @@ -1894,6 +1894,19 @@ bull@^4.10.4: semver "^7.3.2" uuid "^8.3.0" +bullmq@^5.7.1: + version "5.7.1" + resolved "https://registry.yarnpkg.com/bullmq/-/bullmq-5.7.1.tgz#18bbfd404fcc1cb007d0b9b255f04a52cdc76128" + integrity sha512-t7FhF2mCGgmjZ1rHuBYIcLwzONm4QFGrO1+9mF7hpjWtXalGfy+nGciVcb69L7aPcdJMR2XTe6bNMWHGbKy8mQ== + dependencies: + cron-parser "^4.6.0" + ioredis "^5.3.2" + msgpackr "^1.10.1" + node-abort-controller "^3.1.1" + semver "^7.5.4" + tslib "^2.0.0" + uuid "^9.0.0" + busboy@^1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/busboy/-/busboy-1.6.0.tgz#966ea36a9502e43cdb9146962523b92f531f6893" @@ -2376,6 +2389,13 @@ cron-parser@^4.2.1: dependencies: luxon "^3.2.1" +cron-parser@^4.6.0: + version "4.9.0" + resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-4.9.0.tgz#0340694af3e46a0894978c6f52a6dbb5c0f11ad5" + integrity sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q== + dependencies: + luxon "^3.2.1" + cross-fetch@^3.1.5: version "3.1.8" resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.8.tgz#0327eba65fd68a7d119f8fb2bf9334a1a7956f82" @@ -3868,6 +3888,21 @@ ioredis@^5.0.0, ioredis@^5.3.2: redis-parser "^3.0.0" standard-as-callback "^2.1.0" +ioredis@^5.4.1: + version "5.4.1" + resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-5.4.1.tgz#1c56b70b759f01465913887375ed809134296f40" + integrity sha512-2YZsvl7jopIa1gaePkeMtd9rAcSjOOjPtpcLlOeusyO+XH2SK5ZcT+UCrElPP+WVIInh2TzeI4XW9ENaSLVVHA== + dependencies: + "@ioredis/commands" "^1.1.1" + cluster-key-slot "^1.1.0" + debug "^4.3.4" + denque "^2.1.0" + lodash.defaults "^4.2.0" + lodash.isarguments "^3.1.0" + redis-errors "^1.2.0" + redis-parser "^3.0.0" + standard-as-callback "^2.1.0" + ipaddr.js@1.9.1: version "1.9.1" resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.9.1.tgz#bff38543eeb8984825079ff3a2a8e6cbd46781b3" @@ -4810,6 +4845,13 @@ msgpackr-extract@^3.0.2: "@msgpackr-extract/msgpackr-extract-linux-x64" "3.0.2" "@msgpackr-extract/msgpackr-extract-win32-x64" "3.0.2" +msgpackr@^1.10.1: + version "1.10.1" + resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.10.1.tgz#51953bb4ce4f3494f0c4af3f484f01cfbb306555" + integrity sha512-r5VRLv9qouXuLiIBrLpl2d5ZvPt8svdQTl5/vMvE4nzDMyEX4sgW5yWhuBBj5UmgwOTWj8CIdSXn5sAfsHAWIQ== + optionalDependencies: + msgpackr-extract "^3.0.2" + msgpackr@^1.5.2: version "1.9.4" resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.9.4.tgz#039ef7c46efb6bc17612a1819ae7386b2861c321" @@ -4844,6 +4886,11 @@ node-abi@^3.3.0: dependencies: semver "^7.3.5" +node-abort-controller@^3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/node-abort-controller/-/node-abort-controller-3.1.1.tgz#a94377e964a9a37ac3976d848cb5c765833b8548" + integrity sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ== + node-addon-api@^6.1.0: version "6.1.0" resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-6.1.0.tgz#ac8470034e58e67d0c6f1204a18ae6995d9c0d76" @@ -6577,6 +6624,11 @@ ts-node@^10.9.1: v8-compile-cache-lib "^3.0.1" yn "3.1.1" +tslib@^2.0.0: + version "2.6.2" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae" + integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== + tslib@^2.1.0, tslib@^2.5.0: version "2.5.3" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.5.3.tgz#24944ba2d990940e6e982c4bea147aba80209913" From c4ca301eb1e421015b1a96f5695cf848f63f390b Mon Sep 17 00:00:00 2001 From: n4ze3m Date: Thu, 18 Apr 2024 00:44:25 +0530 Subject: [PATCH 03/16] Update package versions and fix error handling in data source deletion --- app/ui/package.json | 2 +- app/ui/src/components/Bot/DS/DsTable.tsx | 17 +++-- package.json | 2 +- .../handlers/api/v1/bot/bot/api.handler.ts | 14 +++- .../handlers/api/v1/bot/bot/delete.handler.ts | 19 +++-- .../handlers/api/v1/bot/bot/post.handler.ts | 75 +++++++++++++------ .../handlers/api/v1/bot/bot/upload.handler.ts | 49 +++++++++--- 7 files changed, 131 insertions(+), 47 deletions(-) diff --git a/app/ui/package.json b/app/ui/package.json index 054d0c11..3ad61cd2 100644 --- a/app/ui/package.json +++ b/app/ui/package.json @@ -1,7 +1,7 @@ { "name": "app", "private": true, - "version": "1.7.9", + "version": "1.8.0", "type": "module", "scripts": { "dev": "vite", diff --git a/app/ui/src/components/Bot/DS/DsTable.tsx b/app/ui/src/components/Bot/DS/DsTable.tsx index 36c61440..7b17193d 100644 --- a/app/ui/src/components/Bot/DS/DsTable.tsx +++ b/app/ui/src/components/Bot/DS/DsTable.tsx @@ -79,9 +79,12 @@ export const DsTable = ({ message: "Data source deleted successfully", }); }, - onError: () => { + onError: (err: any) => { notification.error({ - message: "Error while deleting data source", + message: + err?.response?.data?.message || + err?.response?.data?.error || + "Error while deleting data source", }); }, } @@ -96,9 +99,12 @@ export const DsTable = ({ message: "Data source updated successfully", }); }, - onError: () => { + onError: (err: any) => { notification.error({ - message: "Error while updating data source", + message: + err?.response?.data?.message || + err?.response?.data?.error || + "Error while deleting data source", }); }, } @@ -109,7 +115,8 @@ export const DsTable = ({

- Data Sources

+ Data Sources +

List of data sources that are currently being used by your bot.

diff --git a/package.json b/package.json index d7804c00..7a4a121c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dialoqbase", - "version": "1.7.9", + "version": "1.8.0", "description": "Create chatbots with ease", "scripts": { "ui:dev": "pnpm run --filter ui dev", diff --git a/server/src/handlers/api/v1/bot/bot/api.handler.ts b/server/src/handlers/api/v1/bot/bot/api.handler.ts index a32309b6..eb9dbb5a 100644 --- a/server/src/handlers/api/v1/bot/bot/api.handler.ts +++ b/server/src/handlers/api/v1/bot/bot/api.handler.ts @@ -200,9 +200,21 @@ export const addNewSourceByIdBulkHandler = async ( maxDepth: source.maxDepth, maxLinks: source.maxLinks, options: source.options, + id: botSource.id, }); } - await request.server.queue.add('process', queueSource); + + await request.server.queue.addBulk( + queueSource.map((source) => ({ + data: [source], + name: "process", + opts: { + jobId: source.id, + removeOnComplete: true, + removeOnFail: true, + }, + })) + ); return { success: true, diff --git a/server/src/handlers/api/v1/bot/bot/delete.handler.ts b/server/src/handlers/api/v1/bot/bot/delete.handler.ts index f1f01c26..99396f9b 100644 --- a/server/src/handlers/api/v1/bot/bot/delete.handler.ts +++ b/server/src/handlers/api/v1/bot/bot/delete.handler.ts @@ -5,7 +5,7 @@ import TelegramBot from "../../../../../integration/telegram"; export const deleteSourceByIdHandler = async ( request: FastifyRequest, - reply: FastifyReply, + reply: FastifyReply ) => { const prisma = request.server.prisma; const bot_id = request.params.id; @@ -38,9 +38,16 @@ export const deleteSourceByIdHandler = async ( } if (botSource.isPending) { - return reply.status(400).send({ - message: "Source is in progress", - }); + const job = await request.server.queue.getJob(botSource.id); + if (job) { + const isActive = await job.isActive(); + if (isActive) { + return reply.status(400).send({ + message: "Source is being processed, cannot delete it now", + }); + } + await job.remove(); + } } await prisma.botDocument.deleteMany({ @@ -63,7 +70,7 @@ export const deleteSourceByIdHandler = async ( export const deleteBotByIdHandler = async ( request: FastifyRequest, - reply: FastifyReply, + reply: FastifyReply ) => { const prisma = request.server.prisma; const id = request.params.id; @@ -130,7 +137,7 @@ export const deleteBotByIdHandler = async ( await prisma.botPlayground.deleteMany({ where: { botId: bot.id, - } + }, }); await prisma.bot.delete({ diff --git a/server/src/handlers/api/v1/bot/bot/post.handler.ts b/server/src/handlers/api/v1/bot/bot/post.handler.ts index 5717ea8b..238b7133 100644 --- a/server/src/handlers/api/v1/bot/bot/post.handler.ts +++ b/server/src/handlers/api/v1/bot/bot/post.handler.ts @@ -133,15 +133,23 @@ export const createBotHandler = async ( }, }); - await request.server.queue.add("process", [ + await request.server.queue.add( + "process", + [ + { + ...botSource, + embedding, + maxDepth: request.body.maxDepth, + maxLinks: request.body.maxLinks, + options: request.body.options, + }, + ], { - ...botSource, - embedding, - maxDepth: request.body.maxDepth, - maxLinks: request.body.maxLinks, - options: request.body.options, - }, - ]); + jobId: botSource.id, + removeOnComplete: true, + removeOnFail: true, + } + ); return { id: bot.id, }; @@ -211,15 +219,21 @@ export const addNewSourceByIdHandler = async ( }); } - await request.server.queue.add('process',[ + await request.server.queue.add( + "process", + [ + { + ...botSource, + embedding: bot.embedding, + maxDepth: request.body.maxDepth, + maxLinks: request.body.maxLinks, + options: request.body.options, + }, + ], { - ...botSource, - embedding: bot.embedding, - maxDepth: request.body.maxDepth, - maxLinks: request.body.maxLinks, - options: request.body.options, - }, - ]); + jobId: botSource.id, + } + ); return { id: bot.id, }; @@ -258,6 +272,16 @@ export const refreshSourceByIdHandler = async ( message: "Source not found", }); } + // check if the source is already in pending state + + const isInJob = await request.server.queue.getJob(botSource.id); + if (isInJob) { + if (isInJob.isWaiting() || isInJob.isActive()) { + return reply.status(400).send({ + message: "Source is already in pending state", + }); + } + } await prisma.botSource.update({ where: { @@ -275,13 +299,20 @@ export const refreshSourceByIdHandler = async ( sourceId: source_id, }, }); - await request.server.queue.add('process',[ + await request.server.queue.add( + "process", + [ + { + ...botSource, + embedding: bot.embedding, + }, + ], { - ...botSource, - embedding: bot.embedding, - }, - ]); - + jobId: botSource.id, + removeOnComplete: true, + removeOnFail: true, + } + ); return { id: bot.id, }; diff --git a/server/src/handlers/api/v1/bot/bot/upload.handler.ts b/server/src/handlers/api/v1/bot/bot/upload.handler.ts index 816ed107..cf0f1b9a 100644 --- a/server/src/handlers/api/v1/bot/bot/upload.handler.ts +++ b/server/src/handlers/api/v1/bot/bot/upload.handler.ts @@ -138,12 +138,20 @@ export const createBotFileHandler = async ( }, }); - await request.server.queue.add("process", [ + await request.server.queue.add( + "process", + [ + { + ...botSource, + embedding: bot.embedding, + }, + ], { - ...botSource, - embedding: bot.embedding, - }, - ]); + jobId: botSource.id, + removeOnComplete: true, + removeOnFail: true, + } + ); } return reply.status(200).send({ @@ -210,12 +218,20 @@ export const addNewSourceFileByIdHandler = async ( }); } - await request.server.queue.add("process", [ + await request.server.queue.add( + "process", + [ + { + ...botSource, + embedding: bot.embedding, + }, + ], { - ...botSource, - embedding: bot.embedding, - }, - ]); + jobId: botSource.id, + removeOnComplete: true, + removeOnFail: true, + } + ); } return { @@ -274,10 +290,21 @@ export const addNewSourceFileByIdBulkHandler = async ( queueSource.push({ ...botSource, embedding: bot.embedding, + id: botSource.id, }); } - await request.server.queue.add("process", queueSource); + await request.server.queue.addBulk( + queueSource.map((source) => ({ + data: [source], + name: "process", + opts: { + jobId: source.id, + removeOnComplete: true, + removeOnFail: true, + }, + })) + ); return { source_ids: queueSource.map((source) => source.id), From a41ebb7f095ff11aa04486570469e8f2b9ac2048 Mon Sep 17 00:00:00 2001 From: n4ze3m Date: Sun, 21 Apr 2024 23:23:39 +0530 Subject: [PATCH 04/16] Add new migration files for q_12_1, q_25, and q_12_2 --- server/prisma/migrations/q_12_1/migration.sql | 3 + server/prisma/migrations/q_12_2/migration.sql | 7 + server/prisma/migrations/q_25/migration.sql | 10 + server/prisma/schema.prisma | 26 +- server/prisma/seed.ts | 789 +++++++++--------- 5 files changed, 434 insertions(+), 401 deletions(-) create mode 100644 server/prisma/migrations/q_12_1/migration.sql create mode 100644 server/prisma/migrations/q_12_2/migration.sql create mode 100644 server/prisma/migrations/q_25/migration.sql diff --git a/server/prisma/migrations/q_12_1/migration.sql b/server/prisma/migrations/q_12_1/migration.sql new file mode 100644 index 00000000..b4dfe730 --- /dev/null +++ b/server/prisma/migrations/q_12_1/migration.sql @@ -0,0 +1,3 @@ +-- AlterTable +ALTER TABLE "DialoqbaseSettings" ADD COLUMN "hidePlayground" BOOLEAN NOT NULL DEFAULT false, +ADD COLUMN "legacyMode" BOOLEAN NOT NULL DEFAULT false; diff --git a/server/prisma/migrations/q_12_2/migration.sql b/server/prisma/migrations/q_12_2/migration.sql new file mode 100644 index 00000000..c64fe2f8 --- /dev/null +++ b/server/prisma/migrations/q_12_2/migration.sql @@ -0,0 +1,7 @@ +-- AlterTable +ALTER TABLE "DialoqbaseSettings" ADD COLUMN "defaultChatModel" TEXT NOT NULL DEFAULT 'gpt-3.5-turbo-dbase', +ADD COLUMN "defaultChunkOverlap" INTEGER DEFAULT 200, +ADD COLUMN "defaultChunkSize" INTEGER DEFAULT 1000, +ADD COLUMN "defaultEmbeddingModel" TEXT NOT NULL DEFAULT 'dialoqbase_eb_text-embedding-ada-002', +ADD COLUMN "dynamicallyFetchOllamaModels" BOOLEAN DEFAULT false, +ADD COLUMN "hideDefaultModels" BOOLEAN DEFAULT false; diff --git a/server/prisma/migrations/q_25/migration.sql b/server/prisma/migrations/q_25/migration.sql new file mode 100644 index 00000000..b39b0f66 --- /dev/null +++ b/server/prisma/migrations/q_25/migration.sql @@ -0,0 +1,10 @@ +-- CreateTable +CREATE TABLE "DbSettings" ( + "id" SERIAL NOT NULL, + "key" TEXT NOT NULL, + "value" TEXT NOT NULL, + "family" TEXT NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "DbSettings_pkey" PRIMARY KEY ("id") +); diff --git a/server/prisma/schema.prisma b/server/prisma/schema.prisma index 292bbb72..cf484875 100644 --- a/server/prisma/schema.prisma +++ b/server/prisma/schema.prisma @@ -82,11 +82,19 @@ model User { } model DialoqbaseSettings { - id Int @id @default(autoincrement()) - noOfBotsPerUser Int @default(10) - allowUserToCreateBots Boolean @default(true) - allowUserToRegister Boolean @default(false) - numberOfDocuments Int @default(10) + id Int @id @default(autoincrement()) + noOfBotsPerUser Int @default(10) + allowUserToCreateBots Boolean @default(true) + allowUserToRegister Boolean @default(false) + numberOfDocuments Int @default(10) + legacyMode Boolean @default(false) + hidePlayground Boolean @default(false) + defaultChunkSize Int? @default(1000) + defaultChunkOverlap Int? @default(200) + dynamicallyFetchOllamaModels Boolean? @default(false) + hideDefaultModels Boolean? @default(false) + defaultChatModel String @default("gpt-3.5-turbo-dbase") + defaultEmbeddingModel String @default("dialoqbase_eb_text-embedding-ada-002") } model BotIntegration { @@ -218,3 +226,11 @@ model UserApiKey { User User @relation(fields: [user_id], references: [user_id]) createdAt DateTime @default(now()) } + +model DbSettings { + id Int @id @default(autoincrement()) + key String + value String + family String + createdAt DateTime @default(now()) +} diff --git a/server/prisma/seed.ts b/server/prisma/seed.ts index a9a39087..16654e30 100644 --- a/server/prisma/seed.ts +++ b/server/prisma/seed.ts @@ -10,322 +10,322 @@ const LLMS: { local_model?: boolean; config?: string; }[] = [ - { - name: 'GPT-3.5 Turbo (OpenAI)', - model_id: 'gpt-3.5-turbo-dbase', - model_type: 'chat', - model_provider: 'OpenAI', - stream_available: true, - local_model: false, - config: '{}' - }, - { - name: 'GPT-3.5 Turbo 16K (OpenAI)', - model_id: 'gpt-3.5-turbo-16k-dbase', - model_type: 'chat', - model_provider: 'OpenAI', - stream_available: true, - local_model: false, - config: '{}' - }, - { - name: 'GPT-4 (OpenAI)', - model_id: 'gpt-4-dbase', - model_type: 'chat', - model_provider: 'OpenAI', - stream_available: true, - local_model: false, - config: '{}' - }, - { - name: 'GPT-4 0613 (OpenAI)', - model_id: 'gpt-4-0613-dbase', - model_type: 'chat', - model_provider: 'OpenAI', - stream_available: true, - local_model: false, - config: '{}' - }, - { - name: 'GPT-3.5 Turbo Instruct (OpenAI)', - model_id: 'gpt-3.5-turbo-instruct-dbase', - model_type: 'instruct', - model_provider: 'openai-instruct', - stream_available: true, - local_model: false, - config: '{}' - }, - { - name: 'Claude 1 (Anthropic)', - model_id: 'claude-1-dbase', - model_type: 'chat', - model_provider: 'Anthropic', - stream_available: true, - local_model: false, - config: '{}' - }, - { - name: 'Claude 2 (Anthropic)', - model_id: 'claude-2-dbase', - model_type: 'chat', - model_provider: 'Anthropic', - stream_available: true, - local_model: false, - config: '{}' - }, - { - name: 'Claude Instant (Anthropic)', - model_id: 'claude-instant-1-dbase', - model_type: 'chat', - model_provider: 'Anthropic', - stream_available: true, - local_model: false, - config: '{}' - }, - { - name: 'Google chat-bison-001', - model_id: 'google-bison-dbase', - model_type: 'chat', - model_provider: 'Google', - stream_available: false, - local_model: false, - config: '{}' - }, - { - name: 'Llama v2 7B (Fireworks)', - model_id: 'accounts/fireworks/models/llama-v2-7b-chat', - model_type: 'chat', - model_provider: 'Fireworks', - stream_available: true, - local_model: false, - config: '{}' - }, - { - name: 'Llama v2 13B (Fireworks)', - model_id: 'accounts/fireworks/models/llama-v2-13b-chat', - model_type: 'chat', - model_provider: 'Fireworks', - stream_available: true, - local_model: false, - config: '{}' - }, - { - name: 'Llama v2 70B (Fireworks)', - model_id: 'accounts/fireworks/models/llama-v2-70b-chat', - model_type: 'chat', - model_provider: 'Fireworks', - stream_available: true, - local_model: false, - config: '{}' - }, - { - name: 'Llama v2 7B Chat int8 (Fireworks)', - model_id: 'accounts/fireworks/models/llama-v2-7b-chat-w8a16', - model_type: 'chat', - model_provider: 'Fireworks', - stream_available: true, - local_model: false, - config: '{}' - }, - { - name: 'Llama v2 13B Chat int8 (Fireworks)', - model_id: 'accounts/fireworks/models/llama-v2-13b-chat-w8a16', - model_type: 'chat', - model_provider: 'Fireworks', - stream_available: true, - local_model: false, - config: '{}' - }, - { - name: 'Llama v2 13B Code Instruct (Fireworks)', - model_id: 'accounts/fireworks/models/llama-v2-13b-code-instruct', - model_type: 'instruct', - model_provider: 'Fireworks', - stream_available: true, - local_model: false, - config: '{}' - }, - { - name: 'Llama v2 34B Code Instruct int8 (Fireworks)', - model_id: 'accounts/fireworks/models/llama-v2-34b-code-instruct-w8a16', - model_type: 'instruct', - model_provider: 'Fireworks', - stream_available: true, - local_model: false, - config: '{}' - }, - { - name: 'Mistral 7B Instruct 4K (Fireworks)', - model_id: 'accounts/fireworks/models/mistral-7b-instruct-4k', - model_type: 'instruct', - model_provider: 'Fireworks', - stream_available: true, - local_model: false, - config: '{}' - }, - { - name: 'GPT-3.5 Turbo 1106 (OpenAI)', - model_id: 'gpt-3.5-turbo-1106-dbase', - model_type: 'chat', - model_provider: 'OpenAI', - stream_available: true, - local_model: false, - config: '{}' - }, - { - name: 'GPT-4 Turbo (OpenAI)', - model_id: 'gpt-4-1106-preview-dbase', - model_type: 'chat', - model_provider: 'OpenAI', - stream_available: true, - local_model: false, - config: '{}' - }, - { - name: 'GPT-4 Turbo with vision (OpenAI)', - model_id: 'gpt-4-vision-preview-dbase', - model_type: 'chat', - model_provider: 'OpenAI', - stream_available: true, - local_model: false, - config: '{}' - }, - { - name: "Claude 2.1 (Anthropic)", - model_id: "claude-2.1-dbase", - model_type: "chat", - model_provider: "Anthropic", - stream_available: true, - local_model: false, - config: "{}", - }, - { - name: "Claude Instant 1.2 (Anthropic)", - model_id: "claude-instant-1.2-dbase", - model_type: "chat", - model_provider: "Anthropic", - stream_available: true, - local_model: false, - config: "{}", - }, - { - name: "Yi 34B 200k (Fireworks)", - model_id: "accounts/fireworks/models/yi-34b-200k", - model_type: "instruct", - stream_available: true, - model_provider: "Fireworks", - local_model: false, - config: "{}", - }, - { - model_id: "accounts/fireworks/models/yi-34b-200k-capybara", - name: "Capybara 34B (Fireworks)", - model_type: "chat", - stream_available: true, - model_provider: "Fireworks", - local_model: false, - config: "{}", - }, - { - model_id: "accounts/fireworks/models/zephyr-7b-beta", - name: "Zephyr 7B Beta (Fireworks)", - model_type: "instruct", - stream_available: true, - model_provider: "Fireworks", - local_model: false, - config: "{}", - }, - { - model_id: "accounts/fireworks/models/mixtral-8x7b-instruct", - name: "Mixtral MoE 8x7B Instruct (Fireworks)", - model_type: "instruct", - stream_available: true, - model_provider: "Fireworks", - local_model: false, - config: "{}", - }, - { - model_id: "gemini-pro", - name: "Gemini Pro (Google)", - model_type: "chat", - stream_available: true, - model_provider: "Google", - local_model: false, - config: "{}", - }, - { - model_id: "accounts/fireworks/models/qwen-72b-chat", - name: "Qwen 72b chat (Fireworks)", - model_type: "chat", - stream_available: true, - model_provider: "Fireworks", - local_model: false, - config: "{}", - }, - { - model_id: "gpt-3.5-turbo-0125-dbase", - name: "GPT-3.5 Turbo 0125 (OpenAI)", - model_type: "chat", - stream_available: true, - model_provider: "OpenAI", - local_model: false, - config: "{}", - }, - { - name: "Claude 3 Opus 20240229 (Anthropic)", - model_id: "claude-3-opus-20240229-dbase", - model_type: "chat", - model_provider: "Anthropic", - stream_available: true, - local_model: false, - config: "{}", - }, - { - name: "Claude 3 Sonnet 20240229 (Anthropic)", - model_id: "claude-3-sonnet-20240229-dbase", - model_type: "chat", - model_provider: "Anthropic", - stream_available: true, - local_model: false, - config: "{}", - }, - { - name: "LLaMA2-70b (Groq)", - model_id: "llama2-70b-4096-dbase", - model_type: "chat", - model_provider: "Groq", - stream_available: true, - local_model: false, - config: "{}", - }, - { - name: "Mixtral-8x7b (Groq)", - model_id: "mixtral-8x7b-32768-dbase", - model_type: "chat", - model_provider: "Groq", - stream_available: true, - local_model: false, - config: "{}", - }, - { - name: "Gemma-7b-it (Groq)", - model_id: "gemma-7b-it-dbase", - model_type: "chat", - model_provider: "Groq", - stream_available: true, - local_model: false, - config: "{}", - }, - { - model_id: "claude-3-haiku-20240307-dbase", - name: "Claude 3 Haiku 20240307 (Anthropic)", - model_type: "chat", - stream_available: true, - model_provider: "Anthropic", - local_model: false, - config: "{}", - } - ]; + { + name: "GPT-3.5 Turbo (OpenAI)", + model_id: "gpt-3.5-turbo-dbase", + model_type: "chat", + model_provider: "OpenAI", + stream_available: true, + local_model: false, + config: "{}", + }, + { + name: "GPT-3.5 Turbo 16K (OpenAI)", + model_id: "gpt-3.5-turbo-16k-dbase", + model_type: "chat", + model_provider: "OpenAI", + stream_available: true, + local_model: false, + config: "{}", + }, + { + name: "GPT-4 (OpenAI)", + model_id: "gpt-4-dbase", + model_type: "chat", + model_provider: "OpenAI", + stream_available: true, + local_model: false, + config: "{}", + }, + { + name: "GPT-4 0613 (OpenAI)", + model_id: "gpt-4-0613-dbase", + model_type: "chat", + model_provider: "OpenAI", + stream_available: true, + local_model: false, + config: "{}", + }, + { + name: "GPT-3.5 Turbo Instruct (OpenAI)", + model_id: "gpt-3.5-turbo-instruct-dbase", + model_type: "instruct", + model_provider: "openai-instruct", + stream_available: true, + local_model: false, + config: "{}", + }, + { + name: "Claude 1 (Anthropic)", + model_id: "claude-1-dbase", + model_type: "chat", + model_provider: "Anthropic", + stream_available: true, + local_model: false, + config: "{}", + }, + { + name: "Claude 2 (Anthropic)", + model_id: "claude-2-dbase", + model_type: "chat", + model_provider: "Anthropic", + stream_available: true, + local_model: false, + config: "{}", + }, + { + name: "Claude Instant (Anthropic)", + model_id: "claude-instant-1-dbase", + model_type: "chat", + model_provider: "Anthropic", + stream_available: true, + local_model: false, + config: "{}", + }, + { + name: "Google chat-bison-001", + model_id: "google-bison-dbase", + model_type: "chat", + model_provider: "Google", + stream_available: false, + local_model: false, + config: "{}", + }, + { + name: "Llama v2 7B (Fireworks)", + model_id: "accounts/fireworks/models/llama-v2-7b-chat", + model_type: "chat", + model_provider: "Fireworks", + stream_available: true, + local_model: false, + config: "{}", + }, + { + name: "Llama v2 13B (Fireworks)", + model_id: "accounts/fireworks/models/llama-v2-13b-chat", + model_type: "chat", + model_provider: "Fireworks", + stream_available: true, + local_model: false, + config: "{}", + }, + { + name: "Llama v2 70B (Fireworks)", + model_id: "accounts/fireworks/models/llama-v2-70b-chat", + model_type: "chat", + model_provider: "Fireworks", + stream_available: true, + local_model: false, + config: "{}", + }, + { + name: "Llama v2 7B Chat int8 (Fireworks)", + model_id: "accounts/fireworks/models/llama-v2-7b-chat-w8a16", + model_type: "chat", + model_provider: "Fireworks", + stream_available: true, + local_model: false, + config: "{}", + }, + { + name: "Llama v2 13B Chat int8 (Fireworks)", + model_id: "accounts/fireworks/models/llama-v2-13b-chat-w8a16", + model_type: "chat", + model_provider: "Fireworks", + stream_available: true, + local_model: false, + config: "{}", + }, + { + name: "Llama v2 13B Code Instruct (Fireworks)", + model_id: "accounts/fireworks/models/llama-v2-13b-code-instruct", + model_type: "instruct", + model_provider: "Fireworks", + stream_available: true, + local_model: false, + config: "{}", + }, + { + name: "Llama v2 34B Code Instruct int8 (Fireworks)", + model_id: "accounts/fireworks/models/llama-v2-34b-code-instruct-w8a16", + model_type: "instruct", + model_provider: "Fireworks", + stream_available: true, + local_model: false, + config: "{}", + }, + { + name: "Mistral 7B Instruct 4K (Fireworks)", + model_id: "accounts/fireworks/models/mistral-7b-instruct-4k", + model_type: "instruct", + model_provider: "Fireworks", + stream_available: true, + local_model: false, + config: "{}", + }, + { + name: "GPT-3.5 Turbo 1106 (OpenAI)", + model_id: "gpt-3.5-turbo-1106-dbase", + model_type: "chat", + model_provider: "OpenAI", + stream_available: true, + local_model: false, + config: "{}", + }, + { + name: "GPT-4 Turbo (OpenAI)", + model_id: "gpt-4-1106-preview-dbase", + model_type: "chat", + model_provider: "OpenAI", + stream_available: true, + local_model: false, + config: "{}", + }, + { + name: "GPT-4 Turbo with vision (OpenAI)", + model_id: "gpt-4-vision-preview-dbase", + model_type: "chat", + model_provider: "OpenAI", + stream_available: true, + local_model: false, + config: "{}", + }, + { + name: "Claude 2.1 (Anthropic)", + model_id: "claude-2.1-dbase", + model_type: "chat", + model_provider: "Anthropic", + stream_available: true, + local_model: false, + config: "{}", + }, + { + name: "Claude Instant 1.2 (Anthropic)", + model_id: "claude-instant-1.2-dbase", + model_type: "chat", + model_provider: "Anthropic", + stream_available: true, + local_model: false, + config: "{}", + }, + { + name: "Yi 34B 200k (Fireworks)", + model_id: "accounts/fireworks/models/yi-34b-200k", + model_type: "instruct", + stream_available: true, + model_provider: "Fireworks", + local_model: false, + config: "{}", + }, + { + model_id: "accounts/fireworks/models/yi-34b-200k-capybara", + name: "Capybara 34B (Fireworks)", + model_type: "chat", + stream_available: true, + model_provider: "Fireworks", + local_model: false, + config: "{}", + }, + { + model_id: "accounts/fireworks/models/zephyr-7b-beta", + name: "Zephyr 7B Beta (Fireworks)", + model_type: "instruct", + stream_available: true, + model_provider: "Fireworks", + local_model: false, + config: "{}", + }, + { + model_id: "accounts/fireworks/models/mixtral-8x7b-instruct", + name: "Mixtral MoE 8x7B Instruct (Fireworks)", + model_type: "instruct", + stream_available: true, + model_provider: "Fireworks", + local_model: false, + config: "{}", + }, + { + model_id: "gemini-pro", + name: "Gemini Pro (Google)", + model_type: "chat", + stream_available: true, + model_provider: "Google", + local_model: false, + config: "{}", + }, + { + model_id: "accounts/fireworks/models/qwen-72b-chat", + name: "Qwen 72b chat (Fireworks)", + model_type: "chat", + stream_available: true, + model_provider: "Fireworks", + local_model: false, + config: "{}", + }, + { + model_id: "gpt-3.5-turbo-0125-dbase", + name: "GPT-3.5 Turbo 0125 (OpenAI)", + model_type: "chat", + stream_available: true, + model_provider: "OpenAI", + local_model: false, + config: "{}", + }, + { + name: "Claude 3 Opus 20240229 (Anthropic)", + model_id: "claude-3-opus-20240229-dbase", + model_type: "chat", + model_provider: "Anthropic", + stream_available: true, + local_model: false, + config: "{}", + }, + { + name: "Claude 3 Sonnet 20240229 (Anthropic)", + model_id: "claude-3-sonnet-20240229-dbase", + model_type: "chat", + model_provider: "Anthropic", + stream_available: true, + local_model: false, + config: "{}", + }, + { + name: "LLaMA2-70b (Groq)", + model_id: "llama2-70b-4096-dbase", + model_type: "chat", + model_provider: "Groq", + stream_available: true, + local_model: false, + config: "{}", + }, + { + name: "Mixtral-8x7b (Groq)", + model_id: "mixtral-8x7b-32768-dbase", + model_type: "chat", + model_provider: "Groq", + stream_available: true, + local_model: false, + config: "{}", + }, + { + name: "Gemma-7b-it (Groq)", + model_id: "gemma-7b-it-dbase", + model_type: "chat", + model_provider: "Groq", + stream_available: true, + local_model: false, + config: "{}", + }, + { + model_id: "claude-3-haiku-20240307-dbase", + name: "Claude 3 Haiku 20240307 (Anthropic)", + model_type: "chat", + stream_available: true, + model_provider: "Anthropic", + local_model: false, + config: "{}", + }, +]; const EMBEDDING_MODELS: { name: string; @@ -336,73 +336,73 @@ const EMBEDDING_MODELS: { local_model?: boolean; config?: string; }[] = [ - { - model_id: "dialoqbase_eb_text-embedding-ada-002", - name: "text-embedding-ada-002", - model_provider: "OpenAI", - model_type: "embedding", - }, - { - model_id: "dialoqbase_eb_small", - name: "Cohere", - model_provider: "Cohere", - model_type: "embedding", - }, - { - model_id: "dialoqbase_eb_Xenova/all-MiniLM-L6-v2", - name: "all-MiniLM-L6-v2 (cpu)", - model_type: "embedding", - model_provider: "Transformer", - }, - { - model_id: "dialoqbase_eb_dialoqbase-ollama", - name: "Ollama Embeddings", - model_type: "embedding", - model_provider: "Ollama", - }, - { - model_id: "dialoqbase_eb_models/embedding-gecko-001", - name: "Google text-gecko-001", - model_type: "embedding", - model_provider: "Google PaLM", - }, - { - model_id: "dialoqbase_eb_jina-embeddings-v2-base-en", - name: "jina-embeddings-v2-base-en (API)", - model_type: "embedding", - model_provider: "Jina", - }, - { - model_id: "dialoqbase_eb_Xenova/jina-embeddings-v2-small-en", - name: "jina-embeddings-v2-small-en (cpu)", - model_type: "embedding", - model_provider: "Transformer", - }, - { - model_id: "dialoqbase_eb_embedding-001", - name: "Google embedding-001", - model_type: "embedding", - model_provider: "Google", - }, - { - model_id: "dialoqbase_eb_text-embedding-3-small", - name: "text-embedding-3-small (OpenAI)", - model_type: "embedding", - model_provider: "OpenAI", - }, - { - model_id: "dialoqbase_eb_text-embedding-3-large", - name: "text-embedding-3-large (OpenAI)", - model_type: "embedding", - model_provider: "OpenAI", - }, - { - model_provider: "Fireworks", - model_type: "embedding", - model_id: "dialoqbase_eb_nomic-ai/nomic-embed-text-v1.5", - name: "nomic-ai/nomic-embed-text-v1.5 (Fireworks)", - } - ]; + { + model_id: "dialoqbase_eb_text-embedding-ada-002", + name: "text-embedding-ada-002", + model_provider: "OpenAI", + model_type: "embedding", + }, + { + model_id: "dialoqbase_eb_small", + name: "Cohere", + model_provider: "Cohere", + model_type: "embedding", + }, + { + model_id: "dialoqbase_eb_Xenova/all-MiniLM-L6-v2", + name: "all-MiniLM-L6-v2 (cpu)", + model_type: "embedding", + model_provider: "Transformer", + }, + { + model_id: "dialoqbase_eb_dialoqbase-ollama", + name: "Ollama Embeddings", + model_type: "embedding", + model_provider: "Ollama", + }, + { + model_id: "dialoqbase_eb_models/embedding-gecko-001", + name: "Google text-gecko-001", + model_type: "embedding", + model_provider: "Google PaLM", + }, + { + model_id: "dialoqbase_eb_jina-embeddings-v2-base-en", + name: "jina-embeddings-v2-base-en (API)", + model_type: "embedding", + model_provider: "Jina", + }, + { + model_id: "dialoqbase_eb_Xenova/jina-embeddings-v2-small-en", + name: "jina-embeddings-v2-small-en (cpu)", + model_type: "embedding", + model_provider: "Transformer", + }, + { + model_id: "dialoqbase_eb_embedding-001", + name: "Google embedding-001", + model_type: "embedding", + model_provider: "Google", + }, + { + model_id: "dialoqbase_eb_text-embedding-3-small", + name: "text-embedding-3-small (OpenAI)", + model_type: "embedding", + model_provider: "OpenAI", + }, + { + model_id: "dialoqbase_eb_text-embedding-3-large", + name: "text-embedding-3-large (OpenAI)", + model_type: "embedding", + model_provider: "OpenAI", + }, + { + model_provider: "Fireworks", + model_type: "embedding", + model_id: "dialoqbase_eb_nomic-ai/nomic-embed-text-v1.5", + name: "nomic-ai/nomic-embed-text-v1.5 (Fireworks)", + }, +]; const newModels = async () => { console.log("Seeding new models..."); @@ -446,7 +446,7 @@ const replaceOldEmbeddings = async () => { data: { embedding: "dialoqbase_eb_text-embedding-ada-002", }, - }) + }); await prisma.bot.updateMany({ where: { @@ -455,8 +455,7 @@ const replaceOldEmbeddings = async () => { data: { embedding: "dialoqbase_eb_small", }, - }) - + }); await prisma.bot.updateMany({ where: { @@ -465,7 +464,7 @@ const replaceOldEmbeddings = async () => { data: { embedding: "dialoqbase_eb_Xenova/all-MiniLM-L6-v2", }, - }) + }); await prisma.bot.updateMany({ where: { @@ -474,7 +473,7 @@ const replaceOldEmbeddings = async () => { data: { embedding: "dialoqbase_eb_models/embedding-gecko-001", }, - }) + }); await prisma.bot.updateMany({ where: { @@ -483,7 +482,7 @@ const replaceOldEmbeddings = async () => { data: { embedding: "dialoqbase_eb_jina-embeddings-v2-base-en", }, - }) + }); await prisma.bot.updateMany({ where: { @@ -492,7 +491,7 @@ const replaceOldEmbeddings = async () => { data: { embedding: "dialoqbase_eb_Xenova/jina-embeddings-v2-small-en", }, - }) + }); await prisma.bot.updateMany({ where: { @@ -501,10 +500,8 @@ const replaceOldEmbeddings = async () => { data: { embedding: "dialoqbase_eb_embedding-001", }, - }) - -} - + }); +}; const updateGeminiStreamingToTrue = async () => { await prisma.dialoqbaseModels.update({ @@ -515,13 +512,13 @@ const updateGeminiStreamingToTrue = async () => { stream_available: true, }, }); -} +}; const main = async () => { await newModels(); await removeTensorflowSupport(); await replaceOldEmbeddings(); - await updateGeminiStreamingToTrue() + await updateGeminiStreamingToTrue(); }; main() From ef5b7d9b7e6860ccfc5a7a6438c3923308803ecd Mon Sep 17 00:00:00 2001 From: n4ze3m Date: Sun, 21 Apr 2024 23:23:56 +0530 Subject: [PATCH 05/16] Refactor DsTable component styling and fix data source deletion error handling --- app/ui/src/components/Bot/DS/DsTable.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/ui/src/components/Bot/DS/DsTable.tsx b/app/ui/src/components/Bot/DS/DsTable.tsx index 7b17193d..abda3f59 100644 --- a/app/ui/src/components/Bot/DS/DsTable.tsx +++ b/app/ui/src/components/Bot/DS/DsTable.tsx @@ -132,9 +132,9 @@ export const DsTable = ({
-
-
-
+
+
+
{data.length === 0 && ( )} @@ -170,7 +170,7 @@ export const DsTable = ({ - + {data.map((source) => ( From dfaa6b8dbe4ba1fecfa96ecedd29032e43acc721 Mon Sep 17 00:00:00 2001 From: n4ze3m Date: Sun, 21 Apr 2024 23:24:04 +0530 Subject: [PATCH 06/16] Fix initial values in BotForm component --- app/ui/src/components/Common/BotForm.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/ui/src/components/Common/BotForm.tsx b/app/ui/src/components/Common/BotForm.tsx index 9d9eb6e6..90898e4d 100644 --- a/app/ui/src/components/Common/BotForm.tsx +++ b/app/ui/src/components/Common/BotForm.tsx @@ -480,7 +480,7 @@ export const BotForm = ({ onFinish={createBot} form={form} className="space-y-6" - initialValues={{ + initialValues={{ embedding: "dialoqbase_eb_text-embedding-ada-002", model: "gpt-3.5-turbo-dbase", maxDepth: 2, From 86a1b61994af7c37996a4d823ce321f6dc243d77 Mon Sep 17 00:00:00 2001 From: n4ze3m Date: Sun, 21 Apr 2024 23:24:28 +0530 Subject: [PATCH 07/16] Update styling in teams.tsx and add ApplicationCard component --- .../Settings/Application/ApplicationCard.tsx | 24 ++ app/ui/src/routes/settings/application.tsx | 220 ++++++++----- app/ui/src/routes/settings/model.tsx | 307 +++++++++--------- app/ui/src/routes/settings/teams.tsx | 2 +- 4 files changed, 325 insertions(+), 228 deletions(-) create mode 100644 app/ui/src/components/Settings/Application/ApplicationCard.tsx diff --git a/app/ui/src/components/Settings/Application/ApplicationCard.tsx b/app/ui/src/components/Settings/Application/ApplicationCard.tsx new file mode 100644 index 00000000..995663d7 --- /dev/null +++ b/app/ui/src/components/Settings/Application/ApplicationCard.tsx @@ -0,0 +1,24 @@ +export const ApplicationCard = ({ + children, + description, + title, +}: { + children: React.ReactNode; + title: string; + description: string; +}) => { + return ( +
+

+ {title} +

+

+ {description} +

+ +
+
{children}
+
+
+ ); +}; diff --git a/app/ui/src/routes/settings/application.tsx b/app/ui/src/routes/settings/application.tsx index ff14bfff..c38c9c13 100644 --- a/app/ui/src/routes/settings/application.tsx +++ b/app/ui/src/routes/settings/application.tsx @@ -6,6 +6,7 @@ import { useMutation, useQuery } from "@tanstack/react-query"; import { SettingsLayout } from "../../Layout/SettingsLayout"; import { SkeletonLoading } from "../../components/Common/SkeletonLoading"; import { useNavigate } from "react-router-dom"; +import { ApplicationCard } from "../../components/Settings/Application/ApplicationCard"; export default function SettingsApplicationRoot() { const [form] = Form.useForm(); @@ -17,6 +18,8 @@ export default function SettingsApplicationRoot() { noOfBotsPerUser: number; allowUserToCreateBots: boolean; allowUserToRegister: boolean; + defaultChunkSize: number; + defaultChunkOverlap: number; }; }); @@ -31,6 +34,11 @@ export default function SettingsApplicationRoot() { return response.data; }; + const onRagApplicationUpdate = async (values: any) => { + const response = await api.post("/admin/rag-settings", values); + return response.data; + }; + const { mutateAsync: updateApplicationSettings, isLoading } = useMutation( onUpdateApplicatoon, { @@ -41,7 +49,6 @@ export default function SettingsApplicationRoot() { }); }, onError: (error: any) => { - notification.error({ message: "Error", description: error?.response?.data?.message || "Something went wrong", @@ -50,87 +57,144 @@ export default function SettingsApplicationRoot() { } ); + const { mutateAsync: updateRagSettings, isLoading: isRagLoading } = + useMutation(onRagApplicationUpdate, { + onSuccess: (data) => { + notification.success({ + message: "Success", + description: data.message, + }); + }, + onError: (error: any) => { + notification.error({ + message: "Error", + description: error?.response?.data?.message || "Something went wrong", + }); + }, + }); + return ( {status === "success" && ( - <> -
-

- Application Settings -

-

- Configure your application settings -

- -
-
-
-
-
- - - - - - - - - -
-
- -
-
-
+
+ +
+
+
+ + + + + + + + + +
+
+ +
-
-
- + + + +
+
+
+ + + + + + + +
+
+ +
+
+
+ +
)} {status === "loading" && } diff --git a/app/ui/src/routes/settings/model.tsx b/app/ui/src/routes/settings/model.tsx index 0ca78f3f..7e314374 100644 --- a/app/ui/src/routes/settings/model.tsx +++ b/app/ui/src/routes/settings/model.tsx @@ -110,7 +110,7 @@ export default function SettingsModelRoot() {

-
+
- {modelType === "llm" && ( - text || "Untitled Model", - }, - { - dataIndex: "model_id", - title: "Model ID", - key: "model_id", - className: "text-gray-500", - }, - { - dataIndex: "model_provider", - title: "Provider", - key: "model_provider", - }, - { - dataIndex: "stream_available", - title: "Stream", - key: "stream_available", - render: (value) => ( - - {value ? "Available" : "Unavailable"} - - ), - }, - { - title: "Action", - render: (record) => - record.local_model ? ( -
+
+ {modelType === "llm" && ( +
text || "Untitled Model", + }, + { + dataIndex: "model_id", + title: "Model ID", + key: "model_id", + render: (text: string) => + text + .replace("-dbase", "") + .replace(/_dialoqbase_[0-9]+$/, ""), + className: "text-gray-500", + }, + { + dataIndex: "model_provider", + title: "Provider", + key: "model_provider", + }, + { + dataIndex: "stream_available", + title: "Stream", + key: "stream_available", + render: (value) => ( + + {value ? "Available" : "Unavailable"} + + ), + }, + { + title: "Action", + render: (record) => + record.local_model ? ( +
+ + + + + + +
+ ) : (
text || "Untitled Model", + }, + { + dataIndex: "model_id", + title: "Model ID", + key: "model_id", + className: "text-gray-500", + render: (text) => + text + .replace("dialoqbase_eb_", "") + .replace(/_dialoqbase_[0-9]+$/, ""), + }, + { + dataIndex: "model_provider", + title: "Provider", + key: "model_provider", + }, + { + title: "Action", + render: (record) => + record.local_model ? ( +
+ + + + + - -
- ) : ( - - - - ), - }, - ]} - /> - )} - {modelType === "embedding" && ( -
text || "Untitled Model", - }, - { - dataIndex: "model_id", - title: "Model ID", - key: "model_id", - className: "text-gray-500", - render: (text) => text.replace("dialoqbase_eb_", ""), - }, - { - dataIndex: "model_provider", - title: "Provider", - key: "model_provider", - }, - { - title: "Action", - render: (record) => - record.local_model ? ( -
+ if (confirm) { + del(record.id); + } + }} + className="text-red-400 hover:text-red-500" + > + + + +
+ ) : ( - - - ) : ( - - - - ), - }, - ]} - /> - )} + ), + }, + ]} + /> + )} + diff --git a/app/ui/src/routes/settings/teams.tsx b/app/ui/src/routes/settings/teams.tsx index ad2d9149..a03a2a39 100644 --- a/app/ui/src/routes/settings/teams.tsx +++ b/app/ui/src/routes/settings/teams.tsx @@ -131,7 +131,7 @@ export default function SettingsTeamsRoot() { Manage all users in your Dialoqbase Application

-
+
Date: Sun, 21 Apr 2024 23:24:37 +0530 Subject: [PATCH 08/16] Update model_id in saveModelFromInputedUrlHandler --- .../handlers/api/v1/admin/model.handler.ts | 26 +++++++------------ 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/server/src/handlers/api/v1/admin/model.handler.ts b/server/src/handlers/api/v1/admin/model.handler.ts index 7aa856c8..bd917ef3 100644 --- a/server/src/handlers/api/v1/admin/model.handler.ts +++ b/server/src/handlers/api/v1/admin/model.handler.ts @@ -245,10 +245,11 @@ export const saveModelFromInputedUrlHandler = async ( }); } + let newModelId = model_id.trim() + `_custom_${new Date().getTime()}`; await prisma.dialoqbaseModels.create({ data: { name: isModelExist.name, - model_id: model_id, + model_id: newModelId, stream_available: stream_available, local_model: true, model_provider: "replicate", @@ -263,25 +264,12 @@ export const saveModelFromInputedUrlHandler = async ( message: "success", }; } - - const modelExist = await prisma.dialoqbaseModels.findFirst({ - where: { - model_id: model_id, - hide: false, - deleted: false, - }, - }); - - if (modelExist) { - return reply.status(400).send({ - message: "Model already exist", - }); - } + let newModelId = model_id.trim() + `_dialoqbase_${new Date().getTime()}`; await prisma.dialoqbaseModels.create({ data: { name: name, - model_id: model_id, + model_id: newModelId, stream_available: stream_available, local_model: true, model_provider: api_type === "openai" ? "local" : "ollama", @@ -443,10 +431,14 @@ export const saveEmbedddingModelFromInputedUrlHandler = async ( transformer: "transformer", }; + let newModelId = + `dialoqbase_eb_${model_id}`.trim() + + `_dialoqbase_${new Date().getTime()}`; + await prisma.dialoqbaseModels.create({ data: { name: model_name, - model_id: `dialoqbase_eb_${model_id}`.trim(), + model_id: newModelId, local_model: true, model_type: "embedding", model_provider: model_provider[api_type], From 43c8f27256798979a8bc5d7119110e646c6084c9 Mon Sep 17 00:00:00 2001 From: n4ze3m Date: Sun, 21 Apr 2024 23:24:56 +0530 Subject: [PATCH 09/16] Add updateDialoqbaseRAGSettingsHandler to admin post handler --- .../src/handlers/api/v1/admin/post.handler.ts | 27 +++++++++++++++++++ server/src/handlers/api/v1/admin/type.ts | 15 +++++++---- .../api/v1/bot/playground/chat.handler.ts | 27 +++++++++---------- 3 files changed, 49 insertions(+), 20 deletions(-) diff --git a/server/src/handlers/api/v1/admin/post.handler.ts b/server/src/handlers/api/v1/admin/post.handler.ts index fa7746b3..a26aaecc 100644 --- a/server/src/handlers/api/v1/admin/post.handler.ts +++ b/server/src/handlers/api/v1/admin/post.handler.ts @@ -3,6 +3,7 @@ import { RegisterUserbyAdminRequestBody, ResetUserPasswordByAdminRequest, UpdateDialoqbaseSettingsRequest, + UpdateDialoqbaseRAGSettingsRequest } from "./type"; import { getSettings } from "../../../../utils/common"; import * as bcrypt from "bcryptjs"; @@ -136,3 +137,29 @@ export const registerUserByAdminHandler = async ( }); } }; + + +export const updateDialoqbaseRAGSettingsHandler = async ( + request: FastifyRequest, + reply: FastifyReply, +) => { + const prisma = request.server.prisma; + const user = request.user; + + if (!user.is_admin) { + return reply.status(403).send({ + message: "Forbidden", + }); + } + + await prisma.dialoqbaseSettings.update({ + where: { + id: 1, + }, + data: request.body, + }); + + return { + message: "RAG settings updated successfully", + }; +} \ No newline at end of file diff --git a/server/src/handlers/api/v1/admin/type.ts b/server/src/handlers/api/v1/admin/type.ts index 5fece7ce..6d627f54 100644 --- a/server/src/handlers/api/v1/admin/type.ts +++ b/server/src/handlers/api/v1/admin/type.ts @@ -26,7 +26,7 @@ export type FetchModelFromInputedUrlRequest = { Body: { url?: string; api_key?: string; - api_type: "openai" | "ollama", + api_type: "openai" | "ollama"; ollama_url?: string; }; }; @@ -47,8 +47,6 @@ export type ToogleModelRequest = { }; }; - - export interface SaveEmbeddingModelRequest { Body: { url?: string; @@ -56,5 +54,12 @@ export interface SaveEmbeddingModelRequest { api_type: "openai" | "ollama" | "transformer"; model_id: string; model_name: string; - } -} \ No newline at end of file + }; +} + +export type UpdateDialoqbaseRAGSettingsRequest = { + Body: { + defaultChunkSize: number; + defaultChunkOverlap: number; + }; +}; diff --git a/server/src/handlers/api/v1/bot/playground/chat.handler.ts b/server/src/handlers/api/v1/bot/playground/chat.handler.ts index 18837717..ab950f97 100644 --- a/server/src/handlers/api/v1/bot/playground/chat.handler.ts +++ b/server/src/handlers/api/v1/bot/playground/chat.handler.ts @@ -6,10 +6,7 @@ import { chatModelProvider } from "../../../../../utils/models"; import { DialoqbaseHybridRetrival } from "../../../../../utils/hybrid"; import { BaseRetriever } from "@langchain/core/retrievers"; import { Document } from "langchain/document"; -import { - createChain, - groupMessagesByConversation, -} from "../../../../../chain"; +import { createChain, groupMessagesByConversation } from "../../../../../chain"; export const chatRequestHandler = async ( request: FastifyRequest, @@ -447,6 +444,17 @@ export const chatRequestStreamHandler = async ( // close the model }); + let historyId = history_id; + if (!historyId) { + const newHistory = await prisma.botPlayground.create({ + data: { + botId: bot.id, + title: message, + }, + }); + historyId = newHistory.id; + } + const chain = createChain({ llm: streamedModel, question_llm: nonStreamingModel, @@ -476,19 +484,8 @@ export const chatRequestStreamHandler = async ( response += token; } - let historyId = history_id; const documents = await documentPromise; - if (!historyId) { - const newHistory = await prisma.botPlayground.create({ - data: { - botId: bot.id, - title: message, - }, - }); - historyId = newHistory.id; - } - await prisma.botPlaygroundMessage.create({ data: { type: "human", From 5033776b316bbd022c8f2d1d05acc42ad4bb5cb9 Mon Sep 17 00:00:00 2001 From: n4ze3m Date: Sun, 21 Apr 2024 23:25:22 +0530 Subject: [PATCH 10/16] Refactor text splitting logic in queue controllers --- .../src/queue/controllers/audio.controller.ts | 5 ++-- .../src/queue/controllers/crawl.controller.ts | 26 ++++++++----------- .../src/queue/controllers/csv.controller.ts | 4 +-- .../src/queue/controllers/docx.controller.ts | 4 +-- .../queue/controllers/github.controller.ts | 4 +-- .../src/queue/controllers/pdf.controller.ts | 4 +-- .../queue/controllers/sitemap.controller.ts | 20 ++++++++------ .../src/queue/controllers/text.controller.ts | 4 +-- .../src/queue/controllers/txt.controller.ts | 4 +-- .../src/queue/controllers/video.controller.ts | 4 +-- .../queue/controllers/website.controller.ts | 7 ++--- .../queue/controllers/youtube.controller.ts | 18 +++++-------- server/src/queue/index.ts | 9 +++++-- server/src/queue/type.ts | 2 ++ 14 files changed, 57 insertions(+), 58 deletions(-) diff --git a/server/src/queue/controllers/audio.controller.ts b/server/src/queue/controllers/audio.controller.ts index 7feccd5b..86f6b07c 100644 --- a/server/src/queue/controllers/audio.controller.ts +++ b/server/src/queue/controllers/audio.controller.ts @@ -1,4 +1,3 @@ -// import { PDFLoader } from "langchain/document_loaders/fs/pdf"; import { QSource } from "../type"; import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"; import { DialoqbaseVectorStore } from "../../utils/store"; @@ -21,8 +20,8 @@ export const audioQueueController = async ( const docs = await loader.load(); const textSplitter = new RecursiveCharacterTextSplitter({ - chunkSize: 1000, - chunkOverlap: 200, + chunkSize: source.chunkSize, + chunkOverlap: source.chunkOverlap, }); const chunks = await textSplitter.splitDocuments(docs); diff --git a/server/src/queue/controllers/crawl.controller.ts b/server/src/queue/controllers/crawl.controller.ts index 9b82ebfa..bfb9e16e 100644 --- a/server/src/queue/controllers/crawl.controller.ts +++ b/server/src/queue/controllers/crawl.controller.ts @@ -4,19 +4,10 @@ import { crawl } from "../../utils/crawl"; import { websiteQueueController } from "./website.controller"; const prisma = new PrismaClient(); -export const crawlQueueController = async ( - source: QSource, -) => { +export const crawlQueueController = async (source: QSource) => { let maxDepth = source.maxDepth || 1; let maxLinks = source.maxLinks || 1; - const links = Array.from( - await crawl( - source.content!, - maxDepth, - 0, - maxLinks, - ), - ); + const links = Array.from(await crawl(source.content!, maxDepth, 0, maxLinks)); for (const link of links) { const newSource = await prisma.botSource.create({ @@ -29,10 +20,15 @@ export const crawlQueueController = async ( }, }); - await websiteQueueController({ - ...newSource, - embedding: source.embedding, - }, prisma); + await websiteQueueController( + { + ...newSource, + embedding: source.embedding, + chunkOverlap: source.chunkOverlap, + chunkSize: source.chunkSize, + }, + prisma + ); await prisma.botSource.update({ where: { diff --git a/server/src/queue/controllers/csv.controller.ts b/server/src/queue/controllers/csv.controller.ts index b7d230e1..8e159e5b 100644 --- a/server/src/queue/controllers/csv.controller.ts +++ b/server/src/queue/controllers/csv.controller.ts @@ -16,8 +16,8 @@ export const csvQueueController = async ( const docs = await loader.load(); const textSplitter = new RecursiveCharacterTextSplitter({ - chunkSize: 1000, - chunkOverlap: 200, + chunkSize: source.chunkSize, + chunkOverlap: source.chunkOverlap, }); const chunks = await textSplitter.splitDocuments(docs); diff --git a/server/src/queue/controllers/docx.controller.ts b/server/src/queue/controllers/docx.controller.ts index 3facfea2..185a8e26 100644 --- a/server/src/queue/controllers/docx.controller.ts +++ b/server/src/queue/controllers/docx.controller.ts @@ -17,8 +17,8 @@ export const DocxQueueController = async ( const docs = await loader.load(); const textSplitter = new RecursiveCharacterTextSplitter({ - chunkSize: 1000, - chunkOverlap: 200, + chunkSize: source.chunkSize, + chunkOverlap: source.chunkOverlap, }); const chunks = await textSplitter.splitDocuments(docs); diff --git a/server/src/queue/controllers/github.controller.ts b/server/src/queue/controllers/github.controller.ts index 5f4f43da..7300c90f 100644 --- a/server/src/queue/controllers/github.controller.ts +++ b/server/src/queue/controllers/github.controller.ts @@ -20,8 +20,8 @@ export const githubQueueController = async ( const docs = await loader.load(); const textSplitter = new RecursiveCharacterTextSplitter({ - chunkSize: 1000, - chunkOverlap: 200, + chunkSize: source.chunkSize, + chunkOverlap: source.chunkOverlap, }); const chunks = await textSplitter.splitDocuments(docs); diff --git a/server/src/queue/controllers/pdf.controller.ts b/server/src/queue/controllers/pdf.controller.ts index 89d76ec2..2bfcaa65 100644 --- a/server/src/queue/controllers/pdf.controller.ts +++ b/server/src/queue/controllers/pdf.controller.ts @@ -17,8 +17,8 @@ export const pdfQueueController = async ( const docs = await loader.load(); const textSplitter = new RecursiveCharacterTextSplitter({ - chunkSize: 1000, - chunkOverlap: 200, + chunkSize: source.chunkSize, + chunkOverlap: source.chunkOverlap, }); const chunks = await textSplitter.splitDocuments(docs); diff --git a/server/src/queue/controllers/sitemap.controller.ts b/server/src/queue/controllers/sitemap.controller.ts index f9ddc5a6..93137bd2 100644 --- a/server/src/queue/controllers/sitemap.controller.ts +++ b/server/src/queue/controllers/sitemap.controller.ts @@ -4,9 +4,7 @@ import { websiteQueueController } from "./website.controller"; const prisma = new PrismaClient(); import Sitemapper from "sitemapper"; -export const sitemapQueueController = async ( - source: QSource, -) => { +export const sitemapQueueController = async (source: QSource) => { const url = source.content!; const sitemapper = new Sitemapper({ @@ -19,7 +17,8 @@ export const sitemapQueueController = async ( ? parseInt(process.env.SITEMAPPER_MAX_RETRIES) : 1, requestHeaders: { - "User-Agent": process.env.SITEMAPPER_USER_AGENT || + "User-Agent": + process.env.SITEMAPPER_USER_AGENT || "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:81.0) Gecko/20100101 Firefox/81.0'", }, }); @@ -44,10 +43,15 @@ export const sitemapQueueController = async ( }, }); - await websiteQueueController({ - ...newSource, - embedding: source.embedding, - }, prisma); + await websiteQueueController( + { + ...newSource, + embedding: source.embedding, + chunkSize: source.chunkSize, + chunkOverlap: source.chunkOverlap, + }, + prisma + ); await prisma.botSource.update({ where: { diff --git a/server/src/queue/controllers/text.controller.ts b/server/src/queue/controllers/text.controller.ts index 319e66f6..11e8c2a3 100644 --- a/server/src/queue/controllers/text.controller.ts +++ b/server/src/queue/controllers/text.controller.ts @@ -9,8 +9,8 @@ export const textQueueController = async ( prisma: PrismaClient ) => { const textSplitter = new RecursiveCharacterTextSplitter({ - chunkSize: 1000, - chunkOverlap: 200, + chunkSize: source.chunkSize, + chunkOverlap: source.chunkOverlap, }); const chunks = await textSplitter.splitDocuments([ { diff --git a/server/src/queue/controllers/txt.controller.ts b/server/src/queue/controllers/txt.controller.ts index 7ed84cf3..baf894fd 100644 --- a/server/src/queue/controllers/txt.controller.ts +++ b/server/src/queue/controllers/txt.controller.ts @@ -16,8 +16,8 @@ export const txtQueueController = async ( const docs = await loader.load(); const textSplitter = new RecursiveCharacterTextSplitter({ - chunkSize: 1000, - chunkOverlap: 200, + chunkSize: source.chunkSize, + chunkOverlap: source.chunkOverlap, }); const chunks = await textSplitter.splitDocuments(docs); const embeddingInfo = await prisma.dialoqbaseModels.findFirst({ diff --git a/server/src/queue/controllers/video.controller.ts b/server/src/queue/controllers/video.controller.ts index deb07c26..c0646a3a 100644 --- a/server/src/queue/controllers/video.controller.ts +++ b/server/src/queue/controllers/video.controller.ts @@ -21,8 +21,8 @@ export const videoQueueController = async ( const docs = await loader.load(); const textSplitter = new RecursiveCharacterTextSplitter({ - chunkSize: 1000, - chunkOverlap: 200, + chunkSize: source.chunkSize, + chunkOverlap: source.chunkOverlap, }); const chunks = await textSplitter.splitDocuments(docs); diff --git a/server/src/queue/controllers/website.controller.ts b/server/src/queue/controllers/website.controller.ts index 1c83ac3e..d0843f75 100644 --- a/server/src/queue/controllers/website.controller.ts +++ b/server/src/queue/controllers/website.controller.ts @@ -13,9 +13,6 @@ export const websiteQueueController = async ( source: QSource, prisma: PrismaClient ) => { - // check if url is html or pdf or other - // if html, use cheerio - const response = await axios.get(source.content!); const type = response.headers["content-type"]; @@ -35,8 +32,8 @@ export const websiteQueueController = async ( const docs = await loader.load(); const textSplitter = new RecursiveCharacterTextSplitter({ - chunkSize: 1000, - chunkOverlap: 200, + chunkSize: source.chunkSize, + chunkOverlap: source.chunkOverlap, }); const chunks = await textSplitter.splitDocuments(docs); diff --git a/server/src/queue/controllers/youtube.controller.ts b/server/src/queue/controllers/youtube.controller.ts index f94d362f..fdb0e12a 100644 --- a/server/src/queue/controllers/youtube.controller.ts +++ b/server/src/queue/controllers/youtube.controller.ts @@ -11,15 +11,12 @@ export const youtubeQueueController = async ( source: QSource, prisma: PrismaClient ) => { - const { - language_code, - youtube_mode - } = source.options as { + const { language_code, youtube_mode } = source.options as { language_code: string; youtube_mode: "whisper" | "transcript"; - } + }; if (youtube_mode === "transcript") { - console.log("Using Youtube Transcript Mode") + console.log("Using Youtube Transcript Mode"); const loader = new DialoqbaseYoutubeTranscript({ url: source.content!, language_code, @@ -27,8 +24,8 @@ export const youtubeQueueController = async ( const docs = await loader.load(); const textSplitter = new RecursiveCharacterTextSplitter({ - chunkSize: 1000, - chunkOverlap: 200, + chunkSize: source.chunkSize, + chunkOverlap: source.chunkOverlap, }); const chunks = await textSplitter.splitDocuments(docs); @@ -56,9 +53,8 @@ export const youtubeQueueController = async ( sourceId: source.id, } ); - } - else { - console.log("Using Youtube Whisper Mode") + } else { + console.log("Using Youtube Whisper Mode"); const loader = new DialoqbaseYoutube({ url: source.content!, }); diff --git a/server/src/queue/index.ts b/server/src/queue/index.ts index 0de1f98a..a0d12338 100644 --- a/server/src/queue/index.ts +++ b/server/src/queue/index.ts @@ -14,6 +14,7 @@ import { youtubeQueueController } from "./controllers/youtube.controller"; import { restQueueController } from "./controllers/rest.controller"; import { sitemapQueueController } from "./controllers/sitemap.controller"; import { SandboxedJob } from "bullmq"; +import { getRagSettings } from "../utils/rag-settings"; const prisma = new PrismaClient(); @@ -22,8 +23,9 @@ export default async function queueHandler(job: SandboxedJob) { await prisma.$connect(); console.log("Processing queue"); try { - for (const source of data) { + for (const sourceData of data) { try { + let source = sourceData; await prisma.botSource.update({ where: { id: source.id, @@ -32,6 +34,9 @@ export default async function queueHandler(job: SandboxedJob) { status: "PROCESSING", }, }); + const { chunkOverlap, chunkSize } = await getRagSettings(prisma); + source.chunkOverlap = chunkOverlap; + source.chunkSize = chunkSize; switch (source.type.toLowerCase()) { case "website": await websiteQueueController(source, prisma); @@ -93,7 +98,7 @@ export default async function queueHandler(job: SandboxedJob) { console.log(e); await prisma.botSource.update({ where: { - id: source.id, + id: sourceData.id, }, data: { status: "FAILED", diff --git a/server/src/queue/type.ts b/server/src/queue/type.ts index aae64eb3..f74f4286 100644 --- a/server/src/queue/type.ts +++ b/server/src/queue/type.ts @@ -4,4 +4,6 @@ export interface QSource extends BotSource { embedding: string; maxDepth?: number; maxLinks?: number; + chunkSize: number; + chunkOverlap: number; } From 2746354136d389bd4e523345cebaa85cd248773e Mon Sep 17 00:00:00 2001 From: n4ze3m Date: Sun, 21 Apr 2024 23:25:35 +0530 Subject: [PATCH 11/16] Add updateDialoqbaseRAGSettingsHandler and updateDialoqbaseRAGSettings schema to admin post handler --- server/src/routes/api/v1/admin/root.ts | 15 +++++++++-- server/src/schema/api/v1/admin/index.ts | 36 +++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/server/src/routes/api/v1/admin/root.ts b/server/src/routes/api/v1/admin/root.ts index 64855ebd..5f7e9a87 100644 --- a/server/src/routes/api/v1/admin/root.ts +++ b/server/src/routes/api/v1/admin/root.ts @@ -10,7 +10,8 @@ import { saveModelFromInputedUrlHandler, deleteModelHandler, hideModelHandler, - saveEmbedddingModelFromInputedUrlHandler + saveEmbedddingModelFromInputedUrlHandler, + updateDialoqbaseRAGSettingsHandler } from "../../../../handlers/api/v1/admin"; import { dialoqbaseSettingsSchema, @@ -18,6 +19,7 @@ import { getAllUsersSchema, registerUserByAdminSchema, resetUserPasswordByAdminSchema, + updateDialoqbaseRAGSettings } from "../../../../schema/api/v1/admin"; import { @@ -25,7 +27,7 @@ import { getAllModelsSchema, saveModelFromInputedUrlSchema, toogleModelSchema, - saveEmbeddingModelSchema + saveEmbeddingModelSchema, } from "../../../../schema/api/v1/admin/model"; const root: FastifyPluginAsync = async (fastify, _): Promise => { @@ -47,6 +49,15 @@ const root: FastifyPluginAsync = async (fastify, _): Promise => { updateDialoqbaseSettingsHandler ); + fastify.post( + "/rag-settings", + { + schema: updateDialoqbaseRAGSettings, + onRequest: [fastify.authenticate], + }, + updateDialoqbaseRAGSettingsHandler + ); + fastify.get( "/users", { diff --git a/server/src/schema/api/v1/admin/index.ts b/server/src/schema/api/v1/admin/index.ts index 8a63e915..6f4c9599 100644 --- a/server/src/schema/api/v1/admin/index.ts +++ b/server/src/schema/api/v1/admin/index.ts @@ -15,6 +15,11 @@ export const dialoqbaseSettingsSchema: FastifySchema = { noOfBotsPerUser: { type: "number" }, allowUserToCreateBots: { type: "boolean" }, allowUserToRegister: { type: "boolean" }, + defaultChunkSize: { type: "number" }, + defaultChunkOverlap: { type: "number" }, + defaultChatModel: { type: "string" }, + defaultEmbeddingModel: { type: "string" }, + dynamicallyFetchOllamaModels: { type: "boolean" }, }, }, }; @@ -35,6 +40,9 @@ export const updateDialoqbaseSettingsSchema: FastifySchema = { noOfBotsPerUser: { type: "number" }, allowUserToCreateBots: { type: "boolean" }, allowUserToRegister: { type: "boolean" }, + dynamicallyFetchOllamaModels: { type: "boolean" }, + defaultChatModel: { type: "string" }, + defaultEmbeddingModel: { type: "string" }, }, required: [ "noOfBotsPerUser", @@ -136,3 +144,31 @@ export const registerUserByAdminSchema: FastifySchema = { }, }, }; + +export const updateDialoqbaseRAGSettings: FastifySchema = { + tags: ["Admin"], + summary: "API to update dialoqbase RAG settings", + headers: { + type: "object", + properties: { + Authorization: { type: "string" }, + }, + required: ["Authorization"], + }, + body: { + type: "object", + properties: { + defaultChunkSize: { type: "number" }, + defaultChunkOverlap: { type: "number" }, + }, + required: ["defaultChunkSize", "defaultChunkOverlap"], + }, + response: { + 200: { + type: "object", + properties: { + message: { type: "string" }, + }, + }, + }, +}; From 3f9ebbf5af23251a80a0d219f66a0cfd7a1f0586 Mon Sep 17 00:00:00 2001 From: n4ze3m Date: Sun, 21 Apr 2024 23:25:43 +0530 Subject: [PATCH 12/16] Refactor model name replacement logic in embeddings.ts and models.ts --- server/src/utils/embeddings.ts | 1 + server/src/utils/models.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/server/src/utils/embeddings.ts b/server/src/utils/embeddings.ts index 11fab640..050a9b73 100644 --- a/server/src/utils/embeddings.ts +++ b/server/src/utils/embeddings.ts @@ -12,6 +12,7 @@ export const embeddings = ( otherFields: any ) => { modelName = modelName.replace("dialoqbase_eb_", ""); + modelName = modelName.replace(/_dialoqbase_[0-9]+$/, ""); switch (provider.toLocaleLowerCase()) { case "openai": return new OpenAIEmbeddings({ diff --git a/server/src/utils/models.ts b/server/src/utils/models.ts index 3a9f64c8..0037b00e 100644 --- a/server/src/utils/models.ts +++ b/server/src/utils/models.ts @@ -16,6 +16,7 @@ export const chatModelProvider = ( otherFields?: any ) => { modelName = modelName.replace("-dbase", ""); + modelName = modelName.replace(/_dialoqbase_[0-9]+$/, ""); switch (provider.toLowerCase()) { case "openai": From 15b38c9d8f03ce7d57a3b9eb1643737573a5931f Mon Sep 17 00:00:00 2001 From: n4ze3m Date: Sun, 21 Apr 2024 23:25:50 +0530 Subject: [PATCH 13/16] Add getRagSettings function to rag-settings.ts --- server/src/utils/rag-settings.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 server/src/utils/rag-settings.ts diff --git a/server/src/utils/rag-settings.ts b/server/src/utils/rag-settings.ts new file mode 100644 index 00000000..d27c31e9 --- /dev/null +++ b/server/src/utils/rag-settings.ts @@ -0,0 +1,15 @@ +import { PrismaClient } from "@prisma/client"; + +export const getRagSettings = async (prisma: PrismaClient) => { + const data = await prisma.dialoqbaseSettings.findFirst({ + select: { + defaultChunkSize: true, + defaultChunkOverlap: true, + }, + }); + + return { + chunkSize: data?.defaultChunkSize || 1000, + chunkOverlap: data?.defaultChunkOverlap || 200, + }; +}; From a05bbb02bda6a71c06f223fdcd9e139230c03e82 Mon Sep 17 00:00:00 2001 From: n4ze3m Date: Mon, 22 Apr 2024 10:47:52 +0530 Subject: [PATCH 14/16] Add Ability to configure default chat and embedding model --- app/ui/src/components/Common/BotForm.tsx | 11 +-- app/ui/src/hooks/useCreateConfig.tsx | 2 + app/ui/src/routes/settings/application.tsx | 70 ++++++++++++++++++- .../handlers/api/v1/bot/bot/get.handler.ts | 5 ++ 4 files changed, 82 insertions(+), 6 deletions(-) diff --git a/app/ui/src/components/Common/BotForm.tsx b/app/ui/src/components/Common/BotForm.tsx index 90898e4d..a2837662 100644 --- a/app/ui/src/components/Common/BotForm.tsx +++ b/app/ui/src/components/Common/BotForm.tsx @@ -480,9 +480,11 @@ export const BotForm = ({ onFinish={createBot} form={form} className="space-y-6" - initialValues={{ - embedding: "dialoqbase_eb_text-embedding-ada-002", - model: "gpt-3.5-turbo-dbase", + initialValues={{ + embedding: + botConfig?.defaultEmbeddingModel || + "dialoqbase_eb_text-embedding-ada-002", + model: botConfig?.defaultChatModel || "gpt-3.5-turbo-dbase", maxDepth: 2, maxLinks: 10, options: { @@ -646,7 +648,7 @@ export const BotForm = ({ .toLowerCase() .localeCompare((optionB?.label ?? "").toLowerCase()) } - placeholder="Select a chat model" + placeholder="Select a Chat Model" options={botConfig.chatModel} /> @@ -675,6 +677,7 @@ export const BotForm = ({ .localeCompare((optionB?.label ?? "").toLowerCase()) } options={botConfig.embeddingModel} + placeholder="Select an Embedding Model" /> diff --git a/app/ui/src/hooks/useCreateConfig.tsx b/app/ui/src/hooks/useCreateConfig.tsx index 1a090407..7f1493c9 100644 --- a/app/ui/src/hooks/useCreateConfig.tsx +++ b/app/ui/src/hooks/useCreateConfig.tsx @@ -16,6 +16,8 @@ export const useCreateConfig = () => { label: string; value: string; }[]; + defaultChatModel?: string; + defaultEmbeddingModel?: string; }; } ); diff --git a/app/ui/src/routes/settings/application.tsx b/app/ui/src/routes/settings/application.tsx index c38c9c13..ff70e79f 100644 --- a/app/ui/src/routes/settings/application.tsx +++ b/app/ui/src/routes/settings/application.tsx @@ -1,16 +1,19 @@ -import { Form, InputNumber, Switch, notification } from "antd"; +import { Form, InputNumber, Switch, notification, Select } from "antd"; import React from "react"; import api from "../../services/api"; -import { useMutation, useQuery } from "@tanstack/react-query"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { SettingsLayout } from "../../Layout/SettingsLayout"; import { SkeletonLoading } from "../../components/Common/SkeletonLoading"; import { useNavigate } from "react-router-dom"; import { ApplicationCard } from "../../components/Settings/Application/ApplicationCard"; +import { useCreateConfig } from "../../hooks/useCreateConfig"; export default function SettingsApplicationRoot() { const [form] = Form.useForm(); const navigate = useNavigate(); + const { data: models, status: modeStatus } = useCreateConfig(); + const queryClient = useQueryClient(); const { data, status } = useQuery(["fetchApplicationSettings"], async () => { const response = await api.get("/admin/dialoqbase-settings"); @@ -20,6 +23,8 @@ export default function SettingsApplicationRoot() { allowUserToRegister: boolean; defaultChunkSize: number; defaultChunkOverlap: number; + defaultChatModel: string; + defaultEmbeddingModel: string; }; }); @@ -43,6 +48,7 @@ export default function SettingsApplicationRoot() { onUpdateApplicatoon, { onSuccess: (data) => { + queryClient.invalidateQueries(["fetchBotCreateConfig"]); notification.success({ message: "Success", description: data.message, @@ -129,6 +135,66 @@ export default function SettingsApplicationRoot() { > + + + (option?.label?.toLowerCase() ?? "").includes( + input?.toLowerCase() + ) || + (option?.value?.toLowerCase() ?? "").includes( + input?.toLowerCase() + ) + } + filterSort={(optionA, optionB) => + (optionA?.label ?? "") + .toLowerCase() + .localeCompare((optionB?.label ?? "").toLowerCase()) + } + placeholder="Select a Embedding Model" + options={models?.embeddingModel || []} + loading={modeStatus === "loading"} + /> +