Skip to content

Commit 97e57ff

Browse files
n4ze3mn4ze3m
authored
Merge pull request #290 from n4ze3m/next
v1.9.5
2 parents 43a09e5 + 10f544b commit 97e57ff

File tree

25 files changed

+623
-78
lines changed

25 files changed

+623
-78
lines changed

‎app/ui/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"name": "app",
33
"private": true,
4-
"version": "1.9.4",
4+
"version": "1.9.5",
55
"type": "module",
66
"scripts": {
77
"dev": "vite",

‎app/ui/src/@types/bot.ts

+1
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ export type BotSettings = {
2121
semanticSearchSimilarityScore: string;
2222
inactivityTimeout: number;
2323
autoResetSession: boolean;
24+
autoSyncDataSources: boolean;
2425
};
2526
chatModel: {
2627
label: string;

‎app/ui/src/components/Bot/Settings/SettingsBody.tsx

+9
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,7 @@ export const SettingsBody: React.FC<BotSettings> = ({
168168
semanticSearchSimilarityScore: data.semanticSearchSimilarityScore,
169169
autoResetSession: data.autoResetSession,
170170
inactivityTimeout: data.inactivityTimeout,
171+
autoSyncDataSources: data.autoSyncDataSources,
171172
}}
172173
form={form}
173174
requiredMark={false}
@@ -453,6 +454,14 @@ export const SettingsBody: React.FC<BotSettings> = ({
453454
placeholder="Enter inactivity timeout"
454455
/>
455456
</Form.Item>
457+
458+
<Form.Item
459+
name="autoSyncDataSources"
460+
label="Auto Sync Data Source(s)"
461+
tooltip="This will automatically re-fetch the URL-based data sources at a certain interval."
462+
>
463+
<Switch />
464+
</Form.Item>
456465
</div>
457466
</div>
458467

‎app/ui/src/components/Common/BotForm.tsx

+9-6
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ type Props = {
3535
form: FormInstance<any>;
3636
showEmbeddingAndModels: boolean;
3737
newSelectedSource?: any;
38-
botConfig: BotConfig
38+
botConfig: BotConfig;
3939
};
4040
function classNames(...classes: string[]) {
4141
return classes.filter(Boolean).join(" ");
@@ -47,7 +47,7 @@ export const BotForm = ({
4747
setSelectedSource,
4848
form,
4949
showEmbeddingAndModels,
50-
botConfig
50+
botConfig,
5151
}: Props) => {
5252
const youtubeMode = Form.useWatch(["options", "youtube_mode"], form);
5353
const url = Form.useWatch(["content"], form);
@@ -117,7 +117,7 @@ export const BotForm = ({
117117
}}
118118
>
119119
<Upload.Dragger
120-
accept={`.pdf,.docx,.csv,.txt,.mp3,.mp4`}
120+
accept={`.pdf,.docx,.csv,.txt,.mp3,.mp4,.zip`}
121121
multiple={true}
122122
maxCount={botConfig?.fileUploadSizeLimit}
123123
beforeUpload={(file) => {
@@ -130,6 +130,8 @@ export const BotForm = ({
130130
"audio/mp4",
131131
"video/mp4",
132132
"video/mpeg",
133+
"application/zip",
134+
"application/x-zip-compressed",
133135
]
134136
.map((type) => type.toLowerCase())
135137
.join(", ");
@@ -161,12 +163,13 @@ export const BotForm = ({
161163
<InboxIcon className="h-10 w-10 text-gray-400" />
162164
</p>
163165
<p className="ant-upload-text">
164-
Click or drag PDF, Docx, CSV , TXT, MP3, MP4 files to this
166+
Click or drag PDF, Docx, CSV , TXT, MP3, MP4, Zip files to
167+
this
165168
</p>
166169
<p className="ant-upload-hint">
167-
Support is available for a single or bulk upload of up to 10
170+
{`Support is available for a single or bulk upload of up to ${botConfig?.fileUploadSizeLimit}
168171
files. Please note that file upload is in beta, so if you
169-
encounter any issues, kindly report them.
172+
encounter any issues, kindly report them.`}
170173
</p>
171174
</div>
172175
</Upload.Dragger>

‎app/ui/src/routes/settings/application.tsx

+9-2
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ export default function SettingsApplicationRoot() {
2929
dynamicallyFetchOllamaModels: boolean;
3030
ollamaURL: string;
3131
fileUploadSizeLimit: number;
32+
refetchDatasource: boolean;
3233
};
3334
});
3435

@@ -173,9 +174,15 @@ export default function SettingsApplicationRoot() {
173174
]}
174175
tooltip="Default is 10"
175176
>
176-
177177
<InputNumber size="large" style={{ width: "100%" }} />
178-
178+
</Form.Item>
179+
<Form.Item
180+
label="Refetch Data Source"
181+
name="refetchDatasource"
182+
valuePropName="checked"
183+
help="This will refetch the data source at a specific interval."
184+
>
185+
<Switch />
179186
</Form.Item>
180187
</div>
181188
<div className="bg-gray-50 border-x border-b rounded-b-md rounded-x-md px-4 py-3 text-right sm:px-6 dark:bg-[#141414] dark:border-gray-600">

‎package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "dialoqbase",
3-
"version": "1.9.4",
3+
"version": "1.9.5",
44
"description": "Create chatbots with ease",
55
"scripts": {
66
"ui:dev": "pnpm run --filter ui dev",

‎server/package.json

+5
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@
6161
"cohere-ai": "^6.2.1",
6262
"concurrently": "^7.0.0",
6363
"copyfiles": "^2.4.1",
64+
"cron": "^3.1.7",
6465
"d3-dsv": "2",
6566
"date-fns": "^3.6.0",
6667
"discord.js": "^14.11.0",
@@ -92,6 +93,8 @@
9293
"turndown": "^7.1.3",
9394
"unique-names-generator": "^4.7.1",
9495
"wavefile": "^11.0.0",
96+
"yauzl": "^3.1.3",
97+
"yauzl-promise": "^4.0.0",
9598
"yt-transcript": "^0.0.2",
9699
"ytdl-core": "^4.11.5"
97100
},
@@ -103,6 +106,8 @@
103106
"@types/pubsub-js": "^1.8.3",
104107
"@types/tap": "^15.0.5",
105108
"@types/turndown": "^5.0.4",
109+
"@types/yauzl": "^2.10.3",
110+
"@types/yauzl-promise": "^4.0.1",
106111
"c8": "9.0.0",
107112
"fastify-tsconfig": "2.0.0",
108113
"prisma": "^5.9.1",

‎server/prisma/migrations/q_14_3/migration.sql

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
-- AlterTable
2+
ALTER TABLE "DialoqbaseSettings" ADD COLUMN "refetchDatasource" BOOLEAN NOT NULL DEFAULT false;

‎server/prisma/migrations/q_14_4/migration.sql

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
-- AlterTable
2+
ALTER TABLE "Bot" ADD COLUMN "autoSyncDataSources" BOOLEAN DEFAULT false;

‎server/prisma/schema.prisma

+2
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ model Bot {
4444
bot_api_key String?
4545
bot_model_api_key String?
4646
options Json? @default("{}") @db.Json
47+
autoSyncDataSources Boolean? @default(false)
4748
BotAppearance BotAppearance[]
4849
document BotDocument[]
4950
BotIntegration BotIntegration[]
@@ -106,6 +107,7 @@ model DialoqbaseSettings {
106107
defaultEmbeddingModel String @default("dialoqbase_eb_text-embedding-ada-002")
107108
ollamaURL String? @default("http://host.docker.internal:11434")
108109
usePuppeteerFetch Boolean? @default(false)
110+
refetchDatasource Boolean @default(false)
109111
}
110112

111113
model BotIntegration {

‎server/src/app.ts

+17-1
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,9 @@ import swaggerUi from "@fastify/swagger-ui";
1313
import { pathToFileURL } from "url";
1414
import { Worker } from "bullmq";
1515
import { parseRedisUrl } from "./utils/redis";
16+
import { CronJob } from 'cron';
17+
import { processDatasourceCron } from "./cron/index";
18+
1619
declare module "fastify" {
1720
interface Session {
1821
is_bot_allowed: boolean;
@@ -31,7 +34,11 @@ const app: FastifyPluginAsync<AppOptions> = async (
3134

3235
void fastify.register(FastifySSEPlugin);
3336

34-
void fastify.register(fastifyMultipart);
37+
void fastify.register(fastifyMultipart, {
38+
limits: {
39+
fileSize: 1 * 1024 * 1024 * 1024,
40+
}
41+
});
3542

3643
void fastify.register(swagger);
3744

@@ -99,8 +106,17 @@ const worker = new Worker("vector", workerUrl, {
99106
useWorkerThreads: workerThreads === "true",
100107
});
101108

109+
const job = new CronJob(
110+
process.env.DB_CRON_TIME || '0 0 0 * * *',
111+
processDatasourceCron,
112+
null,
113+
true,
114+
process.env.DB_CRON_TIMEZONE
115+
);
116+
102117
process.on("SIGINT", async () => {
103118
await worker.close();
119+
job.stop();
104120
process.exit();
105121
});
106122

‎server/src/chain/index.ts

+17-1
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,17 @@ type RetrievalChainInput = {
2121
question: string;
2222
};
2323

24+
const updateTemplateVariables = (template: string) => {
25+
// replace template {time} with current time
26+
template = template.replace("{time}", new Date().toLocaleTimeString());
27+
// replace template {date} with current date
28+
template = template.replace("{date}", new Date().toLocaleDateString());
29+
// replace template {day} with current day
30+
template = template.replace("{day}", new Date().toLocaleString('en-us', { weekday: 'long' }));
31+
32+
return template;
33+
}
34+
2435
export function groupMessagesByConversation(messages: any[]) {
2536
// check if messages are in even numbers if not remove the last message
2637
if (messages.length % 2 !== 0) {
@@ -109,12 +120,17 @@ export const createChain = ({
109120
retriever,
110121
response_template,
111122
}: {
112-
llm: BaseLanguageModel<any> | BaseChatModel<any> ;
123+
llm: BaseLanguageModel<any> | BaseChatModel<any>;
113124
question_llm: BaseLanguageModel<any> | BaseChatModel<any>;
114125
retriever: Runnable;
115126
question_template: string;
116127
response_template: string;
117128
}) => {
129+
130+
question_template = updateTemplateVariables(question_template);
131+
132+
response_template = updateTemplateVariables(response_template);
133+
118134
const retrieverChain = createRetrieverChain(
119135
question_llm,
120136
retriever,

‎server/src/cron/index.ts

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
import { PrismaClient } from "@prisma/client";
2+
import { getSettings } from "../utils/common";
3+
import { queue } from "../queue/q";
4+
const prisma = new PrismaClient();
5+
6+
async function processDatasourceCron() {
7+
try {
8+
await prisma.$connect();
9+
const setting = await getSettings(prisma);
10+
11+
if (!setting.refetchDatasource) {
12+
return;
13+
}
14+
15+
console.log("[CRON] Processing datasource cron");
16+
17+
18+
const dataSources = await prisma.botSource.findMany({
19+
where: {
20+
bot: {
21+
autoSyncDataSources: true
22+
},
23+
type: {
24+
in: [
25+
"website",
26+
"crawl",
27+
"sitemap",
28+
]
29+
}
30+
},
31+
include: {
32+
bot: true
33+
}
34+
})
35+
36+
for (const dataSource of dataSources) {
37+
38+
await prisma.botDocument.deleteMany({
39+
where: {
40+
botId: dataSource.botId,
41+
sourceId: dataSource.id,
42+
},
43+
});
44+
await queue.add(
45+
"process",
46+
[
47+
{
48+
...dataSource,
49+
embedding: dataSource.bot.embedding,
50+
},
51+
],
52+
{
53+
jobId: dataSource.id,
54+
removeOnComplete: true,
55+
removeOnFail: true,
56+
}
57+
);
58+
}
59+
60+
61+
console.log("[CRON] Finished processing datasource cron");
62+
63+
} catch (error) {
64+
console.error(error);
65+
} finally {
66+
await prisma.$disconnect();
67+
}
68+
}
69+
70+
export { processDatasourceCron };

‎server/src/handlers/api/v1/admin/type.ts

+1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ export type UpdateDialoqbaseSettingsRequest = {
55
allowUserToRegister: boolean;
66
usePuppeteerFetch: boolean;
77
fileUploadSizeLimit: number;
8+
refetchDatasource: boolean;
89
};
910
};
1011

‎server/src/handlers/api/v1/bot/bot/get.handler.ts

+3-4
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ export const getDatasourceByBotId = async (
6363
where: {
6464
botId: id,
6565
type: {
66-
notIn: ["crawl", "sitemap"],
66+
notIn: ["crawl", "sitemap", "zip"],
6767
},
6868
},
6969
orderBy: {
@@ -77,7 +77,7 @@ export const getDatasourceByBotId = async (
7777
where: {
7878
botId: id,
7979
type: {
80-
notIn: ["crawl", "sitemap"],
80+
notIn: ["crawl", "sitemap", "zip"],
8181
},
8282
},
8383
});
@@ -119,10 +119,9 @@ export const getAllBotsHandler = async (
119119
reply: FastifyReply
120120
) => {
121121
const prisma = request.server.prisma;
122-
123122
const bots = await prisma.bot.findMany({
124123
where: {
125-
user_id: request.user?.is_admin ? undefined : request.user?.user_id
124+
user_id: request.user?.user_id
126125
},
127126
orderBy: {
128127
createdAt: "desc",

‎server/src/handlers/api/v1/bot/bot/upload.handler.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ export const createBotFileHandler = async (
124124
const path = `./uploads/${fileName}`;
125125
await fs.promises.mkdir("./uploads", { recursive: true });
126126
await pump(
127-
file.file,
127+
file.file,
128128
fs.createWriteStream(path) as any
129129
);
130130
const type = fileTypeFinder(file.mimetype);

‎server/src/plugins/bull.ts

+1-11
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import fp from "fastify-plugin";
22
import { FastifyPluginAsync } from "fastify";
33
import { Queue } from "bullmq";
4-
import { parseRedisUrl } from "../utils/redis";
4+
import { queue } from "../queue/q";
55
declare module "fastify" {
66
interface FastifyInstance {
77
queue: Queue<any>;
@@ -13,16 +13,6 @@ const bullPlugin: FastifyPluginAsync = fp(async (server, options) => {
1313
if (!redis_url) {
1414
throw new Error("Redis url is not defined");
1515
}
16-
const { host, port, password } = parseRedisUrl(redis_url);
17-
18-
const queue = new Queue("vector", {
19-
connection: {
20-
host,
21-
port,
22-
password,
23-
username: process?.env?.DB_REDIS_USERNAME,
24-
},
25-
});
2616

2717
server.decorate("queue", queue);
2818

0 commit comments

Comments
 (0)