Merge pull request #163 from n4ze3m/next

v1.4.5

README.md

@@ -93,6 +93,8 @@ _Important: After the first login, remember to change the default credentials._
 
 - Utilize PostgreSQL for vector search and storing the knowledge base.
 
+- Use any language models or embedding models you want
+
 ## Stack 📚
 
 - [React](https://reactjs.org/)

app/ui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "app",
   "private": true,
-  "version": "1.4.4",
+  "version": "1.4.5",
   "type": "module",
   "scripts": {
     "dev": "vite",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "dialoqbase",
-  "version": "1.4.4",
+  "version": "1.4.5",
   "description": "Create chatbots with ease",
   "scripts": {
     "ui:dev": "pnpm run --filter ui dev",

server/package.json

@@ -20,7 +20,7 @@
     "dev:start": "fastify start --ignore-watch=uploads,.ts$ -w  -P dist/app.js"
   },
   "keywords": [],
-  "author": "", 
+  "author": "",
   "license": "MIT",
   "dependencies": {
     "@fastify/autoload": "^5.0.0",
@@ -34,6 +34,7 @@
     "@ffmpeg.wasm/core-mt": "^0.12.0",
     "@ffmpeg.wasm/main": "^0.12.0",
     "@google-ai/generativelanguage": "^0.2.0",
+    "@grammyjs/files": "^1.0.4",
     "@huggingface/inference": "1",
     "@prisma/client": "5",
     "@slack/bolt": "^3.13.2",
@@ -70,9 +71,9 @@
     "pdfjs-dist": "^3.7.107",
     "pubsub-js": "^1.9.4",
     "sitemapper": "^3.2.6",
+    "ts-node": "^10.9.1",
     "unique-names-generator": "^4.7.1",
     "wavefile": "^11.0.0",
-    "ts-node": "^10.9.1",
     "ytdl-core": "^4.11.5"
   },
   "devDependencies": {
@@ -88,4 +89,4 @@
   "prisma": {
     "seed": "ts-node prisma/seed.ts"
   }
-}
+}

server/prisma/seed.ts

@@ -1,7 +1,15 @@
 import { PrismaClient } from "@prisma/client";
 const prisma = new PrismaClient();
 
-const MODELS = [
+const MODELS: {
+  name: string;
+  model_id: string;
+  model_type: string;
+  model_provider?: string;
+  stream_available?: boolean;
+  local_model?: boolean;
+  config?: string;
+}[] = [
   {
     name: "Claude 2.1 (Anthropic)",
     model_id: "claude-2.1-dbase",
@@ -20,6 +28,24 @@ const MODELS = [
     local_model: false,
     config: "{}",
   },
+  {
+    name: "Yi 34B 200k (Fireworks)",
+    model_id: "accounts/fireworks/models/yi-34b-200k",
+    model_type: "instruct",
+    stream_available: true,
+    model_provider: "Fireworks",
+    local_model: false,
+    config: "{}",
+  },
+  {
+    model_id: "accounts/fireworks/models/yi-34b-200k-capybara",
+    name: "Capybara 34B (Fireworks)",
+    model_type: "chat",
+    stream_available: true,
+    model_provider: "Fireworks",
+    local_model: false,
+    config: "{}",
+  },
 ];
 
 const newModels = async () => {

server/src/integration/handlers/telegram.handler.ts

@@ -38,7 +38,7 @@ export const telegramBotHandler = async (
       human: message.human,
       ai: message.bot,
     }));
-    
+
     if (history.length > 20) {
       history.splice(0, history.length - 20);
     }

server/src/integration/handlers/utils/audio-to-text.ts

@@ -0,0 +1,36 @@
+import {
+  piplelineTransformer,
+  //@ts-ignore
+} from "../../../utils/pipleline.js";
+import { WaveFile } from "wavefile";
+
+function _convert(audio: Buffer) {
+  const wav = new WaveFile(audio);
+  wav.toBitDepth("32f");
+  wav.toSampleRate(16000);
+  let audioData = wav.getSamples();
+  if (Array.isArray(audioData)) {
+    console.log(
+      "Multiple channels detected. Selecting the first channel only."
+    );
+    audioData = audioData[0];
+  }
+  return audioData;
+}
+
+export const convertTextToAudio = async (audioData: Buffer) => {
+  const pipeline = await piplelineTransformer();
+  const transcriber = await pipeline(
+    "automatic-speech-recognition",
+    process.env.WHISPER_MODEL || "distil-whisper/distil-medium.en"
+  );
+
+  const audio = _convert(Buffer.from(audioData));
+  let output = (await transcriber(audio, {
+    chunk_length_s: 30,
+  })) as {
+    text: string;
+  };
+
+  return output;
+};

server/src/integration/telegram.ts

@@ -1,17 +1,21 @@
-import { Bot } from "grammy";
+import { Bot, Context } from "grammy";
 import axios from "axios";
 import {
   deleteTelegramChatHistory,
   telegramBotHandler,
   welcomeMessage,
 } from "./handlers/telegram.handler";
-
+import { convertTextToAudio } from "./handlers/utils/audio-to-text";
+import { FileFlavor, hydrateFiles } from "@grammyjs/files";
+import * as fs from "fs/promises";
+import { convertOggToWave } from "../utils/ffmpeg";
+type DialoqBaseContext = FileFlavor<Context>;
 export default class TelegramBot {
   static get clients() {
     return this._clients.values();
   }
 
-  private static _clients: Map<string, Bot> = new Map();
+  private static _clients: Map<string, any> = new Map();
 
   static totolClients() {
     return this._clients.size;
@@ -23,7 +27,9 @@ export default class TelegramBot {
         await this.disconnect(identifier);
       }
 
-      const bot = new Bot(token);
+      const bot = new Bot<DialoqBaseContext>(token);
+      bot.api.config.use(hydrateFiles(bot.token));
+
       await bot.api.setMyCommands([
         { command: "start", description: "Start the bot" },
         { command: "ping", description: "Ping the bot" },
@@ -70,6 +76,37 @@ export default class TelegramBot {
         return await ctx.reply(message);
       });
 
+      bot.on("message:voice", async (ctx) => {
+        try {
+          if (ctx.chat.type !== "private") {
+            return ctx.reply("I can only work in private chats.");
+          }
+
+          await ctx.replyWithChatAction("typing");
+
+          const file = await ctx.getFile();
+          const path = await file.download();
+
+          const audioWav = await convertOggToWave(path);
+          const audio = await fs.readFile(audioWav);
+
+          const response = await convertTextToAudio(audio);
+
+          const user_id = ctx.from.id;
+
+          const message = await telegramBotHandler(
+            identifier,
+            response.text,
+            user_id
+          );
+
+          return await ctx.reply(message);
+        } catch (error) {
+          console.log(error);
+          return await ctx.reply("Opps! Something went wrong");
+        }
+      });
+
       bot.start();
       bot.catch((err) => console.log(`${identifier} error: ${err}`));
       this._clients.set(identifier, bot);

server/src/utils/ffmpeg.ts

@@ -22,7 +22,7 @@ export const convertMp4ToWave = async (file_path: string) => {
     "1",
     "-ar",
     "16000",
-    "./audio.wav",
+    "./audio.wav"
   );
   const data = ffmpeg.FS("readFile", "./audio.wav");
   const new_file_path = file_path.replace(".mp4", ".wav");
@@ -44,7 +44,7 @@ export const convertMp3ToWave = async (file_path: string) => {
     "1",
     "-ar",
     "16000",
-    "./audio.wav",
+    "./audio.wav"
   );
   const data = ffmpeg.FS("readFile", "./audio.wav");
   const new_file_path = file_path.replace(".mp3", ".wav");
@@ -53,6 +53,28 @@ export const convertMp3ToWave = async (file_path: string) => {
   return new_file_path;
 };
 
+export const convertOggToWave = async (file_path: string) => {
+  await _init();
+  const audio = await fs.readFile(file_path);
+  ffmpeg.FS("writeFile", "./audio.ogg", audio);
+  await ffmpeg.run(
+    "-i",
+    "./audio.ogg",
+    "-acodec",
+    "pcm_s16le",
+    "-ac",
+    "1",
+    "-ar",
+    "16000",
+    "./audio.wav"
+  );
+
+  const data = ffmpeg.FS("readFile", "./audio.wav");
+  const new_file_path = file_path.replace(".ogg", ".wav");
+  await fs.writeFile(new_file_path, data);
+  ffmpeg.exit();
+  return new_file_path;
+};
 
 export const convertMp3ToWaveFromBuffer = async (audio: Buffer) => {
   await _init();
@@ -66,9 +88,9 @@ export const convertMp3ToWaveFromBuffer = async (audio: Buffer) => {
     "1",
     "-ar",
     "16000",
-    "./audio.wav",
+    "./audio.wav"
   );
   const data = ffmpeg.FS("readFile", "./audio.wav");
   ffmpeg.exit();
   return data;
-};
+};

server/yarn.lock

@@ -497,6 +497,11 @@
   dependencies:
     google-gax "^3.5.8"
 
+"@grammyjs/files@^1.0.4":
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/@grammyjs/files/-/files-1.0.4.tgz#255d8e92bde0f43768840b80a230394f0a97dba6"
+  integrity sha512-GT7JWRH9cs5kIrBtUGwMwVEhb1w7Vtnc93UT3zM/HYigmeVd93wWaIqyuSYsZq1VIB+9CSk0oz+ZsnxK83z7tg==
+
 "@grammyjs/[email protected]":
   version "3.1.1"
   resolved "https://registry.yarnpkg.com/@grammyjs/types/-/types-3.1.1.tgz#dd80c018ffaf44bd89c56160edb3d9e86b84251f"