Merge pull request #103 from n4ze3m/next

v1.0.3 - rest api as data source

Author: n4ze3m

README.md

@@ -119,7 +119,6 @@ Dialoqbase is a side project and is not ready for production. It is still in the
 - [x] PDF (beta)
 - [x] Web crawler (beta)
 - [x] Microsoft Word documents (beta)
-- [ ] Sitemap
 - [x] Github repository
 - [x] mp3
 - [x] mp4
@@ -132,7 +131,8 @@ and more...
 - [x] OpenAI
 - [x] Anthropic
 - [x] Google chat-bison-001
-- [x] fireworks.ai's llama 2 models api
+- [x] fireworks.ai's llama 2 models
+- [x] fireworks.ai's mistral
 
 ### Embedding models
 
@@ -152,11 +152,9 @@ need more ? create an issue...
 - [x] Generate responses
 - [x] Vector search
 - [x] Bot embed script for websites
-- [ ] Supabase Integration
 - [x] Prompt Editor
 - [ ] Chatbot API
 - [x] Chatbot theme editor
-- [ ] Chatbot analytics
 - [x] Streaming responses
 
 ### Integrations 
@@ -166,8 +164,6 @@ need more ? create an issue...
 - [x] Discord (beta)
 - [ ] Slack
 - [X] Whatsapp (experimental)
-- [ ] Facebook Messenger
-- [ ] Instagram
 
 need more ? create an issue...
 

app/ui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "app",
   "private": true,
-  "version": "1.0.2",
+  "version": "1.0.3",
   "type": "module",
   "scripts": {
     "dev": "vite",

app/ui/src/components/Bot/DS/DsTable.tsx

@@ -13,8 +13,9 @@ import api from "../../../services/api";
 import { useParams } from "react-router-dom";
 import { NewDsForm } from "./NewDsForm";
 import React from "react";
-import { GithubIcon } from "../../Common/GithubIcon";
-import { YoutubeIcon } from "../../Common/Youtube";
+import { GithubIcon } from "../../Icons/GithubIcon";
+import { YoutubeIcon } from "../../Icons/YoutubeIcon";
+import { ApiIcon } from "../../Icons/ApiIcon";
 
 export const DsTable = ({
   data,
@@ -57,6 +58,8 @@ export const DsTable = ({
         return <PlayCircleIcon className="h-10 w-10 text-gray-400" />;
       case "youtube":
         return <YoutubeIcon className="h-10 w-10 text-gray-400" />;
+      case "rest":
+        return <ApiIcon className="h-10 w-10 text-gray-400" />;
       default:
         return <DocumentTextIcon className="h-10 w-10 text-gray-400" />;
     }

app/ui/src/components/Common/BotForm.tsx

@@ -19,9 +19,10 @@ import {
 import React from "react";
 import { availableEmbeddingTypes } from "../../utils/embeddings";
 import { availableChatModels } from "../../utils/chatModels";
-import { SpiderIcon } from "./SpiderIcon";
-import { GithubIcon } from "./GithubIcon";
-import { YoutubeIcon } from "./Youtube";
+import { SpiderIcon } from "../Icons/SpiderIcon";
+import { GithubIcon } from "../Icons/GithubIcon";
+import { YoutubeIcon } from "../Icons/YoutubeIcon";
+import { ApiIcon } from "../Icons/ApiIcon";
 
 type Props = {
   createBot: (values: any) => void;
@@ -42,6 +43,8 @@ export const BotForm = ({
   form,
   showEmbeddingAndModels,
 }: Props) => {
+  const embeddingType = Form.useWatch("embedding", form);
+
   const [availableSources] = React.useState([
     {
       id: 1,
@@ -365,14 +368,70 @@ export const BotForm = ({
         </>
       ),
     },
+    {
+      id: 8,
+      value: "rest",
+      title: "REST API",
+      icon: ApiIcon,
+      formComponent: (
+        <>
+          <Row gutter={24}>
+            <Col span={6}>
+              <Form.Item
+                name={["options", "method"]}
+                rules={[
+                  {
+                    required: true,
+                    message: "Please select a method",
+                  },
+                ]}
+              >
+                <Select
+                  size="large"
+                  options={[
+                    {
+                      label: "GET",
+                      value: "get",
+                    },
+                    {
+                      label: "POST",
+                      value: "post",
+                    },
+                  ]}
+                />
+              </Form.Item>
+            </Col>
+            <Col span={18}>
+              <Form.Item
+                name="content"
+                rules={[
+                  {
+                    required: true,
+                    message: "Please enter a valid REST API URL",
+                  },
+                  {
+                    pattern: new RegExp(/^(https?:\/\/)?(www\.)?(.+)\.(.+)$/),
+                    message: "Please enter a valid REST API URL",
+                  },
+                ]}
+              >
+                <input
+                  type="url"
+                  placeholder="Enter the REST API URL"
+                  className=" block w-full shadow-sm sm:text-sm focus:ring-indigo-500 focus:border-indigo-500 border-gray-300 rounded-md"
+                />
+              </Form.Item>
+            </Col>
+          </Row>
+        </>
+      ),
+    },
   ]);
 
   const [selectedSource, _setSelectedSource] = React.useState<any>(
     showEmbeddingAndModels ? null : availableSources[0]
   );
 
-  const embeddingType = Form.useWatch("embedding", form);
-
   return (
     <Form
       layout="vertical"
@@ -387,6 +446,9 @@ export const BotForm = ({
         options: {
           branch: "main",
           is_private: false,
+          method: "get",
+          headers: "{}",
+          body: "{}",
         },
       }}
     >
@@ -446,6 +508,39 @@ export const BotForm = ({
 
       {selectedSource && selectedSource.formComponent}
 
+      {selectedSource && selectedSource.value === "rest" && (
+        <Row gutter={24}>
+          <Col span={12}>
+            <Form.Item
+              name={["options", "headers"]}
+              label="Headers"
+            >
+              <textarea
+                placeholder="Enter the headers"
+                className=" block w-full shadow-sm sm:text-sm focus:ring-indigo-500 focus:border-indigo-500 border-gray-300 rounded-md"
+              />
+            </Form.Item>
+          </Col>
+          <Col span={12}>
+            <Form.Item
+              name={["options", "body"]}
+              label="Body (JSON)"
+              rules={[
+                {
+                  required: true,
+                  message: "Please enter a valid JSON",
+                },
+              ]}
+            >
+              <textarea
+                placeholder="Enter the body"
+                className=" block w-full shadow-sm sm:text-sm focus:ring-indigo-500 focus:border-indigo-500 border-gray-300 rounded-md"
+              />
+            </Form.Item>
+          </Col>
+        </Row>
+      )}
+
       <Form.Item hidden={!showEmbeddingAndModels} noStyle>
         <Divider />
       </Form.Item>
@@ -462,7 +557,6 @@ export const BotForm = ({
           options={availableChatModels}
         />
       </Form.Item>
-
       <Form.Item
         hidden={!showEmbeddingAndModels}
         label={

app/ui/src/components/Icons/ApiIcon.tsx

@@ -0,0 +1,21 @@
+import React from "react";
+
+export const ApiIcon = React.forwardRef<
+  SVGSVGElement,
+  React.SVGProps<SVGSVGElement>
+>((props, ref) => {
+  return (
+    <svg
+      xmlns="http://www.w3.org/2000/svg"
+      version="1.1"
+      viewBox="0 0 511 511"
+      xmlSpace="preserve"
+      ref={ref}
+      {...props}
+    >
+      <path d="M492.168 309.579l-17.626-10.177c2.96-14.723 4.458-29.466 4.458-43.902 0-14.646-1.474-29.403-4.384-43.946l17.552-10.134c5.436-3.138 9.325-8.206 10.949-14.269s.791-12.396-2.348-17.832l-48-83.139A23.345 23.345 0 00438.5 75.231a23.349 23.349 0 00-17.833 2.348l-17.566 10.142C380.912 68.2 354.798 53.092 327 43.692V23.5C327 10.542 316.458 0 303.5 0h-96C194.542 0 184 10.542 184 23.5v20.193c-27.65 9.362-53.728 24.49-75.999 44.088L90.332 77.579a23.345 23.345 0 00-17.833-2.348A23.349 23.349 0 0058.23 86.18l-48 83.139c-3.139 5.436-3.972 11.769-2.348 17.832s5.513 11.131 10.949 14.269l17.626 10.177C33.499 226.32 32 241.063 32 255.5c0 14.644 1.474 29.401 4.384 43.945l-17.552 10.134c-11.222 6.479-15.08 20.879-8.602 32.102l48 83.139c6.479 11.221 20.879 15.08 32.102 8.601l17.565-10.142c22.19 19.521 48.303 34.629 76.103 44.03V487.5c0 12.958 10.542 23.5 23.5 23.5h96c12.958 0 23.5-10.542 23.5-23.5v-20.193c27.651-9.362 53.729-24.49 76-44.087l17.668 10.201c11.221 6.479 25.623 2.62 32.102-8.601l48-83.139c6.478-11.223 2.62-25.623-8.602-32.102zm-4.389 24.602l-48 83.138c-2.343 4.06-7.552 5.455-11.611 3.111l-22.392-12.928a7.5 7.5 0 00-8.842.989c-22.893 21.173-50.437 37.148-79.653 46.199a7.5 7.5 0 00-5.281 7.164V487.5c0 4.687-3.813 8.5-8.5 8.5h-96c-4.687 0-8.5-3.813-8.5-8.5v-25.645a7.501 7.501 0 00-5.28-7.164c-29.396-9.107-56.974-25.062-79.755-46.139a7.494 7.494 0 00-8.843-.99l-22.29 12.868c-4.06 2.343-9.268.948-11.611-3.111l-48-83.138c-2.343-4.059-.947-9.268 3.111-11.612l22.272-12.859a7.499 7.499 0 003.566-8.146C48.739 286.357 47 270.858 47 255.5c0-15.1 1.765-30.584 5.247-46.022a7.5 7.5 0 00-3.566-8.145L26.332 188.43a8.445 8.445 0 01-3.96-5.161 8.448 8.448 0 01.849-6.45l48-83.139a8.44 8.44 0 015.162-3.96 8.444 8.444 0 016.45.849l22.392 12.928a7.498 7.498 0 008.842-.989c22.894-21.173 50.437-37.148 79.653-46.199a7.5 7.5 0 005.281-7.164V23.5c0-4.687 3.813-8.5 8.5-8.5h96c4.687 0 8.5 3.813 8.5 8.5v25.645a7.501 7.501 0 005.28 7.164c29.395 9.106 56.973 25.061 79.755 46.139a7.5 7.5 0 008.843.99l22.29-12.869a8.445 8.445 0 016.45-.849 8.45 8.45 0 015.162 3.96l48 83.139a8.444 8.444 0 01.849 6.45 8.445 8.445 0 01-3.96 5.161l-22.272 12.859a7.5 7.5 0 00-3.566 8.146c3.431 15.206 5.17 30.704 5.17 46.065 0 15.1-1.765 30.584-5.247 46.021a7.501 7.501 0 003.566 8.145l22.349 12.903c4.057 2.344 5.453 7.553 3.109 11.612z"></path>
+      <path d="M255.5 104C171.962 104 104 171.963 104 255.5S171.962 407 255.5 407 407 339.037 407 255.5 339.038 104 255.5 104zm0 288C180.234 392 119 330.766 119 255.5S180.234 119 255.5 119 392 180.234 392 255.5 330.766 392 255.5 392z"></path>
+      <path d="M283.5 216h-28a7.5 7.5 0 00-7.5 7.5v64a7.5 7.5 0 0015 0V271h20.5c15.164 0 27.5-12.336 27.5-27.5S298.664 216 283.5 216zm0 40H263v-25h20.5c6.893 0 12.5 5.607 12.5 12.5s-5.607 12.5-12.5 12.5zM214.522 220.867A7.5 7.5 0 00207.5 216h-8a7.498 7.498 0 00-7.022 4.867l-24 64a7.5 7.5 0 0014.045 5.267L186.697 279h33.605l4.175 11.133a7.503 7.503 0 007.023 4.869 7.5 7.5 0 007.021-10.135l-23.999-64zM192.322 264l11.178-29.807L214.678 264h-22.356zM327.5 216a7.5 7.5 0 00-7.5 7.5v64a7.5 7.5 0 0015 0v-64a7.5 7.5 0 00-7.5-7.5zM309.152 87.3a175.962 175.962 0 0115.421 5.726 7.473 7.473 0 002.934.601 7.5 7.5 0 002.941-14.402 191.008 191.008 0 00-16.742-6.216 7.5 7.5 0 00-4.554 14.291zM100.45 339.904a7.5 7.5 0 00-13.168 7.184 192.173 192.173 0 0010.239 16.677 7.492 7.492 0 006.189 3.255 7.5 7.5 0 006.176-11.746 177.217 177.217 0 01-9.436-15.37zM240.14 431.341c-40.189-3.463-78.337-20.879-107.416-49.041a7.5 7.5 0 10-10.435 10.775c31.55 30.555 72.947 49.452 116.563 53.21a7.501 7.501 0 008.116-6.828 7.501 7.501 0 00-6.828-8.116zM363.425 97.287a7.5 7.5 0 00-8.464 12.384c34.844 23.813 60.049 59.248 70.972 99.776a7.503 7.503 0 007.237 5.55 7.5 7.5 0 007.246-9.453c-11.852-43.98-39.195-82.426-76.991-108.257z"></path>
+    </svg>
+  );
+});

app/ui/src/components/Common/GithubIcon.tsx renamed to app/ui/src/components/Icons/GithubIcon.tsx

@@ -1,11 +1,15 @@
 export const availableEmbeddingTypes = [
   { value: "openai", label: "OpenAI" },
-  { value: "tensorflow", label: "Tensorflow" },
+  { value: "tensorflow", label: "Tensorflow (cpu)" },
   { value: "cohere", label: "Cohere" },
   { value: "huggingface-api", label: "HuggingFace (Inference)" },
   {
     value: "transformer",
-    label: "all-MiniLM-L6-v2 (xenova/transformers)",
+    label: "Xenova/all-MiniLM-L6-v2 (cpu)",
+  },
+  {
+    value: "supabase",
+    label: "Supabase/gte-small (cpu)",
   },
   {
     value: "google-gecko",

app/ui/src/components/Common/SpiderIcon.tsx renamed to app/ui/src/components/Icons/SpiderIcon.tsx

@@ -1,6 +1,7 @@
-import { GithubIcon } from "../components/Common/GithubIcon";
-import { SpiderIcon } from "../components/Common/SpiderIcon";
-import { YoutubeIcon } from "../components/Common/Youtube";
+import { ApiIcon } from "../components/Icons/ApiIcon";
+import { GithubIcon } from "../components/Icons/GithubIcon";
+import { SpiderIcon } from "../components/Icons/SpiderIcon";
+import { YoutubeIcon } from "../components/Icons/YoutubeIcon";
 
 export const sources = {
   website: (
@@ -148,4 +149,5 @@ export const sources = {
       <rect width="8" height="6" x="2" y="12" rx="1"></rect>
     </svg>
   ),
+  rest: <ApiIcon className="h-4 w-4" />,
 };

app/ui/src/components/Common/Youtube.tsx renamed to app/ui/src/components/Icons/YoutubeIcon.tsx

@@ -1,6 +1,6 @@
 {
   "name": "dialoqbase",
-  "version": "1.0.2",
+  "version": "1.0.3",
   "description": "Create chatbots with ease",
   "scripts": {
     "ui:dev": "pnpm run --filter ui dev",

app/ui/src/utils/embeddings.ts

@@ -40,7 +40,7 @@
     "@tensorflow/tfjs-converter": "^4.7.0",
     "@tensorflow/tfjs-core": "^4.7.0",
     "@waylaidwanderer/fastify-sse-v2": "^3.1.0",
-    "@xenova/transformers": "^2.3.0",
+    "@xenova/transformers": "^2.6.2",
     "axios": "^1.4.0",
     "bcryptjs": "^2.4.3",
     "bull": "^4.10.4",
@@ -59,7 +59,7 @@
     "grammy": "^1.16.2",
     "ignore": "^5.2.4",
     "ioredis": "^5.3.2",
-    "langchain": "^0.0.151",
+    "langchain": "^0.0.160",
     "mammoth": "^1.6.0",
     "pdf-parse": "^1.1.1",
     "pdfjs-dist": "^3.7.107",

app/ui/src/utils/sources.tsx

@@ -18,26 +18,27 @@ export default class DiscordBot {
   static async connect(
     identifier: string,
     token: string,
-    slashCommands: string = "/hey",
+    command: string = "hey",
     slashCommandsDescription: string = "Say hey to the bot",
   ) {
     try {
       if (this._clients.has(identifier)) {
         await this.disconnect(identifier);
       }
-
       const bot = new Client({ intents: [GatewayIntentBits.Guilds] });
 
       this._clients.set(identifier, bot);
 
+      let slashCommands = command.replace(/[^a-zA-Z0-9]/g, "");
+
       bot.on("ready", async () => {
         console.log(`Logged in as ${bot.user?.tag}!`);
         const clientId = bot.user?.id;
         if (clientId) {
           await this.setCommand(
             token,
             clientId,
-            slashCommands,
+            slashCommands.replace(/[^a-zA-Z0-9]/g, ""),
             slashCommandsDescription,
           );
         }
@@ -131,7 +132,7 @@ ${bot_response}
   static async setCommand(
     token: string,
     clientId: string,
-    slashCommands: string = "/hey",
+    slashCommands: string = "hey",
     slashCommandsDescription: string = "Say hey to the bot",
   ) {
     try {