Merge pull request #89 from n4ze3m/next

v0.0.32
n4ze3m · Sep 28, 2023 · 5eefd1e · 5eefd1e
2 parents fe5fcea + 4c5dcd4
commit 5eefd1e
Show file tree

Hide file tree

Showing 33 changed files with 619 additions and 31 deletions.
diff --git a/app/ui/package.json b/app/ui/package.json
@@ -1,7 +1,7 @@
 {
   "name": "app",
   "private": true,
-  "version": "0.0.31",
+  "version": "0.0.32",
   "type": "module",
   "scripts": {
     "dev": "vite",

diff --git a/app/ui/src/components/Bot/DS/DsTable.tsx b/app/ui/src/components/Bot/DS/DsTable.tsx
@@ -14,6 +14,7 @@ import { useParams } from "react-router-dom";
 import { NewDsForm } from "./NewDsForm";
 import React from "react";
 import { GithubIcon } from "../../Common/GithubIcon";
+import { YoutubeIcon } from "../../Common/Youtube";
 
 export const DsTable = ({
   data,
@@ -54,6 +55,8 @@ export const DsTable = ({
         return <VideoCameraIcon className="h-10 w-10 text-gray-400" />;
       case "mp3":
         return <PlayCircleIcon className="h-10 w-10 text-gray-400" />;
+      case "youtube":
+        return <YoutubeIcon className="h-10 w-10 text-gray-400" />;
       default:
         return <DocumentTextIcon className="h-10 w-10 text-gray-400" />;
     }

diff --git a/app/ui/src/components/Bot/Playground/Settings.tsx b/app/ui/src/components/Bot/Playground/Settings.tsx
@@ -33,6 +33,7 @@ export default function PlaygroundSettings({ close }: { close: () => void }) {
 
     setDefaultWebTextToSpeechLanguageWebAPI,
     setElevenLabsDefaultVoice,
+    setDefaultWebTextToSpeechLanguageType,
   } = useStoreMessage();
 
   React.useEffect(() => {
@@ -65,7 +66,7 @@ export default function PlaygroundSettings({ close }: { close: () => void }) {
       type: values.textToSpeechType,
       enabled: values.textToSpeechEnabled,
     });
-    setTextToSpeechEnabled(textToSpeechEnabled);
+    setDefaultWebTextToSpeechLanguageType(textToSpeechType);
     if (textToSpeechType === "web_api") {
       setDefaultWebTextToSpeechLanguageWebAPI(values.webApiDefaultVoice);
       localStorage.setItem(
@@ -79,6 +80,7 @@ export default function PlaygroundSettings({ close }: { close: () => void }) {
         values.elevenLabsDefaultVoice
       );
     }
+    setTextToSpeechEnabled(textToSpeechEnabled);
   };
 
   const { mutate: saveTextToSpeech, isLoading: isSavingTextToSpeech } =

diff --git a/app/ui/src/components/Bot/Settings/SettingsCard.tsx b/app/ui/src/components/Bot/Settings/SettingsCard.tsx
@@ -24,6 +24,7 @@ export const SettingsCard = ({
     questionGeneratorPrompt: string;
     streaming: boolean;
     showRef: boolean;
+    use_hybrid_search: boolean;
   };
 }) => {
   const [form] = Form.useForm();
@@ -127,6 +128,7 @@ export const SettingsCard = ({
             questionGeneratorPrompt: data.questionGeneratorPrompt,
             streaming: data.streaming,
             showRef: data.showRef,
+            use_hybrid_search: data.use_hybrid_search,
           }}
           form={form}
           requiredMark={false}
@@ -257,6 +259,36 @@ export const SettingsCard = ({
                     placeholder=""
                   />
                 </Form.Item>
+
+                <Form.Item
+                  label={
+                    <span className="font-medium text-gray-800 text-sm">
+                      Question Generator Prompt 
+                    </span>
+                  }
+                  name="questionGeneratorPrompt"
+                  rules={[
+                    {
+                      required: true,
+                      message: "Please input a prompt!",
+                    },
+                  ]}
+                >
+                  <textarea
+                    className="mt-1 block w-full shadow-sm sm:text-sm focus:ring-indigo-500 focus:border-indigo-500 border-gray-300 rounded-md"
+                    rows={5}
+                    placeholder=""
+                  />
+                </Form.Item>
+
+                <Form.Item
+                  name="use_hybrid_search"
+                  label="Use Hybrid Search Retrieval (Beta)"
+                  valuePropName="checked"
+                  tooltip="This will use the hybrid search retrieval method instead of the default semantic search retrieval method. Only work on playground ui."
+                >
+                  <Switch />
+                </Form.Item>
               </div>
             </div>
 

diff --git a/app/ui/src/components/Common/BotForm.tsx b/app/ui/src/components/Common/BotForm.tsx
@@ -21,6 +21,7 @@ import { availableEmbeddingTypes } from "../../utils/embeddings";
 import { availableChatModels } from "../../utils/chatModels";
 import { SpiderIcon } from "./SpiderIcon";
 import { GithubIcon } from "./GithubIcon";
+import { YoutubeIcon } from "./Youtube";
 
 type Props = {
   createBot: (values: any) => void;
@@ -90,7 +91,7 @@ export const BotForm = ({
     {
       id: 2,
       value: "file",
-      title: "File (beta)",
+      title: "File",
       icon: DocumentArrowUpIcon,
       formComponent: (
         <>
@@ -183,7 +184,7 @@ export const BotForm = ({
     {
       id: 4,
       value: "crawl",
-      title: "Crawler (beta)",
+      title: "Crawler",
       icon: SpiderIcon,
       formComponent: (
         <>
@@ -254,7 +255,7 @@ export const BotForm = ({
     {
       id: 6,
       value: "github",
-      title: "GitHub (beta)",
+      title: "GitHub",
       icon: GithubIcon,
       formComponent: (
         <>
@@ -323,6 +324,50 @@ export const BotForm = ({
         </>
       ),
     },
+    {
+      id: 7,
+      value: "youtube",
+      title: "Youtube",
+      icon: YoutubeIcon,
+      formComponent: (
+        <>
+          <Form.Item
+            name="content"
+            rules={[
+              {
+                required: true,
+                message: "Please enter a valid youtube URL",
+              },
+              {
+                pattern: new RegExp(
+                  /^(https?:\/\/)?(www\.)?(youtube\.com|youtu\.be)\/.+$/
+                ),
+                message: "Please enter a valid youtube URL",
+              },
+            ]}
+          >
+            <input
+              type="url"
+              placeholder="Enter the youtube URL"
+              className=" block w-full shadow-sm sm:text-sm focus:ring-indigo-500 focus:border-indigo-500 border-gray-300 rounded-md"
+            />
+          </Form.Item>
+
+          <p className="text-sm text-gray-500">
+            If you find any issues, please report them on{" "}
+            <a
+              href="https://github.com/n4ze3m/dialoqbase/issues/new?title=Github%20issue&labels=bug"
+              target="_blank"
+              rel="noreferrer"
+              className="font-medium text-indigo-600 hover:text-indigo-500"
+            >
+              GitHub
+            </a>
+            .
+          </p>
+        </>
+      ),
+    }
   ]);
 
   const [selectedSource, _setSelectedSource] = React.useState<any>(
@@ -359,7 +404,7 @@ export const BotForm = ({
           Select a data source
         </RadioGroup.Label>
 
-        <div className="mt-4 grid grid-cols-1 gap-y-6 sm:grid-cols-2 sm:gap-x-4">
+        <div className="mt-4 grid grid-cols-1 gap-y-6 sm:grid-cols-3 sm:gap-x-4">
           {availableSources.map((source) => (
             <RadioGroup.Option
               key={source.id}

diff --git a/app/ui/src/components/Common/Youtube.tsx b/app/ui/src/components/Common/Youtube.tsx
@@ -0,0 +1,25 @@
+import React from "react";
+
+export const YoutubeIcon = React.forwardRef<
+  SVGSVGElement,
+  React.SVGProps<SVGSVGElement>
+>((props, ref) => {
+  return (
+    <svg
+      xmlns="http://www.w3.org/2000/svg"
+      width="24"
+      height="24"
+      fill="none"
+      stroke="currentColor"
+      strokeLinecap="round"
+      strokeLinejoin="round"
+      strokeWidth="2"
+      viewBox="0 0 24 24"
+      ref={ref}
+      {...props}
+    >
+      <path d="M2.5 17a24.12 24.12 0 010-10 2 2 0 011.4-1.4 49.56 49.56 0 0116.2 0A2 2 0 0121.5 7a24.12 24.12 0 010 10 2 2 0 01-1.4 1.4 49.55 49.55 0 01-16.2 0A2 2 0 012.5 17"></path>
+      <path d="M10 15l5-3-5-3z"></path>
+    </svg>
+  );
+});
diff --git a/app/ui/src/routes/bot/settings.tsx b/app/ui/src/routes/bot/settings.tsx
@@ -25,6 +25,7 @@ export default function BotSettingsRoot() {
           questionGeneratorPrompt: string;
           streaming: boolean;
           showRef: boolean;
+          use_hybrid_search: boolean;
         };
       };
     },

diff --git a/app/ui/src/routes/new/root.tsx b/app/ui/src/routes/new/root.tsx
@@ -69,7 +69,7 @@ export default function NewRoot() {
             Create a new bot
           </h2>
         </div>
-        <div className="mt-8 sm:mx-auto sm:w-full sm:max-w-md">
+        <div className="mt-8 sm:mx-auto sm:w-full sm:max-w-lg">
           <div className="bg-white py-8 px-4 shadow sm:rounded-lg sm:px-10">
             <BotForm
               showEmbeddingAndModels={true}

diff --git a/app/ui/src/utils/chatModels.ts b/app/ui/src/utils/chatModels.ts
@@ -42,6 +42,10 @@ export const availableChatModels = [
     value: "llama-v2-34b-code-instruct-w8a16",
     label: "Llama v2 34B Code Instruct int8 (Fireworks)",
   },
+  {
+    value: "mistral-7b-instruct-4k",
+    label: "Mistral 7B Instruct 4K (Fireworks)",
+  }
   // {
   //   value: "falcon-7b-instruct-inference",
   //   label: "Falcon 7B Instruct (HuggingFace Inference API)",
@@ -63,7 +67,8 @@ export const streamingSupportedModels = [
   "llama-v2-13b-chat-w8a16",
   "llama-v2-13b-code-instruct",
   "llama-v2-34b-code-instruct-w8a16",
-  "gpt-3.5-turbo-instruct"
+  "gpt-3.5-turbo-instruct",
+  "mistral-7b-instruct-4k",
 ];
 
 export const isStreamingSupported = (model: string) => {

diff --git a/app/ui/src/utils/filename.ts b/app/ui/src/utils/filename.ts
@@ -1,3 +1,3 @@
-export function removeUUID(filename:string) {
-    return filename.replace(/^\w{8}-\w{4}-\w{4}-\w{4}-\w{12}-/, '');
-  }
+export function removeUUID(filename: string) {
+  return filename.replace(/^\w{8}-\w{4}-\w{4}-\w{4}-\w{12}-/, "");
+}
diff --git a/app/ui/src/utils/sources.tsx b/app/ui/src/utils/sources.tsx
@@ -1,5 +1,6 @@
 import { GithubIcon } from "../components/Common/GithubIcon";
 import { SpiderIcon } from "../components/Common/SpiderIcon";
+import { YoutubeIcon } from "../components/Common/Youtube";
 
 export const sources = {
   website: (
@@ -108,6 +109,7 @@ export const sources = {
   ),
   crawl: <SpiderIcon className="h-4 w-4" />,
   github: <GithubIcon className="h-4 w-4" />,
+  youtube: <YoutubeIcon className="h-4 w-4" />,
   mp3: (
     <svg
       xmlns="http://www.w3.org/2000/svg"

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "dialoqbase",
-  "version": "0.0.31",
+  "version": "0.0.32",
   "description": "Create chatbots with ease",
   "scripts": {
     "ui:dev": "pnpm run --filter ui dev",

diff --git a/server/package.json b/server/package.json
@@ -64,7 +64,8 @@
     "pdfjs-dist": "^3.7.107",
     "pubsub-js": "^1.9.4",
     "unique-names-generator": "^4.7.1",
-    "wavefile": "^11.0.0"
+    "wavefile": "^11.0.0",
+    "ytdl-core": "^4.11.5"
   },
   "devDependencies": {
     "@types/bcryptjs": "^2.4.2",

diff --git a/server/prisma/migrations/q_11/migration.sql b/server/prisma/migrations/q_11/migration.sql
@@ -0,0 +1,42 @@
+-- New Hybrid Search Query
+create or replace function kw_match_documents(query_text text, bot_id text, match_count int)
+returns table (id int, content text, metadata jsonb, similarity real)
+as $$
+begin
+    return query execute
+    format('select bt.id, bt.content, bt.metadata, ts_rank(to_tsvector(bt.content), plainto_tsquery($1)) as similarity
+    from "BotDocument" AS bt
+    where to_tsvector(bt.content) @@ plainto_tsquery($1) and bt."botId" = $2::text
+    order by similarity desc
+    limit $3')
+    using query_text, bot_id, match_count;
+end;
+$$ language plpgsql;
+-- New Search Query
+CREATE OR REPLACE FUNCTION similarity_search_v2 (
+  query_embedding vector,
+  botId text,
+  match_count int DEFAULT null
+) RETURNS TABLE (
+  id int,
+  content text,
+  metadata jsonb,
+  similarity float
+)
+LANGUAGE plpgsql
+AS $$
+BEGIN
+  RETURN QUERY
+  SELECT
+    bt.id,
+    bt.content,
+    bt.metadata,
+    1 - (bt.embedding <=> query_embedding) AS similarity
+  FROM "BotDocument" AS bt
+  WHERE bt."botId" = similarity_search_v2.botId -- Check botId
+  ORDER BY bt.embedding <=> query_embedding
+  LIMIT match_count;
+END;
+$$;
+-- AlterTable
+ALTER TABLE "Bot" ADD COLUMN "use_hybrid_search" BOOLEAN NOT NULL DEFAULT false;
diff --git a/server/prisma/schema.prisma b/server/prisma/schema.prisma
@@ -27,6 +27,7 @@ model Bot {
   text_to_voice_enabled       Boolean          @default(false)
   text_to_voice_type          String           @default("web_api")
   text_to_voice_type_metadata Json             @default("{}") @db.Json
+  use_hybrid_search           Boolean          @default(false)
   BotAppearance               BotAppearance[]
   document                    BotDocument[]
   BotIntegration              BotIntegration[]

diff --git a/server/src/integration/handlers/discord.handler.ts b/server/src/integration/handlers/discord.handler.ts
@@ -89,6 +89,7 @@ export const discordBotHandler = async (
 
     return bot_response;
   } catch (error) {
+    console.log(error);
     return "Opps! Something went wrong";
   }
 };
@@ -123,6 +124,7 @@ export const clearDiscordChatHistory = async (
 
     return "Chat history cleared";
   } catch (error) {
+    console.log(error);
     return "Opps! Something went wrong";
   }
 };
diff --git a/server/src/integration/handlers/telegram.handler.ts b/server/src/integration/handlers/telegram.handler.ts
@@ -89,6 +89,7 @@ export const telegramBotHandler = async (
 
     return bot_response;
   } catch (error) {
+    console.log(error);
     return "Opps! Something went wrong";
   }
 };
diff --git a/server/src/integration/handlers/whatsapp.handler.ts b/server/src/integration/handlers/whatsapp.handler.ts
@@ -99,6 +99,7 @@ export const whatsappBotHandler = async (
 
     return bot_response;
   } catch (error) {
+    console.log(error);
     return "Opps! Something went wrong";
   }
 };
@@ -132,6 +133,7 @@ export const clearHistoryWhatsapp = async (
 
     return "Chat history cleared";
   } catch (error) {
+    console.log(error);
     return "Opps! Something went wrong";
   }
 };