From 509e474dd60fa662b446715e83f1a7be9f4cd18b Mon Sep 17 00:00:00 2001 From: Gregor Adams <1148334+pixelass@users.noreply.github.com> Date: Tue, 28 May 2024 14:22:04 +0200 Subject: [PATCH] feat: assistant improvements (#324) ## Motivation - overall code section improvement - adds dark/light syntax highlighter theme - warp lines in code blocks - automatic chat titles - prepare the code for further extractions - improved markdown images - --- .../apps/assistant/components/chat-list.tsx | 172 ++++ .../apps/assistant/components/chat-name.tsx | 37 + .../components/data-type-selector.tsx | 43 + .../assistant/components/message-stack.tsx | 56 ++ .../apps/assistant/components/messages.tsx | 102 +++ .../assistant/components/model-selector.tsx | 77 ++ .../assistant/components/sticky-chat-list.tsx | 24 + .../components/streaming-message.tsx | 46 + src/client/apps/assistant/constants.ts | 27 + src/client/apps/assistant/hooks/assistant.tsx | 193 +++++ src/client/apps/assistant/icons.tsx | 13 + src/client/apps/assistant/index.tsx | 783 +++--------------- src/client/apps/assistant/types.ts | 6 + src/client/apps/assistant/utils.ts | 36 + src/client/apps/shared/markdown.tsx | 53 +- src/client/apps/shared/styled.ts | 6 +- src/electron/helpers/ipc/sdk/assistant.ts | 134 +-- 17 files changed, 1057 insertions(+), 751 deletions(-) create mode 100644 src/client/apps/assistant/components/chat-list.tsx create mode 100644 src/client/apps/assistant/components/chat-name.tsx create mode 100644 src/client/apps/assistant/components/data-type-selector.tsx create mode 100644 src/client/apps/assistant/components/message-stack.tsx create mode 100644 src/client/apps/assistant/components/messages.tsx create mode 100644 src/client/apps/assistant/components/model-selector.tsx create mode 100644 src/client/apps/assistant/components/sticky-chat-list.tsx create mode 100644 src/client/apps/assistant/components/streaming-message.tsx create mode 100644 src/client/apps/assistant/hooks/assistant.tsx create mode 100644 src/client/apps/assistant/icons.tsx create mode 100644 src/client/apps/assistant/types.ts create mode 100644 src/client/apps/assistant/utils.ts diff --git a/src/client/apps/assistant/components/chat-list.tsx b/src/client/apps/assistant/components/chat-list.tsx new file mode 100644 index 000000000..22b4eb4e5 --- /dev/null +++ b/src/client/apps/assistant/components/chat-list.tsx @@ -0,0 +1,172 @@ +import { useSDK } from "@captn/react/use-sdk"; +import DeleteForeverIcon from "@mui/icons-material/DeleteForever"; +import MoreVertIcon from "@mui/icons-material/MoreVert"; +import PushPinIcon from "@mui/icons-material/PushPin"; +import StarIcon from "@mui/icons-material/Star"; +import Dropdown from "@mui/joy/Dropdown"; +import IconButton from "@mui/joy/IconButton"; +import List from "@mui/joy/List"; +import ListItem from "@mui/joy/ListItem"; +import ListItemButton from "@mui/joy/ListItemButton"; +import ListItemDecorator from "@mui/joy/ListItemDecorator"; +import Menu from "@mui/joy/Menu"; +import MenuButton from "@mui/joy/MenuButton"; +import MenuItem from "@mui/joy/MenuItem"; +import type { ColorPaletteProp } from "@mui/joy/styles"; +import Typography from "@mui/joy/Typography"; +import { useTranslation } from "next-i18next"; +import type { Except } from "type-fest"; + +import { APP_ID } from "@/client/apps/assistant/constants"; +import type { ChatModel } from "@/shared/types/assistant"; + +export function ChatList({ + chats, + chatId, + color, + onDelete, + onChatSelect, +}: { + chatId: string; + chats: (Except & { id: string })[]; + color: ColorPaletteProp; + onDelete?(chatId: string): void; + onChatSelect(chatId: string): void; +}) { + const { t } = useTranslation(["labels", "common"]); + const { send } = useSDK(APP_ID, {}); + return ( + + {chats + .sort((a, b) => { + if (a.pinned && b.pinned) { + return 0; + } + + return a.pinned ? -1 : 1; + }) + .map(chat => ( + + + + + + { + send({ + action: "assistant:update", + payload: { + id: chat.id, + update: { pinned: !chat.pinned }, + }, + }); + }} + > + ({ + "--Icon-color": chat.pinned + ? theme.vars.palette.primary["500"] + : "currentColor", + })} + > + + {" "} + {t("labels:pin")} + + { + send({ + action: "assistant:update", + payload: { + id: chat.id, + update: { + favorite: !chat.favorite, + }, + }, + }); + }} + > + ({ + "--Icon-color": chat.favorite + ? theme.vars.palette.primary["500"] + : "currentColor", + })} + > + + {" "} + {t("labels:favorite")} + + { + if (onDelete) { + onDelete(chat.id); + } + }} + > + ({ + "--Icon-color": theme.vars.palette.red["500"], + })} + > + + {" "} + {t("labels:delete")} + + + + } + > + { + if (onChatSelect) { + onChatSelect(chat.id); + } + }} + > + {chat.label} + + + ))} + + ); +} diff --git a/src/client/apps/assistant/components/chat-name.tsx b/src/client/apps/assistant/components/chat-name.tsx new file mode 100644 index 000000000..34aa6a4e3 --- /dev/null +++ b/src/client/apps/assistant/components/chat-name.tsx @@ -0,0 +1,37 @@ +import Box from "@mui/joy/Box"; +import Input from "@mui/joy/Input"; +import type { ColorPaletteProp } from "@mui/joy/styles"; +import { useEffect, useState } from "react"; + +export function ChatName({ + name, + onUpdate, + color, +}: { + name: string; + onUpdate(_label: string): void; + color: ColorPaletteProp; +}) { + const [currentName, setCurrentName] = useState(name); + useEffect(() => { + setCurrentName(name); + }, [name]); + return ( + + { + setCurrentName(event.target.value); + }} + onBlur={() => { + if (onUpdate && currentName !== name) { + onUpdate(currentName); + } + }} + /> + + ); +} diff --git a/src/client/apps/assistant/components/data-type-selector.tsx b/src/client/apps/assistant/components/data-type-selector.tsx new file mode 100644 index 000000000..0c7c890e8 --- /dev/null +++ b/src/client/apps/assistant/components/data-type-selector.tsx @@ -0,0 +1,43 @@ +import FindInPageIcon from "@mui/icons-material/FindInPage"; +import List from "@mui/joy/List"; +import ListItem from "@mui/joy/ListItem"; +import ListItemButton from "@mui/joy/ListItemButton"; +import ListItemContent from "@mui/joy/ListItemContent"; +import ListItemDecorator from "@mui/joy/ListItemDecorator"; +import Switch from "@mui/joy/Switch"; +import { useTranslation } from "next-i18next"; +import type { ChangeEvent as ReactChangeEvent } from "react"; + +import { PopupButton } from "@/client/apps/shared/components"; +import type { DataTypeItem } from "@/shared/types/assistant"; + +export function DataTypeSelector({ + dataTypes, + onChange, +}: { + dataTypes: DataTypeItem[]; + onChange(id: string, checked?: boolean): void; +}) { + const { t } = useTranslation(["labels"]); + return ( + }> + + {dataTypes.map(dataType => ( + + + + ) => { + onChange(dataType.id, event.target.checked); + }} + /> + + {t(`labels:${dataType.type}`)} + + + ))} + + + ); +} diff --git a/src/client/apps/assistant/components/message-stack.tsx b/src/client/apps/assistant/components/message-stack.tsx new file mode 100644 index 000000000..2bd878028 --- /dev/null +++ b/src/client/apps/assistant/components/message-stack.tsx @@ -0,0 +1,56 @@ +import CloseIcon from "@mui/icons-material/Close"; +import Box from "@mui/joy/Box"; +import IconButton from "@mui/joy/IconButton"; +import Snackbar from "@mui/joy/Snackbar"; +import Typography from "@mui/joy/Typography"; +import { useTranslation } from "next-i18next"; + +export function MessageStack({ + provider, + isWarningOpen, + error, + onClose, +}: { + provider: string; + isWarningOpen: boolean; + error: string; + onClose(): void; +}) { + const { t } = useTranslation(["labels", "texts"]); + return ( + + + + + + } + > + {error} + + + {t("texts:enterCommonAPIKey", { provider })} + + + + ); +} diff --git a/src/client/apps/assistant/components/messages.tsx b/src/client/apps/assistant/components/messages.tsx new file mode 100644 index 000000000..ce79e292a --- /dev/null +++ b/src/client/apps/assistant/components/messages.tsx @@ -0,0 +1,102 @@ +import AssistantIcon from "@mui/icons-material/Assistant"; +import PersonIcon from "@mui/icons-material/Person"; +import Avatar from "@mui/joy/Avatar"; +import List from "@mui/joy/List"; +import ListItem from "@mui/joy/ListItem"; +import ListItemContent from "@mui/joy/ListItemContent"; +import ListItemDecorator from "@mui/joy/ListItemDecorator"; +import Sheet from "@mui/joy/Sheet"; +import type { ColorPaletteProp } from "@mui/joy/styles"; +import Tooltip from "@mui/joy/Tooltip"; +import { useTranslation } from "next-i18next"; + +import { StreamingMessage } from "@/client/apps/assistant/components/streaming-message"; +import { modelColors, modelNames } from "@/client/apps/assistant/constants"; +import { assistantIcons } from "@/client/apps/assistant/icons"; +import type { MessageModel } from "@/client/apps/assistant/types"; +import { Markdown } from "@/client/apps/shared/markdown"; + +export function Messages({ + messages, + model, + modelPrefix, + message, + color, +}: { + messages: MessageModel[]; + model: string; + modelPrefix: string; + message: string; + color: ColorPaletteProp; +}) { + const { t } = useTranslation(["labels"]); + return ( + + {messages.map(message_ => { + const [modelPrefix_] = (message_.model ?? "unknown").split("-"); + return ( + + ({ + position: "sticky", + top: theme.spacing(0.5), + })} + > + + + {message_.role === "user" ? ( + + ) : ( + assistantIcons[modelPrefix_] ?? + )} + + + + + + + + + + ); + })} + + + ); +} diff --git a/src/client/apps/assistant/components/model-selector.tsx b/src/client/apps/assistant/components/model-selector.tsx new file mode 100644 index 000000000..e67d899c2 --- /dev/null +++ b/src/client/apps/assistant/components/model-selector.tsx @@ -0,0 +1,77 @@ +import AssistantIcon from "@mui/icons-material/Assistant"; +import Avatar from "@mui/joy/Avatar"; +import Box from "@mui/joy/Box"; +import ListItemContent from "@mui/joy/ListItemContent"; +import ListItemDecorator from "@mui/joy/ListItemDecorator"; +import Option from "@mui/joy/Option"; +import Select from "@mui/joy/Select"; +import { useTranslation } from "next-i18next"; + +import { modelColors, modelNames, models } from "@/client/apps/assistant/constants"; +import { assistantIcons } from "@/client/apps/assistant/icons"; + +export function ModelSelector({ + model, + modelPrefix, + onModelSelect, +}: { + model: string; + modelPrefix: string; + onModelSelect(model: string): void; +}) { + const { t } = useTranslation(["labels", "common"]); + return ( + + ); +} diff --git a/src/client/apps/assistant/components/sticky-chat-list.tsx b/src/client/apps/assistant/components/sticky-chat-list.tsx new file mode 100644 index 000000000..560b08796 --- /dev/null +++ b/src/client/apps/assistant/components/sticky-chat-list.tsx @@ -0,0 +1,24 @@ +import AddCommentIcon from "@mui/icons-material/AddComment"; +import ListItem from "@mui/joy/ListItem"; +import ListItemButton from "@mui/joy/ListItemButton"; +import ListItemContent from "@mui/joy/ListItemContent"; +import ListItemDecorator from "@mui/joy/ListItemDecorator"; +import type { ColorPaletteProp } from "@mui/joy/styles"; +import Typography from "@mui/joy/Typography"; +import { useTranslation } from "next-i18next"; + +export function StickyChatList({ onNew, color }: { onNew(): void; color: ColorPaletteProp }) { + const { t } = useTranslation(["labels", "common"]); + return ( + + + + + + + {t("labels:newChat")} + + + + ); +} diff --git a/src/client/apps/assistant/components/streaming-message.tsx b/src/client/apps/assistant/components/streaming-message.tsx new file mode 100644 index 000000000..7ed5b63a3 --- /dev/null +++ b/src/client/apps/assistant/components/streaming-message.tsx @@ -0,0 +1,46 @@ +import AssistantIcon from "@mui/icons-material/Assistant"; +import Avatar from "@mui/joy/Avatar"; +import ListItem from "@mui/joy/ListItem"; +import ListItemContent from "@mui/joy/ListItemContent"; +import ListItemDecorator from "@mui/joy/ListItemDecorator"; +import Sheet from "@mui/joy/Sheet"; + +import { modelColors } from "@/client/apps/assistant/constants"; +import { assistantIcons } from "@/client/apps/assistant/icons"; +import { Markdown } from "@/client/apps/shared/markdown"; + +export function StreamingMessage({ + message, + model, + modelPrefix, +}: { + message: string; + model: string; + modelPrefix: string; +}) { + return ( + message && ( + + + + {assistantIcons[modelPrefix] ?? } + + + + + + + + + ) + ); +} diff --git a/src/client/apps/assistant/constants.ts b/src/client/apps/assistant/constants.ts index 232883553..216772a8c 100644 --- a/src/client/apps/assistant/constants.ts +++ b/src/client/apps/assistant/constants.ts @@ -1 +1,28 @@ +import type { ColorPaletteProp } from "@mui/joy/styles"; + export const APP_ID = "assistant"; +export const modelColors: Record = { + "gpt-3.5-turbo": "neutral", + "gpt-4": "teal", + "gpt-4-turbo": "violet", + "gpt-4o": "blue", + "claude-3-haiku-20240307": "neutral", + "claude-3-sonnet-20240229": "teal", + "claude-3-opus-20240229": "blue", + "mistral-small-latest": "neutral", + "mistral-large-latest": "teal", + "models/gemini-1.5-pro-latest": "neutral", +}; +export const modelNames: Record = { + "gpt-3.5-turbo": "GPT-3.5 Turbo", + "gpt-4": "GPT-4", + "gpt-4-turbo": "GPT-4 Turbo", + "gpt-4o": "GPT-4o", + "claude-3-haiku-20240307": "Claude Haiku", + "claude-3-sonnet-20240229": "Claude Sonnet", + "claude-3-opus-20240229": "Claude Opus", + "mistral-small-latest": "Mistral Small", + "mistral-large-latest": "Mistral Large", + "models/gemini-1.5-pro-latest": "Gemini 1.5 PRO", +}; +export const models = Object.entries(modelNames).map(([id, label]) => ({ id, label })); diff --git a/src/client/apps/assistant/hooks/assistant.tsx b/src/client/apps/assistant/hooks/assistant.tsx new file mode 100644 index 000000000..4435188ea --- /dev/null +++ b/src/client/apps/assistant/hooks/assistant.tsx @@ -0,0 +1,193 @@ +import { useSDK } from "@captn/react/use-sdk"; +import { useEffect, useRef, useState } from "react"; +import type { Except } from "type-fest"; +import { v4 } from "uuid"; + +import { APP_ID, models } from "@/client/apps/assistant/constants"; +import type { MessageModel } from "@/client/apps/assistant/types"; +import { getModelInfo } from "@/client/apps/assistant/utils"; +import { availableDataTypes } from "@/shared/assistant"; +import type { ChatModel } from "@/shared/types/assistant"; + +export function useAssistant() { + const [messages, setMessages] = useState([]); + const [message, setMesssage] = useState(""); + const [chatName, setChatName] = useState(""); + const [isGenerating, setIsGenerating] = useState(false); + const [input, setInput] = useState(""); + const [chatId, setChatId] = useState(""); + const [model, setModel] = useState(models[0].id); + const [error, setError] = useState(""); + const [chats, setChats] = useState<(Except & { id: string })[]>([]); + const [isWarningOpen, setIsWarningOpen] = useState(false); + const [dataTypes, setDataTypes] = useState(availableDataTypes); + + const firstRunReference = useRef(true); + const scrollContainerReference = useRef(null); + + const [modelPrefix] = model.split("-"); + + const { apiKeyGetter, provider } = getModelInfo(modelPrefix); + + const { send } = useSDK(APP_ID, { + onMessage(message) { + switch (message.action) { + case "assistant:nameSuggestion": { + const { chatName: chatName_ } = message.payload as { + chatName: string; + }; + if (chatName_) { + setChatName(chatName_); + } + + break; + } + + case "assistant:response": { + const { + output, + done, + model: model_, + label: chatLabel, + } = message.payload as { + output: string; + done: boolean; + model: string; + label?: string; + }; + if (done) { + setMesssage(""); + setMessages(previousState => [ + ...previousState, + { + role: "assistant", + content: output, + id: v4(), + model: model_, + }, + ]); + setIsGenerating(false); + if (chatLabel) { + setChatName(chatLabel); + } + } else { + setIsGenerating(true); + setMesssage(output); + } + + break; + } + + case "assistant:messages": { + const { chat } = message.payload as { + chat: Except & { messages: MessageModel[] }; + }; + firstRunReference.current = true; + setMessages(chat.messages); + setDataTypes( + availableDataTypes.map(dataType => { + const chatDataTypes = chat.dataTypes ?? []; + const overwrite = chatDataTypes.find( + dataType_ => dataType_.id === dataType.id + ); + return overwrite ?? dataType; + }) + ); + setModel(chat.model); + setChatName(chat.label); + + break; + } + + case "assistant:error": { + const { error: error_ } = message.payload as { + error: string; + }; + setError(error_); + + break; + } + + case "assistant:chats": { + const { chats: allChats } = message.payload as { + chats: (Except & { id: string })[]; + }; + setChats(allChats); + + break; + } + + default: { + break; + } + } + }, + }); + + useEffect(() => { + const container = scrollContainerReference.current; + if (container) { + const threshold = 200; + const isNearBottom = + container.scrollHeight - container.scrollTop - container.clientHeight < threshold; + + if (isNearBottom || firstRunReference.current) { + container.scrollTop = container.scrollHeight; + firstRunReference.current = false; + } + } + }, [messages, message]); + + useEffect(() => { + const container = scrollContainerReference.current; + if (container) { + container.scrollTop = container.scrollHeight; + } + + if (chatId) { + send({ action: "assistant:history", payload: { id: chatId } }); + } else { + setChatName("New Chat"); + setChatId(v4()); + } + }, [send, chatId]); + + useEffect(() => { + if (!apiKeyGetter) { + return; + } + + window.ipc.send(apiKeyGetter); + const unsubscribe = window.ipc.on(apiKeyGetter, (hasKey: boolean) => { + setIsWarningOpen(!hasKey); + }); + return () => { + unsubscribe(); + }; + }, [apiKeyGetter]); + + return { + error, + provider, + isWarningOpen, + setError, + setChatName, + setChatId, + chatId, + chats, + model, + setModel, + dataTypes, + setDataTypes, + modelPrefix, + scrollContainerReference, + chatName, + isGenerating, + setIsGenerating, + input, + setInput, + messages, + message, + setMessages, + }; +} diff --git a/src/client/apps/assistant/icons.tsx b/src/client/apps/assistant/icons.tsx new file mode 100644 index 000000000..5fda00a6c --- /dev/null +++ b/src/client/apps/assistant/icons.tsx @@ -0,0 +1,13 @@ +import GoogleIcon from "@mui/icons-material/Google"; +import type { ReactNode } from "react"; + +import AnthropicIcon from "@/client/atoms/icons/anthropic"; +import ChatGPTIcon from "@/client/atoms/icons/chat-gpt"; +import MistralIcon from "@/client/atoms/icons/mistral"; + +export const assistantIcons: Record = { + gpt: , + claude: , + mistral: , + "models/gemini": , +}; diff --git a/src/client/apps/assistant/index.tsx b/src/client/apps/assistant/index.tsx index 2d5ec820b..4bad32cbe 100644 --- a/src/client/apps/assistant/index.tsx +++ b/src/client/apps/assistant/index.tsx @@ -1,325 +1,86 @@ import { useSDK } from "@captn/react/use-sdk"; -import AddCommentIcon from "@mui/icons-material/AddComment"; import ArrowDownwardIcon from "@mui/icons-material/ArrowDownward"; -import AssistantIcon from "@mui/icons-material/Assistant"; -import CloseIcon from "@mui/icons-material/Close"; -import DeleteForeverIcon from "@mui/icons-material/DeleteForever"; -import FindInPageIcon from "@mui/icons-material/FindInPage"; -import GoogleIcon from "@mui/icons-material/Google"; -import MoreVertIcon from "@mui/icons-material/MoreVert"; -import PersonIcon from "@mui/icons-material/Person"; -import PushPinIcon from "@mui/icons-material/PushPin"; import SendIcon from "@mui/icons-material/Send"; -import StarIcon from "@mui/icons-material/Star"; -import Avatar from "@mui/joy/Avatar"; import Box from "@mui/joy/Box"; -import Dropdown from "@mui/joy/Dropdown"; import IconButton from "@mui/joy/IconButton"; -import Input from "@mui/joy/Input"; import List from "@mui/joy/List"; -import ListItem from "@mui/joy/ListItem"; -import ListItemButton from "@mui/joy/ListItemButton"; -import ListItemContent from "@mui/joy/ListItemContent"; -import ListItemDecorator from "@mui/joy/ListItemDecorator"; -import Menu from "@mui/joy/Menu"; -import MenuButton from "@mui/joy/MenuButton"; -import MenuItem from "@mui/joy/MenuItem"; -import Option from "@mui/joy/Option"; -import Select from "@mui/joy/Select"; import Sheet from "@mui/joy/Sheet"; -import Snackbar from "@mui/joy/Snackbar"; import type { ColorPaletteProp } from "@mui/joy/styles"; -import Switch from "@mui/joy/Switch"; import Textarea from "@mui/joy/Textarea"; -import Tooltip from "@mui/joy/Tooltip"; -import Typography from "@mui/joy/Typography"; import { useTranslation } from "next-i18next"; -import type { ChangeEvent as ReactChangeEvent, ReactNode } from "react"; -import { useEffect, useRef, useState } from "react"; -import type { Except } from "type-fest"; +import { useState } from "react"; import { v4 } from "uuid"; import { APP_ID } from "./constants"; -import { PopupButton } from "@/client/apps/shared/components"; -import { Markdown } from "@/client/apps/shared/markdown"; -import AnthropicIcon from "@/client/atoms/icons/anthropic"; -import ChatGPTIcon from "@/client/atoms/icons/chat-gpt"; -import MistralIcon from "@/client/atoms/icons/mistral"; -import { availableDataTypes } from "@/shared/assistant"; -import type { ChatModel } from "@/shared/types/assistant"; - -interface MessageModel { - id: string; - content: string; - role: "assistant" | "user"; - model?: string; -} - -export const modelColors: Record = { - "gpt-3.5-turbo": "neutral", - "gpt-4": "teal", - "gpt-4-turbo": "violet", - "gpt-4o": "blue", - "claude-3-haiku-20240307": "neutral", - "claude-3-sonnet-20240229": "teal", - "claude-3-opus-20240229": "blue", - "mistral-small-latest": "neutral", - "mistral-large-latest": "teal", - "models/gemini-1.5-pro-latest": "neutral", -}; - -export const modelNames: Record = { - "gpt-3.5-turbo": "GPT-3.5 Turbo", - "gpt-4": "GPT-4", - "gpt-4-turbo": "GPT-4 Turbo", - "gpt-4o": "GPT-4o", - "claude-3-haiku-20240307": "Claude Haiku", - "claude-3-sonnet-20240229": "Claude Sonnet", - "claude-3-opus-20240229": "Claude Opus", - "mistral-small-latest": "Mistral Small", - "mistral-large-latest": "Mistral Large", - "models/gemini-1.5-pro-latest": "Gemini 1.5 PRO", -}; - -const assistantIcons: Record = { - gpt: , - claude: , - mistral: , - "models/gemini": , -}; - -const models = Object.entries(modelNames).map(([id, label]) => ({ id, label })); - -export function ChatName({ - name, - onUpdate, - color, -}: { - name: string; - onUpdate(_label: string): void; - color: ColorPaletteProp; -}) { - const [currentName, setCurrentName] = useState(name); - useEffect(() => { - setCurrentName(name); - }, [name]); - return ( - - { - setCurrentName(event.target.value); - }} - onBlur={() => { - if (onUpdate && currentName !== name) { - onUpdate(currentName); - } - }} - /> - - ); -} +import { ChatList } from "@/client/apps/assistant/components/chat-list"; +import { ChatName } from "@/client/apps/assistant/components/chat-name"; +import { DataTypeSelector } from "@/client/apps/assistant/components/data-type-selector"; +import { MessageStack } from "@/client/apps/assistant/components/message-stack"; +import { Messages } from "@/client/apps/assistant/components/messages"; +import { ModelSelector } from "@/client/apps/assistant/components/model-selector"; +import { StickyChatList } from "@/client/apps/assistant/components/sticky-chat-list"; +import { useAssistant } from "@/client/apps/assistant/hooks/assistant"; export function Assistant({ color }: { color: ColorPaletteProp }) { const { t } = useTranslation(["labels", "common"]); - const [messages, setMessages] = useState([]); - const [message, setMesssage] = useState(""); - const [chatName, setChatName] = useState(""); - const [isGenerating, setIsGenerating] = useState(false); - const [input, setInput] = useState(""); - const [chatId, setChatId] = useState(""); - const [model, setModel] = useState(models[0].id); - const [error, setError] = useState(""); - const [chats, setChats] = useState<(Except & { id: string })[]>([]); - const [isWarningOpen, setIsWarningOpen] = useState(false); const [showScrollToBottom, setShowScrollToBottom] = useState(false); - const [dataTypes, setDataTypes] = useState(availableDataTypes); - - const firstRunReference = useRef(true); - const scrollContainerReference = useRef(null); - - const [modelPrefix] = model.split("-"); - - let apiKeyGetter = ""; - let provider = ""; - - switch (modelPrefix) { - case "gpt": { - apiKeyGetter = "hasOpenAiApiKey"; - provider = "OpenAI"; - break; - } - - case "claude": { - apiKeyGetter = "hasAnthropicApiKey"; - provider = "Anthropic"; - break; - } - - case "mistral": { - apiKeyGetter = "hasMistralApiKey"; - provider = "Mistral"; - break; - } - - case "models/gemini": { - apiKeyGetter = "hasGoogleGenerativeAiApiKey"; - provider = "Google"; - break; - } - - default: { - break; - } - } - - const { send } = useSDK(APP_ID, { - onMessage(message) { - switch (message.action) { - case "assistant:response": { - const { - output, - done, - model: model_, - } = message.payload as { - output: string; - done: boolean; - model: string; - }; - if (done) { - setMesssage(""); - setMessages(previousState => [ - ...previousState, - { - role: "assistant", - content: output, - id: v4(), - model: model_, - }, - ]); - setIsGenerating(false); - } else { - setIsGenerating(true); - setMesssage(output); - } - - break; - } - - case "assistant:messages": { - const { chat } = message.payload as { - chat: Except & { messages: MessageModel[] }; - }; - firstRunReference.current = true; - setMessages(chat.messages); - setDataTypes( - availableDataTypes.map(dataType => { - const chatDataTypes = chat.dataTypes ?? []; - const overwrite = chatDataTypes.find( - dataType_ => dataType_.id === dataType.id - ); - return overwrite ?? dataType; - }) - ); - setModel(chat.model); - setChatName(chat.label); - - break; - } - - case "assistant:error": { - const { error: error_ } = message.payload as { - error: string; - }; - setError(error_); - - break; - } - - case "assistant:chats": { - const { chats: allChats } = message.payload as { - chats: (Except & { id: string })[]; - }; - setChats(allChats); - - break; - } - - default: { - break; - } - } - }, - }); - - useEffect(() => { - const container = scrollContainerReference.current; - if (container) { - const threshold = 200; - const isNearBottom = - container.scrollHeight - container.scrollTop - container.clientHeight < threshold; - if (isNearBottom || firstRunReference.current) { - container.scrollTop = container.scrollHeight; - firstRunReference.current = false; - } - } - }, [messages, message]); - - useEffect(() => { - const container = scrollContainerReference.current; - if (container) { - container.scrollTop = container.scrollHeight; - } - - if (chatId) { - send({ action: "assistant:history", payload: { id: chatId } }); - } else { - setChatName("New Chat"); - setChatId(v4()); - } - }, [send, chatId]); - - useEffect(() => { - if (!apiKeyGetter) { - return; - } - - window.ipc.send(apiKeyGetter); - const unsubscribe = window.ipc.on(apiKeyGetter, (hasKey: boolean) => { - setIsWarningOpen(!hasKey); + const { + error, + model, + setModel, + dataTypes, + setDataTypes, + provider, + isWarningOpen, + setError, + setChatName, + setChatId, + chatName, + chatId, + chats, + scrollContainerReference, + modelPrefix, + input, + setInput, + setIsGenerating, + isGenerating, + messages, + message, + setMessages, + } = useAssistant(); + const { send } = useSDK(APP_ID, {}); + + function sendMessage() { + setIsGenerating(true); + setInput(""); + setMessages(previousState => [ + ...previousState, + { role: "user", content: input.trim(), id: v4() }, + ]); + send({ + action: "assistant:chat", + payload: { + input: input.trim(), + id: chatId, + model, + maxHistory: 10, + dataTypes, + }, }); - return () => { - unsubscribe(); - }; - }, [apiKeyGetter]); + } return ( - { - setError(""); - }} - > - - - } - > - {error} - - - {t("texts:enterCommonAPIKey", { provider })} - - + { + setError(""); + }} + /> - - { + { + setChatName("New Chat"); + setChatId(v4()); + }} + /> + { + setChatId(chatId_); + }} + onDelete={chatId_ => { + send({ + action: "assistant:delete", + payload: { id: chatId_ }, + }); + if (chatId_ === chatId) { setChatName("New Chat"); setChatId(v4()); - }} - > - - - - - {t("labels:newChat")} - - - - - - {chats - .sort((a, b) => { - if (a.pinned && b.pinned) { - return 0; } - - return a.pinned ? -1 : 1; - }) - .map(chat => ( - - - - - - { - send({ - action: "assistant:update", - payload: { - id: chat.id, - update: { pinned: !chat.pinned }, - }, - }); - }} - > - ({ - "--Icon-color": chat.pinned - ? theme.vars.palette.primary["500"] - : "currentColor", - })} - > - - {" "} - {t("labels:pin")} - - { - send({ - action: "assistant:update", - payload: { - id: chat.id, - update: { - favorite: !chat.favorite, - }, - }, - }); - }} - > - ({ - "--Icon-color": chat.favorite - ? theme.vars.palette.primary["500"] - : "currentColor", - })} - > - - {" "} - {t("labels:favorite")} - - { - send({ - action: "assistant:delete", - payload: { id: chat.id }, - }); - - if (chat.id === chatId) { - setChatName("New Chat"); - setChatId(v4()); - } - }} - > - ({ - "--Icon-color": - theme.vars.palette.red["500"], - })} - > - - {" "} - {t("labels:delete")} - - - - } - > - { - setChatId(chat.id); - }} - > - {chat.label} - - - ))} + }} + /> - + /> - }> - - {dataTypes.map(dataType => ( - - - - - ) => { - setDataTypes(previousState => - previousState.map(dataType_ => - dataType_.id === dataType.id - ? { - ...dataType_, - active: event.target - .checked, - } - : dataType_ - ) - ); - }} - /> - - - {t(`labels:${dataType.type}`)} - - - - ))} - - + { + console.log(""); + setDataTypes(previousState => + previousState.map(dataType_ => + dataType_.id === dataTypeId + ? { + ...dataType_, + active: isChecked, + } + : dataType_ + ) + ); + }} + /> - - {messages.map(message_ => { - const [modelPrefix_] = (message_.model ?? "unknown").split("-"); - return ( - - ({ - position: "sticky", - top: theme.spacing(0.5), - })} - > - - - {message_.role === "user" ? ( - - ) : ( - assistantIcons[modelPrefix_] ?? ( - - ) - )} - - - - - - - - - - ); - })} - {message && ( - - - - {assistantIcons[modelPrefix] ?? } - - - - - - - - - )} - + { - setIsGenerating(true); - setInput(""); - setMessages(previousState => [ - ...previousState, - { role: "user", content: input.trim(), id: v4() }, - ]); - send({ - action: "assistant:chat", - payload: { - input: input.trim(), - id: chatId, - model, - maxHistory: 10, - dataTypes, - }, - }); + sendMessage(); }} > @@ -790,22 +248,7 @@ export function Assistant({ color }: { color: ColorPaletteProp }) { onKeyDown={event => { if (event.key === "Enter" && !event.shiftKey && !isGenerating) { event.preventDefault(); - setIsGenerating(true); - setInput(""); - setMessages(previousState => [ - ...previousState, - { role: "user", content: input.trim(), id: v4() }, - ]); - send({ - action: "assistant:chat", - payload: { - input: input.trim(), - id: chatId, - model, - maxHistory: 10, - dataTypes, - }, - }); + sendMessage(); } }} /> diff --git a/src/client/apps/assistant/types.ts b/src/client/apps/assistant/types.ts new file mode 100644 index 000000000..f38f1ba4a --- /dev/null +++ b/src/client/apps/assistant/types.ts @@ -0,0 +1,6 @@ +export interface MessageModel { + id: string; + content: string; + role: "assistant" | "user"; + model?: string; +} diff --git a/src/client/apps/assistant/utils.ts b/src/client/apps/assistant/utils.ts new file mode 100644 index 000000000..5fc0f9d69 --- /dev/null +++ b/src/client/apps/assistant/utils.ts @@ -0,0 +1,36 @@ +export function getModelInfo(modelPrefix: string) { + let apiKeyGetter = ""; + let provider = ""; + + switch (modelPrefix) { + case "gpt": { + apiKeyGetter = "hasOpenAiApiKey"; + provider = "OpenAI"; + break; + } + + case "claude": { + apiKeyGetter = "hasAnthropicApiKey"; + provider = "Anthropic"; + break; + } + + case "mistral": { + apiKeyGetter = "hasMistralApiKey"; + provider = "Mistral"; + break; + } + + case "models/gemini": { + apiKeyGetter = "hasGoogleGenerativeAiApiKey"; + provider = "Google"; + break; + } + + default: { + break; + } + } + + return { apiKeyGetter, provider }; +} diff --git a/src/client/apps/shared/markdown.tsx b/src/client/apps/shared/markdown.tsx index c9fe9c37e..82f90b3e2 100644 --- a/src/client/apps/shared/markdown.tsx +++ b/src/client/apps/shared/markdown.tsx @@ -1,5 +1,7 @@ import { localFile } from "@captn/utils/string"; import CopyAllIcon from "@mui/icons-material/CopyAll"; +import DarkModeIcon from "@mui/icons-material/DarkMode"; +import LightModeIcon from "@mui/icons-material/LightMode"; import OpenInNewIcon from "@mui/icons-material/OpenInNew"; import Box from "@mui/joy/Box"; import Chip from "@mui/joy/Chip"; @@ -7,14 +9,17 @@ import IconButton from "@mui/joy/IconButton"; import type { LinkProps } from "@mui/joy/Link"; import JoyLink from "@mui/joy/Link"; import Sheet from "@mui/joy/Sheet"; +import Switch from "@mui/joy/Switch"; import Typography from "@mui/joy/Typography"; +import { useAtom } from "jotai"; +import { atomWithStorage } from "jotai/utils"; import { useTranslation } from "next-i18next"; -import type { ReactNode } from "react"; +import type { ChangeEvent, ReactNode } from "react"; import type { Components } from "react-markdown"; import ReactMarkdown from "react-markdown"; import type { SyntaxHighlighterProps } from "react-syntax-highlighter"; import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; -import { materialOceanic } from "react-syntax-highlighter/dist/cjs/styles/prism"; +import { oneLight, oneDark } from "react-syntax-highlighter/dist/cjs/styles/prism"; import rehypeFormat from "rehype-format"; import rehypeRaw from "rehype-raw"; import rehypeStringify from "rehype-stringify"; @@ -36,12 +41,16 @@ const codeMappers: Record = { svelte: "html", vue: "html", }; + +const codeModeAtom = atomWithStorage("code-mode", "dark"); + export function Code({ language, code, ...rest }: { code: string } & Except) { const { t } = useTranslation(["labels"]); + const [mode, setMode] = useAtom(codeModeAtom); return ( )} - { - navigator.clipboard.writeText(code); - }} - > - - + + } + endDecorator={} + onChange={(event: ChangeEvent) => { + setMode(event.target.checked ? "dark" : "light"); + }} + /> + { + navigator.clipboard.writeText(code); + }} + > + + + {code} @@ -189,6 +209,9 @@ export const components: Partial = { ); }, + pre({ children }) { + return
{children}
; + }, img({ src, alt }: { src?: string; alt?: string }) { if (!src) { return null; @@ -215,11 +238,10 @@ export const components: Partial = { position: "absolute", top: 0, right: 0, - margin: 0.5, }} > { event.preventDefault(); @@ -254,8 +276,9 @@ export const components: Partial = { my: 1, mx: "auto", width: "100%", - height: "auto", - maxWidth: 512, + maxHeight: 640, + objectFit: "contain", + objectPosition: "center", }} /> diff --git a/src/client/apps/shared/styled.ts b/src/client/apps/shared/styled.ts index 44c64d534..f9cc7493f 100644 --- a/src/client/apps/shared/styled.ts +++ b/src/client/apps/shared/styled.ts @@ -11,16 +11,14 @@ export const StyledColorInput = styled("input")({ cursor: "pointer", }); -export const StyledRenderingAreaWrapper = styled("div")(({ theme }) => ({ +export const StyledRenderingAreaWrapper = styled("div")({ minWidth: "min-content", position: "relative", zIndex: 0, display: "flex", alignItems: "center", justifyContent: "center", - // - // padding: theme.spacing(1), -})); +}); export const StyledDrawingAreaWrapper = styled("div")(({ theme }) => ({ minWidth: "min-content", diff --git a/src/electron/helpers/ipc/sdk/assistant.ts b/src/electron/helpers/ipc/sdk/assistant.ts index 515098535..0a90cbfb5 100644 --- a/src/electron/helpers/ipc/sdk/assistant.ts +++ b/src/electron/helpers/ipc/sdk/assistant.ts @@ -69,76 +69,67 @@ export function extendCaptainData(content: string) { ); } -export function createVectorSearchPrompt() { - return `You return a single search query using several tags or words that help in a vector search (e.g. "illustrations of cute animals, like kittens or puppies playful and cute"). You receive a history of user assistant messages and find the perfect search query for it. You never interact or continue this discussion. Your task is to return a query to help find results. **The user never sees your answer**. Your answer will only be used to run the query through a search. Remember. Your answer is a perfect search query, 1 - 2 sentences maximum. No additional comments or explanation, no user interaction.`; +export function createVectorSearchPrompt({ dataTypes }: { dataTypes: string[] }) { + return `Based on the history of user-assistant messages, generate a single, highly relevant search query that includes several tags or keywords for a vector search. The query should be 1-2 sentences long and tailored to find specific results within the following data types: ${JSON.stringify(dataTypes)}. Do not interact with the user or provide any additional comments. The user will not see your query; it is only used for the search process.`; +} + +export function createChatLabelPrompt() { + return `Your task is to generate a short and concise label or title for the chat based on the user's initial question. The label should accurately reflect the main topic or intent of the user's inquiry, be in the same language as the user's question, and must be no more than four words. Ensure the label is clear and specific, capturing the essence of the conversation that is about to start. Yur answer is just the label, no comments or explanation, the user will not see your message or be able to interact with you.`; } export function createSystemPrompt() { return `# You are Captain | The AI Platform ## You are: -- an open source app developed by Blibla, a german startup from Mainz. -- always nice to the user -- a great support for any problem -- always encouraging -- the user's best friend +- An open source app developed by Blibla, a German startup from Mainz. +- Always nice to the user, encouraging, and a bit quirky but honest. +- The user's best friend and a great support for any problem. +- Extremely precise when precision is required. ## You can: -- store data locally and keep personal data private. -- retrieve data based on user queries by utilizing the integrated local vector store (powered by Qdrant and local embeddings) -- access files from the user's computer, but only those files that are registered in Captain (e.g. every file generated in captain is automatically available) -- link to certain files and/or display them inline if asked to do so. (links or images will open in the integrated Explorer if clicked). -- speak 11 languages (German, English, Spanish, French,Italian, Japanese, Dutch, Polish, Portuguese, Russian, Chinese) and answer in that language if the user uses it +- Store data locally and keep personal data private. +- Retrieve data based on user queries using the integrated local vector store (powered by Qdrant and local embeddings). +- Access files from the user's computer, but only those files registered in Captain. +- Link to or display certain files inline if asked to do so. +- Speak 11 languages (German, English, Spanish, French, Italian, Japanese, Dutch, Polish, Portuguese, Russian, Chinese) and respond in the user's language. > [!NOTE] -> At any time you act as Captain. -> Each user query includes context retrieved by a vector search. The score(0 - 1) determines how well it matches the user's query. - -## You follow this Guide: - -Feature: User Interaction - As a User - I want to get personalized help from Captain - So that I am assisted when using the Captain AI Platform - -Scenario: user speaks x-language - When the user asks in {language} - Then you respond in {language} - And your tone is simple, using common language that is clear - And your response is easy to understand - And you use simple vocabulary to include non-native speakers of given language -Scenario: context is empty - When the context is empty - Then there might not be any related data. -Scenario: context has entries - When the context contains entries - Then they might be related to the user's query -Scenario: context has images - When the context includes images - Then you answer with image tags so that they can be rendered e.g. ![alt value](${PATH_PLACEHOLDER}/path/to/image.ext) -Scenario: user wants a story - When asked to write a story - And the context contains images - Then you select 1 - 3 images, the ones, that best fit the storyline. - And you inline the images within the story - And you return the story including the images. -Scenario: user wants lists and/or links - When asked for links or lists - Then you include the reference to found entries as links e.g. [Link label](${PATH_PLACEHOLDER}/path/to/file.ext) -Scenario: user has a question related to Captain - When the user asks about Captain - Then you explain what you are and how you work - And you offer things to try out -Scenario: misc user input - When the user asks a question - Then you try to make sense of it within the context - And you use what is useful to answer the questions ar perform the given tasks - And you respond accordingly. - And you always make sure to add references as applicable -Scenario: user wants responses based on your last message - When the user asks to write a story about one of the images from your last response - Then you ignore the context - And you use the previous messages as context +> You always act as Captain. Each user query includes context retrieved by a vector search. To activate the context, the user must select the datatypes in the menu. Only files in the selected category are included. + +## Interaction Guide: + +### User Interaction +- **Scenario: User speaks {language}** + - Respond in {language} using simple, clear language. + +- **Scenario: Context is empty** + - Indicate there may not be related data. + +- **Scenario: Context has entries** + - Indicate they might be related to the user's query. + +- **Scenario: Context includes images** + - Answer with image tags (e.g., ![Image content](${PATH_PLACEHOLDER}/path/to/image.ext)). + +- **Scenario: User requests a story** + - Select 1-3 fitting images from the context. + - Return the story including the image as inline image tags. + +- **Scenario: User requests links or lists** + - Include references to found entries as links (e.g., [Link label](${PATH_PLACEHOLDER}/path/to/file.ext)). + +- **Scenario: User has a question about Captain** + - Explain Captain's features and offer things to try out. + +- **Scenario: Miscellaneous user input** + - Use context to answer questions or perform tasks, and always add references if applicable. + +- **Scenario: User wants responses based on the last message** + - Ignore the context and use previous messages as context. + +### Error Handling +- **Scenario: An error occurs** + - Apologize and provide guidance on how to proceed or troubleshoot. `; } @@ -349,21 +340,39 @@ ipcMain.on( } const lastMessage = { role: "user" as const, content: userInput }; + if (chat.messages.length === 0) { + // LastMessage is first message. Let AI create a label / title + chat.label = await chatCompletion({ + model: assistant(model), + system: createChatLabelPrompt(), + messages: [lastMessage], + }); + event.sender.send(channel, { + action: "assistant:nameSuggestion", + payload: { + chatName: chat.label, + }, + }); + } + let optimizedContext = "\n\n"; if (includedDataTypes.length > 0) { logger.info(`Assistant uses data: ${includedDataTypes.join(", ")}`); const minorContent = [ - ...chat.messages.slice(-3), + ...chat.messages.slice(-2), { role: "user" as const, content: userInput + " => ONLY ANSWER WITH A SEARCH QUERY!!!", }, ].filter(Boolean); + if (minorContent[0] && minorContent[0].role !== "user") { + minorContent.shift(); + } const query = await chatCompletion({ model: assistant(model), - system: createVectorSearchPrompt(), + system: createVectorSearchPrompt({ dataTypes: includedDataTypes }), messages: minorContent, }); @@ -480,6 +489,7 @@ ipcMain.on( output: extendCaptainData(fullResponse), done: true, model, + label: chat.label, }, }); const chats = Object.entries(assistantStore.store).map(([id, value]) => ({