Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[FR-315] feat: integrate image generation model
Browse files Browse the repository at this point in the history
agatha197 authored and yomybaby committed Jan 6, 2025

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent da3ef24 commit d52ef12
Showing 22 changed files with 166 additions and 58 deletions.
161 changes: 124 additions & 37 deletions react/src/components/lablupTalkativotUI/LLMChatCard.tsx
Original file line number Diff line number Diff line change
@@ -13,6 +13,7 @@ import {
DeleteOutlined,
LinkOutlined,
MoreOutlined,
PictureOutlined,
RocketOutlined,
} from '@ant-design/icons';
import { Attachments, AttachmentsProps, Sender } from '@ant-design/x';
@@ -31,6 +32,7 @@ import {
MenuProps,
Tag,
theme,
Tooltip,
Typography,
} from 'antd';
import _ from 'lodash';
@@ -92,6 +94,8 @@ const LLMChatCard: React.FC<LLMChatCardProps> = ({
}) => {
const webuiNavigate = useWebUINavigate();
const [isOpenAttachments, setIsOpenAttachments] = useState(false);
const [isImageGeneration, setIsImageGeneration] = useState(false);
const [loadingImageGeneration, setLoadingImageGeneration] = useState(false);

const [modelId, setModelId] = useControllableValue(cardProps, {
valuePropName: 'modelId',
@@ -214,6 +218,34 @@ const LLMChatCard: React.FC<LLMChatCardProps> = ({
},
]);

const generateImage = async (prompt: string, accessKey: string) => {
setLoadingImageGeneration(true);
try {
const response = await fetch(
'https://flux-inference.asia03.app.backend.ai/generate-flux-image',
{
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
prompt: prompt,
access_key: accessKey,
}),
},
);

if (response.ok) {
const responseData = await response.json();
return responseData.image_base64;
} else {
throw new Error('Error generating image');
}
} finally {
setLoadingImageGeneration(false);
}
};

return (
<Card
ref={cardRef}
@@ -244,6 +276,7 @@ const LLMChatCard: React.FC<LLMChatCardProps> = ({
value={modelId}
onChange={setModelId}
allowCustomModel={allowCustomModel}
disabled={isImageGeneration}
/>
<Dropdown menu={{ items }} trigger={['click']}>
<Button
@@ -318,70 +351,124 @@ const LLMChatCard: React.FC<LLMChatCardProps> = ({
/>
</Sender.Header>
}
styles={{
prefix: {
alignSelf: 'center',
},
}}
prefix={
<Attachments
beforeUpload={() => false}
getDropContainer={() => cardRef.current}
accept="image/*,text/*"
items={files}
onChange={({ fileList }) => {
setFiles(fileList);
setIsOpenAttachments(true);
}}
placeholder={(type) =>
type === 'drop'
? {
title: t('chatui.DropFileHere'),
}
: {
icon: <CloudUploadOutlined />,
title: t('chatui.UploadFiles'),
description: t('chatui.UploadFilesDescription'),
}
}
>
<Badge dot={!_.isEmpty(files) && !isOpenAttachments}>
<Button type="text" icon={<LinkOutlined />} />
</Badge>
</Attachments>
isImageGeneration ? (
<Tag
title={t('chatui.Picture')}
closable
onClose={() => setIsImageGeneration(false)}
color="blue"
>
{t('chatui.Picture')}
</Tag>
) : (
<>
<Attachments
beforeUpload={() => false}
getDropContainer={() => cardRef.current}
accept="image/*,text/*"
items={files}
onChange={({ fileList }) => {
setFiles(fileList);
setIsOpenAttachments(true);
}}
placeholder={(type) =>
type === 'drop'
? {
title: t('chatui.DropFileHere'),
}
: {
icon: <CloudUploadOutlined />,
title: t('chatui.UploadFiles'),
description: t('chatui.UploadFilesDescription'),
}
}
>
<Badge dot={!_.isEmpty(files) && !isOpenAttachments}>
<Button type="text" icon={<LinkOutlined />} />
</Badge>
</Attachments>
<Tooltip title={t('chatui.Picture')}>
<Button
type="text"
icon={<PictureOutlined />}
onClick={() => {
setIsImageGeneration(true);
}}
/>
</Tooltip>
</>
)
}
onChange={(v: string) => {
setInput(v);
if (onInputChange) {
onInputChange(v);
}
}}
loading={isLoading}
loading={isLoading || loadingImageGeneration}
onStop={() => {
stop();
}}
onSend={() => {
onSend={async () => {
if (input || !_.isEmpty(files)) {
const fileList = _.map(
files,
(item) => item.originFileObj as File,
);
// Filter after converting to `File`
const fileListArray = _.filter(fileList, Boolean);
const dataTransfer = new DataTransfer();
_.forEach(fileListArray, (file) => {
dataTransfer.items.add(file);
});

append(
{
role: 'user',
content: input,
},
{
experimental_attachments: dataTransfer.files,
},
);
if (isImageGeneration) {
try {
const imageBase64 = await generateImage(input, 'accessKey');
setMessages((prevMessages) => [
...prevMessages,
{
id: _.uniqueId(),
role: 'user',
content: input,
},
{
id: _.uniqueId(),
role: 'assistant',
content: '',
experimental_attachments: [
{
contentType: 'image/png',
url: imageBase64,
},
],
},
]);
} catch (error) {
console.error(error);
}
} else {
append(
{
role: 'user',
content: input,
},
{
experimental_attachments: dataTransfer.files,
},
);
}

setTimeout(() => {
setInput('');
setFiles([]);
setIsOpenAttachments(false);
setIsImageGeneration(false);
}, 0);
onSubmitChange?.();
}
3 changes: 2 additions & 1 deletion resources/i18n/de.json
Original file line number Diff line number Diff line change
@@ -1747,7 +1747,8 @@
"Attachments": "Anhänge",
"DropFileHere": "Datei hier ablegen",
"UploadFiles": "Dateien hochladen",
"UploadFilesDescription": "Klicken oder ziehen Sie Dateien zum Hochladen in diesen Bereich"
"UploadFilesDescription": "Klicken oder ziehen Sie Dateien zum Hochladen in diesen Bereich",
"Picture": "Bild"
},
"time": {
"ms": "ms",
3 changes: 2 additions & 1 deletion resources/i18n/el.json
Original file line number Diff line number Diff line change
@@ -1747,7 +1747,8 @@
"Attachments": "Συνημμένα",
"DropFileHere": "Αποθέστε το αρχείο εδώ",
"UploadFiles": "Μεταφόρτωση αρχείων",
"UploadFilesDescription": "Κάντε κλικ ή σύρετε αρχεία σε αυτήν την περιοχή για μεταφόρτωση"
"UploadFilesDescription": "Κάντε κλικ ή σύρετε αρχεία σε αυτήν την περιοχή για μεταφόρτωση",
"Picture": "Εικόνα"
},
"time": {
"ms": "ms",
3 changes: 2 additions & 1 deletion resources/i18n/en.json
Original file line number Diff line number Diff line change
@@ -1751,7 +1751,8 @@
"Attachments": "Attachments",
"DropFileHere": "Drop file here",
"UploadFiles": "Upload files",
"UploadFilesDescription": "Click or drag files to this area to upload"
"UploadFilesDescription": "Click or drag files to this area to upload",
"Picture": "Picture"
},
"time": {
"ms": "ms",
3 changes: 2 additions & 1 deletion resources/i18n/es.json
Original file line number Diff line number Diff line change
@@ -1749,7 +1749,8 @@
"Attachments": "Adjuntos",
"DropFileHere": "Suelta el archivo aquí",
"UploadFiles": "Subir archivos",
"UploadFilesDescription": "Haga clic o arrastre archivos a esta área para cargarlos"
"UploadFilesDescription": "Haga clic o arrastre archivos a esta área para cargarlos",
"Picture": "Imagen"
},
"time": {
"ms": "ms",
3 changes: 2 additions & 1 deletion resources/i18n/fi.json
Original file line number Diff line number Diff line change
@@ -1746,7 +1746,8 @@
"Attachments": "Liitteet",
"DropFileHere": "Pudota tiedosto tähän",
"UploadFiles": "Lataa tiedostoja",
"UploadFilesDescription": "Napsauta tai vedä tiedostoja tälle alueelle ladataksesi"
"UploadFilesDescription": "Napsauta tai vedä tiedostoja tälle alueelle ladataksesi",
"Picture": "Kuva"
},
"time": {
"ms": "ms",
3 changes: 2 additions & 1 deletion resources/i18n/fr.json
Original file line number Diff line number Diff line change
@@ -1747,7 +1747,8 @@
"Attachments": "Pièces jointes",
"DropFileHere": "Déposez le fichier ici",
"UploadFiles": "Télécharger des fichiers",
"UploadFilesDescription": "Cliquez ou faites glisser les fichiers vers cette zone pour les télécharger"
"UploadFilesDescription": "Cliquez ou faites glisser les fichiers vers cette zone pour les télécharger",
"Picture": "Image"
},
"time": {
"ms": "ms",
3 changes: 2 additions & 1 deletion resources/i18n/id.json
Original file line number Diff line number Diff line change
@@ -1747,7 +1747,8 @@
"Attachments": "Lampiran",
"DropFileHere": "Letakkan file di sini",
"UploadFiles": "Unggah file",
"UploadFilesDescription": "Klik atau seret file ke area ini untuk diunggah"
"UploadFilesDescription": "Klik atau seret file ke area ini untuk diunggah",
"Picture": "Gambar"
},
"time": {
"ms": "ms",
3 changes: 2 additions & 1 deletion resources/i18n/it.json
Original file line number Diff line number Diff line change
@@ -1747,7 +1747,8 @@
"Attachments": "Allegati",
"DropFileHere": "Rilascia il file qui",
"UploadFiles": "Carica file",
"UploadFilesDescription": "Fare clic o trascinare i file in quest'area per caricarli"
"UploadFilesDescription": "Fare clic o trascinare i file in quest'area per caricarli",
"Picture": "Immagine"
},
"time": {
"ms": "ms",
3 changes: 2 additions & 1 deletion resources/i18n/ja.json
Original file line number Diff line number Diff line change
@@ -1747,7 +1747,8 @@
"Attachments": "添付ファイル",
"DropFileHere": "ここにファイルをドロップします",
"UploadFiles": "ファイルをアップロードする",
"UploadFilesDescription": "ファイルをクリックまたはこの領域にドラッグしてアップロードします"
"UploadFilesDescription": "ファイルをクリックまたはこの領域にドラッグしてアップロードします",
"Picture": "写真"
},
"time": {
"ms": "ms",
3 changes: 2 additions & 1 deletion resources/i18n/ko.json
Original file line number Diff line number Diff line change
@@ -1749,7 +1749,8 @@
"Attachments": "첨부파일",
"DropFileHere": "파일을 여기에 놓아주세요.",
"UploadFiles": "파일 업로드",
"UploadFilesDescription": "파일을 여기 끌어다 놓거나 이 영역을 클릭하세요"
"UploadFilesDescription": "파일을 여기 끌어다 놓거나 이 영역을 클릭하세요",
"Picture": "그림"
},
"time": {
"ms": "밀리초",
3 changes: 2 additions & 1 deletion resources/i18n/mn.json
Original file line number Diff line number Diff line change
@@ -1747,7 +1747,8 @@
"Attachments": "Хавсралтууд",
"DropFileHere": "Файлыг энд буулгана уу",
"UploadFiles": "Файлуудыг байршуулах",
"UploadFilesDescription": "Байршуулахын тулд файлуудыг энэ хэсэгт товшиж эсвэл чирнэ үү"
"UploadFilesDescription": "Байршуулахын тулд файлуудыг энэ хэсэгт товшиж эсвэл чирнэ үү",
"Picture": "Зураг"
},
"time": {
"ms": "ms",
3 changes: 2 additions & 1 deletion resources/i18n/ms.json
Original file line number Diff line number Diff line change
@@ -1747,7 +1747,8 @@
"Attachments": "Lampiran",
"DropFileHere": "Lepaskan fail di sini",
"UploadFiles": "Muat naik fail",
"UploadFilesDescription": "Klik atau seret fail ke kawasan ini untuk memuat naik"
"UploadFilesDescription": "Klik atau seret fail ke kawasan ini untuk memuat naik",
"Picture": "Gambar"
},
"time": {
"ms": "ms",
3 changes: 2 additions & 1 deletion resources/i18n/pl.json
Original file line number Diff line number Diff line change
@@ -1747,7 +1747,8 @@
"Attachments": "Załączniki",
"DropFileHere": "Upuść plik tutaj",
"UploadFiles": "Prześlij pliki",
"UploadFilesDescription": "Kliknij lub przeciągnij pliki do tego obszaru, aby je przesłać"
"UploadFilesDescription": "Kliknij lub przeciągnij pliki do tego obszaru, aby je przesłać",
"Picture": "Zdjęcie"
},
"time": {
"ms": "ms",
3 changes: 2 additions & 1 deletion resources/i18n/pt-BR.json
Original file line number Diff line number Diff line change
@@ -1747,7 +1747,8 @@
"Attachments": "Anexos",
"DropFileHere": "Solte o arquivo aqui",
"UploadFiles": "Carregar arquivos",
"UploadFilesDescription": "Clique ou arraste os arquivos para esta área para fazer upload"
"UploadFilesDescription": "Clique ou arraste os arquivos para esta área para fazer upload",
"Picture": "Foto"
},
"time": {
"ms": "ms",
3 changes: 2 additions & 1 deletion resources/i18n/pt.json
Original file line number Diff line number Diff line change
@@ -1747,7 +1747,8 @@
"Attachments": "Anexos",
"DropFileHere": "Solte o arquivo aqui",
"UploadFiles": "Carregar arquivos",
"UploadFilesDescription": "Clique ou arraste os arquivos para esta área para fazer upload"
"UploadFilesDescription": "Clique ou arraste os arquivos para esta área para fazer upload",
"Picture": "Foto"
},
"time": {
"ms": "ms",
3 changes: 2 additions & 1 deletion resources/i18n/ru.json
Original file line number Diff line number Diff line change
@@ -1747,7 +1747,8 @@
"Attachments": "Вложения",
"DropFileHere": "Перетащите файл сюда",
"UploadFiles": "Загрузить файлы",
"UploadFilesDescription": "Нажмите или перетащите файлы в эту область, чтобы загрузить"
"UploadFilesDescription": "Нажмите или перетащите файлы в эту область, чтобы загрузить",
"Picture": "Картина"
},
"time": {
"ms": "ms",
3 changes: 2 additions & 1 deletion resources/i18n/th.json
Original file line number Diff line number Diff line change
@@ -1727,7 +1727,8 @@
"Attachments": "ไฟล์แนบ",
"DropFileHere": "วางไฟล์ที่นี่",
"UploadFiles": "อัพโหลดไฟล์",
"UploadFilesDescription": "คลิกหรือลากไฟล์ไปยังบริเวณนี้เพื่ออัปโหลด"
"UploadFilesDescription": "คลิกหรือลากไฟล์ไปยังบริเวณนี้เพื่ออัปโหลด",
"Picture": "รูปภาพ"
},
"time": {
"ms": "ms",
Loading

0 comments on commit d52ef12

Please sign in to comment.