From b995dc5e8624d67a698e30ce7ec0b2d78172fe4e Mon Sep 17 00:00:00 2001 From: Jan Martiska Date: Thu, 4 Jan 2024 10:58:59 +0100 Subject: [PATCH 1/2] Dev UI support for working with image models --- .../quarkus-langchain4j-huggingface.adoc | 38 ++++- .../includes/quarkus-langchain4j-openai.adoc | 136 ++++++++++++++++++ .../devui/OpenAiDevUIImagePageProcessor.java | 30 ++++ .../src/main/resources/dev-ui/qwc-images.js | 129 +++++++++++++++++ .../devui/OpenAiImagesJsonRPCService.java | 67 +++++++++ 5 files changed, 398 insertions(+), 2 deletions(-) create mode 100644 openai/openai-vanilla/deployment/src/main/java/io/quarkiverse/langchain4j/openai/deployment/devui/OpenAiDevUIImagePageProcessor.java create mode 100644 openai/openai-vanilla/deployment/src/main/resources/dev-ui/qwc-images.js create mode 100644 openai/openai-vanilla/runtime/src/main/java/io/quarkiverse/langchain4j/openai/runtime/devui/OpenAiImagesJsonRPCService.java diff --git a/docs/modules/ROOT/pages/includes/quarkus-langchain4j-huggingface.adoc b/docs/modules/ROOT/pages/includes/quarkus-langchain4j-huggingface.adoc index c4f92df12..d394ff8a7 100644 --- a/docs/modules/ROOT/pages/includes/quarkus-langchain4j-huggingface.adoc +++ b/docs/modules/ROOT/pages/includes/quarkus-langchain4j-huggingface.adoc @@ -254,6 +254,40 @@ endif::add-copy-button-to-env-var[] | +a| [[quarkus-langchain4j-huggingface_quarkus.langchain4j.huggingface.chat-model.log-requests]]`link:#quarkus-langchain4j-huggingface_quarkus.langchain4j.huggingface.chat-model.log-requests[quarkus.langchain4j.huggingface.chat-model.log-requests]` + + +[.description] +-- +Whether chat model requests should be logged + +ifdef::add-copy-button-to-env-var[] +Environment variable: env_var_with_copy_button:+++QUARKUS_LANGCHAIN4J_HUGGINGFACE_CHAT_MODEL_LOG_REQUESTS+++[] +endif::add-copy-button-to-env-var[] +ifndef::add-copy-button-to-env-var[] +Environment variable: `+++QUARKUS_LANGCHAIN4J_HUGGINGFACE_CHAT_MODEL_LOG_REQUESTS+++` +endif::add-copy-button-to-env-var[] +--|boolean +|`false` + + +a| [[quarkus-langchain4j-huggingface_quarkus.langchain4j.huggingface.chat-model.log-responses]]`link:#quarkus-langchain4j-huggingface_quarkus.langchain4j.huggingface.chat-model.log-responses[quarkus.langchain4j.huggingface.chat-model.log-responses]` + + +[.description] +-- +Whether chat model responses should be logged + +ifdef::add-copy-button-to-env-var[] +Environment variable: env_var_with_copy_button:+++QUARKUS_LANGCHAIN4J_HUGGINGFACE_CHAT_MODEL_LOG_RESPONSES+++[] +endif::add-copy-button-to-env-var[] +ifndef::add-copy-button-to-env-var[] +Environment variable: `+++QUARKUS_LANGCHAIN4J_HUGGINGFACE_CHAT_MODEL_LOG_RESPONSES+++` +endif::add-copy-button-to-env-var[] +--|boolean +|`false` + + a| [[quarkus-langchain4j-huggingface_quarkus.langchain4j.huggingface.embedding-model.inference-endpoint-url]]`link:#quarkus-langchain4j-huggingface_quarkus.langchain4j.huggingface.embedding-model.inference-endpoint-url[quarkus.langchain4j.huggingface.embedding-model.inference-endpoint-url]` @@ -298,7 +332,7 @@ a| [[quarkus-langchain4j-huggingface_quarkus.langchain4j.huggingface.log-request [.description] -- -Whether the OpenAI client should log requests +Whether the HuggingFace client should log requests ifdef::add-copy-button-to-env-var[] Environment variable: env_var_with_copy_button:+++QUARKUS_LANGCHAIN4J_HUGGINGFACE_LOG_REQUESTS+++[] @@ -315,7 +349,7 @@ a| [[quarkus-langchain4j-huggingface_quarkus.langchain4j.huggingface.log-respons [.description] -- -Whether the OpenAI client should log responses +Whether the HuggingFace client should log responses ifdef::add-copy-button-to-env-var[] Environment variable: env_var_with_copy_button:+++QUARKUS_LANGCHAIN4J_HUGGINGFACE_LOG_RESPONSES+++[] diff --git a/docs/modules/ROOT/pages/includes/quarkus-langchain4j-openai.adoc b/docs/modules/ROOT/pages/includes/quarkus-langchain4j-openai.adoc index 20740a991..c06832e1f 100644 --- a/docs/modules/ROOT/pages/includes/quarkus-langchain4j-openai.adoc +++ b/docs/modules/ROOT/pages/includes/quarkus-langchain4j-openai.adoc @@ -300,6 +300,40 @@ endif::add-copy-button-to-env-var[] |`0` +a| [[quarkus-langchain4j-openai_quarkus.langchain4j.openai.chat-model.log-requests]]`link:#quarkus-langchain4j-openai_quarkus.langchain4j.openai.chat-model.log-requests[quarkus.langchain4j.openai.chat-model.log-requests]` + + +[.description] +-- +Whether chat model requests should be logged + +ifdef::add-copy-button-to-env-var[] +Environment variable: env_var_with_copy_button:+++QUARKUS_LANGCHAIN4J_OPENAI_CHAT_MODEL_LOG_REQUESTS+++[] +endif::add-copy-button-to-env-var[] +ifndef::add-copy-button-to-env-var[] +Environment variable: `+++QUARKUS_LANGCHAIN4J_OPENAI_CHAT_MODEL_LOG_REQUESTS+++` +endif::add-copy-button-to-env-var[] +--|boolean +|`false` + + +a| [[quarkus-langchain4j-openai_quarkus.langchain4j.openai.chat-model.log-responses]]`link:#quarkus-langchain4j-openai_quarkus.langchain4j.openai.chat-model.log-responses[quarkus.langchain4j.openai.chat-model.log-responses]` + + +[.description] +-- +Whether chat model responses should be logged + +ifdef::add-copy-button-to-env-var[] +Environment variable: env_var_with_copy_button:+++QUARKUS_LANGCHAIN4J_OPENAI_CHAT_MODEL_LOG_RESPONSES+++[] +endif::add-copy-button-to-env-var[] +ifndef::add-copy-button-to-env-var[] +Environment variable: `+++QUARKUS_LANGCHAIN4J_OPENAI_CHAT_MODEL_LOG_RESPONSES+++` +endif::add-copy-button-to-env-var[] +--|boolean +|`false` + + a| [[quarkus-langchain4j-openai_quarkus.langchain4j.openai.embedding-model.model-name]]`link:#quarkus-langchain4j-openai_quarkus.langchain4j.openai.embedding-model.model-name[quarkus.langchain4j.openai.embedding-model.model-name]` @@ -317,6 +351,40 @@ endif::add-copy-button-to-env-var[] |`text-embedding-ada-002` +a| [[quarkus-langchain4j-openai_quarkus.langchain4j.openai.embedding-model.log-requests]]`link:#quarkus-langchain4j-openai_quarkus.langchain4j.openai.embedding-model.log-requests[quarkus.langchain4j.openai.embedding-model.log-requests]` + + +[.description] +-- +Whether embedding model requests should be logged + +ifdef::add-copy-button-to-env-var[] +Environment variable: env_var_with_copy_button:+++QUARKUS_LANGCHAIN4J_OPENAI_EMBEDDING_MODEL_LOG_REQUESTS+++[] +endif::add-copy-button-to-env-var[] +ifndef::add-copy-button-to-env-var[] +Environment variable: `+++QUARKUS_LANGCHAIN4J_OPENAI_EMBEDDING_MODEL_LOG_REQUESTS+++` +endif::add-copy-button-to-env-var[] +--|boolean +|`false` + + +a| [[quarkus-langchain4j-openai_quarkus.langchain4j.openai.embedding-model.log-responses]]`link:#quarkus-langchain4j-openai_quarkus.langchain4j.openai.embedding-model.log-responses[quarkus.langchain4j.openai.embedding-model.log-responses]` + + +[.description] +-- +Whether embedding model responses should be logged + +ifdef::add-copy-button-to-env-var[] +Environment variable: env_var_with_copy_button:+++QUARKUS_LANGCHAIN4J_OPENAI_EMBEDDING_MODEL_LOG_RESPONSES+++[] +endif::add-copy-button-to-env-var[] +ifndef::add-copy-button-to-env-var[] +Environment variable: `+++QUARKUS_LANGCHAIN4J_OPENAI_EMBEDDING_MODEL_LOG_RESPONSES+++` +endif::add-copy-button-to-env-var[] +--|boolean +|`false` + + a| [[quarkus-langchain4j-openai_quarkus.langchain4j.openai.moderation-model.model-name]]`link:#quarkus-langchain4j-openai_quarkus.langchain4j.openai.moderation-model.model-name[quarkus.langchain4j.openai.moderation-model.model-name]` @@ -334,6 +402,40 @@ endif::add-copy-button-to-env-var[] |`text-moderation-latest` +a| [[quarkus-langchain4j-openai_quarkus.langchain4j.openai.moderation-model.log-requests]]`link:#quarkus-langchain4j-openai_quarkus.langchain4j.openai.moderation-model.log-requests[quarkus.langchain4j.openai.moderation-model.log-requests]` + + +[.description] +-- +Whether moderation model requests should be logged + +ifdef::add-copy-button-to-env-var[] +Environment variable: env_var_with_copy_button:+++QUARKUS_LANGCHAIN4J_OPENAI_MODERATION_MODEL_LOG_REQUESTS+++[] +endif::add-copy-button-to-env-var[] +ifndef::add-copy-button-to-env-var[] +Environment variable: `+++QUARKUS_LANGCHAIN4J_OPENAI_MODERATION_MODEL_LOG_REQUESTS+++` +endif::add-copy-button-to-env-var[] +--|boolean +|`false` + + +a| [[quarkus-langchain4j-openai_quarkus.langchain4j.openai.moderation-model.log-responses]]`link:#quarkus-langchain4j-openai_quarkus.langchain4j.openai.moderation-model.log-responses[quarkus.langchain4j.openai.moderation-model.log-responses]` + + +[.description] +-- +Whether moderation model responses should be logged + +ifdef::add-copy-button-to-env-var[] +Environment variable: env_var_with_copy_button:+++QUARKUS_LANGCHAIN4J_OPENAI_MODERATION_MODEL_LOG_RESPONSES+++[] +endif::add-copy-button-to-env-var[] +ifndef::add-copy-button-to-env-var[] +Environment variable: `+++QUARKUS_LANGCHAIN4J_OPENAI_MODERATION_MODEL_LOG_RESPONSES+++` +endif::add-copy-button-to-env-var[] +--|boolean +|`false` + + a| [[quarkus-langchain4j-openai_quarkus.langchain4j.openai.image-model.model-name]]`link:#quarkus-langchain4j-openai_quarkus.langchain4j.openai.image-model.model-name[quarkus.langchain4j.openai.image-model.model-name]` @@ -504,6 +606,40 @@ endif::add-copy-button-to-env-var[] --|string | + +a| [[quarkus-langchain4j-openai_quarkus.langchain4j.openai.image-model.log-requests]]`link:#quarkus-langchain4j-openai_quarkus.langchain4j.openai.image-model.log-requests[quarkus.langchain4j.openai.image-model.log-requests]` + + +[.description] +-- +Whether image model requests should be logged + +ifdef::add-copy-button-to-env-var[] +Environment variable: env_var_with_copy_button:+++QUARKUS_LANGCHAIN4J_OPENAI_IMAGE_MODEL_LOG_REQUESTS+++[] +endif::add-copy-button-to-env-var[] +ifndef::add-copy-button-to-env-var[] +Environment variable: `+++QUARKUS_LANGCHAIN4J_OPENAI_IMAGE_MODEL_LOG_REQUESTS+++` +endif::add-copy-button-to-env-var[] +--|boolean +|`false` + + +a| [[quarkus-langchain4j-openai_quarkus.langchain4j.openai.image-model.log-responses]]`link:#quarkus-langchain4j-openai_quarkus.langchain4j.openai.image-model.log-responses[quarkus.langchain4j.openai.image-model.log-responses]` + + +[.description] +-- +Whether image model responses should be logged + +ifdef::add-copy-button-to-env-var[] +Environment variable: env_var_with_copy_button:+++QUARKUS_LANGCHAIN4J_OPENAI_IMAGE_MODEL_LOG_RESPONSES+++[] +endif::add-copy-button-to-env-var[] +ifndef::add-copy-button-to-env-var[] +Environment variable: `+++QUARKUS_LANGCHAIN4J_OPENAI_IMAGE_MODEL_LOG_RESPONSES+++` +endif::add-copy-button-to-env-var[] +--|boolean +|`false` + |=== ifndef::no-duration-note[] [NOTE] diff --git a/openai/openai-vanilla/deployment/src/main/java/io/quarkiverse/langchain4j/openai/deployment/devui/OpenAiDevUIImagePageProcessor.java b/openai/openai-vanilla/deployment/src/main/java/io/quarkiverse/langchain4j/openai/deployment/devui/OpenAiDevUIImagePageProcessor.java new file mode 100644 index 000000000..60478dd12 --- /dev/null +++ b/openai/openai-vanilla/deployment/src/main/java/io/quarkiverse/langchain4j/openai/deployment/devui/OpenAiDevUIImagePageProcessor.java @@ -0,0 +1,30 @@ +package io.quarkiverse.langchain4j.openai.deployment.devui; + +import io.quarkiverse.langchain4j.openai.deployment.Langchain4jOpenAiBuildConfig; +import io.quarkiverse.langchain4j.openai.runtime.devui.OpenAiImagesJsonRPCService; +import io.quarkus.deployment.IsDevelopment; +import io.quarkus.deployment.annotations.BuildProducer; +import io.quarkus.deployment.annotations.BuildStep; +import io.quarkus.devui.spi.JsonRPCProvidersBuildItem; +import io.quarkus.devui.spi.page.CardPageBuildItem; +import io.quarkus.devui.spi.page.Page; + +public class OpenAiDevUIImagePageProcessor { + + @BuildStep(onlyIf = IsDevelopment.class) + CardPageBuildItem cardPage( + BuildProducer producers, + Langchain4jOpenAiBuildConfig config) { + if (config.imageModel().enabled().orElse(true)) { + CardPageBuildItem card = new CardPageBuildItem(); + card.addPage(Page.webComponentPageBuilder().title("Images") + .componentLink("qwc-images.js") + .icon("font-awesome-solid:palette")); + producers.produce(new JsonRPCProvidersBuildItem(OpenAiImagesJsonRPCService.class)); + return card; + } else { + return null; + } + } + +} diff --git a/openai/openai-vanilla/deployment/src/main/resources/dev-ui/qwc-images.js b/openai/openai-vanilla/deployment/src/main/resources/dev-ui/qwc-images.js new file mode 100644 index 000000000..ed7f915ae --- /dev/null +++ b/openai/openai-vanilla/deployment/src/main/resources/dev-ui/qwc-images.js @@ -0,0 +1,129 @@ +import {html, LitElement} from 'lit'; +import '@vaadin/grid'; +import '@vaadin/grid/vaadin-grid-column.js'; +import '@vaadin/text-area'; +import '@vaadin/button'; +import { JsonRpc } from 'jsonrpc'; + +export class QwcImages extends LitElement { + + jsonRpc = new JsonRpc(this); + + supportedModels = [ + { label: "dall-e-2", value: "dall-e-2"}, + { label: "dall-e-3", value: "dall-e-3"}] + + supportedSizes = [ + { label: "256x256", value: "256x256"}, + { label: "512x512", value: "512x512"}, + { label: "1024x1024", value: "1024x1024"}, + { label: "1024x1792", value: "1024x1792"}, + { label: "1792x1024", value: "1792x1024"} + ] + + supportedQualities = [ + { label: "standard", value: "standard"}, + { label: "hd", value: "hd"}] + + supportedStyles = [ + { label: "vivid", value: "vivid"}, + { label: "natural", value: "natural"}] + + static properties = { + "_generatedImages": {state: true}, + "_statusInfo": {state: true}, + } + + constructor() { + super(); + this._generatedImages = []; + } + + render() { + return html` +

Model configuration

+ + + + + + +
+ ${this._statusInfo} + this._doGenerate( + this.shadowRoot.getElementById('model-name').value, + this.shadowRoot.getElementById('image-prompt').value, + this.shadowRoot.getElementById('size').value, + this.shadowRoot.getElementById('quality').value, + this.shadowRoot.getElementById('style').value + )}>Generate image + +
+

Generated images

+ ${this._renderImages()} + `; + } + + _doGenerate(modelName, prompt, size, quality, style) { + this._statusInfo = html`Generating image...
`; + this.jsonRpc.generate({modelName: modelName, prompt: prompt, size: size, quality: quality, style: style}).then((jsonRpcResponse) => { + this._statusInfo = html` + Image generated successfully. + `; + this._generatedImages = [jsonRpcResponse.result].concat(this._generatedImages) + }).catch((error) => { + this._statusInfo = html` + + ${JSON.stringify(error.error)} + ` + }); + } + + _renderImages() { + if(this._generatedImages.length === 0) { + return html`Nothing yet`; + } else { + return html` + ${this._generatedImages.map((image) => { + return html` + ${image.prompt}
+ ${image.prompt}
+ `; + } + )} + `; + } + } + +} + +customElements.define('qwc-images', QwcImages); \ No newline at end of file diff --git a/openai/openai-vanilla/runtime/src/main/java/io/quarkiverse/langchain4j/openai/runtime/devui/OpenAiImagesJsonRPCService.java b/openai/openai-vanilla/runtime/src/main/java/io/quarkiverse/langchain4j/openai/runtime/devui/OpenAiImagesJsonRPCService.java new file mode 100644 index 000000000..a9adb5b7c --- /dev/null +++ b/openai/openai-vanilla/runtime/src/main/java/io/quarkiverse/langchain4j/openai/runtime/devui/OpenAiImagesJsonRPCService.java @@ -0,0 +1,67 @@ +package io.quarkiverse.langchain4j.openai.runtime.devui; + +import java.time.Duration; +import java.util.Optional; + +import jakarta.inject.Inject; + +import org.eclipse.microprofile.config.inject.ConfigProperty; + +import dev.langchain4j.data.image.Image; +import dev.langchain4j.model.image.ImageModel; +import io.quarkiverse.langchain4j.openai.QuarkusOpenAiImageModel; +import io.vertx.core.json.JsonObject; + +public class OpenAiImagesJsonRPCService { + + @Inject + @ConfigProperty(name = "quarkus.langchain4j.openai.base-url") + String baseUrl; + + @Inject + @ConfigProperty(name = "quarkus.langchain4j.openai.api-key") + String apiKey; + + @Inject + @ConfigProperty(name = "quarkus.langchain4j.openai.timeout") + Duration timeout; + + @Inject + @ConfigProperty(name = "quarkus.langchain4j.openai.image-model.user") + Optional user; + + @Inject + @ConfigProperty(name = "quarkus.langchain4j.openai.max-retries") + Integer maxRetries; + + public JsonObject generate(String modelName, String size, String prompt, String quality) { + ImageModel model = QuarkusOpenAiImageModel.builder() + .baseUrl(baseUrl) + .apiKey(apiKey) + .timeout(timeout) + .user(user) + .maxRetries(maxRetries) + .persistDirectory(Optional.empty()) + .modelName(modelName) + .quality(quality) + .size(size) + .build(); + Image image = model.generate(prompt).content(); + JsonObject result = new JsonObject(); + result.put("prompt", prompt); + // there's either URL or base64Data present in the response, depending + // on `quarkus.langchain4j.openai.image-model.response-format` + if (image.url() != null) { + result.put("url", image.url().toString()); + } else { + result.put("url", null); + } + if (image.base64Data() != null && !image.base64Data().isEmpty()) { + result.put("base64Data", image.base64Data()); + } else { + result.put("base64Data", null); + } + return result; + } + +} From 2b489decc9f2528693c530e374844c1bf6f603a7 Mon Sep 17 00:00:00 2001 From: Jan Martiska Date: Fri, 5 Jan 2024 10:38:41 +0100 Subject: [PATCH 2/2] Some basic documentation about provided Dev UI features --- docs/modules/ROOT/nav.adoc | 3 +++ docs/modules/ROOT/pages/dev-ui.adoc | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 docs/modules/ROOT/pages/dev-ui.adoc diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc index cf76078fd..05f8a4763 100644 --- a/docs/modules/ROOT/nav.adoc +++ b/docs/modules/ROOT/nav.adoc @@ -19,5 +19,8 @@ ** xref:in-process-embedding.adoc[In-Process Embeddings] ** xref:csv.adoc[Loading CSV files] +* Additional tools +** xref:dev-ui.adoc[Dev UI] + * Advanced topics ** xref:fault-tolerance.adoc[Fault Tolerance] diff --git a/docs/modules/ROOT/pages/dev-ui.adoc b/docs/modules/ROOT/pages/dev-ui.adoc new file mode 100644 index 000000000..a9b739093 --- /dev/null +++ b/docs/modules/ROOT/pages/dev-ui.adoc @@ -0,0 +1,19 @@ += Dev UI + +include::./includes/attributes.adoc[] + +If you use the Dev mode, the `quarkus-langchain4j` project provides several pages +in the Dev UI to facilitate development: + +* *AI Services* page: provides a table of all AI Services detected in the application along +with a list of tools that they are declared to use. + +* *Tools* page: provides a list of tools detected in the application. + +* *Chat* page: allows you to manually hold a conversation with a chat model. This +page is only available if the application contains a chat model. + +* *Images* page: allows you to test the outputs of image models and tune its parameters. +This page is provided specifically by the `openai-vanilla` extension and is currently specific +to OpenAI's image models. It appears if the application uses the `openai-vanilla` extension +and doesn't have image models explicitly disabled. \ No newline at end of file