Skip to content

Commit

Permalink
Merge branch 'quarkiverse:main' into main
Browse files Browse the repository at this point in the history
  • Loading branch information
@humcqc
humcqc authored Jun 24, 2024
2 parents 3fe6b8d + 11b1c91 commit c6d0d18
Show file tree
Hide file tree
Showing 43 changed files with 1,223 additions and 101 deletions.
75 changes: 46 additions & 29 deletions core/deployment/src/main/resources/dev-ui/qwc-chat.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,10 +124,10 @@ export class QwcChat extends LitElement {
this._streamingChatEnabled = this._streamingChatEnabled && !this._ragEnabled;
this.render();
}}"/></div>
<div><vaadin-checkbox label="Enable Streaming Chat (if supported by model & rag disabled)"
<div><vaadin-checkbox label="Enable Streaming Chat"
class="${this._streamingChatSupported ? 'show' : 'hide'}"
?checked="${this._streamingChatEnabled}"
?disabled="${!this._streamingChatSupported || this._ragEnabled}"
?disabled="${!this._streamingChatSupported}"
@change="${(event) => {
this._streamingChatEnabled = event.target.checked;
this.render();
Expand Down Expand Up @@ -189,36 +189,53 @@ export class QwcChat extends LitElement {

_handleSendChat(e) {
let message = e.detail.value;
if(message && message.trim().length>0){
if (message && message.trim().length > 0) {
this._cementSystemMessage();
this._addUserMessage(message);
var indexUserMessage = this._addUserMessage(message);
this._showProgressBar();
if (this._streamingChatEnabled) {

if (this._streamingChatEnabled) {
var msg = "";
var index = this._addBotMessage(msg);
try {
this._observer = this.jsonRpc.streamingChat({message: message, ragEnabled: this._ragEnabled})
.onNext(jsonRpcResponse => {
if (jsonRpcResponse.result.error) {
this._showError(jsonRpcResponse.result.error);
this._hideProgressBar();
} else if (jsonRpcResponse.result.message) {
this._updateMessage(index, jsonRpcResponse.result.message);
this._hideProgressBar();
} else {
msg += jsonRpcResponse.result.token;
this._updateMessage(index, msg);
}
})
.onError((error) => {
var index = null;
try {
this._observer = this.jsonRpc.streamingChat({message: message, ragEnabled: this._ragEnabled})
.onNext(jsonRpcResponse => {
if (jsonRpcResponse.result.error) {
this._showError(jsonRpcResponse.result.error);
this._hideProgressBar();
} else if (jsonRpcResponse.result.augmentedMessage) {
// replace the last user message with the augmented message
this._updateMessage(indexUserMessage, jsonRpcResponse.result.augmentedMessage);
} else if (jsonRpcResponse.result.toolExecutionRequest) {
var item = jsonRpcResponse.result.toolExecutionRequest;
this._addToolMessage(`Request to execute the following tool:
Request ID = ${item.id},
tool name = ${item.name},
arguments = ${item.arguments}`);
} else if (jsonRpcResponse.result.toolExecutionResult) {
var item = jsonRpcResponse.result.toolExecutionResult;
this._addToolMessage(`Tool execution result for request ID = ${item.id},
tool name = ${item.toolName},
status = ${item.text}`);
} else if (jsonRpcResponse.result.message) {
this._updateMessage(index, jsonRpcResponse.result.message);
this._hideProgressBar();
} else { // a new token from the stream
if(index === null) {
index = this._addBotMessage(msg);
}
msg += jsonRpcResponse.result.token;
this._updateMessage(index, msg);
}
})
.onError((error) => {
this._showError(error);
this._hideProgressBar();
});
} catch (error) {
this._showError(error);
this._hideProgressBar();
});
} catch(error) {
this._showError(error);
this._hideProgressBar();
}
}
} else {
this.jsonRpc.chat({message: message, ragEnabled: this._ragEnabled}).then(jsonRpcResponse => {
this._showResponse(jsonRpcResponse);
Expand All @@ -229,7 +246,7 @@ export class QwcChat extends LitElement {
}
}

}
}

_showResponse(jsonRpcResponse) {
if (jsonRpcResponse.result === false) {
Expand Down Expand Up @@ -318,7 +335,7 @@ status = ${item.toolExecutionResult.text}`);
}

_addUserMessage(message){
this._addMessage(message, "Me", 1);
return this._addMessage(message, "Me", 1);
}

_addStyledMessage(message, user, colorIndex, className){
Expand Down
114 changes: 92 additions & 22 deletions core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/devui/ChatJsonRPCService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,23 +27,29 @@
import dev.langchain4j.model.chat.StreamingChatLanguageModel;
import dev.langchain4j.model.output.Response;
import dev.langchain4j.model.output.TokenUsage;
import dev.langchain4j.rag.AugmentationRequest;
import dev.langchain4j.rag.RetrievalAugmentor;
import dev.langchain4j.rag.query.Metadata;
import io.quarkiverse.langchain4j.runtime.ToolsRecorder;
import io.quarkiverse.langchain4j.runtime.devui.json.ChatMessagePojo;
import io.quarkiverse.langchain4j.runtime.devui.json.ChatResultPojo;
import io.quarkiverse.langchain4j.runtime.devui.json.ToolExecutionRequestPojo;
import io.quarkiverse.langchain4j.runtime.devui.json.ToolExecutionResultPojo;
import io.quarkiverse.langchain4j.runtime.tool.QuarkusToolExecutor;
import io.quarkiverse.langchain4j.runtime.tool.QuarkusToolExecutorFactory;
import io.quarkiverse.langchain4j.runtime.tool.ToolMethodCreateInfo;
import io.quarkus.arc.All;
import io.quarkus.arc.Arc;
import io.quarkus.logging.Log;
import io.smallrye.mutiny.Multi;
import io.smallrye.mutiny.infrastructure.Infrastructure;
import io.smallrye.mutiny.subscription.MultiEmitter;
import io.vertx.core.json.JsonObject;

@ActivateRequestContext
public class ChatJsonRPCService {

public static final int MAX_SEQUENTIAL_TOOL_EXECUTIONS = 20;
private final ChatLanguageModel model;
private final Optional<StreamingChatLanguageModel> streamingModel;

Expand Down Expand Up @@ -144,38 +150,47 @@ public Multi<JsonObject> streamingChat(String message, boolean ragEnabled) {
// removing single messages
List<ChatMessage> chatMemoryBackup = memory.messages();

return Multi.createFrom().emitter(em -> {
Multi<JsonObject> stream = Multi.createFrom().emitter(em -> {
try {
// invoke RAG is applicable
if (retrievalAugmentor != null && ragEnabled) {
UserMessage userMessage = UserMessage.from(message);
Metadata metadata = Metadata.from(userMessage, currentMemoryId.get(), memory.messages());
memory.add(retrievalAugmentor.augment(userMessage, metadata));
AugmentationRequest augmentationRequest = new AugmentationRequest(userMessage, metadata);
ChatMessage augmentedMessage = retrievalAugmentor.augment(augmentationRequest).chatMessage();
memory.add(augmentedMessage);
em.emit(new JsonObject().put("augmentedMessage", augmentedMessage.text()));
} else {

memory.add(new UserMessage(message));
}

StreamingChatLanguageModel streamingModel = this.streamingModel.orElseThrow(IllegalStateException::new);

streamingModel.generate(memory.messages(), new StreamingResponseHandler<AiMessage>() {
@Override
public void onComplete(Response<AiMessage> response) {
memory.add(response.content());
String message = response.content().text();
em.emit(new JsonObject().put("message", message));
em.complete();
}
// invoke tools if applicable
Response<AiMessage> modelResponse;
if (toolSpecifications.isEmpty()) {
streamingModel.generate(memory.messages(), new StreamingResponseHandler<AiMessage>() {
@Override
public void onComplete(Response<AiMessage> response) {
memory.add(response.content());
String message = response.content().text();
em.emit(new JsonObject().put("message", message));
em.complete();
}

@Override
public void onNext(String token) {
em.emit(new JsonObject().put("token", token));
}
@Override
public void onNext(String token) {
em.emit(new JsonObject().put("token", token));
}

@Override
public void onError(Throwable error) {
em.fail(error);
}
});
@Override
public void onError(Throwable error) {
em.fail(error);
}
});
} else {
executeWithToolsAndStreaming(memory, em, MAX_SEQUENTIAL_TOOL_EXECUTIONS);
}
} catch (Throwable t) {
// restore the memory from the backup
memory.clear();
Expand All @@ -184,6 +199,8 @@ public void onError(Throwable error) {
em.fail(t);
}
});
// run on a worker thread because the retrieval augmentor might be blocking
return stream.runSubscriptionOn(Infrastructure.getDefaultWorkerPool());
}

public ChatResultPojo chat(String message, boolean ragEnabled) {
Expand All @@ -201,7 +218,9 @@ public ChatResultPojo chat(String message, boolean ragEnabled) {
if (retrievalAugmentor != null && ragEnabled) {
UserMessage userMessage = UserMessage.from(message);
Metadata metadata = Metadata.from(userMessage, currentMemoryId.get(), memory.messages());
memory.add(retrievalAugmentor.augment(userMessage, metadata));
AugmentationRequest augmentationRequest = new AugmentationRequest(userMessage, metadata);
ChatMessage augmentedMessage = retrievalAugmentor.augment(augmentationRequest).chatMessage();
memory.add(augmentedMessage);
} else {
memory.add(new UserMessage(message));
}
Expand All @@ -226,7 +245,7 @@ public ChatResultPojo chat(String message, boolean ragEnabled) {

// FIXME: this was basically copied from `dev.langchain4j.service.DefaultAiServices`,
// maybe it could be extracted into a reusable piece of code
public Response<AiMessage> executeWithTools(ChatMemory memory) {
private Response<AiMessage> executeWithTools(ChatMemory memory) {
Response<AiMessage> response = model.generate(memory.messages(), toolSpecifications);
int MAX_SEQUENTIAL_TOOL_EXECUTIONS = 20;
int executionsLeft = MAX_SEQUENTIAL_TOOL_EXECUTIONS;
Expand All @@ -253,4 +272,55 @@ public Response<AiMessage> executeWithTools(ChatMemory memory) {
return Response.from(response.content(), new TokenUsage(), response.finishReason());
}

private void executeWithToolsAndStreaming(ChatMemory memory,
MultiEmitter<? super JsonObject> em,
int toolExecutionsLeft) {
toolExecutionsLeft--;
if (toolExecutionsLeft == 0) {
throw new RuntimeException(
"Something is wrong, exceeded " + MAX_SEQUENTIAL_TOOL_EXECUTIONS + " sequential tool executions");
}
int finalToolExecutionsLeft = toolExecutionsLeft;
streamingModel.get().generate(memory.messages(), toolSpecifications, new StreamingResponseHandler<AiMessage>() {
@Override
public void onComplete(Response<AiMessage> response) {
// run on a worker thread because the tool might be blocking
Infrastructure.getDefaultExecutor().execute(() -> {
AiMessage aiMessage = response.content();
memory.add(aiMessage);
if (!aiMessage.hasToolExecutionRequests()) {
em.emit(new JsonObject().put("message", aiMessage.text()));
em.complete();
} else {
for (ToolExecutionRequest toolExecutionRequest : aiMessage.toolExecutionRequests()) {
ToolExecutor toolExecutor = toolExecutors.get(toolExecutionRequest.name());
ToolExecutionRequestPojo toolExecutionRequestPojo = new ToolExecutionRequestPojo(
toolExecutionRequest.id(), toolExecutionRequest.name(), toolExecutionRequest.arguments());
em.emit(new JsonObject().put("toolExecutionRequest", toolExecutionRequestPojo));
String toolExecutionResult = toolExecutor.execute(toolExecutionRequest, currentMemoryId.get());
ToolExecutionResultMessage toolExecutionResultMessage = ToolExecutionResultMessage
.from(toolExecutionRequest, toolExecutionResult);
memory.add(toolExecutionResultMessage);
ToolExecutionResultPojo toolExecutionResultPojo = new ToolExecutionResultPojo(
toolExecutionResultMessage.id(), toolExecutionResultMessage.toolName(),
toolExecutionResultMessage.text());
em.emit(new JsonObject().put("toolExecutionResult", toolExecutionResultPojo));
}
executeWithToolsAndStreaming(memory, em, finalToolExecutionsLeft);
}
});
}

@Override
public void onNext(String token) {
em.emit(new JsonObject().put("token", token));
}

@Override
public void onError(Throwable error) {
throw new RuntimeException(error);
}
});
}

}
34 changes: 34 additions & 0 deletions docs/modules/ROOT/pages/includes/quarkus-langchain4j-chroma.adoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,40 @@ endif::add-copy-button-to-env-var[]
|


a| [[quarkus-langchain4j-chroma_quarkus-langchain4j-chroma-log-requests]]`link:#quarkus-langchain4j-chroma_quarkus-langchain4j-chroma-log-requests[quarkus.langchain4j.chroma.log-requests]`


[.description]
--
Whether requests to Chroma should be logged

ifdef::add-copy-button-to-env-var[]
Environment variable: env_var_with_copy_button:+++QUARKUS_LANGCHAIN4J_CHROMA_LOG_REQUESTS+++[]
endif::add-copy-button-to-env-var[]
ifndef::add-copy-button-to-env-var[]
Environment variable: `+++QUARKUS_LANGCHAIN4J_CHROMA_LOG_REQUESTS+++`
endif::add-copy-button-to-env-var[]
--|boolean
|`false`


a| [[quarkus-langchain4j-chroma_quarkus-langchain4j-chroma-log-responses]]`link:#quarkus-langchain4j-chroma_quarkus-langchain4j-chroma-log-responses[quarkus.langchain4j.chroma.log-responses]`


[.description]
--
Whether responses from Chroma should be logged

ifdef::add-copy-button-to-env-var[]
Environment variable: env_var_with_copy_button:+++QUARKUS_LANGCHAIN4J_CHROMA_LOG_RESPONSES+++[]
endif::add-copy-button-to-env-var[]
ifndef::add-copy-button-to-env-var[]
Environment variable: `+++QUARKUS_LANGCHAIN4J_CHROMA_LOG_RESPONSES+++`
endif::add-copy-button-to-env-var[]
--|boolean
|`false`


a|icon:lock[title=Fixed at build time] [[quarkus-langchain4j-chroma_quarkus-langchain4j-chroma-devservices-container-env-container-env]]`link:#quarkus-langchain4j-chroma_quarkus-langchain4j-chroma-devservices-container-env-container-env[quarkus.langchain4j.chroma.devservices.container-env]`


Expand Down
2 changes: 2 additions & 0 deletions docs/modules/ROOT/pages/index.adoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ image::llms-big-picture.png[width=600,align="center"]

== Quick Overview

NOTE: If you're interested in a guided tutorial, there is also a https://github.com/quarkusio/quarkus-langchain4j-workshop[workshop] available that walks you through the basics of using Quarkus with LangChain4j.

To incorporate Quarkus LangChain4j into your Quarkus project, add the following Maven dependency:

[source,xml,subs=attributes+]
Expand Down
2 changes: 2 additions & 0 deletions ...c/test/java/io/quarkiverse/langchain4j/chroma/deployment/ChromaEmbeddingStoreCDITest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ class ChromaEmbeddingStoreCDITest extends EmbeddingStoreIT {

@RegisterExtension
static final QuarkusUnitTest unitTest = new QuarkusUnitTest()
// .overrideConfigKey("quarkus.langchain4j.log-requests", "true")
// .overrideConfigKey("quarkus.langchain4j.log-responses", "true")
.setArchiveProducer(() -> ShrinkWrap.create(JavaArchive.class));

@Inject
Expand Down
2 changes: 2 additions & 0 deletions .../src/test/java/io/quarkiverse/langchain4j/chroma/deployment/ChromaEmbeddingStoreTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@ protected ChromaEmbeddingStore embeddingStore() {
embeddingStore = ChromaEmbeddingStore.builder()
.baseUrl(chromaUrl)
.collectionName(randomUUID())
// .logRequests(true)
// .logResponses(true)
.build();
}
return embeddingStore;
Expand Down
Loading

0 comments on commit c6d0d18

Please sign in to comment.