Skip to content

Commit

Permalink
add WIP open/elastic search modules
Browse files Browse the repository at this point in the history
  • Loading branch information
graemerocher committed Jun 27, 2024
1 parent 656edc6 commit 34e890c
Show file tree
Hide file tree
Showing 9 changed files with 198 additions and 2 deletions.
6 changes: 6 additions & 0 deletions gradle/libs.versions.toml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ micronaut-docs = "2.0.0"
micronaut-neo4j = "6.6.0"
micronaut-serde = "2.10.1"
micronaut-sql = "5.7.0"
micronaut-opensearch = "1.1.0"
micronaut-elasticsearch = "5.5.0"
micronaut-redis = "6.4.0"
micronaut-test = "4.2.1"
micronaut-test-resources = "2.5.0"
Expand All @@ -38,6 +40,8 @@ managed-langchain4j = "0.31.0"
[libraries]
micronaut-core = { module = 'io.micronaut:micronaut-core-bom', version.ref = 'micronaut-core' }
micronaut-redis = { module = 'io.micronaut.redis:micronaut-redis-bom', version.ref = 'micronaut-redis' }
micronaut-opensearch = { module = 'io.micronaut.opensearch:micronaut-opensearch-bom', version.ref = 'micronaut-opensearch' }
micronaut-elasticsearch = { module = 'io.micronaut.elasticsearch:micronaut-elasticsearch-bom', version.ref = 'micronaut-elasticsearch' }
micronaut-neo4j = { module = 'io.micronaut.neo4j:micronaut-neo4j-bom', version.ref = 'micronaut-neo4j' }
micronaut-logging = { module = 'io.micronaut.logging:micronaut-logging-bom', version.ref = 'micronaut-logging' }
micronaut-serde = { module = 'io.micronaut.serde:micronaut-serde-bom', version.ref = 'micronaut-serde' }
Expand All @@ -56,6 +60,8 @@ langchain4j-open-ai = { module = 'dev.langchain4j:langchain4j-open-ai', version.
langchain4j-azure-open-ai = { module = 'dev.langchain4j:langchain4j-azure-open-ai', version.ref = "managed-langchain4j" }
langchain4j-redis = { module = 'dev.langchain4j:langchain4j-redis', version.ref = "managed-langchain4j" }
langchain4j-neo4j = { module = 'dev.langchain4j:langchain4j-neo4j', version.ref = "managed-langchain4j" }
langchain4j-opensearch = { module = 'dev.langchain4j:langchain4j-opensearch', version.ref = "managed-langchain4j" }
langchain4j-elasticsearch = { module = 'dev.langchain4j:langchain4j-elasticsearch', version.ref = "managed-langchain4j" }
langchain4j-pgvector = { module = 'dev.langchain4j:langchain4j-pgvector', version.ref = "managed-langchain4j" }
langchain4j-embeddings-all-minilm-l6-v2 = { module = 'dev.langchain4j:langchain4j-embeddings-all-minilm-l6-v2', version.ref = "managed-langchain4j" }

Expand Down
19 changes: 19 additions & 0 deletions micronaut-langchain4j-store-elasticsearch/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
plugins {
id("io.micronaut.build.internal.langchain4j-module")
id("io.micronaut.test-resources")
}

dependencies {
annotationProcessor(projects.micronautLangchain4jProcessor)
annotationProcessor(mn.micronaut.inject.java)
annotationProcessor(mnSourcegen.micronaut.sourcegen.generator.java)
api(projects.micronautLangchain4jCore)
api(mnElasticsearch.micronaut.elasticsearch)
implementation(libs.langchain4j.elasticsearch)
testAnnotationProcessor(mn.micronaut.inject.java)
testImplementation(mnTest.micronaut.test.junit5)
testImplementation(mnSerde.micronaut.serde.jackson)
testImplementation(libs.langchain4j.embeddings.all.minilm.l6.v2)
testRuntimeOnly(mnLogging.logback.classic)
testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine")
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
/*
* Copyright 2017-2024 original authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.micronaut.langchain4j.elasticsearch;

import dev.langchain4j.store.embedding.EmbeddingStore;
import dev.langchain4j.store.embedding.elasticsearch.ElasticsearchEmbeddingStore;
import io.micronaut.langchain4j.annotation.Lang4jConfig;

@Lang4jConfig(
models = @Lang4jConfig.Model(
kind = EmbeddingStore.class,
impl = ElasticsearchEmbeddingStore.class
),
properties = {
@Lang4jConfig.Property(
name = "restClient",
injected = true,
common = true
),
@Lang4jConfig.Property(
name = "apiKey",
common = true
),
@Lang4jConfig.Property(
name = "serverUrl",
common = true
)
}
)
final class ElasticModule {
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
package io.micronaut.langchain4j.elasticsearch;

import dev.langchain4j.data.embedding.Embedding;
import dev.langchain4j.data.segment.TextSegment;
import dev.langchain4j.model.embedding.AllMiniLmL6V2EmbeddingModel;
import dev.langchain4j.model.embedding.EmbeddingModel;
import dev.langchain4j.store.embedding.EmbeddingMatch;
import dev.langchain4j.store.embedding.EmbeddingStore;
import dev.langchain4j.store.embedding.elasticsearch.ElasticsearchEmbeddingStore;
import io.micronaut.test.extensions.junit5.annotation.MicronautTest;
import java.util.List;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;

@MicronautTest
@Disabled("TODO: Fails with dependency conflict NoSuchMethodError: 'co.elastic.clients.elasticsearch._types.mapping.DenseVectorProperty")
public class ElasticTest {
@Test
void testEmbeddingStore(EmbeddingStore<TextSegment> embeddingStore) {
Assertions.assertNotNull(embeddingStore);
Assertions.assertInstanceOf(ElasticsearchEmbeddingStore.class, embeddingStore);
EmbeddingModel embeddingModel = new AllMiniLmL6V2EmbeddingModel();

TextSegment segment1 = TextSegment.from("I like football.");
Embedding embedding1 = embeddingModel.embed(segment1).content();
embeddingStore.add(embedding1, segment1);

TextSegment segment2 = TextSegment.from("The weather is good today.");
Embedding embedding2 = embeddingModel.embed(segment2).content();
embeddingStore.add(embedding2, segment2);

Embedding queryEmbedding = embeddingModel.embed("What is your favourite sport?").content();
List<EmbeddingMatch<TextSegment>> relevant = embeddingStore.findRelevant(queryEmbedding, 1);
EmbeddingMatch<TextSegment> embeddingMatch = relevant.get(0);

System.out.println(embeddingMatch.score()); // 0.8144288608390052
Assertions.assertEquals("I like football.", embeddingMatch.embedded().text());
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
langchain4j.elasticsearch.embedding-store.dimension=384
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
<configuration>

<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%cyan(%d{HH:mm:ss.SSS}) %gray([%thread]) %highlight(%-5level) %magenta(%logger{36}) - %msg%n</pattern>
</encoder>
</appender>

<root level="info">
<appender-ref ref="STDOUT" />
</root>

<logger name="io.micronaut.context.env" level="TRACE" />
</configuration>
19 changes: 19 additions & 0 deletions micronaut-langchain4j-store-opensearch/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
plugins {
id("io.micronaut.build.internal.langchain4j-module")
id("io.micronaut.test-resources")
}

dependencies {
annotationProcessor(projects.micronautLangchain4jProcessor)
annotationProcessor(mn.micronaut.inject.java)
annotationProcessor(mnSourcegen.micronaut.sourcegen.generator.java)
api(projects.micronautLangchain4jCore)
api(mnOpensearch.micronaut.opensearch)
implementation(libs.langchain4j.opensearch)
testAnnotationProcessor(mn.micronaut.inject.java)
testImplementation(mnTest.micronaut.test.junit5)
testImplementation(mnSerde.micronaut.serde.jackson)
testImplementation(libs.langchain4j.embeddings.all.minilm.l6.v2)
testRuntimeOnly(mnLogging.logback.classic)
testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine")
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
/*
* Copyright 2017-2024 original authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.micronaut.langchain4j.opensearch;

import dev.langchain4j.store.embedding.EmbeddingStore;
import dev.langchain4j.store.embedding.opensearch.OpenSearchEmbeddingStore;
import io.micronaut.langchain4j.annotation.Lang4jConfig;

@Lang4jConfig(
models = @Lang4jConfig.Model(
kind = EmbeddingStore.class,
impl = OpenSearchEmbeddingStore.class
),
properties = {
@Lang4jConfig.Property(
name = "openSearchClient",
injected = true,
common = true
),
@Lang4jConfig.Property(
name = "serverUrl",
common = true
),
@Lang4jConfig.Property(
name = "apiKey",
common = true
),
@Lang4jConfig.Property(
name = "options",
injected = true,
common = true
)
}
)
final class OpenSearchModule {
}
8 changes: 6 additions & 2 deletions settings.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ include("micronaut-langchain4j-vertexai")
include("micronaut-langchain4j-mistralai")
include("micronaut-langchain4j-vertexai-gemini")
include("micronaut-langchain4j-ollama-testresource")
include("micronaut-langchain4j-store-elasticsearch")
include("micronaut-langchain4j-store-opensearch")
include("micronaut-langchain4j-store-neo4j")
include("micronaut-langchain4j-store-redis")
include("micronaut-langchain4j-store-pgvector")
Expand All @@ -48,9 +50,11 @@ configure<io.micronaut.build.MicronautBuildSettingsExtension> {
importMicronautCatalog("micronaut-sourcegen")
importMicronautCatalog("micronaut-test-resources")
importMicronautCatalog("micronaut-logging")
importMicronautCatalog("micronaut-elasticsearch")
importMicronautCatalog("micronaut-neo4j")
importMicronautCatalog("micronaut-opensearch")
importMicronautCatalog("micronaut-redis")
importMicronautCatalog("micronaut-serde")
importMicronautCatalog("micronaut-sql")
importMicronautCatalog("micronaut-redis")
importMicronautCatalog("micronaut-neo4j")
// importMicronautCatalog("micronaut-validation")
}

0 comments on commit 34e890c

Please sign in to comment.