Merge pull request #275 from weaviate/add-support-for-multi2vec-jinaa…

…i-module Add support for multi2vec-jinaai module
weaviate · Nov 21, 2024 · e1dbdcb · e1dbdcb
2 parents de54a41 + d1c8071
commit e1dbdcb
Show file tree

Hide file tree

Showing 5 changed files with 27 additions and 2 deletions.
diff --git a/.cicd/test.sh b/.cicd/test.sh
@@ -150,6 +150,8 @@ function check_creates_template() {
   check_modules "--set modules.text2vec-jinaai.enabled=true --set modules.text2vec-jinaai.apiKey=apiKey" "value: text2vec-jinaai"
   check_modules "--set modules.reranker-jinaai.enabled=true" "value: reranker-jinaai"
   check_modules "--set modules.reranker-jinaai.enabled=true --set modules.reranker-jinaai.apiKey=apiKey" "value: reranker-jinaai"
+  check_modules "--set modules.multi2vec-jinaai.enabled=true" "value: multi2vec-jinaai"
+  check_modules "--set modules.multi2vec-jinaai.enabled=true --set modules.multi2vec-jinaai.apiKey=apiKey" "value: multi2vec-jinaai"
   check_modules "--set modules.text2vec-aws.enabled=true" "value: text2vec-aws"
   check_modules "--set modules.generative-aws.enabled=true" "value: generative-aws"
   check_modules "--set modules.text2vec-aws.enabled=true --set modules.generative-aws.enabled=true" "value: generative-aws,text2vec-aws"
@@ -185,6 +187,7 @@ function check_creates_template() {
   check_string_existence "--set modules.qna-openai.enabled=true --set modules.qna-openai.azureApiKey=azureApiKey" "name: AZURE_APIKEY"
   check_string_existence "--set modules.generative-openai.enabled=true --set modules.generative-openai.azureApiKey=azureApiKey" "name: AZURE_APIKEY"
   check_string_existence "--set modules.text2vec-jinaai.enabled=true --set modules.text2vec-jinaai.apiKey=jinaaiApiKey" "name: JINAAI_APIKEY"
+  check_string_existence "--set modules.multi2vec-jinaai.enabled=true --set modules.multi2vec-jinaai.apiKey=jinaaiApiKey" "name: JINAAI_APIKEY"
   check_string_existence "--set grpcService.enabled=true" "containerPort: 50051"
   check_string_existence "--set grpcService.enabled=true --set grpcService.name=weaviate-grpc-service-custom-name" "name: weaviate-grpc-service-custom-name"
   check_string_existence "--set grpcService.name=weaviate-grpc-defult-service-custom-name" "name: weaviate-grpc-defult-service-custom-name"
@@ -219,6 +222,8 @@ function check_creates_template() {
   check_string_existence "--set modules.text2vec-openai.enabled=true --set modules.text2vec-jinaai.enabled=true --set modules.text2vec-jinaai.apiKey=JinaAIApiKey" "name: JINAAI_APIKEY"
   check_string_existence "--set modules.text2vec-openai.enabled=true --set modules.reranker-jinaai.enabled=true --set modules.reranker-jinaai.apiKey=JinaAIApiKey" "name: JINAAI_APIKEY"
   check_string_existence "--set modules.text2vec-openai.enabled=true --set modules.reranker-jinaai.enabled=true --set modules.reranker-jinaai.apiKey=JinaAIApiKey --set modules.text2vec-jinaai.enabled=true --set modules.text2vec-jinaai.apiKey=JinaAIApiKey" "name: JINAAI_APIKEY"
+  check_string_existence "--set modules.text2vec-openai.enabled=true --set modules.multi2vec-jinaai.enabled=true --set modules.multi2vec-jinaai.apiKey=JinaAIApiKey --set modules.text2vec-jinaai.enabled=true --set modules.text2vec-jinaai.apiKey=JinaAIApiKey" "name: JINAAI_APIKEY"
+  check_string_existence "--set modules.text2vec-openai.enabled=true --set modules.multi2vec-jinaai.enabled=true --set modules.multi2vec-jinaai.apiKey=JinaAIApiKey --set modules.reranker-jinaai.enabled=true --set modules.reranker-jinaai.apiKey=JinaAIApiKey --set modules.text2vec-jinaai.enabled=true --set modules.text2vec-jinaai.apiKey=JinaAIApiKey" "name: JINAAI_APIKEY"
   check_string_existence "--set modules.text2vec-openai.enabled=true --set modules.generative-friendliai.enabled=true --set modules.generative-friendliai.token=FriendliToken" "name: FRIENDLI_TOKEN"
   check_string_existence "--set modules.text2vec-databricks.enabled=true --set modules.text2vec-databricks.token=databricksToken" "name: DATABRICKS_TOKEN"
   check_string_existence "--set modules.generative-databricks.enabled=true --set modules.generative-databricks.token=databricksToken" "name: DATABRICKS_TOKEN"

diff --git a/weaviate/templates/_helpers.tpl b/weaviate/templates/_helpers.tpl
@@ -83,6 +83,9 @@
   {{- if (index .Values "modules" "multi2vec-cohere" "enabled") -}}
     {{ $modules = append $modules "multi2vec-cohere" }}
   {{- end -}}
+  {{- if (index .Values "modules" "multi2vec-jinaai" "enabled") -}}
+    {{ $modules = append $modules "multi2vec-jinaai" }}
+  {{- end -}}
   {{- if (index .Values "modules" "text2vec-openai" "enabled") -}}
     {{ $modules = append $modules "text2vec-openai" }}
   {{- end -}}

diff --git a/weaviate/templates/apiKeyJinaAISecret.yaml b/weaviate/templates/apiKeyJinaAISecret.yaml
@@ -1,6 +1,7 @@
 {{- $t2vJinaAI :=  and (index .Values "modules" "text2vec-jinaai" "enabled") (index .Values "modules" "text2vec-jinaai" "apiKey") }}
 {{- $rerankerJinaAI := and (index .Values "modules" "reranker-jinaai" "enabled") (index .Values "modules" "reranker-jinaai" "apiKey") }}
-{{- $jinaai := or ($t2vJinaAI) ($rerankerJinaAI) }}
+{{- $multi2vecJinaAI := and (index .Values "modules" "multi2vec-jinaai" "enabled") (index .Values "modules" "multi2vec-jinaai" "apiKey") }}
+{{- $jinaai := or ($t2vJinaAI) ($rerankerJinaAI) ($multi2vecJinaAI) }}
 {{- if $jinaai }}
 
 {{- $apiKeys := list }}
@@ -10,6 +11,9 @@
 {{- if $rerankerJinaAI }}
   {{- $apiKeys = append $apiKeys (index .Values "modules" "reranker-jinaai" "apiKey") -}}
 {{- end }}
+{{- if $multi2vecJinaAI }}
+  {{- $apiKeys = append $apiKeys (index .Values "modules" "multi2vec-jinaai" "apiKey") -}}
+{{- end }}
 
 {{- if gt (len ($apiKeys | uniq)) 1 -}}
   {{- fail "JinaAI modules activated, but their keys differ. Use the same key on all activated JinaAI modules." }}

diff --git a/weaviate/templates/weaviateStatefulset.yaml b/weaviate/templates/weaviateStatefulset.yaml
@@ -239,7 +239,7 @@ spec:
                 name: weaviate-google
                 key: apiKey
           {{- end  }}
-          {{- if or (and (index .Values "modules" "text2vec-jinaai" "enabled") (index .Values "modules" "text2vec-jinaai" "apiKey")) (and (index .Values "modules" "reranker-jinaai" "enabled") (index .Values "modules" "reranker-jinaai" "apiKey")) }}
+          {{- if or (and (index .Values "modules" "text2vec-jinaai" "enabled") (index .Values "modules" "text2vec-jinaai" "apiKey")) (and (index .Values "modules" "reranker-jinaai" "enabled") (index .Values "modules" "reranker-jinaai" "apiKey")) (and (index .Values "modules" "multi2vec-jinaai" "enabled") (index .Values "modules" "multi2vec-jinaai" "apiKey")) }}
           - name: JINAAI_APIKEY
             valueFrom:
               secretKeyRef:

diff --git a/weaviate/values.yaml b/weaviate/values.yaml
@@ -1237,6 +1237,19 @@ modules:
     # an environment variable
     apiKey: ''
 
+  # The multi2vec-jinaai module uses JinaAI Embeddings API
+  # to dynamically compute vector embeddings.
+  # More information about JinaAI Embeddings API can be found here:
+  # https://jina.ai/embeddings
+  multi2vec-jinaai:
+
+    # enable if you want to use JinaAI module
+    enabled: false
+
+    # Set your JinaAI API Key to be passed to Weaviate pod as
+    # an environment variable
+    apiKey: ''
+
   # The qna-transformers module uses neural networks, such as BERT,
   # DistilBERT, to find an answer in text to a given question
   qna-transformers: