Merge pull request #84 from semi-technologies/add-support-for-hf-module

Add support for text2vec-huggingface module

weaviate/templates/_helpers.tpl

@@ -29,6 +29,9 @@
   {{- if (index .Values "modules" "text2vec-openai" "enabled") -}}
     {{ $modules = append $modules "text2vec-openai" }}
   {{- end -}}
+  {{- if (index .Values "modules" "text2vec-huggingface" "enabled") -}}
+    {{ $modules = append $modules "text2vec-huggingface" }}
+  {{- end -}}
   {{- if gt (len $modules) 0 -}}
           - name: ENABLE_MODULES
             value: {{ join "," $modules }}

weaviate/templates/weaviateSecretHuggingFace.yaml

@@ -0,0 +1,9 @@
+{{ if and (index .Values "modules" "text2vec-huggingface" "enabled") (index .Values "modules" "text2vec-huggingface" "apiKey") }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: weaviate-huggingface
+type: Opaque
+data:
+  apiKey: {{ index .Values "modules" "text2vec-huggingface" "apiKey" | b64enc }}
+{{ end }}

weaviate/templates/weaviateStatefulset.yaml

@@ -130,6 +130,13 @@ spec:
                 name: weaviate-openai
                 key: apiKey
           {{- end  }}
+          {{- if and (index .Values "modules" "text2vec-huggingface" "enabled") (index .Values "modules" "text2vec-huggingface" "apiKey") }}
+          - name: HUGGINGFACE_APIKEY
+            valueFrom:
+              secretKeyRef:
+                name: weaviate-huggingface
+                key: apiKey
+          {{- end  }}
           - name: CLUSTER_GOSSIP_BIND_PORT
             value: "7000"
           - name: CLUSTER_DATA_BIND_PORT

weaviate/values.yaml

@@ -293,6 +293,20 @@ modules:
     # an environment variable
     apiKey: ''
 
+  # The text2vec-huggingface module uses HuggingFace API
+  # to dynamically compute vector embeddings based on the
+  # sentence's context.
+  # More information about HuggingFace API can be found here:
+  # https://huggingface.co/docs/api-inference/detailed_parameters#feature-extraction-task
+  text2vec-huggingface:
+
+    # enable if you want to use HuggingFace module
+    enabled: false
+
+    # Set your HuggingFace API Key to be passed to Weaviate pod as
+    # an environment variable
+    apiKey: ''
+
   # The multi2vec-clip modules uses CLIP transformers to vectorize both images
   # and text in the same vector space. It is typically slow(er) on CPUs and should
   # run with CUDA-enabled GPUs for optimal performance.