Milvus, Chroma, Qdrant, Pinecone: Update CDK (#30820)

Co-authored-by: flash1293 <[email protected]>
airbytehq · Oct 1, 2023 · 14bfc11 · 14bfc11
1 parent ded1d60
commit 14bfc11
Show file tree

Hide file tree

Showing 17 changed files with 29 additions and 14 deletions.
diff --git a/airbyte-integrations/connectors/destination-chroma/Dockerfile b/airbyte-integrations/connectors/destination-chroma/Dockerfile
@@ -41,5 +41,5 @@ COPY destination_chroma ./destination_chroma
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.0.1
+LABEL io.airbyte.version=0.0.2
 LABEL io.airbyte.name=airbyte/destination-chroma
diff --git a/airbyte-integrations/connectors/destination-chroma/destination_chroma/destination.py b/airbyte-integrations/connectors/destination-chroma/destination_chroma/destination.py
@@ -7,7 +7,14 @@
 
 from airbyte_cdk import AirbyteLogger
 from airbyte_cdk.destinations import Destination
-from airbyte_cdk.destinations.vector_db_based.embedder import CohereEmbedder, Embedder, FakeEmbedder, FromFieldEmbedder, OpenAIEmbedder
+from airbyte_cdk.destinations.vector_db_based.embedder import (
+    AzureOpenAIEmbedder,
+    CohereEmbedder,
+    Embedder,
+    FakeEmbedder,
+    FromFieldEmbedder,
+    OpenAIEmbedder,
+)
 from airbyte_cdk.destinations.vector_db_based.indexer import Indexer
 from airbyte_cdk.destinations.vector_db_based.writer import Writer
 from airbyte_cdk.models import (
@@ -26,6 +33,7 @@
 
 embedder_map = {
     "openai": OpenAIEmbedder,
+    "azure_openai": AzureOpenAIEmbedder,
     "cohere": CohereEmbedder,
     "fake": FakeEmbedder,
     "from_field": FromFieldEmbedder,
@@ -39,7 +47,10 @@ class DestinationChroma(Destination):
     embedder: Embedder
 
     def _init_indexer(self, config: ConfigModel):
-        self.embedder = embedder_map[config.embedding.mode](config.embedding)
+        if config.embedding.mode == "azure_openai" or config.embedding.mode == "openai":
+            self.embedder = embedder_map[config.embedding.mode](config.embedding, config.processing.chunk_size)
+        else:
+            self.embedder = embedder_map[config.embedding.mode](config.embedding)
         self.indexer = ChromaIndexer(config.indexing)
 
     def write(

diff --git a/airbyte-integrations/connectors/destination-chroma/metadata.yaml b/airbyte-integrations/connectors/destination-chroma/metadata.yaml
@@ -7,7 +7,7 @@ data:
   connectorSubtype: database
   connectorType: destination
   definitionId: 0b75218b-f702-4a28-85ac-34d3d84c0fc2
-  dockerImageTag: 0.0.1
+  dockerImageTag: 0.0.2
   dockerRepository: airbyte/destination-chroma
   githubIssueLabel: destination-chroma
   icon: chroma.svg

diff --git a/airbyte-integrations/connectors/destination-chroma/setup.py b/airbyte-integrations/connectors/destination-chroma/setup.py
@@ -6,7 +6,7 @@
 from setuptools import find_packages, setup
 
 MAIN_REQUIREMENTS = [
-    "airbyte-cdk[vector-db-based]",
+    "airbyte-cdk[vector-db-based]==0.51.22",
     "chromadb",
 ]
 

diff --git a/airbyte-integrations/connectors/destination-milvus/Dockerfile b/airbyte-integrations/connectors/destination-milvus/Dockerfile
@@ -37,5 +37,5 @@ COPY destination_milvus ./destination_milvus
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.0.2
+LABEL io.airbyte.version=0.0.3
 LABEL io.airbyte.name=airbyte/destination-milvus
diff --git a/airbyte-integrations/connectors/destination-milvus/metadata.yaml b/airbyte-integrations/connectors/destination-milvus/metadata.yaml
@@ -20,7 +20,7 @@ data:
   connectorSubtype: database
   connectorType: destination
   definitionId: 65de8962-48c9-11ee-be56-0242ac120002
-  dockerImageTag: 0.0.2
+  dockerImageTag: 0.0.3
   dockerRepository: airbyte/destination-milvus
   githubIssueLabel: destination-milvus
   icon: milvus.svg

diff --git a/airbyte-integrations/connectors/destination-milvus/setup.py b/airbyte-integrations/connectors/destination-milvus/setup.py
@@ -5,7 +5,7 @@
 
 from setuptools import find_packages, setup
 
-MAIN_REQUIREMENTS = ["airbyte-cdk[vector-db-based]==0.51.20", "pymilvus==2.3.0"]
+MAIN_REQUIREMENTS = ["airbyte-cdk[vector-db-based]==0.51.22", "pymilvus==2.3.0"]
 
 TEST_REQUIREMENTS = ["pytest~=6.2"]
 

diff --git a/airbyte-integrations/connectors/destination-pinecone/Dockerfile b/airbyte-integrations/connectors/destination-pinecone/Dockerfile
@@ -38,6 +38,6 @@ COPY destination_pinecone ./destination_pinecone
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.0.13
+LABEL io.airbyte.version=0.0.14
 
 LABEL io.airbyte.name=airbyte/destination-pinecone
diff --git a/airbyte-integrations/connectors/destination-pinecone/metadata.yaml b/airbyte-integrations/connectors/destination-pinecone/metadata.yaml
@@ -20,7 +20,7 @@ data:
   connectorSubtype: database
   connectorType: destination
   definitionId: 3d2b6f84-7f0d-4e3f-a5e5-7c7d4b50eabd
-  dockerImageTag: 0.0.13
+  dockerImageTag: 0.0.14
   dockerRepository: airbyte/destination-pinecone
   githubIssueLabel: destination-pinecone
   icon: pinecone.svg

diff --git a/airbyte-integrations/connectors/destination-pinecone/setup.py b/airbyte-integrations/connectors/destination-pinecone/setup.py
@@ -6,7 +6,7 @@
 from setuptools import find_packages, setup
 
 MAIN_REQUIREMENTS = [
-    "airbyte-cdk[vector-db-based]==0.51.20",
+    "airbyte-cdk[vector-db-based]==0.51.22",
     "pinecone-client[grpc]",
 ]
 

diff --git a/airbyte-integrations/connectors/destination-qdrant/Dockerfile b/airbyte-integrations/connectors/destination-qdrant/Dockerfile
@@ -41,5 +41,5 @@ COPY destination_qdrant ./destination_qdrant
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.0.2
+LABEL io.airbyte.version=0.0.3
 LABEL io.airbyte.name=airbyte/destination-qdrant
diff --git a/airbyte-integrations/connectors/destination-qdrant/metadata.yaml b/airbyte-integrations/connectors/destination-qdrant/metadata.yaml
@@ -20,7 +20,7 @@ data:
   connectorSubtype: database
   connectorType: destination
   definitionId: 6eb1198a-6d38-43e5-aaaa-dccd8f71db2b
-  dockerImageTag: 0.0.2
+  dockerImageTag: 0.0.3
   dockerRepository: airbyte/destination-qdrant
   githubIssueLabel: destination-qdrant
   icon: qdrant.svg

diff --git a/airbyte-integrations/connectors/destination-qdrant/setup.py b/airbyte-integrations/connectors/destination-qdrant/setup.py
@@ -5,7 +5,7 @@
 
 from setuptools import find_packages, setup
 
-MAIN_REQUIREMENTS = ["airbyte-cdk[vector-db-based]==0.51.20", "qdrant-client", "fastembed"]
+MAIN_REQUIREMENTS = ["airbyte-cdk[vector-db-based]==0.51.22", "qdrant-client", "fastembed"]
 
 TEST_REQUIREMENTS = ["pytest~=6.2"]
 

diff --git a/docs/integrations/destinations/chroma.md b/docs/integrations/destinations/chroma.md
@@ -75,4 +75,5 @@ You should now have all the requirements needed to configure Chroma as a destina
 
 | Version | Date       | Pull Request                                               | Subject                                    |
 | :------ | :--------- | :--------------------------------------------------------- | :----------------------------------------- |
+| 0.0.2   | 2023-09-29 | [#30820](https://github.com/airbytehq/airbyte/pull/30820)     | Update CDK | 
 | 0.0.1   | 2023-09-08 | [#30023](https://github.com/airbytehq/airbyte/pull/30023) | 🎉 New Destination: Chroma (Vector Database)    |
diff --git a/docs/integrations/destinations/milvus.md b/docs/integrations/destinations/milvus.md
@@ -105,5 +105,6 @@ vector_store.similarity_search("test")
 
 | Version | Date       | Pull Request                                                  | Subject                                                                                                                                              |
 |:--------| :--------- |:--------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.0.3   | 2023-09-29 | [#30820](https://github.com/airbytehq/airbyte/pull/30820)     | Update CDK | 
 | 0.0.2   | 2023-08-25 | [#30689](https://github.com/airbytehq/airbyte/pull/30689)     | Update CDK to support azure OpenAI embeddings and text splitting options, make sure primary key field is not accidentally set, promote to certified | 
 | 0.0.1   | 2023-08-12 | [#29442](https://github.com/airbytehq/airbyte/pull/29442)     | Milvus connector with some embedders  | 
diff --git a/docs/integrations/destinations/pinecone.md b/docs/integrations/destinations/pinecone.md
@@ -74,6 +74,7 @@ OpenAI and Fake embeddings produce vectors with 1536 dimensions, and the Cohere
 
 | Version | Date       | Pull Request                                                  | Subject                                                                                                                                              |
 |:--------| :--------- |:--------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.0.14   | 2023-09-29 | [#30820](https://github.com/airbytehq/airbyte/pull/30820)     | Update CDK | 
 | 0.0.13   | 2023-09-26 | [#30649](https://github.com/airbytehq/airbyte/pull/30649)     | Allow more text splitting options | 
 | 0.0.12   | 2023-09-25 | [#30649](https://github.com/airbytehq/airbyte/pull/30649)     | Fix bug with stale documents left on starter pods | 
 | 0.0.11   | 2023-09-22 | [#30649](https://github.com/airbytehq/airbyte/pull/30649)     | Set visible certified flag | 

diff --git a/docs/integrations/destinations/qdrant.md b/docs/integrations/destinations/qdrant.md
@@ -70,5 +70,6 @@ You should now have all the requirements needed to configure Qdrant as a destina
 
 | Version | Date       | Pull Request                                               | Subject                                    |
 | :------ | :--------- | :--------------------------------------------------------- | :----------------------------------------- |
+| 0.0.3   | 2023-09-29 | [#30820](https://github.com/airbytehq/airbyte/pull/30820)     | Update CDK | 
 | 0.0.2   | 2023-09-25 | [#30689](https://github.com/airbytehq/airbyte/pull/30689) | Update CDK to support Azure OpenAI embeddings and text splitting options |
 | 0.0.1   | 2023-09-22 | [#30332](https://github.com/airbytehq/airbyte/pull/30332) | 🎉 New Destination: Qdrant (Vector Database)    |