✨Source SFTP Bulk: avoid error on empty stream when running discover (#…

…38703)
airbytehq · May 28, 2024 · 4f0d12e · 4f0d12e
1 parent f2b6754
commit 4f0d12e
Show file tree

Hide file tree

Showing 15 changed files with 800 additions and 529 deletions.
diff --git a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/acceptance.py
@@ -1,6 +1,8 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
+
+
 import os
 import shutil
 import time

diff --git a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/conftest.py b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/conftest.py
@@ -12,7 +12,7 @@
 import docker
 import paramiko
 import pytest
-from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode
+from airbyte_cdk import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode
 
 from .utils import get_docker_ip, load_config
 

diff --git a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/integration_test.py b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/integration_test.py
@@ -2,11 +2,12 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
+
 import logging
 from copy import deepcopy
 from typing import Any, Mapping
 
-from airbyte_cdk.models import ConfiguredAirbyteCatalog, Status
+from airbyte_cdk import ConfiguredAirbyteCatalog, Status
 from airbyte_cdk.test.entrypoint_wrapper import read
 from source_sftp_bulk import SourceSFTPBulk
 

diff --git a/airbyte-integrations/connectors/source-sftp-bulk/main.py b/airbyte-integrations/connectors/source-sftp-bulk/main.py
@@ -2,6 +2,7 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
+
 from source_sftp_bulk.run import run
 
 if __name__ == "__main__":

diff --git a/airbyte-integrations/connectors/source-sftp-bulk/metadata.yaml b/airbyte-integrations/connectors/source-sftp-bulk/metadata.yaml
@@ -7,7 +7,7 @@ data:
   connectorSubtype: file
   connectorType: source
   definitionId: 31e3242f-dee7-4cdc-a4b8-8e06c5458517
-  dockerImageTag: 1.0.0
+  dockerImageTag: 1.0.1
   dockerRepository: airbyte/source-sftp-bulk
   documentationUrl: https://docs.airbyte.com/integrations/sources/sftp-bulk
   githubIssueLabel: source-sftp-bulk

diff --git a/airbyte-integrations/connectors/source-sftp-bulk/poetry.lock b/airbyte-integrations/connectors/source-sftp-bulk/poetry.lock
diff --git a/airbyte-integrations/connectors/source-sftp-bulk/pyproject.toml b/airbyte-integrations/connectors/source-sftp-bulk/pyproject.toml
@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "1.0.0"
+version = "1.0.1"
 name = "source-sftp-bulk"
 description = "Source implementation for SFTP Bulk."
 authors = [ "Airbyte <[email protected]>",]
@@ -17,7 +17,7 @@ include = "source_sftp_bulk"
 
 [tool.poetry.dependencies]
 python = "^3.9,<3.12"
-airbyte-cdk = {version = "^0", extras = ["file-based"]}
+airbyte-cdk = {version = "^1", extras = ["file-based"]}
 paramiko = "3.4.0"
 
 [tool.poetry.scripts]

diff --git a/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/client.py b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/client.py
@@ -2,14 +2,14 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
+
 import io
 import logging
 from typing import Optional
 
 import backoff
 import paramiko
-from airbyte_cdk.utils import AirbyteTracedException
-from airbyte_protocol.models import FailureType
+from airbyte_cdk import AirbyteTracedException, FailureType
 from paramiko.ssh_exception import AuthenticationException
 
 # set default timeout to 300 seconds

diff --git a/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/run.py b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/run.py
@@ -5,7 +5,7 @@
 
 import sys
 
-from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch
+from airbyte_cdk import AirbyteEntrypoint, launch
 from source_sftp_bulk import SourceSFTPBulk
 
 

diff --git a/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/source.py b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/source.py
@@ -1,13 +1,14 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
+
+
 import logging
 from typing import Any, Mapping, Optional
 
-from airbyte_cdk.models import ConfiguredAirbyteCatalog
+from airbyte_cdk import ConfiguredAirbyteCatalog, TState
 from airbyte_cdk.sources.file_based.file_based_source import FileBasedSource
 from airbyte_cdk.sources.file_based.stream.cursor.default_file_based_cursor import DefaultFileBasedCursor
-from airbyte_cdk.sources.source import TState
 from source_sftp_bulk.spec import SourceSFTPBulkSpec
 from source_sftp_bulk.stream_reader import SourceSFTPBulkStreamReader
 

diff --git a/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/spec.py b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/spec.py
@@ -1,9 +1,10 @@
 # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
 
+
 from typing import Literal, Optional, Union
 
+from airbyte_cdk import OneOfOptionConfig
 from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec
-from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig
 from pydantic import BaseModel, Field
 
 

diff --git a/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/stream_reader.py b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/stream_reader.py
@@ -1,5 +1,6 @@
 # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
 
+
 import datetime
 import logging
 import stat

diff --git a/airbyte-integrations/connectors/source-sftp-bulk/unit_tests/client_test.py b/airbyte-integrations/connectors/source-sftp-bulk/unit_tests/client_test.py
@@ -1,5 +1,6 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 
+
 from unittest.mock import MagicMock, patch
 
 import paramiko

diff --git a/airbyte-integrations/connectors/source-sftp-bulk/unit_tests/stream_reader_test.py b/airbyte-integrations/connectors/source-sftp-bulk/unit_tests/stream_reader_test.py
@@ -1,4 +1,6 @@
 # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+
 import datetime
 import logging
 from unittest.mock import MagicMock, patch

diff --git a/docs/integrations/sources/sftp-bulk.md b/docs/integrations/sources/sftp-bulk.md
@@ -119,8 +119,9 @@ More formats \(e.g. Apache Avro\) will be supported in the future.
 ## Changelog
 
 | Version | Date       | Pull Request                                             | Subject                                                     |
-| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------------------- |
+|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------|
+| 1.0.1   | 2024-05-29 | [38703](https://github.com/airbytehq/airbyte/pull/38703) | Avoid error on empty stream when running discover           |
 | 1.0.0   | 2024-03-22 | [36256](https://github.com/airbytehq/airbyte/pull/36256) | Migrate to File-Based CDK. Manage dependencies with Poetry. |
 | 0.1.2   | 2023-04-19 | [19224](https://github.com/airbytehq/airbyte/pull/19224) | Support custom CSV separators                               |
 | 0.1.1   | 2023-03-17 | [24180](https://github.com/airbytehq/airbyte/pull/24180) | Fix field order                                             |
-| 0.1.0   | 2021-24-05 |                                                          | Initial version                                             |
+| 0.1.0   | 2021-24-05 | [17691](https://github.com/airbytehq/airbyte/pull/17691) | Initial version                                             |
-Original file line number
+Diff line change
@@ Expand Up / @@ -2,6 +2,7 @@ @@
     # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
     #
     from source_sftp_bulk.run import run
     if __name__ == "__main__":
@@ Expand Down @@